3 ###############################################################################
5 # This script assumes the presence of the COVID-19 repo.
7 # It (1) reads in the article list and then (2) calls the Wikimedia API to
8 # fetch view information for each article. Output is to (3) JSON and TSV.
10 ###############################################################################
18 from requests import Request
19 from csv import DictWriter
26 parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.')
27 parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str)
28 parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str)
29 parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=digobs.get_loglevel),
30 parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str),
31 args = parser.parse_args()
37 output_path = args.output_folder
38 article_filename = args.article_file
41 if args.logging_destination:
42 logging.basicConfig(filename=args.logging_destination, filemode='a', level=args.logging_level)
44 logging.basicConfig(level=args.logging_level)
46 export_time = str(datetime.datetime.now())
47 export_date = datetime.datetime.today().strftime("%Y%m%d")
49 logging.info(f"Starting run at {export_time}")
50 logging.info(f"Last commit: {digobs.git_hash()}")
52 json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.json")
53 tsv_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.tsv")
55 api_session = api.Session("https://en.wikipedia.org/w/api.php")
57 # list of properties from the API we want to gather (basically all of
58 # them supported by mediawik-utilities)
60 rv_props = {'revid' : 'ids',
61 'timestamp' : 'timestamp',
66 'contentmodel' : 'contentmodel',
69 'comment' : 'comment',
70 'content' : 'content' }
72 exclude_from_tsv = ['tags', 'comment', 'content', 'flags']
74 # load the list of articles
75 with open(article_filename, 'r') as infile:
76 article_list= list(map(str.strip, infile))
78 def get_revisions_for_page(title):
79 return api_session.revisions.query(properties=rv_props.values(),
83 tsv_fields = ['title', 'pageid', 'namespace']
84 tsv_fields = tsv_fields + list(rv_props.keys())
86 # drop fields that we identified for exclusion
87 tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv]
89 # add special export fields
90 tsv_fields = tsv_fields + ['anon', 'minor', 'url', 'export_timestamp', 'export_commit']
92 export_info = { 'git_commit' : digobs.git_hash(),
93 'timestamp' : export_time }
95 with open(json_output_filename, 'w') as json_output, \
96 open(tsv_output_filename, 'w') as tsv_output:
98 tsv_writer = DictWriter(tsv_output, fieldnames=tsv_fields, delimiter="\t")
99 tsv_writer.writeheader()
101 for article in article_list:
102 logging.info(f"pulling revisions for: {article}")
103 for rev in get_revisions_for_page(article):
104 logging.debug(f"processing raw revision: {rev}")
106 # add export metadata
107 rev['exported'] = export_info
109 # save the json version of the code
110 print(json.dumps(rev), file=json_output)
112 # handle missing data
113 if "sha1" not in rev:
116 if "userhidden" in rev:
120 # recode anon so it's true or false instead of present/missing
126 # let's recode "minor" in the same way
132 # add page title information
133 rev['title'] = rev['page']['title']
134 rev['pageid'] = rev['page']['pageid']
135 rev['namespace'] = rev['page']['ns']
138 rev['url'] = Request('GET', 'https://en.wikipedia.org/w/index.php',
139 params={'title' : rev['title'].replace(" ", "_"),
140 'oldid' : rev['revid']}).prepare().url
142 rev['export_timestamp'] = export_time
143 rev['export_commit'] = digobs.git_hash(short=True)
145 tsv_writer.writerow({k: rev[k] for k in tsv_fields})
147 if __name__ == "__main__":