From: Benjamin Mako Hill Date: Wed, 1 Apr 2020 14:42:38 +0000 (-0700) Subject: added initial version of revision-scraper X-Git-Url: https://code.communitydata.science/covid19.git/commitdiff_plain/6493361fbd95f44a3b27131f4f79329d40e61c90?ds=inline added initial version of revision-scraper Borrows much of the structure from the (patched) version of the dailyview scraper. --- diff --git a/wikipedia_views/scripts/fetch_enwiki_revisions.py b/wikipedia_views/scripts/fetch_enwiki_revisions.py new file mode 100755 index 0000000..77c5664 --- /dev/null +++ b/wikipedia_views/scripts/fetch_enwiki_revisions.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +############################################################################### +# +# This script assumes the presence of the COVID-19 repo. +# +# It (1) reads in the article list and then (2) calls the Wikimedia API to +# fetch view information for each article. Output is to (3) JSON and TSV. +# +############################################################################### + +import argparse +import logging +import os.path +import json +import subprocess +import datetime + +from requests import Request +from csv import DictWriter +from mw import api + + +def parse_args(): + + parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.') + parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia_views/data", type=str) + parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia_views/resources/enwp_wikiproject_covid19_articles.txt", type=str) + parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str) + parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=str), + parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), + args = parser.parse_args() + return(args) + +def main(): + args = parse_args() + + output_path = args.output_folder + article_filename = args.article_file + #handle -d + if args.query_date: + query_date = args.query_date + else: + yesterday = datetime.datetime.today() - datetime.timedelta(days=1) + query_date = yesterday.strftime("%Y%m%d") + + query_data = query_date + "00" #requires specifying hours + + #handle -L + loglevel_mapping = { 'debug' : logging.DEBUG, + 'info' : logging.INFO, + 'warning' : logging.WARNING, + 'error' : logging.ERROR, + 'critical' : logging.CRITICAL } + + if args.logging_level in loglevel_mapping: + loglevel = loglevel_mapping[args.logging_level] + else: + print("Choose a valid log level: debug, info, warning, error, or critical") + exit + + #handle -W + if args.logging_destination: + logging.basicConfig(filename=args.logging_destination, filemode='a', level=loglevel) + else: + logging.basicConfig(level=loglevel) + + export_git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() + export_git_short_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip() + export_time = str(datetime.datetime.now()) + + logging.info(f"Starting run at {export_time}") + logging.info(f"Last commit: {export_git_hash}") + + json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_date}.json") + tsv_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_data}.tsv") + + api_session = api.Session("https://en.wikipedia.org/w/api.php") + + # list of properties from the API we want to gather (basically all of + # them supported by mediawik-utilities) + + rv_props = {'revid' : 'ids', + 'timestamp' : 'timestamp', + 'user' : 'user', + 'userid' : 'userid', + 'size' : 'size', + 'sha1' : 'sha1', + 'contentmodel' : 'contentmodel', + 'tags' : 'tags', + 'comment' : 'comment', + 'content' : 'content' } + + exclude_from_tsv = ['tags', 'comment', 'content'] + + # load the list of articles + with open(article_filename, 'r') as infile: + article_list = [art.strip() for art in list(infile)] + + def get_revisions_for_page(title): + return api_session.revisions.query(properties=rv_props.values(), + titles={title}, + direction="newer") + + tsv_fields = ['title', 'pageid', 'namespace'] + tsv_fields = tsv_fields + list(rv_props.keys()) + + # drop fields that we identified for exclusion + tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv] + + # add special export fields + tsv_fields = tsv_fields + ['url', 'export_timestamp', 'export_commit'] + + export_info = { 'git_commit' : export_git_hash, + 'timestamp' : export_time } + + with open(json_output_filename, 'w') as json_output, \ + open(tsv_output_filename, 'w') as tsv_output: + + tsv_writer = DictWriter(tsv_output, fieldnames=tsv_fields, delimiter="\t") + tsv_writer.writeheader() + + for article in article_list: + logging.info(f"pulling revisiosn for: {article}") + for rev in get_revisions_for_page(article): + logging.debug(f"processing raw revision: {rev}") + + # add export metadata + rev['exported'] = export_info + + # save the json version of the code + print(json.dumps(rev), file=json_output) + + # handle missing data + if "sha1" not in rev: + rev["sha1"] = "" + + # add page title information + rev['title'] = rev['page']['title'] + rev['pageid'] = rev['page']['pageid'] + rev['namespace'] = rev['page']['ns'] + + # construct a URL + rev['url'] = Request('GET', 'https://en.wikipedia.org/w/index.php', + params={'title' : rev['title'].replace(" ", "_"), + 'oldid' : rev['revid']}).prepare().url + + rev['export_timestamp'] = export_time + rev['export_commit'] = export_git_short_hash + + tsv_writer.writerow({k: rev[k] for k in tsv_fields}) + break + +if __name__ == "__main__": + + main()