X-Git-Url: https://code.communitydata.science/covid19.git/blobdiff_plain/6493361fbd95f44a3b27131f4f79329d40e61c90..784458f2067287871dcfeefe8348fcc71a230244:/wikipedia_views/scripts/fetch_enwiki_revisions.py diff --git a/wikipedia_views/scripts/fetch_enwiki_revisions.py b/wikipedia_views/scripts/fetch_enwiki_revisions.py deleted file mode 100755 index 77c5664..0000000 --- a/wikipedia_views/scripts/fetch_enwiki_revisions.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 - -############################################################################### -# -# This script assumes the presence of the COVID-19 repo. -# -# It (1) reads in the article list and then (2) calls the Wikimedia API to -# fetch view information for each article. Output is to (3) JSON and TSV. -# -############################################################################### - -import argparse -import logging -import os.path -import json -import subprocess -import datetime - -from requests import Request -from csv import DictWriter -from mw import api - - -def parse_args(): - - parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.') - parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia_views/data", type=str) - parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia_views/resources/enwp_wikiproject_covid19_articles.txt", type=str) - parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str) - parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=str), - parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), - args = parser.parse_args() - return(args) - -def main(): - args = parse_args() - - output_path = args.output_folder - article_filename = args.article_file - #handle -d - if args.query_date: - query_date = args.query_date - else: - yesterday = datetime.datetime.today() - datetime.timedelta(days=1) - query_date = yesterday.strftime("%Y%m%d") - - query_data = query_date + "00" #requires specifying hours - - #handle -L - loglevel_mapping = { 'debug' : logging.DEBUG, - 'info' : logging.INFO, - 'warning' : logging.WARNING, - 'error' : logging.ERROR, - 'critical' : logging.CRITICAL } - - if args.logging_level in loglevel_mapping: - loglevel = loglevel_mapping[args.logging_level] - else: - print("Choose a valid log level: debug, info, warning, error, or critical") - exit - - #handle -W - if args.logging_destination: - logging.basicConfig(filename=args.logging_destination, filemode='a', level=loglevel) - else: - logging.basicConfig(level=loglevel) - - export_git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() - export_git_short_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip() - export_time = str(datetime.datetime.now()) - - logging.info(f"Starting run at {export_time}") - logging.info(f"Last commit: {export_git_hash}") - - json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_date}.json") - tsv_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_data}.tsv") - - api_session = api.Session("https://en.wikipedia.org/w/api.php") - - # list of properties from the API we want to gather (basically all of - # them supported by mediawik-utilities) - - rv_props = {'revid' : 'ids', - 'timestamp' : 'timestamp', - 'user' : 'user', - 'userid' : 'userid', - 'size' : 'size', - 'sha1' : 'sha1', - 'contentmodel' : 'contentmodel', - 'tags' : 'tags', - 'comment' : 'comment', - 'content' : 'content' } - - exclude_from_tsv = ['tags', 'comment', 'content'] - - # load the list of articles - with open(article_filename, 'r') as infile: - article_list = [art.strip() for art in list(infile)] - - def get_revisions_for_page(title): - return api_session.revisions.query(properties=rv_props.values(), - titles={title}, - direction="newer") - - tsv_fields = ['title', 'pageid', 'namespace'] - tsv_fields = tsv_fields + list(rv_props.keys()) - - # drop fields that we identified for exclusion - tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv] - - # add special export fields - tsv_fields = tsv_fields + ['url', 'export_timestamp', 'export_commit'] - - export_info = { 'git_commit' : export_git_hash, - 'timestamp' : export_time } - - with open(json_output_filename, 'w') as json_output, \ - open(tsv_output_filename, 'w') as tsv_output: - - tsv_writer = DictWriter(tsv_output, fieldnames=tsv_fields, delimiter="\t") - tsv_writer.writeheader() - - for article in article_list: - logging.info(f"pulling revisiosn for: {article}") - for rev in get_revisions_for_page(article): - logging.debug(f"processing raw revision: {rev}") - - # add export metadata - rev['exported'] = export_info - - # save the json version of the code - print(json.dumps(rev), file=json_output) - - # handle missing data - if "sha1" not in rev: - rev["sha1"] = "" - - # add page title information - rev['title'] = rev['page']['title'] - rev['pageid'] = rev['page']['pageid'] - rev['namespace'] = rev['page']['ns'] - - # construct a URL - rev['url'] = Request('GET', 'https://en.wikipedia.org/w/index.php', - params={'title' : rev['title'].replace(" ", "_"), - 'oldid' : rev['revid']}).prepare().url - - rev['export_timestamp'] = export_time - rev['export_commit'] = export_git_short_hash - - tsv_writer.writerow({k: rev[k] for k in tsv_fields}) - break - -if __name__ == "__main__": - - main()