From: Benjamin Mako Hill Date: Wed, 1 Apr 2020 21:39:53 +0000 (-0500) Subject: tweaks to revision export code X-Git-Url: https://code.communitydata.science/covid19.git/commitdiff_plain/d655e1ce9332f039a2bd271776752b3f74863848?ds=inline tweaks to revision export code - flags were not being exported (e.g., minor, anon) - broke with hidden/deleted user names --- diff --git a/wikipedia/scripts/fetch_enwiki_revisions.py b/wikipedia/scripts/fetch_enwiki_revisions.py index 3793c0b..6a3563f 100755 --- a/wikipedia/scripts/fetch_enwiki_revisions.py +++ b/wikipedia/scripts/fetch_enwiki_revisions.py @@ -80,10 +80,11 @@ def main(): 'sha1' : 'sha1', 'contentmodel' : 'contentmodel', 'tags' : 'tags', + 'flags' : 'flags', 'comment' : 'comment', 'content' : 'content' } - exclude_from_tsv = ['tags', 'comment', 'content'] + exclude_from_tsv = ['tags', 'comment', 'content', 'flags'] # load the list of articles with open(article_filename, 'r') as infile: @@ -101,7 +102,7 @@ def main(): tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv] # add special export fields - tsv_fields = tsv_fields + ['url', 'export_timestamp', 'export_commit'] + tsv_fields = tsv_fields + ['anon', 'minor', 'url', 'export_timestamp', 'export_commit'] export_info = { 'git_commit' : export_git_hash, 'timestamp' : export_time } @@ -127,6 +128,22 @@ def main(): if "sha1" not in rev: rev["sha1"] = "" + if "userhidden" in rev: + rev["user"] = "" + rev["userid"] = "" + + # recode anon so it's true or false instead of present/missing + if "anon" in rev: + rev["anon"] = True + else: + rev["anon"] = False + + # let's recode "minor" in the same way + if "minor" in rev: + rev["minor"] = True + else: + rev["minor"] = False + # add page title information rev['title'] = rev['page']['title'] rev['pageid'] = rev['page']['pageid']