Merge pull request #20 from makoshark/master

[covid19.git] / wikipedia / scripts / fetch_enwiki_revisions.py

diff --git a/wikipedia/scripts/fetch_enwiki_revisions.py b/wikipedia/scripts/fetch_enwiki_revisions.py
index 3ecd04f5370f890a04d45f11a39cb8f76b17d84c..2d25e85d0bfbe1a7c4fefa7c346270456bc9c5db 100755 (executable)
--- a/wikipedia/scripts/fetch_enwiki_revisions.py
+++ b/wikipedia/scripts/fetch_enwiki_revisions.py
@@ -1,4 +1,4 @@
-#!yusr/bin/env python3
+#!/usr/bin/env python3

 
 ###############################################################################
 #
 
 ###############################################################################
 #


@@ -13,12 +13,12 @@ import argparse
 import logging
 import os.path
 import json
 import logging
 import os.path
 import json

-import subprocess

 import datetime
 
 from requests import Request
 from csv import DictWriter
 from mw import api
 import datetime
 
 from requests import Request
 from csv import DictWriter
 from mw import api

+import digobs

 
 
 def parse_args():
 
 
 def parse_args():


@@ -26,8 +26,7 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.')
     parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str)
     parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str)
     parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.')
     parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str)
     parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str)

-    parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str)
-    parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=str), 
+    parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=digobs.get_loglevel), 

     parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), 
     args = parser.parse_args()
     return(args)
     parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), 
     args = parser.parse_args()
     return(args)


@@ -37,41 +36,21 @@ def main():
 
     output_path = args.output_folder
     article_filename = args.article_file
 
     output_path = args.output_folder
     article_filename = args.article_file

-    #handle -d
-    if args.query_date:
-        query_date = args.query_date
-    else:
-        yesterday = datetime.datetime.today() - datetime.timedelta(days=1)
-        query_date = yesterday.strftime("%Y%m%d")
-
-    #handle -L
-    loglevel_mapping = { 'debug' : logging.DEBUG,
-                         'info' : logging.INFO,
-                         'warning' : logging.WARNING,
-                         'error' : logging.ERROR,
-                         'critical' : logging.CRITICAL }
-
-    if args.logging_level in loglevel_mapping:
-        loglevel = loglevel_mapping[args.logging_level]
-    else:
-        print("Choose a valid log level: debug, info, warning, error, or critical") 
-        exit

 
     #handle -W
     if args.logging_destination:
 
     #handle -W
     if args.logging_destination:

-        logging.basicConfig(filename=args.logging_destination, filemode='a', level=loglevel)
+        logging.basicConfig(filename=args.logging_destination, filemode='a', level=args.logging_level)

     else:
     else:

-        logging.basicConfig(level=loglevel)
+        logging.basicConfig(level=args.logging_level)

 
 

-    export_git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip()
-    export_git_short_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip()

     export_time = str(datetime.datetime.now())
     export_time = str(datetime.datetime.now())

+    export_date = datetime.datetime.today().strftime("%Y%m%d")

 
     logging.info(f"Starting run at {export_time}")
 
     logging.info(f"Starting run at {export_time}")

-    logging.info(f"Last commit: {export_git_hash}")
+    logging.info(f"Last commit: {digobs.git_hash()}")

 
 

-    json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_date}.json")
-    tsv_output_filename =  os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_date}.tsv")
+    json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.json")
+    tsv_output_filename =  os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.tsv")

     
     api_session = api.Session("https://en.wikipedia.org/w/api.php")
 
     
     api_session = api.Session("https://en.wikipedia.org/w/api.php")
 


@@ -86,14 +65,15 @@ def main():
                  'sha1' : 'sha1',
                  'contentmodel' : 'contentmodel',
                  'tags' : 'tags',
                  'sha1' : 'sha1',
                  'contentmodel' : 'contentmodel',
                  'tags' : 'tags',

+                 'flags' : 'flags',

                  'comment' : 'comment',
                  'content' : 'content' }
 
                  'comment' : 'comment',
                  'content' : 'content' }
 

-    exclude_from_tsv = ['tags', 'comment', 'content']
+    exclude_from_tsv = ['tags', 'comment', 'content', 'flags']

 
     # load the list of articles
     with open(article_filename, 'r') as infile:
 
     # load the list of articles
     with open(article_filename, 'r') as infile:

-        article_list = [art.strip() for art in list(infile)]
+        article_list= list(map(str.strip, infile))

 
     def get_revisions_for_page(title):
         return api_session.revisions.query(properties=rv_props.values(),
 
     def get_revisions_for_page(title):
         return api_session.revisions.query(properties=rv_props.values(),


@@ -107,9 +87,9 @@ def main():
     tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv]
 
     # add special export fields
     tsv_fields = [e for e in tsv_fields if e not in exclude_from_tsv]
 
     # add special export fields

-    tsv_fields = tsv_fields + ['url', 'export_timestamp', 'export_commit']
+    tsv_fields = tsv_fields + ['anon', 'minor', 'url', 'export_timestamp', 'export_commit']

 
 

-    export_info = { 'git_commit' : export_git_hash,
+    export_info = { 'git_commit' : digobs.git_hash(),

                     'timestamp' : export_time }
 
     with open(json_output_filename, 'w') as json_output, \
                     'timestamp' : export_time }
 
     with open(json_output_filename, 'w') as json_output, \


@@ -119,7 +99,7 @@ def main():
         tsv_writer.writeheader()
 
         for article in article_list:
         tsv_writer.writeheader()
 
         for article in article_list:

-            logging.info(f"pulling revisiosn for: {article}")
+            logging.info(f"pulling revisions for: {article}")

             for rev in get_revisions_for_page(article):
                 logging.debug(f"processing raw revision: {rev}")
 
             for rev in get_revisions_for_page(article):
                 logging.debug(f"processing raw revision: {rev}")
 


@@ -133,6 +113,22 @@ def main():
                 if "sha1" not in rev:
                     rev["sha1"] = ""
 
                 if "sha1" not in rev:
                     rev["sha1"] = ""
 

+                if "userhidden" in rev:
+                    rev["user"] = ""
+                    rev["userid"] = ""
+
+                # recode anon so it's true or false instead of present/missing
+                if "anon" in rev:
+                    rev["anon"] = True
+                else:
+                    rev["anon"] = False
+                    
+                # let's recode "minor" in the same way
+                if "minor" in rev:
+                    rev["minor"] = True
+                else:
+                    rev["minor"] = False
+

                 # add page title information
                 rev['title'] = rev['page']['title']
                 rev['pageid'] = rev['page']['pageid']
                 # add page title information
                 rev['title'] = rev['page']['title']
                 rev['pageid'] = rev['page']['pageid']


@@ -144,11 +140,9 @@ def main():
                                             'oldid' : rev['revid']}).prepare().url
 
                 rev['export_timestamp'] = export_time
                                             'oldid' : rev['revid']}).prepare().url
 
                 rev['export_timestamp'] = export_time

-                rev['export_commit'] = export_git_short_hash
+                rev['export_commit'] = digobs.git_hash(short=True)

 
                 tsv_writer.writerow({k: rev[k] for k in tsv_fields})
 
                 tsv_writer.writerow({k: rev[k] for k in tsv_fields})

-            break

 
 if __name__ == "__main__":
 
 if __name__ == "__main__":

-

     main()
     main()