Merge pull request #20 from makoshark/master

[covid19.git] / wikipedia / scripts / fetch_enwiki_daily_views.py

diff --git a/wikipedia/scripts/fetch_enwiki_daily_views.py b/wikipedia/scripts/fetch_enwiki_daily_views.py
index 7015a3b5a2ac03fa4ec1bf5ce4ee06d2958f8a4e..9f147e07a30cc20e78b27ec1afb71c64be582b1e 100755 (executable)
--- a/wikipedia/scripts/fetch_enwiki_daily_views.py
+++ b/wikipedia/scripts/fetch_enwiki_daily_views.py
@@ -26,7 +26,7 @@ def parse_args():
     parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str)
     parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str)
     parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str)
     parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str)
     parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str)
     parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str)

-    parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=str), 
+    parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=digobs.get_loglevel), 

     parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), 
     args = parser.parse_args()
     return(args)
     parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), 
     args = parser.parse_args()
     return(args)


@@ -45,14 +45,11 @@ def main():
         yesterday = datetime.datetime.today() - datetime.timedelta(days=1)
         query_date = yesterday.strftime("%Y%m%d")
 
         yesterday = datetime.datetime.today() - datetime.timedelta(days=1)
         query_date = yesterday.strftime("%Y%m%d")
 

-    #handle -L
-    loglevel = digobs.get_loglevel(args.logging_level)
-

     #handle -W
     if args.logging_destination:
     #handle -W
     if args.logging_destination:

-        logging.basicConfig(filename=args.logging_destination, filemode='a', level=loglevel)
+        logging.basicConfig(filename=args.logging_destination, filemode='a', level=args.logging_level)

     else:
     else:

-        logging.basicConfig(level=loglevel)
+        logging.basicConfig(level=args.logging_level)

 
     export_time = str(datetime.datetime.now())
     export_date = datetime.datetime.today().strftime("%Y%m%d")
 
     export_time = str(datetime.datetime.now())
     export_date = datetime.datetime.today().strftime("%Y%m%d")


@@ -61,8 +58,8 @@ def main():
     logging.info(f"Last commit: {digobs.git_hash()}")
 
     #1 Load up the list of article names
     logging.info(f"Last commit: {digobs.git_hash()}")
 
     #1 Load up the list of article names

-    j_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{export_date}.json")
-    t_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{export_date}.tsv")
+    j_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{query_date}.json")
+    t_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{query_date}.tsv")

 
     with open(articleFile, 'r') as infile:
         articleList = list(map(str.strip, infile))
 
     with open(articleFile, 'r') as infile:
         articleList = list(map(str.strip, infile))


@@ -85,6 +82,7 @@ def main():
             else:
                 failure = failure + 1
                 logging.warning(f"Failure: {response.status_code} from {url}")
             else:
                 failure = failure + 1
                 logging.warning(f"Failure: {response.status_code} from {url}")

+                continue

 
             # start writing the CSV File if it doesn't exist yet
             try:
 
             # start writing the CSV File if it doesn't exist yet
             try: