From: Benjamin Mako Hill Date: Wed, 1 Apr 2020 20:18:50 +0000 (-0500) Subject: Merge branch 'master' of github.com:makoshark/COVID-19_Digital_Observatory X-Git-Url: https://code.communitydata.science/covid19.git/commitdiff_plain/5739d1c4042bc57aa8b3da46158704454ae9692d?hp=4e1b7fbdfe346a1b0546a8afb24f8c2f10138b79 Merge branch 'master' of github.com:makoshark/COVID-19_Digital_Observatory --- diff --git a/cron-wikipedia_revisions.sh b/cron-wikipedia_revisions.sh new file mode 100644 index 0000000..71ee752 --- /dev/null +++ b/cron-wikipedia_revisions.sh @@ -0,0 +1,17 @@ +#!/bin/bash -x + +TZ="UTC" +date_string=$(date +%Y%m%d) + +./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/enwp-wikiproject_scraper-${date_string}.log) + +revs_log="enwp-revisions-${date_string}.log" +./wikipedia/scripts/fetch_enwiki_revisions.py 2> >(tee wikipedia/logs/${rev_log}) +cp wikipedia/logs/${rev_log} /var/www/covid19/wikipedia/logs/ + +revs_tsv="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.tsv" +cp wikipedia/data/${revs_tsv} /var/www/covid19/wikipedia + +revs_json="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.json" +xz wikipedia/data/${revs_json} +cp wikipedia/data/${revs_json}.xz /var/www/covid19/wikipedia diff --git a/cron-wikipedia_views.sh b/cron-wikipedia_views.sh new file mode 100644 index 0000000..ec3d4ec --- /dev/null +++ b/cron-wikipedia_views.sh @@ -0,0 +1,16 @@ +#!/bin/bash -x + +TZ="UTC" +date_string=$(date +%Y%m%d) + +./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/enwp-wikiproject_scraper-${date_string}.log) + +# get the list of files +view_log="enwp-daily_views-${date_string}.log" +./wikipedia/scripts/fetch_enwiki_daily_views.py 2> >(tee wikipedia/logs/${view_log}) +cp wikipedia/logs/${view_log} /var/www/covid19/wikipedia/logs/${view_log} +cp wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.tsv /var/www/covid19/wikipedia/ + +# xz wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json +cp wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json /var/www/covid19/wikipedia/ + diff --git a/wikipedia/scripts/fetch_enwiki_daily_views.py b/wikipedia/scripts/fetch_enwiki_daily_views.py index c250bfe..f766ed8 100755 --- a/wikipedia/scripts/fetch_enwiki_daily_views.py +++ b/wikipedia/scripts/fetch_enwiki_daily_views.py @@ -23,7 +23,6 @@ from csv import DictWriter #import feather #TBD def parse_args(): - parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia view data.') parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str) parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str) @@ -42,12 +41,10 @@ def main(): #handle -d if args.query_date: - queryDate = args.query_date + query_date = args.query_date else: yesterday = datetime.datetime.today() - datetime.timedelta(days=1) - queryDate = yesterday.strftime("%Y%m%d") - - queryDate = queryDate + "00" #requires specifying hours + query_date = yesterday.strftime("%Y%m%d") #handle -L loglevel_mapping = { 'debug' : logging.DEBUG, @@ -71,13 +68,14 @@ def main(): export_git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() export_git_short_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip() export_time = str(datetime.datetime.now()) + export_date = datetime.datetime.today().strftime("%Y%m%d") logging.info(f"Starting run at {export_time}") logging.info(f"Last commit: {export_git_hash}") #1 Load up the list of article names - j_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{queryDate}.json") - t_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{queryDate}.tsv") + j_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{export_date}.json") + t_outfilename = os.path.join(outputPath, f"digobs_covid19-wikipedia-enwiki_dailyviews-{export_date}.tsv") with open(articleFile, 'r') as infile: articleList = list(infile) @@ -92,7 +90,7 @@ def main(): #2 Repeatedly call the API with that list of names for a in articleList: a = a.strip("\"\n") #destringify - url= f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/{a}/daily/{queryDate}/{queryDate}" + url= f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/{a}/daily/{query_date}00/{query_date}00" response = requests.get(url) if response.ok: @@ -117,7 +115,7 @@ def main(): # write out of the csv file dw.writerow(jd) - # f_Out = outputPath + "dailyviews" + queryDate + ".feather" + # f_Out = outputPath + "dailyviews" + query_date + ".feather" # read the json back in and make a feather file? logging.debug(f"Run complete at {datetime.datetime.now()}") logging.info(f"Processed {success} successful URLs and {failure} failures.") diff --git a/wikipedia/scripts/fetch_enwiki_revisions.py b/wikipedia/scripts/fetch_enwiki_revisions.py index fdd4ad5..44528e8 100755 --- a/wikipedia/scripts/fetch_enwiki_revisions.py +++ b/wikipedia/scripts/fetch_enwiki_revisions.py @@ -26,7 +26,6 @@ def parse_args(): parser = argparse.ArgumentParser(description='Call the views API to collect Wikipedia revision data.') parser.add_argument('-o', '--output_folder', help='Where to save output', default="wikipedia/data", type=str) parser.add_argument('-i', '--article_file', help='File listing article names', default="wikipedia/resources/enwp_wikiproject_covid19_articles.txt", type=str) - parser.add_argument('-d', '--query_date', help='Date if not yesterday, in YYYYMMDD format.', type=str) parser.add_argument('-L', '--logging_level', help='Logging level. Options are debug, info, warning, error, critical. Default: info.', default='info', type=str), parser.add_argument('-W', '--logging_destination', help='Logging destination file. (default: standard error)', type=str), args = parser.parse_args() @@ -37,14 +36,6 @@ def main(): output_path = args.output_folder article_filename = args.article_file - #handle -d - if args.query_date: - query_date = args.query_date - else: - yesterday = datetime.datetime.today() - datetime.timedelta(days=1) - query_date = yesterday.strftime("%Y%m%d") - - query_data = query_date + "00" #requires specifying hours #handle -L loglevel_mapping = { 'debug' : logging.DEBUG, @@ -68,12 +59,13 @@ def main(): export_git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() export_git_short_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip() export_time = str(datetime.datetime.now()) + export_date = datetime.datetime.today().strftime("%Y%m%d") logging.info(f"Starting run at {export_time}") logging.info(f"Last commit: {export_git_hash}") - json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_date}.json") - tsv_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{query_data}.tsv") + json_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.json") + tsv_output_filename = os.path.join(output_path, f"digobs_covid19-wikipedia-enwiki_revisions-{export_date}.tsv") api_session = api.Session("https://en.wikipedia.org/w/api.php")