Update reddit comments data with daily dumps.

diff --git a/check_comments_shas.py b/check_comments_shas.py
old mode 100644 (file)
new mode 100755 (executable)
index a2bc89b..199261c
--- a/check_comments_shas.py
+++ b/check_comments_shas.py
@@ -5,8 +5,10 @@ import requests
 from os import path
 import hashlib
 
-shasums = requests.get("https://files.pushshift.io/reddit/comments/sha256sums.txt").text
+shasums1 = requests.get("https://files.pushshift.io/reddit/comments/sha256sum.txt").text
+shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text
 
+shasums = shasums1 + shasums2
 dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
 
 for l in shasums.strip().split('\n'):

diff --git a/comments_2_parquet.sh b/comments_2_parquet.sh
index 92331383a692659d60837cea3e67311be1cdd036..e9818c19cfcd6c7f29f2c7bcc04b564b5471e3e6 100755 (executable)
--- a/comments_2_parquet.sh
+++ b/comments_2_parquet.sh
@@ -1,9 +1,6 @@
 ## needs to be run by hand since i don't have a nice way of waiting on a parallel-sql job to complete 
 
 #!/usr/bin/env bash
-
-
-
 echo "#!/usr/bin/bash" > job_script.sh
 echo "source $(pwd)/../bin/activate" >> job_script.sh
 echo "python3 $(pwd)/comments_2_parquet_part1.py" >> job_script.sh

diff --git a/helper.py b/helper.py
index b401cada8a3ec394c5c0f6985f7954d2d39af1b7..af87f71d265501f1f3ca25ecb64882f4ff9997da 100644 (file)
--- a/helper.py
+++ b/helper.py
@@ -40,6 +40,8 @@ def open_input_file(input_filename):
         cmd = ["xzcat",'-dk', '-T 20',input_filename]
     elif re.match(r'.*\.zst',input_filename):
         cmd = ['zstd','-dck', input_filename]
+    elif re.match(r'.*\.gz',input_filename):
+        cmd = ['gzip','-dc', input_filename]
     try:
         input_file = Popen(cmd, stdout=PIPE).stdout
     except NameError as e:

diff --git a/pull_pushshift_comments.sh b/pull_pushshift_comments.sh
index 243e46480dc6c55374a479e0835663a7317513e1..3f6d2c91b151712fcbca93ff67f9a03524b78774 100755 (executable)
--- a/pull_pushshift_comments.sh
+++ b/pull_pushshift_comments.sh
@@ -4,8 +4,11 @@ user_agent='nathante teblunthuis <nathante@uw.edu>'
 output_dir='/gscratch/comdata/raw_data/reddit_dumps/comments'
 base_url='https://files.pushshift.io/reddit/comments/'
 
-wget -r --no-parent -A 'RC_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url
-wget -r --no-parent -A 'RC_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url
-wget -r --no-parent -A 'RC_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url
+wget -r --no-parent -A 'RC_201*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url
+wget -r --no-parent -A 'RC_201*.xz' -U $user_agent -P $output_dir -nd -nc $base_url
+wget -r --no-parent -A 'RC_201*.zst' -U $user_agent -P $output_dir -nd -nc $base_url
 
-./check_comment_shas.py
+# starting in 2020 we use daily dumps not monthly dumps
+wget -r --no-parent -A 'RC_202*.gz' -U $user_agent -P $output_dir -nd -nc $base_url/daily/
+
+./check_comments_shas.py