From: Nate E TeBlunthuis Date: Tue, 7 Jul 2020 06:31:52 +0000 (-0700) Subject: Check the shas when we download dumps X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/commitdiff_plain/9cd0954288512d002144d02c70b22869a6a4a271 Check the shas when we download dumps --- diff --git a/check_comments_shas.py b/check_comments_shas.py new file mode 100644 index 0000000..a2bc89b --- /dev/null +++ b/check_comments_shas.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# run from a build_machine + +import requests +from os import path +import hashlib + +shasums = requests.get("https://files.pushshift.io/reddit/comments/sha256sums.txt").text + +dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments" + +for l in shasums.strip().split('\n'): + sha256_hash = hashlib.sha256() + parts = l.split(' ') + + correct_sha256 = parts[0] + filename = parts[-1] + print(f"checking {filename}") + fpath = path.join(dumpdir,filename) + if path.isfile(fpath): + with open(fpath,'rb') as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + + if sha256_hash.hexdigest() == correct_sha256: + print(f"{filename} checks out") + else: + print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!") + else: + print(f"Skipping {filename} as it doesn't exist") + diff --git a/pull_pushshift_comments.sh b/pull_pushshift_comments.sh index bfe3938..243e464 100755 --- a/pull_pushshift_comments.sh +++ b/pull_pushshift_comments.sh @@ -8,5 +8,4 @@ wget -r --no-parent -A 'RC_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_ wget -r --no-parent -A 'RC_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url wget -r --no-parent -A 'RC_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url - - +./check_comment_shas.py diff --git a/pull_pushshift_submissions.sh b/pull_pushshift_submissions.sh index f64a09f..5a53c87 100755 --- a/pull_pushshift_submissions.sh +++ b/pull_pushshift_submissions.sh @@ -11,4 +11,4 @@ wget -r --no-parent -A 'RS_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_ wget -r --no-parent -A 'RS_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ wget -r --no-parent -A 'RS_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ - +./check_submissions_shas.py