From c972d828b31b650a028ffb015367479e7fc9eb23 Mon Sep 17 00:00:00 2001 From: Nate E TeBlunthuis Date: Fri, 3 Jul 2020 13:35:46 -0700 Subject: [PATCH] Script for checking shas for submissions. --- check_submission_shas.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 check_submission_shas.py diff --git a/check_submission_shas.py b/check_submission_shas.py new file mode 100755 index 0000000..4950131 --- /dev/null +++ b/check_submission_shas.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# run from a build_machine + +import requests +from os import path +import hashlib + +file1 = requests.get("https://files.pushshift.io/reddit/submissions/sha256sums.txt").text +file2 = requests.get("https://files.pushshift.io/reddit/submissions/old_v1_data/sha256sums.txt").text +dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/submissions" + +for l in file1.strip().split('\n') + file2.strip().split('\n'): + sha256_hash = hashlib.sha256() + parts = l.split(' ') + + correct_sha256 = parts[0] + filename = parts[-1] + print(f"checking {filename}") + fpath = path.join(dumpdir,filename) + if path.isfile(fpath): + with open(fpath,'rb') as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + + if sha256_hash.hexdigest() == correct_sha256: + print(f"{filename} checks out") + else: + print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!") + else: + print(f"Skipping {filename} as it doesn't exist") + -- 2.39.5