--- /dev/null
+#!/usr/bin/env python3
+# run from a build_machine
+
+import requests
+from os import path
+import hashlib
+
+shasums = requests.get("https://files.pushshift.io/reddit/comments/sha256sums.txt").text
+
+dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
+
+for l in shasums.strip().split('\n'):
+ sha256_hash = hashlib.sha256()
+ parts = l.split(' ')
+
+ correct_sha256 = parts[0]
+ filename = parts[-1]
+ print(f"checking {filename}")
+ fpath = path.join(dumpdir,filename)
+ if path.isfile(fpath):
+ with open(fpath,'rb') as f:
+ for byte_block in iter(lambda: f.read(4096),b""):
+ sha256_hash.update(byte_block)
+
+ if sha256_hash.hexdigest() == correct_sha256:
+ print(f"{filename} checks out")
+ else:
+ print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
+ else:
+ print(f"Skipping {filename} as it doesn't exist")
+
wget -r --no-parent -A 'RS_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/
wget -r --no-parent -A 'RS_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/
-
+./check_submissions_shas.py