dumps/check_submission_shas.py

   1 #!/usr/bin/env python3
   2 # run from a build_machine
   3
   4 import requests
   5 from os import path
   6 import hashlib
   7
   8 file1 = requests.get("https://files.pushshift.io/reddit/submissions/sha256sums.txt").text
   9 file2 = requests.get("https://files.pushshift.io/reddit/submissions/old_v1_data/sha256sums.txt").text
  10 dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/submissions"
  11
  12 for l in file1.strip().split('\n') + file2.strip().split('\n'):
  13     sha256_hash = hashlib.sha256()
  14     parts = l.split(' ')
  15
  16     correct_sha256 = parts[0]
  17     filename = parts[-1]
  18     print(f"checking {filename}")
  19     fpath = path.join(dumpdir,filename)
  20     if path.isfile(fpath):
  21         with open(fpath,'rb') as f:
  22             for byte_block in iter(lambda: f.read(4096),b""):
  23                 sha256_hash.update(byte_block)
  24
  25         if sha256_hash.hexdigest() == correct_sha256:
  26             print(f"{filename} checks out")
  27         else:
  28             print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
  29     else:
  30         print(f"Skipping {filename} as it doesn't exist")
  31