]> code.communitydata.science - cdsc_reddit.git/blob - check_submission_shas.py
increase learning rate.
[cdsc_reddit.git] / check_submission_shas.py
1 #!/usr/bin/env python3
2 # run from a build_machine
3
4 import requests
5 from os import path
6 import hashlib
7
8 file1 = requests.get("https://files.pushshift.io/reddit/submissions/sha256sums.txt").text
9 file2 = requests.get("https://files.pushshift.io/reddit/submissions/old_v1_data/sha256sums.txt").text
10 dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/submissions"
11
12 for l in file1.strip().split('\n') + file2.strip().split('\n'):
13     sha256_hash = hashlib.sha256()
14     parts = l.split(' ')
15
16     correct_sha256 = parts[0]
17     filename = parts[-1]
18     print(f"checking {filename}")
19     fpath = path.join(dumpdir,filename)
20     if path.isfile(fpath):
21         with open(fpath,'rb') as f:
22             for byte_block in iter(lambda: f.read(4096),b""):
23                 sha256_hash.update(byte_block)
24
25         if sha256_hash.hexdigest() == correct_sha256:
26             print(f"{filename} checks out")
27         else:
28             print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
29     else:
30         print(f"Skipping {filename} as it doesn't exist")
31

Community Data Science Collective || Want to submit a patch?