X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/a60747292e91a47d122158659182f82bfd2e922a..e6294b5b90135a5163441c8dc62252dd6a188412:/check_comments_shas.py

diff --git a/check_comments_shas.py b/check_comments_shas.py
deleted file mode 100755
index 199261c..0000000
--- a/check_comments_shas.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-# run from a build_machine
-
-import requests
-from os import path
-import hashlib
-
-shasums1 = requests.get("https://files.pushshift.io/reddit/comments/sha256sum.txt").text
-shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text
-
-shasums = shasums1 + shasums2
-dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
-
-for l in shasums.strip().split('\n'):
-    sha256_hash = hashlib.sha256()
-    parts = l.split(' ')
-
-    correct_sha256 = parts[0]
-    filename = parts[-1]
-    print(f"checking {filename}")
-    fpath = path.join(dumpdir,filename)
-    if path.isfile(fpath):
-        with open(fpath,'rb') as f:
-            for byte_block in iter(lambda: f.read(4096),b""):
-                sha256_hash.update(byte_block)
-
-        if sha256_hash.hexdigest() == correct_sha256:
-            print(f"{filename} checks out")
-        else:
-            print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
-    else:
-        print(f"Skipping {filename} as it doesn't exist")
-