X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/811a0d87c4d394c2c7849a613f6aec2d81e49138..07b0dff9bc0dae2ab6f7fb7334007a5269a512ad:/dumps/remove_duplicate_comments.py?ds=sidebyside

diff --git a/dumps/remove_duplicate_comments.py b/dumps/remove_duplicate_comments.py
new file mode 100644
index 0000000..e639586
--- /dev/null
+++ b/dumps/remove_duplicate_comments.py
@@ -0,0 +1,34 @@
+from pathlib import Path
+from itertools import chain, groupby
+
+dumpdir = Path("/gscratch/comdata/raw_data/reddit_dumps/comments")
+
+zst_files = dumpdir.glob("*.zst")
+bz2_files = dumpdir.glob("*.bz2")
+xz_files = dumpdir.glob("*.xz")
+all_files = sorted(list(chain(zst_files, bz2_files, xz_files)))
+groups = groupby(all_files, key = lambda p: p.stem)
+
+kept_paths = []
+removed_paths = []
+
+priority = ['.zst','.xz','.bz2']
+
+for stem, files in groups:
+    keep_file = None
+    remove_files = []
+    for f in files:
+        if keep_file is None:
+            keep_file = f
+        elif priority.index(keep_file.suffix) > priority.index(f.suffix):
+            remove_files.append(keep_file)
+            keep_file = f
+        else:
+            remove_files.append(f)
+    kept_paths.append(keep_file)
+    removed_paths.extend(remove_files)
+
+(dumpdir / "to_remove").mkdir()
+
+for f in removed_paths:
+    f.rename(f.parent / "to_remove" / f.name)