update pushshift dumps.

[cdsc_reddit.git] / similarities / tfidf.py

diff --git a/similarities/tfidf.py b/similarities/tfidf.py
index 002e89f785b37fd9df3c903775ab6f71846909d4..01b0b20a0b94f6348834818b6762f3805cbfa8ea 100644 (file)
--- a/similarities/tfidf.py
+++ b/similarities/tfidf.py
@@ -1,17 +1,17 @@
 import fire
 from pyspark.sql import SparkSession
 from pyspark.sql import functions as f
-from similarities_helper import build_tfidf_dataset, build_weekly_tfidf_dataset, select_topN_subreddits
+from similarities_helper import tfidf_dataset, build_weekly_tfidf_dataset, select_topN_subreddits
 
 def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_subreddits):
-    spark = SparkSession.builder.getOrCreate()
+    spark = SparkSession.builder.getOrCreate()y
 
     df = spark.read.parquet(inpath)
 
     df = df.filter(~ f.col(term_colname).isin(exclude))
 
     if included_subreddits is not None:
-        include_subs = list(open(included_subreddits))
+        include_subs = set(map(str.strip,open(included_subreddits)))
     else:
         include_subs = select_topN_subreddits(topN)
 
@@ -21,54 +21,62 @@ def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_
     spark.stop()
 
 def tfidf(inpath, outpath, topN, term_colname, exclude, included_subreddits):
-    return _tfidf_wrapper(build_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
+    return _tfidf_wrapper(tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
 
 def tfidf_weekly(inpath, outpath, topN, term_colname, exclude, included_subreddits):
     return _tfidf_wrapper(build_weekly_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
 
-def tfidf_authors(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet',
-                  topN=25000):
+def tfidf_authors(inpath="/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
+                  outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet',
+                  topN=None,
+                  included_subreddits=None):
 
-    return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
+    return tfidf(inpath,
                  outpath,
                  topN,
                  'author',
                  ['[deleted]','AutoModerator'],
-                 included_subreddits=None
+                 included_subreddits=included_subreddits
                  )
 
-def tfidf_terms(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_terms.parquet',
-                topN=25000):
+def tfidf_terms(inpath="/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
+                outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_terms.parquet',
+                topN=None,
+                included_subreddits=None):
 
-    return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
+    return tfidf(inpath,
                  outpath,
                  topN,
                  'term',
                  [],
-                 included_subreddits=None
+                 included_subreddits=included_subreddits
                  )
 
-def tfidf_authors_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors.parquet',
-                         topN=25000):
+def tfidf_authors_weekly(inpath="/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
+                         outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors.parquet',
+                         topN=None,
+                         included_subreddits=None):
 
-    return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
+    return tfidf_weekly(inpath,
                         outpath,
                         topN,
                         'author',
                         ['[deleted]','AutoModerator'],
-                        included_subreddits=None
+                        included_subreddits=included_subreddits
                         )
 
-def tfidf_terms_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet',
-                       topN=25000):
+def tfidf_terms_weekly(inpath="/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
+                       outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet',
+                       topN=None,
+                       included_subreddits=None):
 
 
-    return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
+    return tfidf_weekly(inpath,
                         outpath,
                         topN,
                         'term',
                         [],
-                        included_subreddits=None
+                        included_subreddits=included_subreddits
                         )