Merge branch 'excise_reindex' of code:cdsc_reddit into excise_reindex

[cdsc_reddit.git] / similarities / tfidf.py

diff --git a/similarities/tfidf.py b/similarities/tfidf.py
index 110536eeb22b5c13132ff17b33d882fc47da63b7..19d30138457843df62ef440d3a75acc45b41df87 100644 (file)
--- a/similarities/tfidf.py
+++ b/similarities/tfidf.py
@@ -1,7 +1,7 @@
 import fire
 from pyspark.sql import SparkSession
 from pyspark.sql import functions as f
 import fire
 from pyspark.sql import SparkSession
 from pyspark.sql import functions as f

-from similarities_helper import build_tfidf_dataset, build_weekly_tfidf_dataset, select_topN_subreddits
+from similarities_helper import tfidf_dataset, build_weekly_tfidf_dataset, select_topN_subreddits

 
 def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_subreddits):
     spark = SparkSession.builder.getOrCreate()
 
 def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_subreddits):
     spark = SparkSession.builder.getOrCreate()


@@ -11,7 +11,7 @@ def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_
     df = df.filter(~ f.col(term_colname).isin(exclude))
 
     if included_subreddits is not None:
     df = df.filter(~ f.col(term_colname).isin(exclude))
 
     if included_subreddits is not None:

-        include_subs = set(map(str.strip,map(str.lower, open(included_subreddits))))
+        include_subs = set(map(str.strip,open(included_subreddits)))

     else:
         include_subs = select_topN_subreddits(topN)
 
     else:
         include_subs = select_topN_subreddits(topN)
 


@@ -21,13 +21,13 @@ def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_
     spark.stop()
 
 def tfidf(inpath, outpath, topN, term_colname, exclude, included_subreddits):
     spark.stop()
 
 def tfidf(inpath, outpath, topN, term_colname, exclude, included_subreddits):

-    return _tfidf_wrapper(build_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
+    return _tfidf_wrapper(tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)

 
 def tfidf_weekly(inpath, outpath, topN, term_colname, exclude, included_subreddits):
     return _tfidf_wrapper(build_weekly_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
 
 def tfidf_authors(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet',
 
 def tfidf_weekly(inpath, outpath, topN, term_colname, exclude, included_subreddits):
     return _tfidf_wrapper(build_weekly_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
 
 def tfidf_authors(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet',

-                  topN=25000,
+                  topN=None,

                   included_subreddits=None):
 
     return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
                   included_subreddits=None):
 
     return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",


@@ -39,7 +39,7 @@ def tfidf_authors(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comm
                  )
 
 def tfidf_terms(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_terms.parquet',
                  )
 
 def tfidf_terms(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_terms.parquet',

-                topN=25000,
+                topN=None,

                 included_subreddits=None):
 
     return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
                 included_subreddits=None):
 
     return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",


@@ -51,7 +51,7 @@ def tfidf_terms(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/commen
                  )
 
 def tfidf_authors_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors.parquet',
                  )
 
 def tfidf_authors_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors.parquet',

-                         topN=25000,
+                         topN=None,

                          included_subreddits=None):
 
     return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
                          included_subreddits=None):
 
     return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",


@@ -63,7 +63,7 @@ def tfidf_authors_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfi
                         )
 
 def tfidf_terms_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet',
                         )
 
 def tfidf_terms_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet',

-                       topN=25000,
+                       topN=None,

                        included_subreddits=None):
 
 
                        included_subreddits=None):