Merge branch 'excise_reindex' of code:cdsc_reddit into excise_reindex

[cdsc_reddit.git] / similarities / lsi_similarities.py

diff --git a/similarities/lsi_similarities.py b/similarities/lsi_similarities.py
index eb89f55789c5dcb285c78737d8d280b935e2bb72..565e53fa33a9ae8c5af8062ceb892cbca31226c6 100644 (file)
--- a/similarities/lsi_similarities.py
+++ b/similarities/lsi_similarities.py
@@ -5,28 +5,28 @@ from similarities_helper import *
 #from similarities_helper import similarities, lsi_column_similarities
 from functools import partial
 
-inpath = "/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/tfidf/comment_terms_compex.parquet/"
-term_colname='term'
-outfile='/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/similarity/comment_terms_compex_LSI'
-n_components=[10,50,100]
-included_subreddits="/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/included_subreddits.txt"
-n_iter=5
-random_state=1968
-algorithm='arpack'
-topN = None
-from_date=None
-to_date=None
-min_df=None
-max_df=None
+# inpath = "/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/tfidf/comment_authors_compex.parquet"
+# term_colname='authors'
+# outfile='/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/similarity/comment_test_compex_LSI'
+# n_components=[10,50,100]
+# included_subreddits="/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/included_subreddits.txt"
+# n_iter=5
+# random_state=1968
+# algorithm='randomized'
+# topN = None
+# from_date=None
+# to_date=None
+# min_df=None
+# max_df=None
+
 def lsi_similarities(inpath, term_colname, outfile, min_df=None, max_df=None, included_subreddits=None, topN=None, from_date=None, to_date=None, tfidf_colname='tf_idf',n_components=100,n_iter=5,random_state=1968,algorithm='arpack',lsi_model=None):
     print(n_components,flush=True)
-
         
     if lsi_model is None:
         if type(n_components) == list:
-            lsi_model = Path(outfile) / f'{max(n_components)}_{term_colname}_LSIMOD.pkl'
+            lsi_model = Path(outfile) / f'{max(n_components)}_{term_colname}s_LSIMOD.pkl'
         else:
-            lsi_model = Path(outfile) / f'{n_components}_{term_colname}_LSIMOD.pkl'
+            lsi_model = Path(outfile) / f'{n_components}_{term_colname}s_LSIMOD.pkl'
 
     simfunc = partial(lsi_column_similarities,n_components=n_components,n_iter=n_iter,random_state=random_state,algorithm=algorithm,lsi_model_save=lsi_model)
 
@@ -62,7 +62,7 @@ def author_lsi_similarities(inpath='/gscratch/comdata/output/reddit_similarity/t
                             n_components=n_components
                                )
 
-def author_tf_similarities(inpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors_100k.parquet',outfile=None, min_df=2, max_df=None, included_subreddits=None, topN=None, from_date=None, to_date=None,n_components=300,n_iter=5,random_state=1968):
+def author_tf_similarities(inpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors_100k.parquet',outfile=None, min_df=2, max_df=None, included_subreddits=None, topN=None, from_date=None, to_date=None,algorithm='arpack',n_components=300,n_iter=5,random_state=1968):
     return lsi_similarities(inpath,
                             'author',
                             outfile,