bug fix in affinity clustering

diff --git a/clustering/Makefile b/clustering/Makefile
index 7e8cf396c262eab70766cc630dc9715172b353e4..69c6c15ba6280e4cac64dd6d734eb1f7f26bf086 100644 (file)
--- a/clustering/Makefile
+++ b/clustering/Makefile
@@ -4,7 +4,7 @@ similarity_data=/gscratch/comdata/output/reddit_similarity
 clustering_data=/gscratch/comdata/output/reddit_clustering
 kmeans_selection_grid="--max_iters=[3000] --n_inits=[10] --n_clusters=[100,500,1000,1250,1500,1750,2000]"
 hdbscan_selection_grid="--min_cluster_sizes=[2,3,4,5] --min_samples=[2,3,4,5] --cluster_selection_epsilons=[0,0.01,0.05,0.1,0.15,0.2] --cluster_selection_methods=eom,leaf"
-affinity_selection_grid="--dampings=[0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.95,0.97,0.99] --preference_quantiles=[0.1,0.3,0.5,0.7,0.9] --convergence_iters=[30]"
+affinity_selection_grid="--dampings=[0.5,0.6,0.7,0.8,0.95,0.97,0.99] --preference_quantiles=[0.1,0.3,0.5,0.7,0.9] --convergence_iters=[15]"
 
 authors_10k_input=$(similarity_data)/subreddit_comment_authors_10k.feather
 authors_10k_input_lsi=$(similarity_data)/subreddit_comment_authors_10k_LSI

diff --git a/clustering/affinity_clustering.py b/clustering/affinity_clustering.py
index d10628a1ac3f7106ae38ec61fd3a5a888cdc3f13..737967e964b2cad824837be7a6da74bf74c4437b 100644 (file)
--- a/clustering/affinity_clustering.py
+++ b/clustering/affinity_clustering.py
@@ -81,7 +81,7 @@ class affinity_grid_sweep(grid_sweep):
 
         return f"damp-{damping}_maxit-{max_iter}_convit-{convergence_iter}_prefq-{preference_quantile}"
 
-def run_affinity_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_iters=[3000], convergence_iters=[30], preference_quantiles=[0.5]):
+def run_affinity_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_iters=[3000], convergence_iters=[30], preference_quantiles=[0.5],n_cores=10):
     """Run affinity clustering once or more with different parameters.
     
     Usage:
@@ -102,7 +102,7 @@ def run_affinity_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_iters
                          map(int,max_iters),
                          map(int,convergence_iters),
                          map(float,preference_quantiles))
-    obj.run(1)
+    obj.run(n_cores)
     obj.save(savefile)
     
 def test_select_affinity_clustering():

diff --git a/clustering/affinity_clustering_lsi.py b/clustering/affinity_clustering_lsi.py
index f5c794eb01e309279ae660f9baa463c18aa4acff..983e861006e7b745e9721d880bb80e47bbb28278 100644 (file)
--- a/clustering/affinity_clustering_lsi.py
+++ b/clustering/affinity_clustering_lsi.py
@@ -58,7 +58,7 @@ class _affinity_lsi_grid_sweep(grid_sweep):
                          inpath,
                          outpath,
                          self.namer,
-                         self.lsi_dim,
+                         [self.lsi_dim],
                          *args,
                          **kwargs)
 
@@ -67,7 +67,7 @@ class _affinity_lsi_grid_sweep(grid_sweep):
         s += f"_lsi-{self.lsi_dim}"
         return s
                          
-def run_affinity_lsi_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_iters=[3000], convergence_iters=[30], preference_quantiles=[0.5], lsi_dimensions='all'):
+def run_affinity_lsi_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_iters=[3000], convergence_iters=[30], preference_quantiles=[0.5], lsi_dimensions='all',n_cores=30):
     """Run affinity clustering once or more with different parameters.
     
     Usage:
@@ -92,7 +92,7 @@ def run_affinity_lsi_grid_sweep(savefile, inpath, outpath, dampings=[0.8], max_i
                             map(int,convergence_iters),
                             map(float,preference_quantiles))
 
-    obj.run(1)
+    obj.run(n_cores)
     obj.save(savefile)
 
 if __name__ == "__main__":

diff --git a/clustering/clustering_base.py b/clustering/clustering_base.py
index 1d24533b520865d8e3f8bd53bad8a344178d8741..3778fc3fa91259f49a1b1470e7d0901e1f3ee6ba 100644 (file)
--- a/clustering/clustering_base.py
+++ b/clustering/clustering_base.py
@@ -3,6 +3,7 @@ import numpy as np
 import pandas as pd
 from dataclasses import dataclass
 from sklearn.metrics import silhouette_score, silhouette_samples
+from collections import Counter
 
 # this is meant to be an interface, not created directly
 class clustering_job:
@@ -38,9 +39,11 @@ class clustering_job:
         return self.result
 
     def silhouette(self):
-        isolates = self.clustering.labels_ == -1
+        counts = Counter(self.clustering.labels_)
+        singletons = [key for key, value in counts.items() if value == 1]
+        isolates = (self.clustering.labels_ == -1) | (np.isin(self.clustering.labels_,np.array(singletons)))
         scoremat = self.mat[~isolates][:,~isolates]
-        if scoremat.shape[0] > 0:
+        if self.n_clusters > 1:
             score = silhouette_score(scoremat, self.clustering.labels_[~isolates], metric='precomputed')
             silhouette_samp = silhouette_samples(self.mat, self.clustering.labels_, metric='precomputed')
             silhouette_samp = pd.DataFrame({'subreddit':self.subreddits,'score':silhouette_samp})
@@ -80,8 +83,9 @@ class clustering_job:
 
         print(f"{n_isolates1} clusters have 1 member")
 
-        n_isolates2 = (cluster_sizes.loc[cluster_sizes.cluster==-1,['subreddit']])
-
+        n_isolates2 = cluster_sizes.loc[cluster_sizes.cluster==-1,:]['subreddit'].to_list()
+        if len(n_isolates2) > 0:
+            n_isloates2 = n_isolates2[0]
         print(f"{n_isolates2} subreddits are in cluster -1",flush=True)
 
         if n_isolates1 == 0:

diff --git a/clustering/fit_tsne.py b/clustering/fit_tsne.py
index c9f45f61320ad8eb2cb88583b27388b663f36b19..55d72394c8fd32a13a4336463f636fc29a6bb4d3 100644 (file)
--- a/clustering/fit_tsne.py
+++ b/clustering/fit_tsne.py
@@ -17,7 +17,7 @@ def fit_tsne(similarities, output, learning_rate=750, perplexity=50, n_iter=1000
     df = pd.read_feather(similarities)
 
     n = df.shape[0]
-    mat = np.array(df.drop('subreddit',1),dtype=np.float64)
+    mat = np.array(df.drop('_subreddit',1),dtype=np.float64)
     mat[range(n),range(n)] = 1
     mat[mat > 1] = 1
     dist = 2*np.arccos(mat)/np.pi
@@ -26,7 +26,7 @@ def fit_tsne(similarities, output, learning_rate=750, perplexity=50, n_iter=1000
 
     tsne_fit_whole = tsne_fit_model.fit_transform(dist)
 
-    plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':df.subreddit})
+    plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], '_subreddit':df['_subreddit']})
 
     plot_data.to_feather(output)
 

diff --git a/clustering/lsi_base.py b/clustering/lsi_base.py
index 45cc49b67060259da15ba650e94f6196fedd320c..f07bca6f01d61f8d6338f4f4adac6a4cf9536046 100644 (file)
--- a/clustering/lsi_base.py
+++ b/clustering/lsi_base.py
@@ -20,9 +20,9 @@ class lsi_grid_sweep(grid_sweep):
         if lsi_dimensions == 'all':
             lsi_paths = list(inpath.glob("*"))
         else:
-            lsi_paths = [inpath / (dim + '.feather') for dim in lsi_dimensions]
+            lsi_paths = [inpath / (str(dim) + '.feather') for dim in lsi_dimensions]
 
-        lsi_nums = [p.stem for p in lsi_paths]
+        lsi_nums = [int(p.stem) for p in lsi_paths]
         self.hasrun = False
         self.subgrids = [self.subsweep(lsi_path, outpath,  lsi_dim, *args, **kwargs) for lsi_dim, lsi_path in zip(lsi_nums, lsi_paths)]
         self.jobs = list(chain(*map(lambda gs: gs.jobs, self.subgrids)))