refactor clustering.py into method-specific files.

diff --git a/clustering/select_affinity.py b/clustering/affinity_clustering.py
similarity index 63%
rename from clustering/select_affinity.py
rename to clustering/affinity_clustering.py
index b8bd13a8c63da809f59e053cf8ce1700115fa7ec..287f7e21e84b7ddf6ad87fda6b96431d9ceec922 100644 (file)
--- a/clustering/select_affinity.py
+++ b/clustering/affinity_clustering.py
@@ -18,7 +18,44 @@ class affinity_clustering_result(clustering_result):
     convergence_iter:int
     preference_quantile:float
 
     convergence_iter:int
     preference_quantile:float
 

-def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
+def affinity_clustering(similarities, output, *args, **kwargs):
+    subreddits, mat = read_similarity_mat(similarities)
+    clustering = _affinity_clustering(mat, *args, **kwargs)
+    cluster_data = process_clustering_result(clustering, subreddits)
+    cluster_data['algorithm'] = 'affinity'
+    return(cluster_data)
+
+def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True):
+    '''
+    similarities: matrix of similarity scores
+    preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits.
+    damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author. 
+    '''
+    print(f"damping:{damping}; convergenceIter:{convergence_iter}; preferenceQuantile:{preference_quantile}")
+
+    preference = np.quantile(mat,preference_quantile)
+
+    print(f"preference is {preference}")
+    print("data loaded")
+    sys.stdout.flush()
+    clustering = AffinityPropagation(damping=damping,
+                                     max_iter=max_iter,
+                                     convergence_iter=convergence_iter,
+                                     copy=False,
+                                     preference=preference,
+                                     affinity='precomputed',
+                                     verbose=verbose,
+                                     random_state=random_state).fit(mat)
+
+    cluster_data = process_clustering_result(clustering, subreddits)
+    output = Path(output)
+    output.parent.mkdir(parents=True,exist_ok=True)
+    cluster_data.to_feather(output)
+    print(f"saved {output}")
+    return clustering
+
+
+def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):

     if name is None:
         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
     print(name)
     if name is None:
         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
     print(name)


@@ -53,41 +90,6 @@ def do_affinity_clustering(damping, convergence_iter, preference_quantile, name,
 
     return res
 
 
     return res
 

-def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
-    if name is None:
-        name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
-    print(name)
-    sys.stdout.flush()
-    outpath = outdir / (str(name) + ".feather")
-    outpath.parent.mkdir(parents=True,exist_ok=True)
-    print(outpath)
-    clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
-    mat = sim_to_dist(clustering.affinity_matrix_)
-
-    try: 
-        score = silhouette_score(mat, clustering.labels_, metric='precomputed')
-    except ValueError:
-        score = None
-
-    if alt_mat is not None:
-        alt_distances = sim_to_dist(alt_mat)
-        try:
-            alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
-        except ValueError:
-            alt_score = None
-    
-    res = clustering_result(outpath=outpath,
-                            damping=damping,
-                            max_iter=max_iter,
-                            convergence_iter=convergence_iter,
-                            preference_quantile=preference_quantile,
-                            silhouette_score=score,
-                            alt_silhouette_score=score,
-                            name=str(name))
-
-    return res
-
-

 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
 
 def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
 
 def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):


@@ -116,7 +118,7 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max
     hyper_grid = product(damping, convergence_iter, preference_quantile)
     hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
 
     hyper_grid = product(damping, convergence_iter, preference_quantile)
     hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
 

-    _do_clustering = partial(do_affinity_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
+    _do_clustering = partial(do_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)

 
     #    similarities = Array('d', mat)
     # call pool.starmap
 
     #    similarities = Array('d', mat)
     # call pool.starmap


@@ -124,8 +126,6 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max
     clustering_data = pool.starmap(_do_clustering, hyper_grid)
     clustering_data = pd.DataFrame(list(clustering_data))
     clustering_data.to_csv(outinfo)
     clustering_data = pool.starmap(_do_clustering, hyper_grid)
     clustering_data = pd.DataFrame(list(clustering_data))
     clustering_data.to_csv(outinfo)

-
-    

     return clustering_data
 
 if __name__ == "__main__":
     return clustering_data
 
 if __name__ == "__main__":

diff --git a/clustering/clustering.py b/clustering/clustering.py
index 85be3fe1a1b42d490731ed67bd7e420a9d0242d1..6ee78420824c0af5cdf59410eff7bda5226e39c1 100755 (executable)
--- a/clustering/clustering.py
+++ b/clustering/clustering.py
@@ -3,7 +3,7 @@
 import sys
 import pandas as pd
 import numpy as np
 import sys
 import pandas as pd
 import numpy as np

-from sklearn.cluster import AffinityPropagation, KMeans
+from sklearn.cluster import AffinityPropagation

 import fire
 from pathlib import Path
 from multiprocessing import cpu_count
 import fire
 from pathlib import Path
 from multiprocessing import cpu_count


@@ -46,24 +46,6 @@ def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000,
     print(f"saved {output}")
     return clustering
 
     print(f"saved {output}")
     return clustering
 

-def kmeans_clustering(similarities, *args, **kwargs):
-    subreddits, mat = read_similarity_mat(similarities)
-    mat = sim_to_dist(mat)
-    clustering = _kmeans_clustering(mat, *args, **kwargs)
-    cluster_data = process_clustering_result(clustering, subreddits)
-    return(cluster_data)
-
-def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True):
-
-    clustering = KMeans(n_clusters=n_clusters,
-                        n_init=n_init,
-                        max_iter=max_iter,
-                        random_state=random_state,
-                        verbose=verbose
-                        ).fit(mat)
-
-    return clustering
-

 
 
 if __name__ == "__main__":
 
 
 if __name__ == "__main__":

diff --git a/clustering/hdbscan_clustering.py b/clustering/hdbscan_clustering.py
index 888554a55255a69e1c883bffa6ee4035a11a845d..4f4e0d6f2c4f18b47d3d96ac0991fbc72fdb6aef 100644 (file)
--- a/clustering/hdbscan_clustering.py
+++ b/clustering/hdbscan_clustering.py
@@ -28,6 +28,13 @@ def test_select_hdbscan_clustering():
     cluster_selection_methods=['eom'];
     lsi_dimensions='all'
 
     cluster_selection_methods=['eom'];
     lsi_dimensions='all'
 

+    df = pd.read_csv("test_hdbscan/selection_data.csv")
+    test_select_hdbscan_clustering()
+    check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather")
+    silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather")
+    c = check_clusters.merge(silscores,on='subreddit')#    fire.Fire(select_hdbscan_clustering)
+
+

 @dataclass
 class hdbscan_clustering_result(clustering_result):
     min_cluster_size:int
 @dataclass
 class hdbscan_clustering_result(clustering_result):
     min_cluster_size:int


@@ -165,8 +172,4 @@ def make_KNN_plots():
     KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png')
 
 if __name__ == "__main__":
     KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png')
 
 if __name__ == "__main__":

-    df = pd.read_csv("test_hdbscan/selection_data.csv")
-    test_select_hdbscan_clustering()
-    check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather")
-    silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather")
-    c = check_clusters.merge(silscores,on='subreddit')#    fire.Fire(select_hdbscan_clustering)
+    fire.Fire(select_hdbscan_clustering)

diff --git a/clustering/select_kmeans.py b/clustering/kmeans_clustering.py
similarity index 77%
rename from clustering/select_kmeans.py
rename to clustering/kmeans_clustering.py
index b07a108a01b999f2d143926bed123afbba1b24ee..8822e9f0cd67c2ece76d9552d7c2c77883bac76f 100644 (file)
--- a/clustering/select_kmeans.py
+++ b/clustering/kmeans_clustering.py
@@ -1,23 +1,32 @@
-from sklearn.metrics import silhouette_score
-from sklearn.cluster import AffinityPropagation
-from functools import partial
-from clustering import _kmeans_clustering, read_similarity_mat, sim_to_dist, process_clustering_result, clustering_result
-from dataclasses import dataclass
-from multiprocessing  import Pool, cpu_count, Array, Process
-from pathlib import Path
-from itertools import product, starmap
-import numpy as np
-import pandas as pd
+from sklearn.cluster import KMeans

 import fire
 import fire

-import sys
+from pathlib import Path
+from multiprocessing import cpu_count
+from dataclasses import dataclass
+from clustering_base import sim_to_dist, process_clustering_result, clustering_result, read_similarity_mat

 
 @dataclass
 class kmeans_clustering_result(clustering_result):
     n_clusters:int
     n_init:int
 
 
 @dataclass
 class kmeans_clustering_result(clustering_result):
     n_clusters:int
     n_init:int
 

+def kmeans_clustering(similarities, *args, **kwargs):
+    subreddits, mat = read_similarity_mat(similarities)
+    mat = sim_to_dist(mat)
+    clustering = _kmeans_clustering(mat, *args, **kwargs)
+    cluster_data = process_clustering_result(clustering, subreddits)
+    return(cluster_data)
+
+def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True):
+
+    clustering = KMeans(n_clusters=n_clusters,
+                        n_init=n_init,
+                        max_iter=max_iter,
+                        random_state=random_state,
+                        verbose=verbose
+                        ).fit(mat)

 
 

-# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. 
+    return clustering

 
 def do_clustering(n_clusters, n_init, name, mat, subreddits,  max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
     if name is None:
 
 def do_clustering(n_clusters, n_init, name, mat, subreddits,  max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
     if name is None: