lsi support for weekly similarities

[cdsc_reddit.git] / clustering / selection.py

diff --git a/clustering/selection.py b/clustering/selection.py
index 520857daf1df62c716c54cc94ec0ff9b5e68a42e..81641db00155389739634075dcb413946da8672c 100644 (file)
--- a/clustering/selection.py
+++ b/clustering/selection.py
@@ -1,101 +1,38 @@
-from sklearn.metrics import silhouette_score
-from sklearn.cluster import AffinityPropagation
-from functools import partial
-from clustering import _affinity_clustering, read_similarity_mat
-from dataclasses import dataclass
-from multiprocessing  import Pool, cpu_count, Array, Process
-from pathlib import Path
-from itertools import product, starmap
-import numpy as np

 import pandas as pd
 import pandas as pd

-import fire
-import sys
-
-# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. 
-
-@dataclass
-class clustering_result:
-    outpath:Path
-    damping:float
-    max_iter:int
-    convergence_iter:int
-    preference_quantile:float
-    silhouette_score:float
-    alt_silhouette_score:float
-    name:str
-
-
-def sim_to_dist(mat):
-    dist = 1-mat
-    dist[dist < 0] = 0
-    np.fill_diagonal(dist,0)
-    return dist
-
-def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
-    if name is None:
-        name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
-    print(name)
-    sys.stdout.flush()
-    outpath = outdir / (str(name) + ".feather")
-    print(outpath)
-    clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
-    mat = sim_to_dist(clustering.affinity_matrix_)
-
-    score = silhouette_score(mat, clustering.labels_, metric='precomputed')
-
-    if alt_mat is not None:
-        alt_distances = sim_to_dist(alt_mat)
-        alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
-    
-    res = clustering_result(outpath=outpath,
-                            damping=damping,
-                            max_iter=max_iter,
-                            convergence_iter=convergence_iter,
-                            preference_quantile=preference_quantile,
-                            silhouette_score=score,
-                            alt_silhouette_score=score,
-                            name=str(name))
-
-    return res
-
-# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
+import plotnine as pn
+from pathlib import Path
+from clustering.fit_tsne import fit_tsne
+from visualization.tsne_vis import build_visualization

 
 

-def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
+df = pd.read_csv("/gscratch/comdata/output/reddit_clustering/subreddit_comment_authors-tf_10k_LSI/hdbscan/selection_data.csv",index_col=0)

 
 

-    damping = list(map(float,damping))
-    convergence_iter = convergence_iter = list(map(int,convergence_iter))
-    preference_quantile = list(map(float,preference_quantile))
+# plot silhouette_score as a function of isolates
+df = df.sort_values("silhouette_score")

 
 

-    if type(outdir) is str:
-        outdir = Path(outdir)
+df['n_isolates'] = df.n_isolates.str.split("\n0").apply(lambda rg: int(rg[1]))
+p = pn.ggplot(df,pn.aes(x='n_isolates',y='silhouette_score')) + pn.geom_point()
+p.save("isolates_x_score.png")

 
 

-    outdir.mkdir(parents=True,exist_ok=True)
+p = pn.ggplot(df,pn.aes(y='n_clusters',x='n_isolates',color='silhouette_score')) + pn.geom_point()
+p.save("clusters_x_isolates.png")

 
 

-    subreddits, mat = read_similarity_mat(similarities,use_threads=True)
+# the best result for hdbscan seems like this one: it has a decent number of 
+# i think I prefer the 'eom' clustering style because larger clusters are less likely to suffer from ommitted variables
+best_eom = df[(df.n_isolates <5000)&(df.silhouette_score>0.4)&(df.cluster_selection_method=='eom')&(df.min_cluster_size==2)].iloc[df.shape[1]]

 
 

-    if alt_similarities is not None:
-        alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
-    else:
-        alt_mat = None
+best_lsi = df[(df.n_isolates <5000)&(df.silhouette_score>0.4)&(df.cluster_selection_method=='leaf')&(df.min_cluster_size==2)].iloc[df.shape[1]]

 
 

-    if J is None:
-        J = cpu_count()
-    pool = Pool(J)
+tsne_data = Path("./clustering/authors-tf_lsi850_tsne.feather")

 
 

-    # get list of tuples: the combinations of hyperparameters
-    hyper_grid = product(damping, convergence_iter, preference_quantile)
-    hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
+if not tnse_data.exists():
+    fit_tsne("/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors-tf_10k_LSI/850.feather",
+             tnse_data)

 
 

-    _do_clustering = partial(do_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
+build_visualization("./clustering/authors-tf_lsi850_tsne.feather",
+                    Path(best_eom.outpath)/(best_eom['name']+'.feather'),
+                    "./authors-tf_lsi850_best_eom.html")

 
 

-    #    similarities = Array('d', mat)
-    # call pool.starmap
-    print("running clustering selection")
-    clustering_data = pool.starmap(_do_clustering, hyper_grid)
-    clustering_data = pd.DataFrame(list(clustering_data))
-    clustering_data.to_csv(outinfo)
-    
-    return clustering_data
+build_visualization("./clustering/authors-tf_lsi850_tsne.feather",
+                    Path(best_leaf.outpath)/(best_leaf['name']+'.feather'),
+                    "./authors-tf_lsi850_best_leaf.html")

 
 

-if __name__ == "__main__":
-    x = fire.Fire(select_affinity_clustering)