X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/f20365c07e93ae71b95af314f869fcdcfb58100f..b7c39a3494ce214f315fd7e3bb0bf99bc58070d1:/clustering/selection.py diff --git a/clustering/selection.py b/clustering/selection.py index 520857d..81641db 100644 --- a/clustering/selection.py +++ b/clustering/selection.py @@ -1,101 +1,38 @@ -from sklearn.metrics import silhouette_score -from sklearn.cluster import AffinityPropagation -from functools import partial -from clustering import _affinity_clustering, read_similarity_mat -from dataclasses import dataclass -from multiprocessing import Pool, cpu_count, Array, Process -from pathlib import Path -from itertools import product, starmap -import numpy as np import pandas as pd -import fire -import sys - -# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. - -@dataclass -class clustering_result: - outpath:Path - damping:float - max_iter:int - convergence_iter:int - preference_quantile:float - silhouette_score:float - alt_silhouette_score:float - name:str - - -def sim_to_dist(mat): - dist = 1-mat - dist[dist < 0] = 0 - np.fill_diagonal(dist,0) - return dist - -def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False): - if name is None: - name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}" - print(name) - sys.stdout.flush() - outpath = outdir / (str(name) + ".feather") - print(outpath) - clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose) - mat = sim_to_dist(clustering.affinity_matrix_) - - score = silhouette_score(mat, clustering.labels_, metric='precomputed') - - if alt_mat is not None: - alt_distances = sim_to_dist(alt_mat) - alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed') - - res = clustering_result(outpath=outpath, - damping=damping, - max_iter=max_iter, - convergence_iter=convergence_iter, - preference_quantile=preference_quantile, - silhouette_score=score, - alt_silhouette_score=score, - name=str(name)) - - return res - -# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering). +import plotnine as pn +from pathlib import Path +from clustering.fit_tsne import fit_tsne +from visualization.tsne_vis import build_visualization -def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None): +df = pd.read_csv("/gscratch/comdata/output/reddit_clustering/subreddit_comment_authors-tf_10k_LSI/hdbscan/selection_data.csv",index_col=0) - damping = list(map(float,damping)) - convergence_iter = convergence_iter = list(map(int,convergence_iter)) - preference_quantile = list(map(float,preference_quantile)) +# plot silhouette_score as a function of isolates +df = df.sort_values("silhouette_score") - if type(outdir) is str: - outdir = Path(outdir) +df['n_isolates'] = df.n_isolates.str.split("\n0").apply(lambda rg: int(rg[1])) +p = pn.ggplot(df,pn.aes(x='n_isolates',y='silhouette_score')) + pn.geom_point() +p.save("isolates_x_score.png") - outdir.mkdir(parents=True,exist_ok=True) +p = pn.ggplot(df,pn.aes(y='n_clusters',x='n_isolates',color='silhouette_score')) + pn.geom_point() +p.save("clusters_x_isolates.png") - subreddits, mat = read_similarity_mat(similarities,use_threads=True) +# the best result for hdbscan seems like this one: it has a decent number of +# i think I prefer the 'eom' clustering style because larger clusters are less likely to suffer from ommitted variables +best_eom = df[(df.n_isolates <5000)&(df.silhouette_score>0.4)&(df.cluster_selection_method=='eom')&(df.min_cluster_size==2)].iloc[df.shape[1]] - if alt_similarities is not None: - alt_mat = read_similarity_mat(alt_similarities,use_threads=True) - else: - alt_mat = None +best_lsi = df[(df.n_isolates <5000)&(df.silhouette_score>0.4)&(df.cluster_selection_method=='leaf')&(df.min_cluster_size==2)].iloc[df.shape[1]] - if J is None: - J = cpu_count() - pool = Pool(J) +tsne_data = Path("./clustering/authors-tf_lsi850_tsne.feather") - # get list of tuples: the combinations of hyperparameters - hyper_grid = product(damping, convergence_iter, preference_quantile) - hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid)) +if not tnse_data.exists(): + fit_tsne("/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors-tf_10k_LSI/850.feather", + tnse_data) - _do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat) +build_visualization("./clustering/authors-tf_lsi850_tsne.feather", + Path(best_eom.outpath)/(best_eom['name']+'.feather'), + "./authors-tf_lsi850_best_eom.html") - # similarities = Array('d', mat) - # call pool.starmap - print("running clustering selection") - clustering_data = pool.starmap(_do_clustering, hyper_grid) - clustering_data = pd.DataFrame(list(clustering_data)) - clustering_data.to_csv(outinfo) - - return clustering_data +build_visualization("./clustering/authors-tf_lsi850_tsne.feather", + Path(best_leaf.outpath)/(best_leaf['name']+'.feather'), + "./authors-tf_lsi850_best_leaf.html") -if __name__ == "__main__": - x = fire.Fire(select_affinity_clustering)