1 from sklearn.metrics import silhouette_score
2 from sklearn.cluster import AffinityPropagation
3 from functools import partial
4 from dataclasses import dataclass
5 from clustering import _affinity_clustering, read_similarity_mat, sim_to_dist, process_clustering_result, clustering_result
6 from multiprocessing import Pool, cpu_count, Array, Process
7 from pathlib import Path
8 from itertools import product, starmap
14 # silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying.
16 class affinity_clustering_result(clustering_result):
19 preference_quantile:float
21 def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
23 name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
26 outpath = outdir / (str(name) + ".feather")
27 outpath.parent.mkdir(parents=True,exist_ok=True)
29 clustering = _affinity_clustering(mat, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
30 cluster_data = process_clustering_result(clustering, subreddits)
31 mat = sim_to_dist(clustering.affinity_matrix_)
34 score = silhouette_score(mat, clustering.labels_, metric='precomputed')
38 if alt_mat is not None:
39 alt_distances = sim_to_dist(alt_mat)
41 alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
45 res = affinity_clustering_result(outpath=outpath,
48 convergence_iter=convergence_iter,
49 preference_quantile=preference_quantile,
50 silhouette_score=score,
51 alt_silhouette_score=score,
56 def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
58 name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
61 outpath = outdir / (str(name) + ".feather")
62 outpath.parent.mkdir(parents=True,exist_ok=True)
64 clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
65 mat = sim_to_dist(clustering.affinity_matrix_)
68 score = silhouette_score(mat, clustering.labels_, metric='precomputed')
72 if alt_mat is not None:
73 alt_distances = sim_to_dist(alt_mat)
75 alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
79 res = clustering_result(outpath=outpath,
82 convergence_iter=convergence_iter,
83 preference_quantile=preference_quantile,
84 silhouette_score=score,
85 alt_silhouette_score=score,
91 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
93 def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
95 damping = list(map(float,damping))
96 convergence_iter = convergence_iter = list(map(int,convergence_iter))
97 preference_quantile = list(map(float,preference_quantile))
99 if type(outdir) is str:
100 outdir = Path(outdir)
102 outdir.mkdir(parents=True,exist_ok=True)
104 subreddits, mat = read_similarity_mat(similarities,use_threads=True)
106 if alt_similarities is not None:
107 alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
115 # get list of tuples: the combinations of hyperparameters
116 hyper_grid = product(damping, convergence_iter, preference_quantile)
117 hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
119 _do_clustering = partial(do_affinity_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
121 # similarities = Array('d', mat)
123 print("running clustering selection")
124 clustering_data = pool.starmap(_do_clustering, hyper_grid)
125 clustering_data = pd.DataFrame(list(clustering_data))
126 clustering_data.to_csv(outinfo)
129 return clustering_data
131 if __name__ == "__main__":
132 x = fire.Fire(select_affinity_clustering)