]> code.communitydata.science - cdsc_reddit.git/blob - clustering/selection.py
bfa1c31d6febbc4a532588df0332df4d9c732f74
[cdsc_reddit.git] / clustering / selection.py
1 from sklearn.metrics import silhouette_score
2 from sklearn.cluster import AffinityPropagation
3 from functools import partial
4 from clustering import _affinity_clustering, read_similarity_mat
5 from dataclasses import dataclass
6 from multiprocessing  import Pool, cpu_count, Array, Process
7 from pathlib import Path
8 from itertools import product, starmap
9 import pandas as pd
10 import fire
11 import sys
12
13 # silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. 
14
15 @dataclass
16 class clustering_result:
17     outpath:Path
18     damping:float
19     max_iter:int
20     convergence_iter:int
21     preference_quantile:float
22     silhouette_score:float
23     alt_silhouette_score:float
24     name:str
25
26 def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat):
27     if name is None:
28         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{convergence_iter}"
29     print(name)
30     sys.stdout.flush()
31     outpath = outdir / (str(name) + ".feather")
32     print(outpath)
33     clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
34     score = silhouette_score(clustering.affinity_matrix_, clustering.labels_, metric='precomputed')
35     alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
36     
37     res = clustering_result(outpath=outpath,
38                             damping=damping,
39                             max_iter=max_iter,
40                             convergence_iter=convergence_iter,
41                             preference_quantile=preference_quantile,
42                             silhouette_score=score,
43                             alt_silhouette_score=score,
44                             name=str(name))
45
46     return res
47
48 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
49
50 def select_affinity_clustering(similarities, outdir, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
51
52     damping = list(map(float,damping))
53     convergence_iter = convergence_iter = list(map(int,convergence_iter))
54     preference_quantile = list(map(float,preference_quantile))
55
56     if type(outdir) is str:
57         outdir = Path(outdir)
58
59     outdir.mkdir(parents=True,exist_ok=True)
60
61     subreddits, mat = read_similarity_mat(similarities,use_threads=True)
62
63     if alt_similarities is not None:
64         alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
65     else:
66         alt_mat = None
67
68     if J is None:
69         J = cpu_count()
70     pool = Pool(J)
71
72     # get list of tuples: the combinations of hyperparameters
73     hyper_grid = product(damping, convergence_iter, preference_quantile)
74     hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
75
76     _do_clustering = partial(do_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
77
78     #    similarities = Array('d', mat)
79     # call pool.starmap
80     print("running clustering selection")
81     clustering_data = pool.starmap(_do_clustering, hyper_grid)
82     clustering_data = pd.DataFrame(list(clustering_data))
83     return clustering_data
84
85
86 if __name__ == "__main__":
87     fire.Fire(select_affinity_clustering)

Community Data Science Collective || Want to submit a patch?