clustering/select_affinity.py

   1 from sklearn.metrics import silhouette_score
   2 from sklearn.cluster import AffinityPropagation
   3 from functools import partial
   4 from dataclasses import dataclass
   5 from clustering import _affinity_clustering, read_similarity_mat, sim_to_dist, process_clustering_result, clustering_result
   6 from multiprocessing  import Pool, cpu_count, Array, Process
   7 from pathlib import Path
   8 from itertools import product, starmap
   9 import numpy as np
  10 import pandas as pd
  11 import fire
  12 import sys
  13
  14 # silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying.
  15 @dataclass
  16 class affinity_clustering_result(clustering_result):
  17     damping:float
  18     convergence_iter:int
  19     preference_quantile:float
  20
  21 def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
  22     if name is None:
  23         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
  24     print(name)
  25     sys.stdout.flush()
  26     outpath = outdir / (str(name) + ".feather")
  27     outpath.parent.mkdir(parents=True,exist_ok=True)
  28     print(outpath)
  29     clustering = _affinity_clustering(mat, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
  30     cluster_data = process_clustering_result(clustering, subreddits)
  31     mat = sim_to_dist(clustering.affinity_matrix_)
  32
  33     try:
  34         score = silhouette_score(mat, clustering.labels_, metric='precomputed')
  35     except ValueError:
  36         score = None
  37
  38     if alt_mat is not None:
  39         alt_distances = sim_to_dist(alt_mat)
  40         try:
  41             alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
  42         except ValueError:
  43             alt_score = None
  44
  45     res = affinity_clustering_result(outpath=outpath,
  46                                      damping=damping,
  47                                      max_iter=max_iter,
  48                                      convergence_iter=convergence_iter,
  49                                      preference_quantile=preference_quantile,
  50                                      silhouette_score=score,
  51                                      alt_silhouette_score=score,
  52                                      name=str(name))
  53
  54     return res
  55
  56 def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
  57     if name is None:
  58         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
  59     print(name)
  60     sys.stdout.flush()
  61     outpath = outdir / (str(name) + ".feather")
  62     outpath.parent.mkdir(parents=True,exist_ok=True)
  63     print(outpath)
  64     clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
  65     mat = sim_to_dist(clustering.affinity_matrix_)
  66
  67     try:
  68         score = silhouette_score(mat, clustering.labels_, metric='precomputed')
  69     except ValueError:
  70         score = None
  71
  72     if alt_mat is not None:
  73         alt_distances = sim_to_dist(alt_mat)
  74         try:
  75             alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
  76         except ValueError:
  77             alt_score = None
  78
  79     res = clustering_result(outpath=outpath,
  80                             damping=damping,
  81                             max_iter=max_iter,
  82                             convergence_iter=convergence_iter,
  83                             preference_quantile=preference_quantile,
  84                             silhouette_score=score,
  85                             alt_silhouette_score=score,
  86                             name=str(name))
  87
  88     return res
  89
  90
  91 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
  92
  93 def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
  94
  95     damping = list(map(float,damping))
  96     convergence_iter = convergence_iter = list(map(int,convergence_iter))
  97     preference_quantile = list(map(float,preference_quantile))
  98
  99     if type(outdir) is str:
 100         outdir = Path(outdir)
 101
 102     outdir.mkdir(parents=True,exist_ok=True)
 103
 104     subreddits, mat = read_similarity_mat(similarities,use_threads=True)
 105
 106     if alt_similarities is not None:
 107         alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
 108     else:
 109         alt_mat = None
 110
 111     if J is None:
 112         J = cpu_count()
 113     pool = Pool(J)
 114
 115     # get list of tuples: the combinations of hyperparameters
 116     hyper_grid = product(damping, convergence_iter, preference_quantile)
 117     hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
 118
 119     _do_clustering = partial(do_affinity_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
 120
 121     #    similarities = Array('d', mat)
 122     # call pool.starmap
 123     print("running clustering selection")
 124     clustering_data = pool.starmap(_do_clustering, hyper_grid)
 125     clustering_data = pd.DataFrame(list(clustering_data))
 126     clustering_data.to_csv(outinfo)
 127
 128
 129     return clustering_data
 130
 131 if __name__ == "__main__":
 132     x = fire.Fire(select_affinity_clustering)