clustering/selection.py

   1 from sklearn.metrics import silhouette_score
   2 from sklearn.cluster import AffinityPropagation
   3 from functools import partial
   4 from clustering import _affinity_clustering, read_similarity_mat
   5 from dataclasses import dataclass
   6 from multiprocessing  import Pool, cpu_count, Array, Process
   7 from pathlib import Path
   8 from itertools import product, starmap
   9 import numpy as np
  10 import pandas as pd
  11 import fire
  12 import sys
  13
  14 # silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying.
  15
  16 @dataclass
  17 class clustering_result:
  18     outpath:Path
  19     damping:float
  20     max_iter:int
  21     convergence_iter:int
  22     preference_quantile:float
  23     silhouette_score:float
  24     alt_silhouette_score:float
  25     name:str
  26
  27
  28 def sim_to_dist(mat):
  29     dist = 1-mat
  30     dist[dist < 0] = 0
  31     np.fill_diagonal(dist,0)
  32     return dist
  33
  34 def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits,  max_iter,  outdir:Path, random_state, verbose, alt_mat, overwrite=False):
  35     if name is None:
  36         name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
  37     print(name)
  38     sys.stdout.flush()
  39     outpath = outdir / (str(name) + ".feather")
  40     print(outpath)
  41     clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
  42     mat = sim_to_dist(clustering.affinity_matrix_)
  43
  44     score = silhouette_score(mat, clustering.labels_, metric='precomputed')
  45
  46     if alt_mat is not None:
  47         alt_distances = sim_to_dist(alt_mat)
  48         alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
  49
  50     res = clustering_result(outpath=outpath,
  51                             damping=damping,
  52                             max_iter=max_iter,
  53                             convergence_iter=convergence_iter,
  54                             preference_quantile=preference_quantile,
  55                             silhouette_score=score,
  56                             alt_silhouette_score=score,
  57                             name=str(name))
  58
  59     return res
  60
  61 # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
  62
  63 def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
  64
  65     damping = list(map(float,damping))
  66     convergence_iter = convergence_iter = list(map(int,convergence_iter))
  67     preference_quantile = list(map(float,preference_quantile))
  68
  69     if type(outdir) is str:
  70         outdir = Path(outdir)
  71
  72     outdir.mkdir(parents=True,exist_ok=True)
  73
  74     subreddits, mat = read_similarity_mat(similarities,use_threads=True)
  75
  76     if alt_similarities is not None:
  77         alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
  78     else:
  79         alt_mat = None
  80
  81     if J is None:
  82         J = cpu_count()
  83     pool = Pool(J)
  84
  85     # get list of tuples: the combinations of hyperparameters
  86     hyper_grid = product(damping, convergence_iter, preference_quantile)
  87     hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
  88
  89     _do_clustering = partial(do_clustering,  mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
  90
  91     #    similarities = Array('d', mat)
  92     # call pool.starmap
  93     print("running clustering selection")
  94     clustering_data = pool.starmap(_do_clustering, hyper_grid)
  95     clustering_data = pd.DataFrame(list(clustering_data))
  96     clustering_data.to_csv(outinfo)
  97
  98     return clustering_data
  99
 100 if __name__ == "__main__":
 101     x = fire.Fire(select_affinity_clustering)