from sklearn.metrics import silhouette_score from sklearn.cluster import AffinityPropagation from functools import partial from clustering import _affinity_clustering, read_similarity_mat from dataclasses import dataclass from multiprocessing import Pool, cpu_count, Array, Process from pathlib import Path from itertools import product, starmap import pandas as pd import fire import sys # silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. @dataclass class clustering_result: outpath:Path damping:float max_iter:int convergence_iter:int preference_quantile:float silhouette_score:float alt_silhouette_score:float name:str def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat): if name is None: name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{convergence_iter}" print(name) sys.stdout.flush() outpath = outdir / (str(name) + ".feather") print(outpath) clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose) score = silhouette_score(clustering.affinity_matrix_, clustering.labels_, metric='precomputed') alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed') res = clustering_result(outpath=outpath, damping=damping, max_iter=max_iter, convergence_iter=convergence_iter, preference_quantile=preference_quantile, silhouette_score=score, alt_silhouette_score=score, name=str(name)) return res # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering). def select_affinity_clustering(similarities, outdir, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None): damping = list(map(float,damping)) convergence_iter = convergence_iter = list(map(int,convergence_iter)) preference_quantile = list(map(float,preference_quantile)) if type(outdir) is str: outdir = Path(outdir) outdir.mkdir(parents=True,exist_ok=True) subreddits, mat = read_similarity_mat(similarities,use_threads=True) if alt_similarities is not None: alt_mat = read_similarity_mat(alt_similarities,use_threads=True) else: alt_mat = None if J is None: J = cpu_count() pool = Pool(J) # get list of tuples: the combinations of hyperparameters hyper_grid = product(damping, convergence_iter, preference_quantile) hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid)) _do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat) # similarities = Array('d', mat) # call pool.starmap print("running clustering selection") clustering_data = pool.starmap(_do_clustering, hyper_grid) clustering_data = pd.DataFrame(list(clustering_data)) return clustering_data if __name__ == "__main__": fire.Fire(select_affinity_clustering)