From 01a4c353588ab1a28f36980157daa5e682ea9edc Mon Sep 17 00:00:00 2001 From: Nate E TeBlunthuis Date: Tue, 20 Apr 2021 11:33:54 -0700 Subject: [PATCH] grid sweep selection for clustering hyperparameters --- clustering/Makefile | 52 ++++++++++++++++-------- clustering/clustering.py | 32 ++++++++++----- clustering/selection.py | 87 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 27 deletions(-) create mode 100644 clustering/selection.py diff --git a/clustering/Makefile b/clustering/Makefile index 20d7808..adaa8fe 100644 --- a/clustering/Makefile +++ b/clustering/Makefile @@ -1,32 +1,52 @@ #srun_cdsc='srun -p comdata-int -A comdata --time=300:00:00 --time-min=00:15:00 --mem=100G --ntasks=1 --cpus-per-task=28' -all:/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather -#all:/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather +srun_singularity=source /gscratch/comdata/users/nathante/cdsc_reddit/bin/activate && srun_singularity.sh +similarity_data=/gscratch/comdata/output/reddit_similarity +clustering_data=/gscratch/comdata/output/reddit_clustering +selection_grid="--max_iter=10000 --convergence_iter=15,30,100 --preference_quantile=0.85 --damping=0.5,0.6,0.7,0.8,0.85,0.9,0.95,0.97,0.99, --preference_quantile=0.1,0.3,0.5,0.7,0.9" +all:$(clustering_data)/subreddit_comment_authors_30k.feather $(clustering_data)/subreddit_authors-tf_similarities_30k.feather $(clustering_data)/subreddit_comment_authors_10k.feather $(clustering_data)/subreddit_authors-tf_similarities_10k.feather $(clustering_data)/subreddit_comment_terms_30k.feather $(clustering_data)/subreddit_comment_terms_10k.feather -/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather -# $srun_cdsc python3 - start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.85 --damping=0.85 +$(clustering_data)/subreddit_comment_authors_10k.feather:selection.py $(similarity_data)/subreddit_comment_authors_10k.feather clustering.py + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors_10k.feather $(clustering_data)/subreddit_comment_authors_10k $(selection_grid) -J 20 -/gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_terms_10000.feather -# $srun_cdsc python3 - start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather ---max_iter=1000 --convergence_iter=15 --preference_quantile=0.9 --damping=0.5 +$(clustering_data)/subreddit_comment_terms_10k.feather:selection.py $(similarity_data)/subreddit_comment_terms_10k.feather clustering.py + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_terms_10k.feather $(clustering_data)/subreddit_comment_terms_10k $(selection_grid) -J 20 + +$(clustering_data)/subreddit_authors-tf_similarities_10k.feather:clustering.py $(similarity_data)/subreddit_comment_authors-tf_10k.feather + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors-tf_10k.feather $(clustering_data)/subreddit_comment_authors-tf_10k $(selection_grid) -J 20 + +$(clustering_data)/subreddit_comment_authors_30k.feather:selection.py $(similarity_data)/subreddit_comment_authors_30k.feather clustering.py + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors_30k.feather $(clustering_data)/subreddit_comment_authors_30k $(selection_grid) -J 10 + +$(clustering_data)/subreddit_comment_terms_30k.feather:selection.py $(similarity_data)/subreddit_comment_terms_30k.feather clustering.py + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_terms_30k.feather $(clustering_data)/subreddit_comment_terms_30k $(selection_grid) -J 10 + +$(clustering_data)/subreddit_authors-tf_similarities_30k.feather:clustering.py $(similarity_data)/subreddit_comment_authors-tf_30k.feather + $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors-tf_30k.feather $(clustering_data)/subreddit_comment_authors-tf_30k $(selection_grid) -J 8 + + +# $(clustering_data)/subreddit_comment_authors_100k.feather:clustering.py $(similarity_data)/subreddit_comment_authors_100k.feather +# $(srun_singularity) python3 clustering.py $(similarity_data)/subreddit_comment_authors_100k.feather $(clustering_data)/subreddit_comment_authors_100k.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.85 --damping=0.85 + +# $(clustering_data)/comment_terms_100k.feather:clustering.py $(similarity_data)/subreddit_comment_terms_100k.feather +# $(srun_singularity) python3 clustering.py $(similarity_data)/comment_terms_10000.feather $(clustering_data)/comment_terms_10000.feather ---max_iter=1000 --convergence_iter=15 --preference_quantile=0.9 --damping=0.5 + +# $(clustering_data)/subreddit_comment_author-tf_100k.feather:clustering.py $(similarity_data)/subreddit_comment_author-tf_100k.feather +# $(srun_singularity) python3 clustering.py $(similarity_data)/subreddit_comment_author-tf_100k.parquet $(clustering_data)/subreddit_comment_author-tf_100k.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.5 --damping=0.85 -/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet -# $srun_cdsc - start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.5 --damping=0.85 # it's pretty difficult to get a result that isn't one huge megacluster. A sign that it's bullcrap # /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather # ./clustering.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.9 --damping=0.85 -/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet +# /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet - start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet --output=/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather +# start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet --output=/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather # /gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather # python3 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather --output=/gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather -/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather -# $srun_cdsc python3 - start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather --output=/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather +# /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather +# # $srun_cdsc python3 +# start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather --output=/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather diff --git a/clustering/clustering.py b/clustering/clustering.py index 4cde717..cac5730 100755 --- a/clustering/clustering.py +++ b/clustering/clustering.py @@ -1,29 +1,36 @@ #!/usr/bin/env python3 - +# TODO: replace prints with logging. +import sys import pandas as pd import numpy as np from sklearn.cluster import AffinityPropagation import fire +from pathlib import Path + +def read_similarity_mat(similarities, use_threads=True): + df = pd.read_feather(similarities, use_threads=use_threads) + mat = np.array(df.drop('_subreddit',1)) + n = mat.shape[0] + mat[range(n),range(n)] = 1 + return (df._subreddit,mat) + +def affinity_clustering(similarities, *args, **kwargs): + subreddits, mat = read_similarity_mat(similarities) + return _affinity_clustering(mat, subreddits, *args, **kwargs) -def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True): +def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True): ''' similarities: feather file with a dataframe of similarity scores preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits. damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author. ''' - - df = pd.read_feather(similarities) - n = df.shape[0] - mat = np.array(df.drop('_subreddit',1)) - mat[range(n),range(n)] = 1 - assert(all(np.diag(mat)==1)) + print(f"damping:{damping}; convergenceIter:{convergence_iter}; preferenceQuantile:{preference_quantilne}") preference = np.quantile(mat,preference_quantile) print(f"preference is {preference}") - print("data loaded") - + sys.stdout.flush() clustering = AffinityPropagation(damping=damping, max_iter=max_iter, convergence_iter=convergence_iter, @@ -39,7 +46,7 @@ def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, conv print(f"found {len(set(clusters))} clusters") - cluster_data = pd.DataFrame({'subreddit': df.subreddit,'cluster':clustering.labels_}) + cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_}) cluster_sizes = cluster_data.groupby("cluster").count() print(f"the largest cluster has {cluster_sizes.subreddit.max()} members") @@ -48,7 +55,10 @@ def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, conv print(f"{(cluster_sizes.subreddit==1).sum()} clusters have 1 member") + sys.stdout.flush() cluster_data.to_feather(output) + print(f"saved {output}") + return clustering if __name__ == "__main__": fire.Fire(affinity_clustering) diff --git a/clustering/selection.py b/clustering/selection.py new file mode 100644 index 0000000..bfa1c31 --- /dev/null +++ b/clustering/selection.py @@ -0,0 +1,87 @@ +from sklearn.metrics import silhouette_score +from sklearn.cluster import AffinityPropagation +from functools import partial +from clustering import _affinity_clustering, read_similarity_mat +from dataclasses import dataclass +from multiprocessing import Pool, cpu_count, Array, Process +from pathlib import Path +from itertools import product, starmap +import pandas as pd +import fire +import sys + +# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. + +@dataclass +class clustering_result: + outpath:Path + damping:float + max_iter:int + convergence_iter:int + preference_quantile:float + silhouette_score:float + alt_silhouette_score:float + name:str + +def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat): + if name is None: + name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{convergence_iter}" + print(name) + sys.stdout.flush() + outpath = outdir / (str(name) + ".feather") + print(outpath) + clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose) + score = silhouette_score(clustering.affinity_matrix_, clustering.labels_, metric='precomputed') + alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed') + + res = clustering_result(outpath=outpath, + damping=damping, + max_iter=max_iter, + convergence_iter=convergence_iter, + preference_quantile=preference_quantile, + silhouette_score=score, + alt_silhouette_score=score, + name=str(name)) + + return res + +# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering). + +def select_affinity_clustering(similarities, outdir, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None): + + damping = list(map(float,damping)) + convergence_iter = convergence_iter = list(map(int,convergence_iter)) + preference_quantile = list(map(float,preference_quantile)) + + if type(outdir) is str: + outdir = Path(outdir) + + outdir.mkdir(parents=True,exist_ok=True) + + subreddits, mat = read_similarity_mat(similarities,use_threads=True) + + if alt_similarities is not None: + alt_mat = read_similarity_mat(alt_similarities,use_threads=True) + else: + alt_mat = None + + if J is None: + J = cpu_count() + pool = Pool(J) + + # get list of tuples: the combinations of hyperparameters + hyper_grid = product(damping, convergence_iter, preference_quantile) + hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid)) + + _do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat) + + # similarities = Array('d', mat) + # call pool.starmap + print("running clustering selection") + clustering_data = pool.starmap(_do_clustering, hyper_grid) + clustering_data = pd.DataFrame(list(clustering_data)) + return clustering_data + + +if __name__ == "__main__": + fire.Fire(select_affinity_clustering) -- 2.39.5