From: Nate E TeBlunthuis Date: Mon, 3 May 2021 18:28:48 +0000 (-0700) Subject: refactor clustering.py into method-specific files. X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/commitdiff_plain/8d1df5b26ee80fee639e5b3ecd057fe8e72f166c?ds=inline;hp=e1c9d9af6fccf3f2de24d192f9678318ad04a4ea refactor clustering.py into method-specific files. --- diff --git a/clustering/select_affinity.py b/clustering/affinity_clustering.py similarity index 63% rename from clustering/select_affinity.py rename to clustering/affinity_clustering.py index b8bd13a..287f7e2 100644 --- a/clustering/select_affinity.py +++ b/clustering/affinity_clustering.py @@ -18,7 +18,44 @@ class affinity_clustering_result(clustering_result): convergence_iter:int preference_quantile:float -def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False): +def affinity_clustering(similarities, output, *args, **kwargs): + subreddits, mat = read_similarity_mat(similarities) + clustering = _affinity_clustering(mat, *args, **kwargs) + cluster_data = process_clustering_result(clustering, subreddits) + cluster_data['algorithm'] = 'affinity' + return(cluster_data) + +def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True): + ''' + similarities: matrix of similarity scores + preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits. + damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author. + ''' + print(f"damping:{damping}; convergenceIter:{convergence_iter}; preferenceQuantile:{preference_quantile}") + + preference = np.quantile(mat,preference_quantile) + + print(f"preference is {preference}") + print("data loaded") + sys.stdout.flush() + clustering = AffinityPropagation(damping=damping, + max_iter=max_iter, + convergence_iter=convergence_iter, + copy=False, + preference=preference, + affinity='precomputed', + verbose=verbose, + random_state=random_state).fit(mat) + + cluster_data = process_clustering_result(clustering, subreddits) + output = Path(output) + output.parent.mkdir(parents=True,exist_ok=True) + cluster_data.to_feather(output) + print(f"saved {output}") + return clustering + + +def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False): if name is None: name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}" print(name) @@ -53,41 +90,6 @@ def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, return res -def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False): - if name is None: - name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}" - print(name) - sys.stdout.flush() - outpath = outdir / (str(name) + ".feather") - outpath.parent.mkdir(parents=True,exist_ok=True) - print(outpath) - clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose) - mat = sim_to_dist(clustering.affinity_matrix_) - - try: - score = silhouette_score(mat, clustering.labels_, metric='precomputed') - except ValueError: - score = None - - if alt_mat is not None: - alt_distances = sim_to_dist(alt_mat) - try: - alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed') - except ValueError: - alt_score = None - - res = clustering_result(outpath=outpath, - damping=damping, - max_iter=max_iter, - convergence_iter=convergence_iter, - preference_quantile=preference_quantile, - silhouette_score=score, - alt_silhouette_score=score, - name=str(name)) - - return res - - # alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering). def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None): @@ -116,7 +118,7 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max hyper_grid = product(damping, convergence_iter, preference_quantile) hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid)) - _do_clustering = partial(do_affinity_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat) + _do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat) # similarities = Array('d', mat) # call pool.starmap @@ -124,8 +126,6 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max clustering_data = pool.starmap(_do_clustering, hyper_grid) clustering_data = pd.DataFrame(list(clustering_data)) clustering_data.to_csv(outinfo) - - return clustering_data if __name__ == "__main__": diff --git a/clustering/clustering.py b/clustering/clustering.py index 85be3fe..6ee7842 100755 --- a/clustering/clustering.py +++ b/clustering/clustering.py @@ -3,7 +3,7 @@ import sys import pandas as pd import numpy as np -from sklearn.cluster import AffinityPropagation, KMeans +from sklearn.cluster import AffinityPropagation import fire from pathlib import Path from multiprocessing import cpu_count @@ -46,24 +46,6 @@ def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, print(f"saved {output}") return clustering -def kmeans_clustering(similarities, *args, **kwargs): - subreddits, mat = read_similarity_mat(similarities) - mat = sim_to_dist(mat) - clustering = _kmeans_clustering(mat, *args, **kwargs) - cluster_data = process_clustering_result(clustering, subreddits) - return(cluster_data) - -def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True): - - clustering = KMeans(n_clusters=n_clusters, - n_init=n_init, - max_iter=max_iter, - random_state=random_state, - verbose=verbose - ).fit(mat) - - return clustering - if __name__ == "__main__": diff --git a/clustering/hdbscan_clustering.py b/clustering/hdbscan_clustering.py index 888554a..4f4e0d6 100644 --- a/clustering/hdbscan_clustering.py +++ b/clustering/hdbscan_clustering.py @@ -28,6 +28,13 @@ def test_select_hdbscan_clustering(): cluster_selection_methods=['eom']; lsi_dimensions='all' + df = pd.read_csv("test_hdbscan/selection_data.csv") + test_select_hdbscan_clustering() + check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather") + silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather") + c = check_clusters.merge(silscores,on='subreddit')# fire.Fire(select_hdbscan_clustering) + + @dataclass class hdbscan_clustering_result(clustering_result): min_cluster_size:int @@ -165,8 +172,4 @@ def make_KNN_plots(): KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png') if __name__ == "__main__": - df = pd.read_csv("test_hdbscan/selection_data.csv") - test_select_hdbscan_clustering() - check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather") - silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather") - c = check_clusters.merge(silscores,on='subreddit')# fire.Fire(select_hdbscan_clustering) + fire.Fire(select_hdbscan_clustering) diff --git a/clustering/select_kmeans.py b/clustering/kmeans_clustering.py similarity index 77% rename from clustering/select_kmeans.py rename to clustering/kmeans_clustering.py index b07a108..8822e9f 100644 --- a/clustering/select_kmeans.py +++ b/clustering/kmeans_clustering.py @@ -1,23 +1,32 @@ -from sklearn.metrics import silhouette_score -from sklearn.cluster import AffinityPropagation -from functools import partial -from clustering import _kmeans_clustering, read_similarity_mat, sim_to_dist, process_clustering_result, clustering_result -from dataclasses import dataclass -from multiprocessing import Pool, cpu_count, Array, Process -from pathlib import Path -from itertools import product, starmap -import numpy as np -import pandas as pd +from sklearn.cluster import KMeans import fire -import sys +from pathlib import Path +from multiprocessing import cpu_count +from dataclasses import dataclass +from clustering_base import sim_to_dist, process_clustering_result, clustering_result, read_similarity_mat @dataclass class kmeans_clustering_result(clustering_result): n_clusters:int n_init:int +def kmeans_clustering(similarities, *args, **kwargs): + subreddits, mat = read_similarity_mat(similarities) + mat = sim_to_dist(mat) + clustering = _kmeans_clustering(mat, *args, **kwargs) + cluster_data = process_clustering_result(clustering, subreddits) + return(cluster_data) + +def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True): + + clustering = KMeans(n_clusters=n_clusters, + n_init=n_init, + max_iter=max_iter, + random_state=random_state, + verbose=verbose + ).fit(mat) -# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying. + return clustering def do_clustering(n_clusters, n_init, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False): if name is None: