]> code.communitydata.science - cdsc_reddit.git/blob - clustering/clustering.py
4cde71787eb5f208a0e51afb68ef57f1f99c1106
[cdsc_reddit.git] / clustering / clustering.py
1 #!/usr/bin/env python3
2
3 import pandas as pd
4 import numpy as np
5 from sklearn.cluster import AffinityPropagation
6 import fire
7
8 def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True):
9     '''
10     similarities: feather file with a dataframe of similarity scores
11     preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits.
12     damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author. 
13     '''
14
15     df = pd.read_feather(similarities)
16     n = df.shape[0]
17     mat = np.array(df.drop('_subreddit',1))
18     mat[range(n),range(n)] = 1
19     assert(all(np.diag(mat)==1))
20
21     preference = np.quantile(mat,preference_quantile)
22
23     print(f"preference is {preference}")
24
25     print("data loaded")
26
27     clustering = AffinityPropagation(damping=damping,
28                                      max_iter=max_iter,
29                                      convergence_iter=convergence_iter,
30                                      copy=False,
31                                      preference=preference,
32                                      affinity='precomputed',
33                                      verbose=verbose,
34                                      random_state=random_state).fit(mat)
35
36
37     print(f"clustering took {clustering.n_iter_} iterations")
38     clusters = clustering.labels_
39
40     print(f"found {len(set(clusters))} clusters")
41
42     cluster_data = pd.DataFrame({'subreddit': df.subreddit,'cluster':clustering.labels_})
43
44     cluster_sizes = cluster_data.groupby("cluster").count()
45     print(f"the largest cluster has {cluster_sizes.subreddit.max()} members")
46
47     print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
48
49     print(f"{(cluster_sizes.subreddit==1).sum()} clusters have 1 member")
50
51     cluster_data.to_feather(output)
52
53 if __name__ == "__main__":
54     fire.Fire(affinity_clustering)

Community Data Science Collective || Want to submit a patch?