2 from pathlib import Path
5 from dataclasses import dataclass
6 from sklearn.metrics import silhouette_score, silhouette_samples
7 from collections import Counter
9 # this is meant to be an interface, not created directly
11 def __init__(self, infile, outpath, name, call, *args, **kwargs):
12 self.outpath = Path(outpath)
16 self.infile = Path(infile)
21 self.subreddits, self.mat = self.read_distance_mat(self.infile)
22 self.clustering = self.call(self.mat, *self.args, **self.kwargs)
23 self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
24 self.score = self.silhouette()
25 self.outpath.mkdir(parents=True, exist_ok=True)
26 self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
31 self.cluster_data = None
40 self.result = clustering_result(outpath=str(self.outpath.resolve()),
41 silhouette_score=self.score,
43 n_clusters=self.n_clusters,
44 n_isolates=self.n_isolates,
45 silhouette_samples = self.silsampout
50 counts = Counter(self.clustering.labels_)
51 singletons = [key for key, value in counts.items() if value == 1]
52 isolates = (self.clustering.labels_ == -1) | (np.isin(self.clustering.labels_,np.array(singletons)))
53 scoremat = self.mat[~isolates][:,~isolates]
54 if self.n_clusters > 1:
55 score = silhouette_score(scoremat, self.clustering.labels_[~isolates], metric='precomputed')
56 silhouette_samp = silhouette_samples(self.mat, self.clustering.labels_, metric='precomputed')
57 silhouette_samp = pd.DataFrame({'subreddit':self.subreddits,'score':silhouette_samp})
58 self.outpath.mkdir(parents=True, exist_ok=True)
59 silsampout = self.outpath / ("silhouette_samples-" + self.name + ".feather")
60 self.silsampout = silsampout.resolve()
61 silhouette_samp.to_feather(self.silsampout)
64 self.silsampout = None
67 def read_distance_mat(self, similarities, use_threads=True):
69 df = pd.read_feather(similarities, use_threads=use_threads)
70 mat = np.array(df.drop('_subreddit',1))
72 mat[range(n),range(n)] = 1
73 return (df._subreddit,1-mat)
75 def process_clustering(self, clustering, subreddits):
77 if hasattr(clustering,'n_iter_'):
78 print(f"clustering took {clustering.n_iter_} iterations")
80 clusters = clustering.labels_
81 self.n_clusters = len(set(clusters))
83 print(f"found {self.n_clusters} clusters")
85 cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
87 cluster_sizes = cluster_data.groupby("cluster").count().reset_index()
88 print(f"the largest cluster has {cluster_sizes.loc[cluster_sizes.cluster!=-1].subreddit.max()} members")
90 print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
91 n_isolates1 = (cluster_sizes.subreddit==1).sum()
93 print(f"{n_isolates1} clusters have 1 member")
95 n_isolates2 = cluster_sizes.loc[cluster_sizes.cluster==-1,:]['subreddit'].to_list()
96 if len(n_isolates2) > 0:
97 n_isloates2 = n_isolates2[0]
98 print(f"{n_isolates2} subreddits are in cluster -1",flush=True)
101 self.n_isolates = n_isolates2
103 self.n_isolates = n_isolates1
107 class twoway_clustering_job(clustering_job):
108 def __init__(self, infile, outpath, name, call1, call2, args1, args2):
109 self.outpath = Path(outpath)
114 self.infile = Path(infile)
117 self.args = args1|args2
120 self.subreddits, self.mat = self.read_distance_mat(self.infile)
121 self.step1 = self.call1(self.mat, **self.args1)
122 self.clustering = self.call2(self.mat, self.step1, **self.args2)
123 self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
129 self.score = self.silhouette()
130 self.outpath.mkdir(parents=True, exist_ok=True)
131 print(self.outpath/(self.name+".feather"))
132 self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
140 class clustering_result:
142 silhouette_score:float
146 silhouette_samples:str