]> code.communitydata.science - cdsc_reddit.git/blob - clustering/clustering_base.py
ced627d2863eb9448126c625020fb7aba530b21c
[cdsc_reddit.git] / clustering / clustering_base.py
1 import pickle
2 from pathlib import Path
3 import numpy as np
4 import pandas as pd
5 from dataclasses import dataclass
6 from sklearn.metrics import silhouette_score, silhouette_samples
7 from collections import Counter
8
9 # this is meant to be an interface, not created directly
10 class clustering_job:
11     def __init__(self, infile, outpath, name, call, *args, **kwargs):
12         self.outpath = Path(outpath)
13         self.call = call
14         self.args = args
15         self.kwargs = kwargs
16         self.infile = Path(infile)
17         self.name = name
18         self.hasrun = False
19
20     def run(self):
21         self.subreddits, self.mat = self.read_distance_mat(self.infile)
22         self.clustering = self.call(self.mat, *self.args, **self.kwargs)
23         self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
24         self.score = self.silhouette()
25         self.outpath.mkdir(parents=True, exist_ok=True)
26         self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
27         self.hasrun = True
28         self.cleanup()
29
30     def cleanup(self):
31         self.cluster_data = None
32         self.mat = None
33         self.clustering=None
34         self.subreddits=None
35         
36     def get_info(self):
37         if not self.hasrun:
38             self.run()
39
40         self.result = clustering_result(outpath=str(self.outpath.resolve()),
41                                         silhouette_score=self.score,
42                                         name=self.name,
43                                         n_clusters=self.n_clusters,
44                                         n_isolates=self.n_isolates,
45                                         silhouette_samples = self.silsampout
46                                         )
47         return self.result
48
49     def silhouette(self):
50         counts = Counter(self.clustering.labels_)
51         singletons = [key for key, value in counts.items() if value == 1]
52         isolates = (self.clustering.labels_ == -1) | (np.isin(self.clustering.labels_,np.array(singletons)))
53         scoremat = self.mat[~isolates][:,~isolates]
54         if self.n_clusters > 1:
55             score = silhouette_score(scoremat, self.clustering.labels_[~isolates], metric='precomputed')
56             silhouette_samp = silhouette_samples(self.mat, self.clustering.labels_, metric='precomputed')
57             silhouette_samp = pd.DataFrame({'subreddit':self.subreddits,'score':silhouette_samp})
58             self.outpath.mkdir(parents=True, exist_ok=True)
59             silsampout = self.outpath / ("silhouette_samples-" + self.name +  ".feather")
60             self.silsampout = silsampout.resolve()
61             silhouette_samp.to_feather(self.silsampout)
62         else:
63             score = None
64             self.silsampout = None
65         return score
66
67     def read_distance_mat(self, similarities, use_threads=True):
68         print(similarities)
69         df = pd.read_feather(similarities, use_threads=use_threads)
70         mat = np.array(df.drop('_subreddit',1))
71         n = mat.shape[0]
72         mat[range(n),range(n)] = 1
73         return (df._subreddit,1-mat)
74
75     def process_clustering(self, clustering, subreddits):
76
77         if hasattr(clustering,'n_iter_'):
78             print(f"clustering took {clustering.n_iter_} iterations")
79
80         clusters = clustering.labels_
81         self.n_clusters = len(set(clusters))
82
83         print(f"found {self.n_clusters} clusters")
84
85         cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
86
87         cluster_sizes = cluster_data.groupby("cluster").count().reset_index()
88         print(f"the largest cluster has {cluster_sizes.loc[cluster_sizes.cluster!=-1].subreddit.max()} members")
89
90         print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
91         n_isolates1 = (cluster_sizes.subreddit==1).sum()
92
93         print(f"{n_isolates1} clusters have 1 member")
94
95         n_isolates2 = cluster_sizes.loc[cluster_sizes.cluster==-1,:]['subreddit'].to_list()
96         if len(n_isolates2) > 0:
97             n_isloates2 = n_isolates2[0]
98         print(f"{n_isolates2} subreddits are in cluster -1",flush=True)
99
100         if n_isolates1 == 0:
101             self.n_isolates = n_isolates2
102         else:
103             self.n_isolates = n_isolates1
104
105         return cluster_data
106
107 class twoway_clustering_job(clustering_job):
108     def __init__(self, infile, outpath, name, call1, call2, args1, args2):
109         self.outpath = Path(outpath)
110         self.call1 = call1
111         self.args1 = args1
112         self.call2 = call2
113         self.args2 = args2
114         self.infile = Path(infile)
115         self.name = name
116         self.hasrun = False
117         self.args = args1|args2
118
119     def run(self):
120         self.subreddits, self.mat = self.read_distance_mat(self.infile)
121         self.step1 = self.call1(self.mat, **self.args1)
122         self.clustering = self.call2(self.mat, self.step1, **self.args2)
123         self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
124         self.hasrun = True
125         self.after_run()
126         self.cleanup()
127
128     def after_run():
129         self.score = self.silhouette()
130         self.outpath.mkdir(parents=True, exist_ok=True)
131         print(self.outpath/(self.name+".feather"))
132         self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
133
134
135     def cleanup(self):
136         super().cleanup()
137         self.step1 = None
138
139 @dataclass
140 class clustering_result:
141     outpath:Path
142     silhouette_score:float
143     name:str
144     n_clusters:int
145     n_isolates:int
146     silhouette_samples:str

Community Data Science Collective || Want to submit a patch?