]> code.communitydata.science - cdsc_reddit.git/blob - clustering/clustering_base.py
lsi support for weekly similarities
[cdsc_reddit.git] / clustering / clustering_base.py
1 from pathlib import Path
2 import numpy as np
3 import pandas as pd
4 from dataclasses import dataclass
5 from sklearn.metrics import silhouette_score, silhouette_samples
6 from collections import Counter
7
8 # this is meant to be an interface, not created directly
9 class clustering_job:
10     def __init__(self, infile, outpath, name, call, *args, **kwargs):
11         self.outpath = Path(outpath)
12         self.call = call
13         self.args = args
14         self.kwargs = kwargs
15         self.infile = Path(infile)
16         self.name = name
17         self.hasrun = False
18
19     def run(self):
20         self.subreddits, self.mat = self.read_distance_mat(self.infile)
21         self.clustering = self.call(self.mat, *self.args, **self.kwargs)
22         self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
23         self.score = self.silhouette()
24         self.outpath.mkdir(parents=True, exist_ok=True)
25         self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
26         self.hasrun = True
27         
28     def get_info(self):
29         if not self.hasrun:
30             self.run()
31
32         self.result = clustering_result(outpath=str(self.outpath.resolve()),
33                                         silhouette_score=self.score,
34                                         name=self.name,
35                                         n_clusters=self.n_clusters,
36                                         n_isolates=self.n_isolates,
37                                         silhouette_samples = self.silsampout
38                                         )
39         return self.result
40
41     def silhouette(self):
42         counts = Counter(self.clustering.labels_)
43         singletons = [key for key, value in counts.items() if value == 1]
44         isolates = (self.clustering.labels_ == -1) | (np.isin(self.clustering.labels_,np.array(singletons)))
45         scoremat = self.mat[~isolates][:,~isolates]
46         if self.n_clusters > 1:
47             score = silhouette_score(scoremat, self.clustering.labels_[~isolates], metric='precomputed')
48             silhouette_samp = silhouette_samples(self.mat, self.clustering.labels_, metric='precomputed')
49             silhouette_samp = pd.DataFrame({'subreddit':self.subreddits,'score':silhouette_samp})
50             self.outpath.mkdir(parents=True, exist_ok=True)
51             silsampout = self.outpath / ("silhouette_samples-" + self.name +  ".feather")
52             self.silsampout = silsampout.resolve()
53             silhouette_samp.to_feather(self.silsampout)
54         else:
55             score = None
56             self.silsampout = None
57         return score
58
59     def read_distance_mat(self, similarities, use_threads=True):
60         df = pd.read_feather(similarities, use_threads=use_threads)
61         mat = np.array(df.drop('_subreddit',1))
62         n = mat.shape[0]
63         mat[range(n),range(n)] = 1
64         return (df._subreddit,1-mat)
65
66     def process_clustering(self, clustering, subreddits):
67
68         if hasattr(clustering,'n_iter_'):
69             print(f"clustering took {clustering.n_iter_} iterations")
70
71         clusters = clustering.labels_
72         self.n_clusters = len(set(clusters))
73
74         print(f"found {self.n_clusters} clusters")
75
76         cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
77
78         cluster_sizes = cluster_data.groupby("cluster").count().reset_index()
79         print(f"the largest cluster has {cluster_sizes.loc[cluster_sizes.cluster!=-1].subreddit.max()} members")
80
81         print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
82         n_isolates1 = (cluster_sizes.subreddit==1).sum()
83
84         print(f"{n_isolates1} clusters have 1 member")
85
86         n_isolates2 = cluster_sizes.loc[cluster_sizes.cluster==-1,:]['subreddit'].to_list()
87         if len(n_isolates2) > 0:
88             n_isloates2 = n_isolates2[0]
89         print(f"{n_isolates2} subreddits are in cluster -1",flush=True)
90
91         if n_isolates1 == 0:
92             self.n_isolates = n_isolates2
93         else:
94             self.n_isolates = n_isolates1
95
96         return cluster_data
97
98 @dataclass
99 class clustering_result:
100     outpath:Path
101     silhouette_score:float
102     name:str
103     n_clusters:int
104     n_isolates:int
105     silhouette_samples:str

Community Data Science Collective || Want to submit a patch?