clustering/clustering_base.py

   1 import pickle
   2 from pathlib import Path
   3 import numpy as np
   4 import pandas as pd
   5 from dataclasses import dataclass
   6 from sklearn.metrics import silhouette_score, silhouette_samples
   7 from collections import Counter
   8
   9 # this is meant to be an interface, not created directly
  10 class clustering_job:
  11     def __init__(self, infile, outpath, name, call, *args, **kwargs):
  12         self.outpath = Path(outpath)
  13         self.call = call
  14         self.args = args
  15         self.kwargs = kwargs
  16         self.infile = Path(infile)
  17         self.name = name
  18         self.hasrun = False
  19
  20     def run(self):
  21         self.subreddits, self.mat = self.read_distance_mat(self.infile)
  22         self.clustering = self.call(self.mat, *self.args, **self.kwargs)
  23         self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
  24         self.score = self.silhouette()
  25         self.outpath.mkdir(parents=True, exist_ok=True)
  26         self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
  27         self.hasrun = True
  28         self.cleanup()
  29
  30     def cleanup(self):
  31         self.cluster_data = None
  32         self.mat = None
  33         self.clustering=None
  34         self.subreddits=None
  35
  36     def get_info(self):
  37         if not self.hasrun:
  38             self.run()
  39
  40         self.result = clustering_result(outpath=str(self.outpath.resolve()),
  41                                         silhouette_score=self.score,
  42                                         name=self.name,
  43                                         n_clusters=self.n_clusters,
  44                                         n_isolates=self.n_isolates,
  45                                         silhouette_samples = self.silsampout
  46                                         )
  47         return self.result
  48
  49     def silhouette(self):
  50         counts = Counter(self.clustering.labels_)
  51         singletons = [key for key, value in counts.items() if value == 1]
  52         isolates = (self.clustering.labels_ == -1) | (np.isin(self.clustering.labels_,np.array(singletons)))
  53         scoremat = self.mat[~isolates][:,~isolates]
  54         if self.n_clusters > 1:
  55             score = silhouette_score(scoremat, self.clustering.labels_[~isolates], metric='precomputed')
  56             silhouette_samp = silhouette_samples(self.mat, self.clustering.labels_, metric='precomputed')
  57             silhouette_samp = pd.DataFrame({'subreddit':self.subreddits,'score':silhouette_samp})
  58             self.outpath.mkdir(parents=True, exist_ok=True)
  59             silsampout = self.outpath / ("silhouette_samples-" + self.name +  ".feather")
  60             self.silsampout = silsampout.resolve()
  61             silhouette_samp.to_feather(self.silsampout)
  62         else:
  63             score = None
  64             self.silsampout = None
  65         return score
  66
  67     def read_distance_mat(self, similarities, use_threads=True):
  68         print(similarities)
  69         df = pd.read_feather(similarities, use_threads=use_threads)
  70         mat = np.array(df.drop('_subreddit',1))
  71         n = mat.shape[0]
  72         mat[range(n),range(n)] = 1
  73         return (df._subreddit,1-mat)
  74
  75     def process_clustering(self, clustering, subreddits):
  76
  77         if hasattr(clustering,'n_iter_'):
  78             print(f"clustering took {clustering.n_iter_} iterations")
  79
  80         clusters = clustering.labels_
  81         self.n_clusters = len(set(clusters))
  82
  83         print(f"found {self.n_clusters} clusters")
  84
  85         cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
  86
  87         cluster_sizes = cluster_data.groupby("cluster").count().reset_index()
  88         print(f"the largest cluster has {cluster_sizes.loc[cluster_sizes.cluster!=-1].subreddit.max()} members")
  89
  90         print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
  91         n_isolates1 = (cluster_sizes.subreddit==1).sum()
  92
  93         print(f"{n_isolates1} clusters have 1 member")
  94
  95         n_isolates2 = cluster_sizes.loc[cluster_sizes.cluster==-1,:]['subreddit'].to_list()
  96         if len(n_isolates2) > 0:
  97             n_isloates2 = n_isolates2[0]
  98         print(f"{n_isolates2} subreddits are in cluster -1",flush=True)
  99
 100         if n_isolates1 == 0:
 101             self.n_isolates = n_isolates2
 102         else:
 103             self.n_isolates = n_isolates1
 104
 105         return cluster_data
 106
 107 class twoway_clustering_job(clustering_job):
 108     def __init__(self, infile, outpath, name, call1, call2, args1, args2):
 109         self.outpath = Path(outpath)
 110         self.call1 = call1
 111         self.args1 = args1
 112         self.call2 = call2
 113         self.args2 = args2
 114         self.infile = Path(infile)
 115         self.name = name
 116         self.hasrun = False
 117         self.args = args1|args2
 118
 119     def run(self):
 120         self.subreddits, self.mat = self.read_distance_mat(self.infile)
 121         self.step1 = self.call1(self.mat, **self.args1)
 122         self.clustering = self.call2(self.mat, self.step1, **self.args2)
 123         self.cluster_data = self.process_clustering(self.clustering, self.subreddits)
 124         self.hasrun = True
 125         self.after_run()
 126         self.cleanup()
 127
 128     def after_run():
 129         self.score = self.silhouette()
 130         self.outpath.mkdir(parents=True, exist_ok=True)
 131         print(self.outpath/(self.name+".feather"))
 132         self.cluster_data.to_feather(self.outpath/(self.name + ".feather"))
 133
 134
 135     def cleanup(self):
 136         super().cleanup()
 137         self.step1 = None
 138
 139 @dataclass
 140 class clustering_result:
 141     outpath:Path
 142     silhouette_score:float
 143     name:str
 144     n_clusters:int
 145     n_isolates:int
 146     silhouette_samples:str