clustering/hdbscan_clustering_lsi.py

   1 from hdbscan_clustering import hdbscan_job, hdbscan_grid_sweep, hdbscan_clustering_result
   2 from lsi_base import lsi_grid_sweep, lsi_mixin, lsi_result_mixin
   3 from grid_sweep import grid_sweep
   4 import fire
   5 from dataclasses import dataclass
   6
   7 @dataclass
   8 class hdbscan_clustering_result_lsi(hdbscan_clustering_result, lsi_result_mixin):
   9     pass
  10
  11 class hdbscan_lsi_job(hdbscan_job, lsi_mixin):
  12     def __init__(self, infile, outpath, name, lsi_dims, *args, **kwargs):
  13         super().__init__(
  14                          infile,
  15                          outpath,
  16                          name,
  17                          *args,
  18                          **kwargs)
  19         super().set_lsi_dims(lsi_dims)
  20
  21     def get_info(self):
  22         partial_result = super().get_info()
  23         self.result = hdbscan_clustering_result_lsi(**partial_result.__dict__,
  24                                                     lsi_dimensions=self.lsi_dims)
  25         return self.result
  26
  27 class hdbscan_lsi_grid_sweep(lsi_grid_sweep):
  28     def __init__(self,
  29                  inpath,
  30                  lsi_dims,
  31                  outpath,
  32                  min_cluster_sizes,
  33                  min_samples,
  34                  cluster_selection_epsilons,
  35                  cluster_selection_methods
  36                  ):
  37
  38         super().__init__(hdbscan_lsi_job,
  39                          _hdbscan_lsi_grid_sweep,
  40                          inpath,
  41                          lsi_dims,
  42                          outpath,
  43                          min_cluster_sizes,
  44                          min_samples,
  45                          cluster_selection_epsilons,
  46                          cluster_selection_methods)
  47
  48
  49
  50 class _hdbscan_lsi_grid_sweep(grid_sweep):
  51     def __init__(self,
  52                  inpath,
  53                  outpath,
  54                  lsi_dim,
  55                  *args,
  56                  **kwargs):
  57         print(args)
  58         print(kwargs)
  59
  60         self.lsi_dim = lsi_dim
  61         self.jobtype = hdbscan_lsi_job
  62         super().__init__(self.jobtype, inpath, outpath, self.namer, self.lsi_dim, *args, **kwargs)
  63
  64
  65     def namer(self, *args, **kwargs):
  66         s = hdbscan_grid_sweep.namer(self, *args[1:], **kwargs)
  67         s += f"_lsi-{self.lsi_dim}"
  68         return s
  69
  70 def run_hdbscan_lsi_grid_sweep(savefile, inpath, outpath,  min_cluster_sizes=[2], min_samples=[1], cluster_selection_epsilons=[0], cluster_selection_methods=['eom'],lsi_dimensions='all'):
  71     """Run hdbscan clustering once or more with different parameters.
  72
  73     Usage:
  74     hdbscan_clustering_lsi --savefile=SAVEFILE --inpath=INPATH --outpath=OUTPATH --min_cluster_sizes=<csv> --min_samples=<csv> --cluster_selection_epsilons=<csv> --cluster_selection_methods=[eom]> --lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH.
  75
  76     Keword arguments:
  77     savefile: path to save the metadata and diagnostics
  78     inpath: path to folder containing feather files with LSI similarity labeled matrices of subreddit similarities.
  79     outpath: path to output fit clusterings.
  80     min_cluster_sizes: one or more integers indicating the minumum cluster size
  81     min_samples: one ore more integers indicating the minimum number of samples used in the algorithm
  82     cluster_selection_epsilons: one or more similarity thresholds for transition from dbscan to hdbscan
  83     cluster_selection_methods: one or more of "eom" or "leaf" eom gives larger clusters.
  84     lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH.
  85     """
  86
  87     obj = hdbscan_lsi_grid_sweep(inpath,
  88                                  lsi_dimensions,
  89                                  outpath,
  90                                  map(int,min_cluster_sizes),
  91                                  map(int,min_samples),
  92                                  map(float,cluster_selection_epsilons),
  93                                  cluster_selection_methods
  94                                  )
  95
  96     obj.run(10)
  97     obj.save(savefile)
  98
  99
 100 if __name__ == "__main__":
 101     fire.Fire(run_hdbscan_lsi_grid_sweep)