clustering/umap_hdbscan_clustering_lsi.py

   1 from umap_hdbscan_clustering import umap_hdbscan_job, umap_hdbscan_grid_sweep, umap_hdbscan_clustering_result
   2 from lsi_base import twoway_lsi_grid_sweep, lsi_mixin, lsi_result_mixin
   3 from grid_sweep import twoway_grid_sweep
   4 import fire
   5 from dataclasses import dataclass
   6
   7 @dataclass
   8 class umap_hdbscan_clustering_result_lsi(umap_hdbscan_clustering_result, lsi_result_mixin):
   9     pass
  10
  11 class umap_hdbscan_lsi_job(umap_hdbscan_job, lsi_mixin):
  12     def __init__(self, infile, outpath, name, umap_args, hdbscan_args, lsi_dims):
  13         super().__init__(
  14             infile,
  15             outpath,
  16             name,
  17             umap_args,
  18             hdbscan_args
  19         )
  20         super().set_lsi_dims(lsi_dims)
  21
  22     def get_info(self):
  23         partial_result = super().get_info()
  24         self.result = umap_hdbscan_clustering_result_lsi(**partial_result.__dict__,
  25                                                          lsi_dimensions=self.lsi_dims)
  26         return self.result
  27
  28 class umap_hdbscan_lsi_grid_sweep(twoway_lsi_grid_sweep):
  29     def __init__(self,
  30                  inpath,
  31                  lsi_dims,
  32                  outpath,
  33                  umap_args,
  34                  hdbscan_args
  35                  ):
  36
  37         super().__init__(umap_hdbscan_lsi_job,
  38                          _umap_hdbscan_lsi_grid_sweep,
  39                          inpath,
  40                          lsi_dims,
  41                          outpath,
  42                          umap_args,
  43                          hdbscan_args
  44                          )
  45
  46
  47
  48 class _umap_hdbscan_lsi_grid_sweep(twoway_grid_sweep):
  49     def __init__(self,
  50                  inpath,
  51                  outpath,
  52                  lsi_dim,
  53                  umap_args,
  54                  hdbscan_args,
  55                  ):
  56
  57         self.lsi_dim = lsi_dim
  58         self.jobtype = umap_hdbscan_lsi_job
  59         super().__init__(self.jobtype, inpath, outpath, self.namer, umap_args, hdbscan_args, lsi_dim)
  60
  61
  62     def namer(self, *args, **kwargs):
  63         s = umap_hdbscan_grid_sweep.namer(self, *args, **kwargs)
  64         s += f"_lsi-{self.lsi_dim}"
  65         return s
  66
  67 def run_umap_hdbscan_lsi_grid_sweep(savefile, inpath, outpath, n_neighbors = [15], n_components=[2], learning_rate=[1], min_dist=[1], local_connectivity=[1],
  68                                 densmap=[False],
  69                                     min_cluster_sizes=[2], min_samples=[1], cluster_selection_epsilons=[0], cluster_selection_methods=['eom'], lsi_dimensions='all'):
  70     """Run hdbscan clustering once or more with different parameters.
  71
  72     Usage:
  73     hdbscan_clustering_lsi --savefile=SAVEFILE --inpath=INPATH --outpath=OUTPATH --min_cluster_sizes=<csv> --min_samples=<csv> --cluster_selection_epsilons=<csv> --cluster_selection_methods=[eom]> --lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH.
  74
  75     Keword arguments:
  76     savefile: path to save the metadata and diagnostics
  77     inpath: path to folder containing feather files with LSI similarity labeled matrices of subreddit similarities.
  78     outpath: path to output fit clusterings.
  79     min_cluster_sizes: one or more integers indicating the minumum cluster size
  80     min_samples: one ore more integers indicating the minimum number of samples used in the algorithm
  81     cluster_selection_epsilons: one or more similarity thresholds for transition from dbscan to hdbscan
  82     cluster_selection_methods: one or more of "eom" or "leaf" eom gives larger clusters.
  83     lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH.
  84     """
  85
  86
  87     umap_args = {'n_neighbors':list(map(int, n_neighbors)),
  88                  'learning_rate':list(map(float,learning_rate)),
  89                  'min_dist':list(map(float,min_dist)),
  90                  'local_connectivity':list(map(int,local_connectivity)),
  91                  'n_components':list(map(int, n_components)),
  92                  'densmap':list(map(bool,densmap))
  93                  }
  94
  95     hdbscan_args = {'min_cluster_size':list(map(int,min_cluster_sizes)),
  96                     'min_samples':list(map(int,min_samples)),
  97                     'cluster_selection_epsilon':list(map(float,cluster_selection_epsilons)),
  98                     'cluster_selection_method':cluster_selection_methods}
  99
 100     obj = umap_hdbscan_lsi_grid_sweep(inpath,
 101                                       lsi_dimensions,
 102                                       outpath,
 103                                       umap_args,
 104                                       hdbscan_args
 105                                       )
 106
 107
 108     obj.run(10)
 109     obj.save(savefile)
 110
 111
 112 if __name__ == "__main__":
 113     fire.Fire(run_umap_hdbscan_lsi_grid_sweep)