visualization/tsne_vis.py

   1 import pyarrow
   2 import altair as alt
   3 alt.data_transformers.disable_max_rows()
   4 alt.data_transformers.enable('default')
   5 from sklearn.neighbors import NearestNeighbors
   6 import pandas as pd
   7 from numpy import random
   8 import fire
   9 import numpy as np
  10
  11 def base_plot(plot_data):
  12
  13 #    base = base.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
  14
  15     cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
  16
  17     #    subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
  18
  19     cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
  20     # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
  21     # cluster_select_and = cluster_click_select & cluster_select
  22     #
  23     #    subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
  24
  25     base_scale = alt.Scale(scheme={"name":'category10',
  26                                    "extent":[0,100],
  27                                    "count":10})
  28
  29     color = alt.condition(cluster_click_select ,
  30                           alt.Color(field='color',type='nominal',scale=base_scale),
  31                           alt.value("lightgray"))
  32
  33
  34     base = alt.Chart(plot_data).mark_text().encode(
  35         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
  36         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
  37         color=color,
  38         text='subreddit')
  39
  40     base = base.add_selection(cluster_click_select)
  41
  42
  43     return base
  44
  45 def zoom_plot(plot_data):
  46     chart = base_plot(plot_data)
  47
  48     chart = chart.interactive()
  49     chart = chart.properties(width=1275,height=800)
  50
  51     return chart
  52
  53 def viewport_plot(plot_data):
  54     selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
  55     selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
  56     selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
  57
  58     base = base_plot(plot_data)
  59
  60     viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
  61         alt.X('x',axis=alt.Axis(grid=False)),
  62         alt.Y('y',axis=alt.Axis(grid=False)),
  63     )
  64
  65     viewport = viewport.properties(width=600,height=400)
  66
  67     viewport1 = viewport.add_selection(selector1)
  68
  69     viewport2 = viewport.encode(
  70         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
  71         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
  72     )
  73
  74     viewport2 = viewport2.add_selection(selectorx2)
  75     viewport2 = viewport2.add_selection(selectory2)
  76
  77     sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
  78                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
  79     )
  80
  81
  82     sr = sr.properties(width=1275,height=600)
  83
  84
  85     chart = (viewport1 | viewport2) & sr
  86
  87
  88     return chart
  89
  90 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
  91     isolate_color = 101
  92
  93     cluster_sizes = clusters.groupby('cluster').count()
  94     singletons = set(cluster_sizes.loc[cluster_sizes.subreddit == 1].reset_index().cluster)
  95
  96     tsne_data = tsne_data.merge(clusters,on='subreddit')
  97
  98     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
  99
 100     color_ids = np.arange(n_colors)
 101
 102     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
 103
 104     groups = tsne_data.groupby('cluster')
 105
 106     points = np.array(tsne_data.loc[:,['x','y']])
 107     centers = np.array(centroids.loc[:,['x','y']])
 108
 109     # point x centroid
 110     point_center_distances = np.linalg.norm((points[:,None,:] - centers[None,:,:]),axis=-1)
 111
 112     # distances is cluster x point
 113     for gid, group in groups:
 114         c_dists = point_center_distances[group.index.values,:].min(axis=0)
 115         distances[group.cluster.values[0],] = c_dists
 116
 117     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids)
 118     # distances, indices = nbrs.kneighbors()
 119
 120     nearest = distances.argpartition(n_neighbors,0)
 121     indices = nearest[:n_neighbors,:].T
 122     # neighbor_distances = np.copy(distances)
 123     # neighbor_distances.sort(0)
 124     # neighbor_distances = neighbor_distances[0:n_neighbors,:]
 125
 126     # nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances)
 127     # distances, indices = nbrs.kneighbors()
 128
 129     color_assignments = np.repeat(-1,len(centroids))
 130
 131     for i in range(len(centroids)):
 132         if (centroids.iloc[i].name == -1) or (i in singletons):
 133             color_assignments[i] = isolate_color
 134         else:
 135             knn = indices[i]
 136             knn_colors = color_assignments[knn]
 137             available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
 138
 139             if(len(available_colors) > 0):
 140                 color_assignments[i] = available_colors[0]
 141             else:
 142                 raise Exception("Can't color this many neighbors with this many colors")
 143
 144     centroids = centroids.reset_index()
 145     colors = centroids.loc[:,['cluster']]
 146     colors['color'] = color_assignments
 147
 148     tsne_data = tsne_data.merge(colors,on='cluster')
 149     return(tsne_data)
 150
 151 def build_visualization(tsne_data, clusters, output):
 152
 153     # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
 154     # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
 155
 156     tsne_data = pd.read_feather(tsne_data)
 157     tsne_data = tsne_data.rename(columns={'_subreddit':'subreddit'})
 158     clusters = pd.read_feather(clusters)
 159
 160     tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
 161
 162     sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
 163     sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
 164
 165     tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
 166
 167     term_zoom_plot = zoom_plot(tsne_data)
 168
 169     term_zoom_plot.save(output)
 170
 171     term_viewport_plot = viewport_plot(tsne_data)
 172
 173     term_viewport_plot.save(output.replace(".html","_viewport.html"))
 174
 175 if __name__ == "__main__":
 176     fire.Fire(build_visualization)
 177
 178 # commenter_data = pd.read_feather("tsne_author_fit.feather")
 179 # clusters = pd.read_feather('author_3000_clusters.feather')
 180 # commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
 181 # commenter_zoom_plot = zoom_plot(commenter_data)
 182 # commenter_viewport_plot = viewport_plot(commenter_data)
 183 # commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
 184 # commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
 185
 186 # chart = chart.properties(width=10000,height=10000)
 187 # chart.save("test_tsne_whole.svg")