visualization/tsne_vis.py

   1 import pyarrow
   2 import altair as alt
   3 alt.data_transformers.disable_max_rows()
   4 alt.data_transformers.enable('default')
   5 from sklearn.neighbors import NearestNeighbors
   6 import pandas as pd
   7 from numpy import random
   8 import numpy as np
   9
  10 def base_plot(plot_data):
  11     base = alt.Chart(plot_data).mark_text().encode(
  12         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
  13         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
  14         text='subreddit')
  15
  16     return base
  17
  18 def zoom_plot(plot_data):
  19     chart = base_plot(plot_data)
  20     chart = chart.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
  21     chart = chart.interactive()
  22     chart = chart.properties(width=1275,height=1000)
  23
  24     return chart
  25
  26 def viewport_plot(plot_data):
  27     selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
  28     selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
  29     selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
  30
  31     base = base_plot(plot_data)
  32
  33     viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
  34         alt.X('x',axis=alt.Axis(grid=False)),
  35         alt.Y('y',axis=alt.Axis(grid=False)),
  36     )
  37
  38     viewport = viewport.properties(width=600,height=400)
  39
  40     viewport1 = viewport.add_selection(selector1)
  41
  42     viewport2 = viewport.encode(
  43         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
  44         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
  45     )
  46
  47     viewport2 = viewport2.add_selection(selectorx2)
  48     viewport2 = viewport2.add_selection(selectory2)
  49
  50     sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
  51                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
  52     )
  53
  54     sr = sr.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
  55     sr = sr.properties(width=1275,height=600)
  56
  57
  58     chart = (viewport1 | viewport2) & sr
  59
  60
  61     return chart
  62
  63 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
  64     tsne_data = tsne_data.merge(clusters,on='subreddit')
  65
  66     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
  67
  68     color_ids = np.arange(n_colors)
  69
  70     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
  71
  72     groups = tsne_data.groupby('cluster')
  73     for centroid in centroids.itertuples():
  74         c_dists = groups.apply(lambda r: min(np.sqrt(np.square(centroid.x - r.x) + np.square(centroid.y-r.y))))
  75         distances[:,centroid.Index] = c_dists
  76
  77     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids)
  78     # distances, indices = nbrs.kneighbors()
  79
  80     nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances)
  81     distances, indices = nbrs.kneighbors()
  82
  83     color_assignments = np.repeat(-1,len(centroids))
  84
  85     for i in range(len(centroids)):
  86         knn = indices[i]
  87         knn_colors = color_assignments[knn]
  88         available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
  89
  90         if(len(available_colors) > 0):
  91             color_assignments[i] = available_colors[0]
  92         else:
  93             raise Exception("Can't color this many neighbors with this many colors")
  94
  95
  96     centroids = centroids.reset_index()
  97     colors = centroids.loc[:,['cluster']]
  98     colors['color'] = color_assignments
  99
 100     tsne_data = tsne_data.merge(colors,on='cluster')
 101     return(tsne_data)
 102
 103 term_data = pd.read_feather("tsne_subreddit_fit.feather")
 104 clusters = pd.read_feather("term_3000_clusters.feather")
 105
 106 tsne_data = assign_cluster_colors(term_data,clusters,10,8)
 107
 108 term_zoom_plot = zoom_plot(tsne_data)
 109
 110 term_zoom_plot.save("subreddit_terms_tsne_3000.html")
 111
 112 term_viewport_plot = viewport_plot(tsne_data)
 113
 114 term_viewport_plot.save("subreddit_terms_tsne_3000_viewport.html")
 115
 116 commenter_data = pd.read_feather("tsne_author_fit.feather")
 117 clusters = pd.read_feather('author_3000_clusters.feather')
 118 commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
 119 commenter_zoom_plot = zoom_plot(commenter_data)
 120 commenter_viewport_plot = viewport_plot(commenter_data)
 121 commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
 122 commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
 123
 124 # chart = chart.properties(width=10000,height=10000)
 125 # chart.save("test_tsne_whole.svg")