3 alt.data_transformers.disable_max_rows()
4 alt.data_transformers.enable('default')
5 from sklearn.neighbors import NearestNeighbors
7 from numpy import random
11 def base_plot(plot_data):
13 # base = base.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
15 cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
17 # subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
19 cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
20 # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
21 # cluster_select_and = cluster_click_select & cluster_select
23 # subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
25 base_scale = alt.Scale(scheme={"name":'category10',
29 color = alt.condition(cluster_click_select ,
30 alt.Color(field='color',type='nominal',scale=base_scale),
31 alt.value("lightgray"))
34 base = alt.Chart(plot_data).mark_text().encode(
35 alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
36 alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
40 base = base.add_selection(cluster_click_select)
45 def zoom_plot(plot_data):
46 chart = base_plot(plot_data)
48 chart = chart.interactive()
49 chart = chart.properties(width=1275,height=800)
53 def viewport_plot(plot_data):
54 selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
55 selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
56 selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
58 base = base_plot(plot_data)
60 viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
61 alt.X('x',axis=alt.Axis(grid=False)),
62 alt.Y('y',axis=alt.Axis(grid=False)),
65 viewport = viewport.properties(width=600,height=400)
67 viewport1 = viewport.add_selection(selector1)
69 viewport2 = viewport.encode(
70 alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
71 alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
74 viewport2 = viewport2.add_selection(selectorx2)
75 viewport2 = viewport2.add_selection(selectory2)
77 sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
78 alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
82 sr = sr.properties(width=1275,height=600)
85 chart = (viewport1 | viewport2) & sr
90 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
93 cluster_sizes = clusters.groupby('cluster').count()
94 singletons = set(cluster_sizes.loc[cluster_sizes.subreddit == 1].reset_index().cluster)
96 tsne_data = tsne_data.merge(clusters,on='subreddit')
98 centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
100 color_ids = np.arange(n_colors)
102 distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
104 groups = tsne_data.groupby('cluster')
106 points = np.array(tsne_data.loc[:,['x','y']])
107 centers = np.array(centroids.loc[:,['x','y']])
110 point_center_distances = np.linalg.norm((points[:,None,:] - centers[None,:,:]),axis=-1)
112 # distances is cluster x point
113 for gid, group in groups:
114 c_dists = point_center_distances[group.index.values,:].min(axis=0)
115 distances[group.cluster.values[0],] = c_dists
117 # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids)
118 # distances, indices = nbrs.kneighbors()
120 nearest = distances.argpartition(n_neighbors,0)
121 indices = nearest[:n_neighbors,:].T
122 # neighbor_distances = np.copy(distances)
123 # neighbor_distances.sort(0)
124 # neighbor_distances = neighbor_distances[0:n_neighbors,:]
126 # nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances)
127 # distances, indices = nbrs.kneighbors()
129 color_assignments = np.repeat(-1,len(centroids))
131 for i in range(len(centroids)):
132 if (centroids.iloc[i].name == -1) or (i in singletons):
133 color_assignments[i] = isolate_color
136 knn_colors = color_assignments[knn]
137 available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
139 if(len(available_colors) > 0):
140 color_assignments[i] = available_colors[0]
142 raise Exception("Can't color this many neighbors with this many colors")
144 centroids = centroids.reset_index()
145 colors = centroids.loc[:,['cluster']]
146 colors['color'] = color_assignments
148 tsne_data = tsne_data.merge(colors,on='cluster')
151 def build_visualization(tsne_data, clusters, output):
153 # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
154 # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
156 tsne_data = pd.read_feather(tsne_data)
157 tsne_data = tsne_data.rename(columns={'_subreddit':'subreddit'})
158 clusters = pd.read_feather(clusters)
160 tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
162 sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
163 sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
165 tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
167 term_zoom_plot = zoom_plot(tsne_data)
169 term_zoom_plot.save(output)
171 term_viewport_plot = viewport_plot(tsne_data)
173 term_viewport_plot.save(output.replace(".html","_viewport.html"))
175 if __name__ == "__main__":
176 fire.Fire(build_visualization)
178 # commenter_data = pd.read_feather("tsne_author_fit.feather")
179 # clusters = pd.read_feather('author_3000_clusters.feather')
180 # commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
181 # commenter_zoom_plot = zoom_plot(commenter_data)
182 # commenter_viewport_plot = viewport_plot(commenter_data)
183 # commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
184 # commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
186 # chart = chart.properties(width=10000,height=10000)
187 # chart.save("test_tsne_whole.svg")