]> code.communitydata.science - cdsc_reddit.git/blob - visualization/tsne_vis.py
add note to try other tf normalization strategies.
[cdsc_reddit.git] / visualization / tsne_vis.py
1 import pyarrow
2 import altair as alt
3 alt.data_transformers.disable_max_rows()
4 alt.data_transformers.enable('default')
5 from sklearn.neighbors import NearestNeighbors
6 import pandas as pd
7 from numpy import random
8 import fire
9 import numpy as np
10
11 def base_plot(plot_data):
12
13 #    base = base.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
14
15     cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
16
17     #    subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
18
19     cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
20     # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
21     # cluster_select_and = cluster_click_select & cluster_select
22     #
23     #    subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
24     
25     color = alt.condition(cluster_click_select ,
26                           alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')),
27                           alt.value("lightgray"))
28   
29     
30     base = alt.Chart(plot_data).mark_text().encode(
31         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
32         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
33         color=color,
34         text='subreddit')
35
36     base = base.add_selection(cluster_click_select)
37  
38
39     return base
40
41 def zoom_plot(plot_data):
42     chart = base_plot(plot_data)
43
44     chart = chart.interactive()
45     chart = chart.properties(width=1275,height=800)
46
47     return chart
48
49 def viewport_plot(plot_data):
50     selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
51     selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
52     selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
53
54     base = base_plot(plot_data)
55
56     viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
57         alt.X('x',axis=alt.Axis(grid=False)),
58         alt.Y('y',axis=alt.Axis(grid=False)),
59     )
60    
61     viewport = viewport.properties(width=600,height=400)
62
63     viewport1 = viewport.add_selection(selector1)
64
65     viewport2 = viewport.encode(
66         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
67         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
68     )
69
70     viewport2 = viewport2.add_selection(selectorx2)
71     viewport2 = viewport2.add_selection(selectory2)
72
73     sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
74                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
75     )
76
77
78     sr = sr.properties(width=1275,height=600)
79
80
81     chart = (viewport1 | viewport2) & sr
82
83
84     return chart
85
86 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
87     tsne_data = tsne_data.merge(clusters,on='subreddit')
88     
89     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
90
91     color_ids = np.arange(n_colors)
92
93     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
94
95     groups = tsne_data.groupby('cluster')
96     
97     points = np.array(tsne_data.loc[:,['x','y']])
98     centers = np.array(centroids.loc[:,['x','y']])
99
100     # point x centroid
101     point_center_distances = np.linalg.norm((points[:,None,:] - centers[None,:,:]),axis=-1)
102     
103     # distances is cluster x point
104     for gid, group in groups:
105         c_dists = point_center_distances[group.index.values,:].min(axis=0)
106         distances[group.cluster.values[0],] = c_dists        
107
108     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids) 
109     # distances, indices = nbrs.kneighbors()
110
111     nearest = distances.argpartition(n_neighbors,0)
112     indices = nearest[:n_neighbors,:].T
113     # neighbor_distances = np.copy(distances)
114     # neighbor_distances.sort(0)
115     # neighbor_distances = neighbor_distances[0:n_neighbors,:]
116     
117     # nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances) 
118     # distances, indices = nbrs.kneighbors()
119
120     color_assignments = np.repeat(-1,len(centroids))
121
122     for i in range(len(centroids)):
123         knn = indices[i]
124         knn_colors = color_assignments[knn]
125         available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
126
127         if(len(available_colors) > 0):
128             color_assignments[i] = available_colors[0]
129         else:
130             raise Exception("Can't color this many neighbors with this many colors")
131
132
133     centroids = centroids.reset_index()
134     colors = centroids.loc[:,['cluster']]
135     colors['color'] = color_assignments
136
137     tsne_data = tsne_data.merge(colors,on='cluster')
138     return(tsne_data)
139
140 def build_visualization(tsne_data, clusters, output):
141
142     # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
143     # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
144
145     tsne_data = pd.read_feather(tsne_data)
146     clusters = pd.read_feather(clusters)
147
148     tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
149
150     # sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
151     # sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
152
153     tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
154
155     term_zoom_plot = zoom_plot(tsne_data)
156
157     term_zoom_plot.save(output)
158
159     term_viewport_plot = viewport_plot(tsne_data)
160
161     term_viewport_plot.save(output.replace(".html","_viewport.html"))
162
163 if __name__ == "__main__":
164     fire.Fire(build_visualization)
165
166 # commenter_data = pd.read_feather("tsne_author_fit.feather")
167 # clusters = pd.read_feather('author_3000_clusters.feather')
168 # commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
169 # commenter_zoom_plot = zoom_plot(commenter_data)
170 # commenter_viewport_plot = viewport_plot(commenter_data)
171 # commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
172 # commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
173
174 # chart = chart.properties(width=10000,height=10000)
175 # chart.save("test_tsne_whole.svg")

Community Data Science Collective || Want to submit a patch?