]> code.communitydata.science - cdsc_reddit.git/blob - visualization/tsne_vis.py
915cd7e9019a095855b5c3bc512cee80acbef49c
[cdsc_reddit.git] / visualization / tsne_vis.py
1 import pyarrow
2 import altair as alt
3 alt.data_transformers.disable_max_rows()
4 alt.data_transformers.enable('default')
5 from sklearn.neighbors import NearestNeighbors
6 import pandas as pd
7 from numpy import random
8 import numpy as np
9
10 def base_plot(plot_data):
11     base = alt.Chart(plot_data).mark_text().encode(
12         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
13         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
14         text='subreddit')
15
16     return base
17
18 def zoom_plot(plot_data):
19     chart = base_plot(plot_data)
20     chart = chart.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
21     chart = chart.interactive()
22     chart = chart.properties(width=1275,height=1000)
23
24     return chart
25
26 def viewport_plot(plot_data):
27     selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
28     selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
29     selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
30
31     base = base_plot(plot_data)
32
33     viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
34         alt.X('x',axis=alt.Axis(grid=False)),
35         alt.Y('y',axis=alt.Axis(grid=False)),
36     )
37
38     viewport = viewport.properties(width=600,height=400)
39
40     viewport1 = viewport.add_selection(selector1)
41
42     viewport2 = viewport.encode(
43         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
44         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
45     )
46
47     viewport2 = viewport2.add_selection(selectorx2)
48     viewport2 = viewport2.add_selection(selectory2)
49
50     sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
51                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
52     )
53
54     sr = sr.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
55     sr = sr.properties(width=1275,height=600)
56
57
58     chart = (viewport1 | viewport2) & sr
59
60
61     return chart
62
63 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
64     tsne_data = tsne_data.merge(clusters,on='subreddit')
65     
66     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
67
68     color_ids = np.arange(n_colors)
69
70     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
71
72     groups = tsne_data.groupby('cluster')
73     for centroid in centroids.itertuples():
74         c_dists = groups.apply(lambda r: min(np.sqrt(np.square(centroid.x - r.x) + np.square(centroid.y-r.y))))
75         distances[:,centroid.Index] = c_dists
76
77     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids) 
78     # distances, indices = nbrs.kneighbors()
79
80     nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances) 
81     distances, indices = nbrs.kneighbors()
82
83     color_assignments = np.repeat(-1,len(centroids))
84
85     for i in range(len(centroids)):
86         knn = indices[i]
87         knn_colors = color_assignments[knn]
88         available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
89
90         if(len(available_colors) > 0):
91             color_assignments[i] = available_colors[0]
92         else:
93             raise Exception("Can't color this many neighbors with this many colors")
94
95
96     centroids = centroids.reset_index()
97     colors = centroids.loc[:,['cluster']]
98     colors['color'] = color_assignments
99
100     tsne_data = tsne_data.merge(colors,on='cluster')
101     return(tsne_data)
102
103 term_data = pd.read_feather("tsne_subreddit_fit.feather")
104 clusters = pd.read_feather("term_3000_clusters.feather")
105
106 tsne_data = assign_cluster_colors(term_data,clusters,10,8)
107
108 term_zoom_plot = zoom_plot(tsne_data)
109
110 term_zoom_plot.save("subreddit_terms_tsne_3000.html")
111
112 term_viewport_plot = viewport_plot(tsne_data)
113
114 term_viewport_plot.save("subreddit_terms_tsne_3000_viewport.html")
115
116 commenter_data = pd.read_feather("tsne_author_fit.feather")
117 clusters = pd.read_feather('author_3000_clusters.feather')
118 commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
119 commenter_zoom_plot = zoom_plot(commenter_data)
120 commenter_viewport_plot = viewport_plot(commenter_data)
121 commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
122 commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
123
124 # chart = chart.properties(width=10000,height=10000)
125 # chart.save("test_tsne_whole.svg")

Community Data Science Collective || Want to submit a patch?