import pyarrow
import altair as alt
alt.data_transformers.disable_max_rows()
-alt.data_transformers.enable('data_server')
+alt.data_transformers.enable('default')
+from sklearn.neighbors import NearestNeighbors
import pandas as pd
from numpy import random
+import fire
import numpy as np
-from sklearn.manifold import TSNE
-df = pd.read_csv("reddit_term_similarity_3000.csv")
-df = df.sort_values(['i','j'])
+def base_plot(plot_data):
-n = max(df.i.max(),df.j.max())
+# base = base.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
-def zero_pad(grp):
- p = grp.shape[0]
- grp = grp.sort_values('j')
- return np.concatenate([np.zeros(n-p),np.zeros(1),np.array(grp.value)])
+ cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
-col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
-first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
-col_names = [first_name] + list(col_names.subreddit_j)
-mat = df.groupby('i').apply(zero_pad)
-mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
-mat = np.stack(mat)
+ # subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
-# plot the matrix using the first and second eigenvalues
-mat = mat + np.tril(mat.transpose(),k=-1)
+ cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
+ # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
+ # cluster_select_and = cluster_click_select & cluster_select
+ #
+ # subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
+
+ color = alt.condition(cluster_click_select ,
+ alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')),
+ alt.value("lightgray"))
+
+
+ base = alt.Chart(plot_data).mark_text().encode(
+ alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
+ alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
+ color=color,
+ text='subreddit')
-tsne_model = TSNE(2,learning_rate=500,perplexity=40,n_iter=2000)
-tsne_fit_model = tsne_model.fit(mat)
-tsne_fit_whole = tsne_fit_model.fit_transform(mat)
+ base = base.add_selection(cluster_click_select)
+
-plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
+ return base
-plot_data.to_feather("tsne_subreddit_fit.feather")
+def zoom_plot(plot_data):
+ chart = base_plot(plot_data)
-slider = alt.binding_range(min=1,max=100,step=1,name='zoom: ')
-selector = alt.selection_single(name='zoomselect',fields=['zoom'],bind='scales',init={'zoom':1})
+ chart = chart.interactive()
+ chart = chart.properties(width=1275,height=800)
-xrange = plot_data.x.max()-plot_data.x.min()
-yrange = plot_data.y.max()-plot_data.y.min()
+ return chart
-chart = alt.Chart(plot_data).mark_text().encode(
- alt.X('x',axis=alt.Axis(grid=False)),
- alt.Y('y',axis=alt.Axis(grid=False)),
- text='subreddit')
+def viewport_plot(plot_data):
+ selector1 = alt.selection_interval(encodings=['x','y'],init={'x':(-65,65),'y':(-65,65)})
+ selectorx2 = alt.selection_interval(encodings=['x'],init={'x':(30,40)})
+ selectory2 = alt.selection_interval(encodings=['y'],init={'y':(-20,0)})
-#chart = chart.add_selection(selector)
+ base = base_plot(plot_data)
-chart = chart.configure_view(
- continuousHeight=xrange/20,
- continuousWidth=yrange/20
-)
+ viewport = base.mark_point(fillOpacity=0.2,opacity=0.2).encode(
+ alt.X('x',axis=alt.Axis(grid=False)),
+ alt.Y('y',axis=alt.Axis(grid=False)),
+ )
+
+ viewport = viewport.properties(width=600,height=400)
-amount_shown = lambda zoom: {'width':xrange/zoom,'height':yrange/zoom}
+ viewport1 = viewport.add_selection(selector1)
-alt.data_transformers.enable('default')
-chart = chart.properties(width=1000,height=1000)
-chart = chart.interactive()
-chart.save("test_tsne_whole.html")
-chart = chart.properties(width=10000,height=10000)
-chart.save("test_tsne_whole.svg")
+ viewport2 = viewport.encode(
+ alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1)),
+ alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selector1))
+ )
+
+ viewport2 = viewport2.add_selection(selectorx2)
+ viewport2 = viewport2.add_selection(selectory2)
+
+ sr = base.encode(alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectorx2)),
+ alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
+ )
+
+
+ sr = sr.properties(width=1275,height=600)
+
+
+ chart = (viewport1 | viewport2) & sr
+
+
+ return chart
+
+def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
+ tsne_data = tsne_data.merge(clusters,on='subreddit')
+
+ centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
+
+ color_ids = np.arange(n_colors)
+
+ distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
+
+ groups = tsne_data.groupby('cluster')
+
+ points = np.array(tsne_data.loc[:,['x','y']])
+ centers = np.array(centroids.loc[:,['x','y']])
+
+ # point x centroid
+ point_center_distances = np.linalg.norm((points[:,None,:] - centers[None,:,:]),axis=-1)
+
+ # distances is cluster x point
+ for gid, group in groups:
+ c_dists = point_center_distances[group.index.values,:].min(axis=0)
+ distances[group.cluster.values[0],] = c_dists
+
+ # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids)
+ # distances, indices = nbrs.kneighbors()
+
+ nearest = distances.argpartition(n_neighbors,0)
+ indices = nearest[:n_neighbors,:].T
+ # neighbor_distances = np.copy(distances)
+ # neighbor_distances.sort(0)
+ # neighbor_distances = neighbor_distances[0:n_neighbors,:]
+
+ # nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances)
+ # distances, indices = nbrs.kneighbors()
+
+ color_assignments = np.repeat(-1,len(centroids))
+
+ for i in range(len(centroids)):
+ knn = indices[i]
+ knn_colors = color_assignments[knn]
+ available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
+
+ if(len(available_colors) > 0):
+ color_assignments[i] = available_colors[0]
+ else:
+ raise Exception("Can't color this many neighbors with this many colors")
+
+
+ centroids = centroids.reset_index()
+ colors = centroids.loc[:,['cluster']]
+ colors['color'] = color_assignments
+
+ tsne_data = tsne_data.merge(colors,on='cluster')
+ return(tsne_data)
+
+def build_visualization(tsne_data, clusters, output):
+
+ # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
+ # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
+
+ tsne_data = pd.read_feather(tsne_data)
+ clusters = pd.read_feather(clusters)
+
+ tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
+
+ # sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
+ # sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
+
+ tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
+
+ term_zoom_plot = zoom_plot(tsne_data)
+
+ term_zoom_plot.save(output)
+
+ term_viewport_plot = viewport_plot(tsne_data)
+
+ term_viewport_plot.save(output.replace(".html","_viewport.html"))
+
+if __name__ == "__main__":
+ fire.Fire(build_visualization)
+
+# commenter_data = pd.read_feather("tsne_author_fit.feather")
+# clusters = pd.read_feather('author_3000_clusters.feather')
+# commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
+# commenter_zoom_plot = zoom_plot(commenter_data)
+# commenter_viewport_plot = viewport_plot(commenter_data)
+# commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
+# commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
+
+# chart = chart.properties(width=10000,height=10000)
+# chart.save("test_tsne_whole.svg")
projects / cdsc_reddit.git / blobdiff