Updates to similarities code for smap project.

[cdsc_reddit.git] / visualization / tsne_vis.py

diff --git a/visualization/tsne_vis.py b/visualization/tsne_vis.py
index a52d812257f1603b9a86c12dfd2a944858923132..eb6a6be840c0497810c49a8c84f0610fb828d9db 100644 (file)
--- a/visualization/tsne_vis.py
+++ b/visualization/tsne_vis.py
@@ -9,18 +9,44 @@ import fire
 import numpy as np
 
 def base_plot(plot_data):
 import numpy as np
 
 def base_plot(plot_data):

+
+#    base = base.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
+
+    cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
+
+    #    subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
+
+    cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
+    # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
+    # cluster_select_and = cluster_click_select & cluster_select
+    #
+    #    subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
+    
+    base_scale = alt.Scale(scheme={"name":'category10',
+                                   "extent":[0,100],
+                                   "count":10})
+
+    color = alt.condition(cluster_click_select ,
+                          alt.Color(field='color',type='nominal',scale=base_scale),
+                          alt.value("lightgray"))
+  
+    

     base = alt.Chart(plot_data).mark_text().encode(
         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
     base = alt.Chart(plot_data).mark_text().encode(
         alt.X('x',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),
         alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=(-65,65))),

+        color=color,

         text='subreddit')
 
         text='subreddit')
 

+    base = base.add_selection(cluster_click_select)
+ 
+

     return base
 
 def zoom_plot(plot_data):
     chart = base_plot(plot_data)
     return base
 
 def zoom_plot(plot_data):
     chart = base_plot(plot_data)

-    chart = chart.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
+

     chart = chart.interactive()
     chart = chart.interactive()

-    chart = chart.properties(width=1275,height=1000)
+    chart = chart.properties(width=1275,height=800)

 
     return chart
 
 
     return chart
 


@@ -35,7 +61,7 @@ def viewport_plot(plot_data):
         alt.X('x',axis=alt.Axis(grid=False)),
         alt.Y('y',axis=alt.Axis(grid=False)),
     )
         alt.X('x',axis=alt.Axis(grid=False)),
         alt.Y('y',axis=alt.Axis(grid=False)),
     )

-
+   

     viewport = viewport.properties(width=600,height=400)
 
     viewport1 = viewport.add_selection(selector1)
     viewport = viewport.properties(width=600,height=400)
 
     viewport1 = viewport.add_selection(selector1)


@@ -52,7 +78,7 @@ def viewport_plot(plot_data):
                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
     )
 
                      alt.Y('y',axis=alt.Axis(grid=False),scale=alt.Scale(domain=selectory2))
     )
 

-    sr = sr.encode(alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')))
+

     sr = sr.properties(width=1275,height=600)
 
 
     sr = sr.properties(width=1275,height=600)
 
 


@@ -62,6 +88,11 @@ def viewport_plot(plot_data):
     return chart
 
 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
     return chart
 
 def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):

+    isolate_color = 101
+
+    cluster_sizes = clusters.groupby('cluster').count()
+    singletons = set(cluster_sizes.loc[cluster_sizes.subreddit == 1].reset_index().cluster)
+

     tsne_data = tsne_data.merge(clusters,on='subreddit')
     
     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
     tsne_data = tsne_data.merge(clusters,on='subreddit')
     
     centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})


@@ -71,28 +102,44 @@ def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
 
     groups = tsne_data.groupby('cluster')
     distances = np.empty(shape=(centroids.shape[0],centroids.shape[0]))
 
     groups = tsne_data.groupby('cluster')

-    for centroid in centroids.itertuples():
-        c_dists = groups.apply(lambda r: min(np.sqrt(np.square(centroid.x - r.x) + np.square(centroid.y-r.y))))
-        distances[:,centroid.Index] = c_dists
+    
+    points = np.array(tsne_data.loc[:,['x','y']])
+    centers = np.array(centroids.loc[:,['x','y']])
+
+    # point x centroid
+    point_center_distances = np.linalg.norm((points[:,None,:] - centers[None,:,:]),axis=-1)
+    
+    # distances is cluster x point
+    for gid, group in groups:
+        c_dists = point_center_distances[group.index.values,:].min(axis=0)
+        distances[group.cluster.values[0],] = c_dists        

 
     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids) 
     # distances, indices = nbrs.kneighbors()
 
 
     # nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(centroids) 
     # distances, indices = nbrs.kneighbors()
 

-    nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances) 
-    distances, indices = nbrs.kneighbors()
+    nearest = distances.argpartition(n_neighbors,0)
+    indices = nearest[:n_neighbors,:].T
+    # neighbor_distances = np.copy(distances)
+    # neighbor_distances.sort(0)
+    # neighbor_distances = neighbor_distances[0:n_neighbors,:]
+    
+    # nbrs = NearestNeighbors(n_neighbors=n_neighbors,metric='precomputed').fit(distances) 
+    # distances, indices = nbrs.kneighbors()

 
     color_assignments = np.repeat(-1,len(centroids))
 
     for i in range(len(centroids)):
 
     color_assignments = np.repeat(-1,len(centroids))
 
     for i in range(len(centroids)):

-        knn = indices[i]
-        knn_colors = color_assignments[knn]
-        available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
-
-        if(len(available_colors) > 0):
-            color_assignments[i] = available_colors[0]
+        if (centroids.iloc[i].name == -1) or (i in singletons):
+            color_assignments[i] = isolate_color

         else:
         else:

-            raise Exception("Can't color this many neighbors with this many colors")
+            knn = indices[i]
+            knn_colors = color_assignments[knn]
+            available_colors = color_ids[list(set(color_ids) - set(knn_colors))]

 
 

+            if(len(available_colors) > 0):
+                color_assignments[i] = available_colors[0]
+            else:
+                raise Exception("Can't color this many neighbors with this many colors")

 
     centroids = centroids.reset_index()
     colors = centroids.loc[:,['cluster']]
 
     centroids = centroids.reset_index()
     colors = centroids.loc[:,['cluster']]


@@ -103,11 +150,20 @@ def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
 
 def build_visualization(tsne_data, clusters, output):
 
 
 def build_visualization(tsne_data, clusters, output):
 

+    # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
+    # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
+

     tsne_data = pd.read_feather(tsne_data)
     tsne_data = pd.read_feather(tsne_data)

+    tsne_data = tsne_data.rename(columns={'_subreddit':'subreddit'})

     clusters = pd.read_feather(clusters)
 
     tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
 
     clusters = pd.read_feather(clusters)
 
     tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
 

+    sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
+    sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
+
+    tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
+

     term_zoom_plot = zoom_plot(tsne_data)
 
     term_zoom_plot.save(output)
     term_zoom_plot = zoom_plot(tsne_data)
 
     term_zoom_plot.save(output)