reindex tfidf in memory instead of using spark

[cdsc_reddit.git] / visualization / tsne_vis.py

diff --git a/visualization/tsne_vis.py b/visualization/tsne_vis.py
index 4b42e8e56547b410a4d190f2ce41252a7b62cfa6..c39a7400e5e5c2ab726eb0a692e4180536d72ce5 100644 (file)
--- a/visualization/tsne_vis.py
+++ b/visualization/tsne_vis.py
@@ -14,9 +14,9 @@ def base_plot(plot_data):
 
     cluster_dropdown = alt.binding_select(options=[str(c) for c in sorted(set(plot_data.cluster))])
 
-    subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
+    #    subreddit_dropdown = alt.binding_select(options=sorted(plot_data.subreddit))
 
-    cluster_click_select = alt.selection_single(on='click,',fields=['cluster'], bind=cluster_dropdown, name=' ')
+    cluster_click_select = alt.selection_single(on='click',fields=['cluster'], bind=cluster_dropdown, name=' ')
     # cluster_select = alt.selection_single(fields=['cluster'], bind=cluster_dropdown, name='cluster')
     # cluster_select_and = cluster_click_select & cluster_select
     #
@@ -42,7 +42,7 @@ def zoom_plot(plot_data):
     chart = base_plot(plot_data)
 
     chart = chart.interactive()
-    chart = chart.properties(width=1275,height=1000)
+    chart = chart.properties(width=1275,height=800)
 
     return chart
 
@@ -139,11 +139,19 @@ def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
 
 def build_visualization(tsne_data, clusters, output):
 
+    # tsne_data = "/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather"
+    # clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
+
     tsne_data = pd.read_feather(tsne_data)
     clusters = pd.read_feather(clusters)
 
     tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
 
+    # sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
+    # sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
+
+    tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')
+
     term_zoom_plot = zoom_plot(tsne_data)
 
     term_zoom_plot.save(output)
@@ -155,13 +163,13 @@ def build_visualization(tsne_data, clusters, output):
 if __name__ == "__main__":
     fire.Fire(build_visualization)
 
-commenter_data = pd.read_feather("tsne_author_fit.feather")
-clusters = pd.read_feather('author_3000_clusters.feather')
-commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
-commenter_zoom_plot = zoom_plot(commenter_data)
-commenter_viewport_plot = viewport_plot(commenter_data)
-commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
-commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
+# commenter_data = pd.read_feather("tsne_author_fit.feather")
+# clusters = pd.read_feather('author_3000_clusters.feather')
+# commenter_data = assign_cluster_colors(commenter_data,clusters,10,8)
+# commenter_zoom_plot = zoom_plot(commenter_data)
+# commenter_viewport_plot = viewport_plot(commenter_data)
+# commenter_zoom_plot.save("subreddit_commenters_tsne_3000.html")
+# commenter_viewport_plot.save("subreddit_commenters_tsne_3000_viewport.html")
 
 # chart = chart.properties(width=10000,height=10000)
 # chart.save("test_tsne_whole.svg")