split fitting and plotting tsne.

diff --git a/fit_tsne.py b/fit_tsne.py
new file mode 100644 (file)
index 0000000..37341d4
--- /dev/null
+++ b/fit_tsne.py
@@ -0,0 +1,35 @@
+import pyarrow
+import pandas as pd
+from numpy import random
+import numpy as np
+from sklearn.manifold import TSNE
+
+df = pd.read_feather("reddit_term_similarity_3000.feather")
+df = df.sort_values(['i','j'])
+
+n = max(df.i.max(),df.j.max())
+
+def zero_pad(grp):
+    p = grp.shape[0]
+    grp = grp.sort_values('j')
+    return np.concatenate([np.zeros(n-p),np.ones(1),np.array(grp.value)])
+
+col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
+first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
+col_names = [first_name] + list(col_names.subreddit_j)
+mat = df.groupby('i').apply(zero_pad)
+mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
+mat = np.stack(mat)
+
+mat = mat + np.tril(mat.transpose(),k=-1)
+dist = 2*np.arccos(mat)/np.pi
+
+tsne_model = TSNE(2,learning_rate=200,perplexity=40,n_iter=5000,metric='precomputed')
+
+tsne_fit_model = tsne_model.fit(dist)
+
+tsne_fit_whole = tsne_fit_model.fit_transform(mat)
+
+plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
+
+plot_data.to_feather("tsne_subreddit_fit.feather")

diff --git a/tsne_subreddit_fit.feather b/tsne_subreddit_fit.feather
new file mode 120000 (symlink)
index 0000000..9521ddc
--- /dev/null
+++ b/tsne_subreddit_fit.feather
@@ -0,0 +1 @@
+.git/annex/objects/1M/PF/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5
\ No newline at end of file

diff --git a/visualization/tsne_vis.py b/visualization/tsne_vis.py
index ca3da3be8e6356d158c75915bbb4681e12d90d62..1e2aeaef3f810277cbea532659ae5d58f8cfe275 100644 (file)
--- a/visualization/tsne_vis.py
+++ b/visualization/tsne_vis.py
@@ -7,33 +7,7 @@ from numpy import random
 import numpy as np
 from sklearn.manifold import TSNE
 
-df = pd.read_csv("reddit_term_similarity_3000.csv")
-df = df.sort_values(['i','j'])
-
-n = max(df.i.max(),df.j.max())
-
-def zero_pad(grp):
-    p = grp.shape[0]
-    grp = grp.sort_values('j')
-    return np.concatenate([np.zeros(n-p),np.zeros(1),np.array(grp.value)])
-
-col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
-first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
-col_names = [first_name] + list(col_names.subreddit_j)
-mat = df.groupby('i').apply(zero_pad)
-mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
-mat = np.stack(mat)
-
-# plot the matrix using the first and second eigenvalues
-mat = mat + np.tril(mat.transpose(),k=-1)
-
-tsne_model = TSNE(2,learning_rate=500,perplexity=40,n_iter=2000)
-tsne_fit_model = tsne_model.fit(mat)
-tsne_fit_whole = tsne_fit_model.fit_transform(mat)
-
-plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
-
-plot_data.to_feather("tsne_subreddit_fit.feather")
+pd.read_feather("tsne_subreddit_fit.feather")
 
 slider = alt.binding_range(min=1,max=100,step=1,name='zoom: ')
 selector = alt.selection_single(name='zoomselect',fields=['zoom'],bind='scales',init={'zoom':1})