From: Nate E TeBlunthuis Date: Thu, 12 Nov 2020 00:38:22 +0000 (-0800) Subject: split fitting and plotting tsne. X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/commitdiff_plain/4447c60265c5c5de3281ca135461d91ab5339f03?ds=inline;hp=-c split fitting and plotting tsne. --- 4447c60265c5c5de3281ca135461d91ab5339f03 diff --git a/fit_tsne.py b/fit_tsne.py new file mode 100644 index 0000000..37341d4 --- /dev/null +++ b/fit_tsne.py @@ -0,0 +1,35 @@ +import pyarrow +import pandas as pd +from numpy import random +import numpy as np +from sklearn.manifold import TSNE + +df = pd.read_feather("reddit_term_similarity_3000.feather") +df = df.sort_values(['i','j']) + +n = max(df.i.max(),df.j.max()) + +def zero_pad(grp): + p = grp.shape[0] + grp = grp.sort_values('j') + return np.concatenate([np.zeros(n-p),np.ones(1),np.array(grp.value)]) + +col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates() +first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0] +col_names = [first_name] + list(col_names.subreddit_j) +mat = df.groupby('i').apply(zero_pad) +mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)]) +mat = np.stack(mat) + +mat = mat + np.tril(mat.transpose(),k=-1) +dist = 2*np.arccos(mat)/np.pi + +tsne_model = TSNE(2,learning_rate=200,perplexity=40,n_iter=5000,metric='precomputed') + +tsne_fit_model = tsne_model.fit(dist) + +tsne_fit_whole = tsne_fit_model.fit_transform(mat) + +plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names}) + +plot_data.to_feather("tsne_subreddit_fit.feather") diff --git a/tsne_subreddit_fit.feather b/tsne_subreddit_fit.feather new file mode 120000 index 0000000..9521ddc --- /dev/null +++ b/tsne_subreddit_fit.feather @@ -0,0 +1 @@ +.git/annex/objects/1M/PF/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5 \ No newline at end of file diff --git a/visualization/tsne_vis.py b/visualization/tsne_vis.py index ca3da3b..1e2aeae 100644 --- a/visualization/tsne_vis.py +++ b/visualization/tsne_vis.py @@ -7,33 +7,7 @@ from numpy import random import numpy as np from sklearn.manifold import TSNE -df = pd.read_csv("reddit_term_similarity_3000.csv") -df = df.sort_values(['i','j']) - -n = max(df.i.max(),df.j.max()) - -def zero_pad(grp): - p = grp.shape[0] - grp = grp.sort_values('j') - return np.concatenate([np.zeros(n-p),np.zeros(1),np.array(grp.value)]) - -col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates() -first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0] -col_names = [first_name] + list(col_names.subreddit_j) -mat = df.groupby('i').apply(zero_pad) -mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)]) -mat = np.stack(mat) - -# plot the matrix using the first and second eigenvalues -mat = mat + np.tril(mat.transpose(),k=-1) - -tsne_model = TSNE(2,learning_rate=500,perplexity=40,n_iter=2000) -tsne_fit_model = tsne_model.fit(mat) -tsne_fit_whole = tsne_fit_model.fit_transform(mat) - -plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names}) - -plot_data.to_feather("tsne_subreddit_fit.feather") +pd.read_feather("tsne_subreddit_fit.feather") slider = alt.binding_range(min=1,max=100,step=1,name='zoom: ') selector = alt.selection_single(name='zoomselect',fields=['zoom'],bind='scales',init={'zoom':1})