fit_tsne.py

   1 import pyarrow
   2 import pandas as pd
   3 from numpy import random
   4 import numpy as np
   5 from sklearn.manifold import TSNE
   6
   7 df = pd.read_feather("reddit_term_similarity_3000.feather")
   8 df = df.sort_values(['i','j'])
   9
  10 n = max(df.i.max(),df.j.max())
  11
  12 def zero_pad(grp):
  13     p = grp.shape[0]
  14     grp = grp.sort_values('j')
  15     return np.concatenate([np.zeros(n-p),np.ones(1),np.array(grp.value)])
  16
  17 col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
  18 first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
  19 col_names = [first_name] + list(col_names.subreddit_j)
  20 mat = df.groupby('i').apply(zero_pad)
  21 mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
  22 mat = np.stack(mat)
  23
  24 mat = mat + np.tril(mat.transpose(),k=-1)
  25 dist = 2*np.arccos(mat)/np.pi
  26
  27 tsne_model = TSNE(2,learning_rate=200,perplexity=40,n_iter=5000,metric='precomputed')
  28
  29 tsne_fit_model = tsne_model.fit(dist)
  30
  31 tsne_fit_whole = tsne_fit_model.fit_transform(dist)
  32
  33 plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
  34
  35 plot_data.to_feather("tsne_subreddit_fit.feather")