]> code.communitydata.science - cdsc_reddit.git/blob - clustering/fit_tsne.py
remove nsfw subs from topN
[cdsc_reddit.git] / clustering / fit_tsne.py
1 import fire
2 import pyarrow
3 import pandas as pd
4 from numpy import random
5 import numpy as np
6 from sklearn.manifold import TSNE
7
8 similarities = "term_similarities_10000.feather"
9
10 def fit_tsne(similarities, output, learning_rate=750, perplexity=50, n_iter=10000, early_exaggeration=20):
11     '''
12     similarities: feather file with a dataframe of similarity scores
13     learning_rate: parameter controlling how fast the model converges. Too low and you get outliers. Too high and you get a ball.
14     perplexity: number of neighbors to use. the default of 50 is often good.
15
16     '''
17     df = pd.read_feather(similarities)
18
19     n = df.shape[0]
20     mat = np.array(df.drop('subreddit',1),dtype=np.float64)
21     mat[range(n),range(n)] = 1
22     mat[mat > 1] = 1
23     dist = 2*np.arccos(mat)/np.pi
24     tsne_model = TSNE(2,learning_rate=750,perplexity=50,n_iter=10000,metric='precomputed',early_exaggeration=20,n_jobs=-1)
25     tsne_fit_model = tsne_model.fit(dist)
26
27     tsne_fit_whole = tsne_fit_model.fit_transform(dist)
28
29     plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':df.subreddit})
30
31     plot_data.to_feather(output)
32
33 if __name__ == "__main__":
34     fire.Fire(fit_tsne)

Community Data Science Collective || Want to submit a patch?