from nltk.util import ngrams
import string
from random import random
-
-# remove urls
-# taken from https://stackoverflow.com/questions/3809401/what-is-a-good-regular-expression-to-match-a-url
-urlregex = re.compile(r"[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)")
+from redditcleaner import clean
# compute term frequencies for comments in each subreddit by week
def weekly_tf(partition, mwe_pass = 'first'):
# lowercase
text = text.lower()
- # remove urls
- text = urlregex.sub("", text)
+ # redditcleaner removes reddit markdown(newlines, quotes, bullet points, links, strikethrough, spoiler, code, superscript, table, headings)
+ text = clean(text)
# sentence tokenize
sentences = sent_tokenize(text)
# remove punctuation
sentences = map(remove_punct, sentences)
-
- # remove sentences with less than 2 words
- sentences = filter(lambda sentence: len(sentence) > 2, sentences)
-
# datta et al. select relatively common phrases from the reddit corpus, but they don't really explain how. We'll try that in a second phase.
# they say that the extract 1-4 grams from 10% of the sentences and then find phrases that appear often relative to the original terms
# here we take a 10 percent sample of sentences
if mwe_pass == 'first':
+
+ # remove sentences with less than 2 words
+ sentences = filter(lambda sentence: len(sentence) > 2, sentences)
sentences = list(sentences)
for sentence in sentences:
if random() <= 0.1:
#all: /gscratch/comdata/output/reddit_similarity/tfidf/comment_terms_130k.parquet /gscratch/comdata/output/reddit_similarity/tfidf/comment_authors_130k.parquet /gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms_130k.parquet /gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors_130k.parquet
srun_singularity=source /gscratch/comdata/users/nathante/cdsc_reddit/bin/activate && srun_singularity.sh
srun_singularity_huge=source /gscratch/comdata/users/nathante/cdsc_reddit/bin/activate && srun_singularity_huge.sh
-base_data=/gscratch/comdata/output/
+base_data=/gscratch/comdata/output
similarity_data=${base_data}/reddit_similarity
tfidf_data=${similarity_data}/tfidf
tfidf_weekly_data=${similarity_data}/tfidf_weekly
start_spark_and_run.sh 4 tfidf.py authors_weekly --topN=100000 --outpath=${tfidf_weekly_data}/comment_authors_100k.parquet
${tfidf_weekly_data}/comment_terms_30k.parquet: /gscratch/comdata/output/reddit_ngrams/comment_terms.parquet ${similarity_data}/subreddits_by_num_comments.csv
- start_spark_and_run.sh 4 tfidf.py terms_weekly --topN=30000 --outpath=${tfidf_weekly_data}/comment_authors_30k.parquet
+ start_spark_and_run.sh 2 tfidf.py terms_weekly --topN=30000 --outpath=${tfidf_weekly_data}/comment_authors_30k.parquet
${tfidf_weekly_data}/comment_authors_30k.parquet: /gscratch/comdata/output/reddit_ngrams/comment_terms.parquet ${similarity_data}/subreddits_by_num_comments.csv
start_spark_and_run.sh 4 tfidf.py authors_weekly --topN=30000 --outpath=${tfidf_weekly_data}/comment_authors_30k.parquet
infile = "/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet"
cache_file = "/gscratch/comdata/users/nathante/cdsc_reddit/similarities/term_tfidf_entries_bak.parquet"
-def termauthor_tfidf(term_tfidf_callable, author_tfidf_callable):
-
-
# subreddits missing after this step don't have any terms that have a high enough idf
# try rewriting without merges
def reindex_tfidf(infile, term_colname, min_df=None, max_df=None, included_subreddits=None, topN=500, week=None, from_date=None, to_date=None, rescale_idf=True, tf_family=tf_weight.MaxTF):
df = df.withColumn("tf_idf", (0.5 + 0.5 * df.relative_tf) * df.idf)
df = df.repartition(400,'subreddit','week')
- dfwriter = df.write.partitionBy("week").sortBy("subreddit")
+ dfwriter = df.write.partitionBy("week")
return dfwriter
def _calc_tfidf(df, term_colname, tf_family):
df = _calc_tfidf(df, term_colname, tf_family)
df = df.repartition('subreddit')
- dfwriter = df.write.sortBy("subreddit","tf")
+ dfwriter = df.write
return dfwriter
def select_topN_subreddits(topN, path="/gscratch/comdata/output/reddit_similarity/subreddits_by_num_comments_nonsfw.csv"):
#
# subreddit_select = alt.selection_single(on='click',fields=['subreddit'],bind=subreddit_dropdown,name='subreddit_click')
+ base_scale = alt.Scale(scheme={"name":'category10',
+ "extent":[0,100],
+ "count":10})
+
color = alt.condition(cluster_click_select ,
- alt.Color(field='color',type='nominal',scale=alt.Scale(scheme='category10')),
+ alt.Color(field='color',type='nominal',scale=base_scale),
alt.value("lightgray"))
return chart
def assign_cluster_colors(tsne_data, clusters, n_colors, n_neighbors = 4):
+ isolate_color = 101
+
+ cluster_sizes = clusters.groupby('cluster').count()
+ singletons = set(cluster_sizes.loc[cluster_sizes.subreddit == 1].reset_index().cluster)
+
tsne_data = tsne_data.merge(clusters,on='subreddit')
centroids = tsne_data.groupby('cluster').agg({'x':np.mean,'y':np.mean})
color_assignments = np.repeat(-1,len(centroids))
for i in range(len(centroids)):
- knn = indices[i]
- knn_colors = color_assignments[knn]
- available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
-
- if(len(available_colors) > 0):
- color_assignments[i] = available_colors[0]
+ if (centroids.iloc[i].name == -1) or (i in singletons):
+ color_assignments[i] = isolate_color
else:
- raise Exception("Can't color this many neighbors with this many colors")
+ knn = indices[i]
+ knn_colors = color_assignments[knn]
+ available_colors = color_ids[list(set(color_ids) - set(knn_colors))]
+ if(len(available_colors) > 0):
+ color_assignments[i] = available_colors[0]
+ else:
+ raise Exception("Can't color this many neighbors with this many colors")
centroids = centroids.reset_index()
colors = centroids.loc[:,['cluster']]
# clusters = "/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather"
tsne_data = pd.read_feather(tsne_data)
+ tsne_data = tsne_data.rename(columns={'_subreddit':'subreddit'})
clusters = pd.read_feather(clusters)
tsne_data = assign_cluster_colors(tsne_data,clusters,10,8)
- # sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
- # sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
+ sr_per_cluster = tsne_data.groupby('cluster').subreddit.count().reset_index()
+ sr_per_cluster = sr_per_cluster.rename(columns={'subreddit':'cluster_size'})
tsne_data = tsne_data.merge(sr_per_cluster,on='cluster')