From 31556005146ac1b9db9c1671195579c003a84af7 Mon Sep 17 00:00:00 2001 From: Nate E TeBlunthuis Date: Mon, 28 Dec 2020 21:11:44 -0800 Subject: [PATCH] remove nsfw subs from topN --- similarities/top_subreddits_by_comments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/similarities/top_subreddits_by_comments.py b/similarities/top_subreddits_by_comments.py index 214c7e0..1197b51 100644 --- a/similarities/top_subreddits_by_comments.py +++ b/similarities/top_subreddits_by_comments.py @@ -5,6 +5,10 @@ from pyspark.sql import Window spark = SparkSession.builder.getOrCreate() conf = spark.sparkContext.getConf() +submissions = spark.read.parquet("/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet") + +prop_nsfw = submissions.select(['subreddit','over_18']).groupby('subreddit').agg(f.mean(f.col('over_18').astype('double')).alias('prop_nsfw')) + df = spark.read.parquet("/gscratch/comdata/output/reddit_comments_by_subreddit.parquet") # remove /u/ pages @@ -12,6 +16,9 @@ df = df.filter(~df.subreddit.like("u_%")) df = df.groupBy('subreddit').agg(f.count('id').alias("n_comments")) +df = df.join(prop_nsfw,on='subreddit') +df = df.filter(df.prop_nsfw < 0.5) + win = Window.orderBy(f.col('n_comments').desc()) df = df.withColumn('comments_rank', f.rank().over(win)) -- 2.39.2