1 from pyspark.sql import functions as f
2 from pyspark.sql import SparkSession
3 from pyspark.sql import Window
4 from datetime import datetime
5 from pathlib import Path
7 spark = SparkSession.builder.getOrCreate()
8 conf = spark.sparkContext.getConf()
10 submissions = spark.read.parquet("../../data/reddit_submissions_by_subreddit.parquet")
12 submissions = submissions.filter(f.col("CreatedAt") <= datetime(2020,4,13))
14 prop_nsfw = submissions.select(['subreddit','over_18']).groupby('subreddit').agg(f.mean(f.col('over_18').astype('double')).alias('prop_nsfw'))
16 df = spark.read.parquet("../../data/reddit_comments_by_subreddit.parquet")
17 df = df.filter(f.col("CreatedAt") <= datetime(2020,4,13))
19 df = df.filter(~df.subreddit.like("u_%"))
21 df = df.groupBy('subreddit').agg(f.count('id').alias("n_comments"))
23 df = df.join(prop_nsfw,on='subreddit')
24 df = df.filter(df.prop_nsfw < 0.5)
26 win = Window.orderBy(f.col('n_comments').desc())
27 df = df.withColumn('comments_rank', f.rank().over(win))
31 df = df.sort_values("n_comments")
33 outpath = Path("../../data/reddit_similarity/subreddits_by_num_comments_nonsfw.csv")
34 outpath.parent.mkdir(exist_ok=True, parents=True)
35 df.to_csv(str(outpath), index=False)