similarities/top_subreddits_by_comments.py

   1 from pyspark.sql import functions as f
   2 from pyspark.sql import SparkSession
   3 from pyspark.sql import Window
   4
   5 spark = SparkSession.builder.getOrCreate()
   6 conf = spark.sparkContext.getConf()
   7
   8 df = spark.read.parquet("/gscratch/comdata/output/reddit_comments_by_subreddit.parquet")
   9
  10 # remove /u/ pages
  11 df = df.filter(~df.subreddit.like("u_%"))
  12
  13 df = df.groupBy('subreddit').agg(f.count('id').alias("n_comments"))
  14
  15 win = Window.orderBy(f.col('n_comments').desc())
  16 df = df.withColumn('comments_rank', f.rank().over(win))
  17
  18 df = df.toPandas()
  19
  20 df = df.sort_values("n_comments")
  21
  22 df.to_csv('/gscratch/comdata/output/reddit_similarity/subreddits_by_num_comments.csv', index=False)