ngrams/sort_tf_comments.py

   1 #!/usr/bin/env python3
   2
   3 from pyspark.sql import functions as f
   4 from pyspark.sql import SparkSession
   5 import fire
   6
   7 def main(inparquet, outparquet, colname):
   8     spark = SparkSession.builder.getOrCreate()
   9     df = spark.read.parquet(inparquet)
  10
  11     df = df.repartition(2000,colname)
  12     df = df.sort([colname,'week','subreddit'])
  13     df = df.sortWithinPartitions([colname,'week','subreddit'])
  14
  15     df.write.parquet(outparquet,mode='overwrite',compression='snappy')
  16
  17 if __name__ == '__main__':
  18     fire.Fire(main)