]> code.communitydata.science - cdsc_reddit.git/blob - sort_tf_comments.py
git-annex in nathante@mox2.hyak.local:/gscratch/comdata/users/nathante/cdsc-reddit
[cdsc_reddit.git] / sort_tf_comments.py
1 #!/usr/bin/env python3
2
3 from pyspark.sql import functions as f
4 from pyspark.sql import SparkSession
5
6 spark = SparkSession.builder.getOrCreate()
7 df = spark.read.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test.parquet_temp/")
8
9 df = df.repartition(2000,'term')
10 df = df.sort(['term','week','subreddit'])
11 df = df.sortWithinPartitions(['term','week','subreddit'])
12
13 df.write.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test_sorted_tf.parquet_temp",mode='overwrite',compression='snappy')

Community Data Science Collective || Want to submit a patch?