X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/e6294b5b90135a5163441c8dc62252dd6a188412..9345f9de9437d5965ad4ee5874bc24199e077d48:/datasets/submissions_2_parquet_part2.py?ds=sidebyside diff --git a/datasets/submissions_2_parquet_part2.py b/datasets/submissions_2_parquet_part2.py index 3a58617..7dc4f74 100644 --- a/datasets/submissions_2_parquet_part2.py +++ b/datasets/submissions_2_parquet_part2.py @@ -29,14 +29,14 @@ df = df.withColumn("Day",f.dayofmonth(f.col("CreatedAt"))) df = df.withColumn("subreddit_hash",f.sha2(f.col("subreddit"), 256)[0:3]) # next we gotta resort it all. -df = df.repartition("subreddit") -df2 = df.sort(["subreddit","CreatedAt","id"],ascending=True) +df = df.repartition(800,"subreddit","Year","Month") +df2 = df.sort(["subreddit","Year","Month","CreatedAt","id"],ascending=True) df2 = df.sortWithinPartitions(["subreddit","CreatedAt","id"],ascending=True) df2.write.parquet("/gscratch/comdata/output/temp/reddit_submissions_by_subreddit.parquet2", mode='overwrite',compression='snappy') # # we also want to have parquet files sorted by author then reddit. -df = df.repartition("author") -df3 = df.sort(["author","CreatedAt","id"],ascending=True) +df = df.repartition(800,"author","subreddit","Year","Month") +df3 = df.sort(["author","Year","Month","CreatedAt","id"],ascending=True) df3 = df.sortWithinPartitions(["author","CreatedAt","id"],ascending=True) df3.write.parquet("/gscratch/comdata/output/temp/reddit_submissions_by_author.parquet2", mode='overwrite',compression='snappy')