X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/4ec9c1424711eb81edf6b6431d5dfae360013174..b3ffaaba1d065614f3f19ee0cbc876185dc220e1:/submissions_2_parquet_part2.py diff --git a/submissions_2_parquet_part2.py b/submissions_2_parquet_part2.py index 1708548..b88764b 100644 --- a/submissions_2_parquet_part2.py +++ b/submissions_2_parquet_part2.py @@ -4,22 +4,20 @@ import pyspark from pyspark.sql import functions as f -from pyspark.sql.types import * -from pyspark import SparkConf, SparkContext -from pyspark.sql import SparkSession, SQLContext +from pyspark.sql import SparkSession import os spark = SparkSession.builder.getOrCreate() sc = spark.sparkContext -conf = SparkConf().setAppName("Reddit submissions to parquet") +conf = pyspark.SparkConf().setAppName("Reddit submissions to parquet") conf = conf.set("spark.sql.shuffle.partitions",2000) conf = conf.set('spark.sql.crossJoin.enabled',"true") conf = conf.set('spark.debug.maxToStringFields',200) sqlContext = pyspark.SQLContext(sc) -df = spark.read.parquet("/gscratch/comdata/output/reddit_submissions.parquet_temp") +df = spark.read.parquet("/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet") df = df.withColumn("subreddit_2", f.lower(f.col('subreddit'))) df = df.drop('subreddit') @@ -31,12 +29,16 @@ df = df.withColumn("Day",f.dayofmonth(f.col("CreatedAt"))) df = df.withColumn("subreddit_hash",f.sha2(f.col("subreddit"), 256)[0:3]) # next we gotta resort it all. -df2 = df.sort(["subreddit","author","id","Year","Month","Day"],ascending=True) -df2.write.parquet("/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet", partitionBy=["Year",'Month'], mode='overwrite') +df = df.repartition("subreddit") +df2 = df.sort(["subreddit","CreatedAt","id"],ascending=True) +df2 = df.sortWithinPartitions(["subreddit","CreatedAt","id"],ascending=True) +df2.write.parquet("/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet2", mode='overwrite',compression='snappy') # # we also want to have parquet files sorted by author then reddit. -df3 = df.sort(["author","CreatedAt","subreddit","id","Year","Month","Day"],ascending=True) -df3.write.parquet("/gscratch/comdata/output/reddit_submissions_by_author.parquet", partitionBy=["Year",'Month'], mode='overwrite') +df = df.repartition("author") +df3 = df.sort(["author","CreatedAt","id"],ascending=True) +df3 = df.sortWithinPartitions(["author","CreatedAt","id"],ascending=True) +df3.write.parquet("/gscratch/comdata/output/reddit_submissions_by_author.parquet2", mode='overwrite',compression='snappy') os.remove("/gscratch/comdata/output/reddit_submissions.parquet_temp")