-# output: term | week | df
-#idf.write.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test_sorted_tf.parquet_temp",mode='overwrite',compression='snappy')
-
-# collect the dictionary to make a pydict of terms to indexes
-terms = idf.select('term').distinct()
-terms = terms.withColumn('term_id',f.monotonically_increasing_id())