-
- with pq.ParquetWriter(f"/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet/{partition}",schema=schema,compression='snappy',flavor='spark') as writer, pq.ParquetWriter(f"/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet/{partition}",schema=author_schema,compression='snappy',flavor='spark') as author_writer:
+
+ termtf_outputdir = (outputdir / "comment_terms")
+ termtf_outputdir.mkdir(parents=True, exist_ok=True)
+ authortf_outputdir = (outputdir / "comment_authors")
+ authortf_outputdir.mkdir(parents=True, exist_ok=True)
+ termtf_path = termtf_outputdir / partition
+ authortf_path = authortf_outputdir / partition
+ with pq.ParquetWriter(termtf_path, schema=schema, compression='snappy', flavor='spark') as writer, \
+ pq.ParquetWriter(authortf_path, schema=author_schema, compression='snappy', flavor='spark') as author_writer: