]> code.communitydata.science - cdsc_reddit.git/blobdiff - submissions_2_parquet.sh
Add code for running tf-idf at the weekly level.
[cdsc_reddit.git] / submissions_2_parquet.sh
index d1c6bce4df0704e6c3924de079b3f89894841845..f133069b64763c2f96a10bbae7133aab0efdfcc8 100644 (file)
@@ -1,8 +1,8 @@
-#!/usr/bin/env bash
+## this should be run manually since we don't have a nice way to wait on parallel_sql jobs
 
-# part2 should be run on one ore more spark nodes
+#!/usr/bin/env bash
 
-./submissions_2_parquet_part1.py
+./parse_submissions.sh
 
 start_spark_and_run.sh 1 $(pwd)/submissions_2_parquet_part2.py
 

Community Data Science Collective || Want to submit a patch?