git-annex in nathante@mox2.hyak.local:/gscratch/comdata/users/nathante/cdsc-reddit

[cdsc_reddit.git] / submissions_2_parquet.sh

diff --git a/submissions_2_parquet.sh b/submissions_2_parquet.sh
index d1c6bce4df0704e6c3924de079b3f89894841845..f133069b64763c2f96a10bbae7133aab0efdfcc8 100644 (file)
--- a/submissions_2_parquet.sh
+++ b/submissions_2_parquet.sh
@@ -1,8 +1,8 @@
-#!/usr/bin/env bash
+## this should be run manually since we don't have a nice way to wait on parallel_sql jobs
 
-# part2 should be run on one ore more spark nodes
+#!/usr/bin/env bash
 
-./submissions_2_parquet_part1.py
+./parse_submissions.sh
 
 start_spark_and_run.sh 1 $(pwd)/submissions_2_parquet_part2.py