X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/40d45637702fb51feb9f99ff7f6d71787af765ed..529b7f051133df6cccd3fbf360f8b196bf5b5996:/comments_2_parquet.sh diff --git a/comments_2_parquet.sh b/comments_2_parquet.sh index 802cc70..096fa06 100755 --- a/comments_2_parquet.sh +++ b/comments_2_parquet.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -echo "!#/usr/bin/bash" > job_script.sh +echo "#!/usr/bin/bash" > job_script.sh echo "source $(pwd)/../bin/activate" >> job_script.sh echo "python3 $(pwd)/comments_2_parquet_part1.py" >> job_script.sh -srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 job_script.sh +srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 --pty job_script.sh start_spark_and_run.sh 1 $(pwd)/comments_2_parquet_part2.py