X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/06fd99e7cd50024f76dc90c96a75171f78208c73..aa84a7df032dcb20bb284892f12cdac4853f31aa:/comments_2_parquet.sh diff --git a/comments_2_parquet.sh b/comments_2_parquet.sh index 802cc70..096fa06 100755 --- a/comments_2_parquet.sh +++ b/comments_2_parquet.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -echo "!#/usr/bin/bash" > job_script.sh +echo "#!/usr/bin/bash" > job_script.sh echo "source $(pwd)/../bin/activate" >> job_script.sh echo "python3 $(pwd)/comments_2_parquet_part1.py" >> job_script.sh -srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 job_script.sh +srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 --pty job_script.sh start_spark_and_run.sh 1 $(pwd)/comments_2_parquet_part2.py