X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/811a0d87c4d394c2c7849a613f6aec2d81e49138..07b0dff9bc0dae2ab6f7fb7334007a5269a512ad:/start_spark_worker.sh

diff --git a/start_spark_worker.sh b/start_spark_worker.sh
new file mode 100755
index 0000000..a343a31
--- /dev/null
+++ b/start_spark_worker.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# runs on worker node
+# instance_name=spark-worker-$(hostname)
+# echo $hostname
+# instance_url="instance://$instance_name"
+# singularity instance list
+# singularity instance stop -F "$instance_name"
+# singularity instance list
+# sleep 5
+# ls $HOME/.singularity/instances/sing/$(hostname)/nathante/$instance_name
+# rm -r $HOME/.singularity/instances/sing/$(hostname)/nathante/$instance_name
+# singularity instance start /gscratch/comdata/users/nathante/cdsc_base.sif $instance_name
+source /gscratch/comdata/env/cdsc_klone_bashrc
+source $SPARK_CONF_DIR/spark-env.sh
+echo $(which python3)
+echo $PYSPARK_PYTHON
+echo "start-worker.sh spark://$1:$SPARK_MASTER_PORT"
+start-worker.sh spark://$1:$SPARK_MASTER_PORT