X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/811a0d87c4d394c2c7849a613f6aec2d81e49138..07b0dff9bc0dae2ab6f7fb7334007a5269a512ad:/start_spark_cluster.sh?ds=sidebyside

diff --git a/start_spark_cluster.sh b/start_spark_cluster.sh
new file mode 100755
index 0000000..c6c0ea4
--- /dev/null
+++ b/start_spark_cluster.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+nodes="$(scontrol show hostnames)"
+
+export SPARK_MASTER_HOST=$(hostname)
+echo $SPARK_MASTER_HOST
+# singularity instance stop spark-boss
+# rm -r $HOME/.singularity/instances/sing/$(hostname)/nathante/spark-boss
+ 
+# for node in $nodes
+# dol
+#     echo $node
+#     ssh $node "singularity instance stop --all -F"
+# done
+
+# singularity instance start /gscratch/comdata/users/nathante/cdsc_base.sif spark-boss
+#apptainer exec /gscratch/comdata/users/nathante/containers/nathante.sif
+start-master.sh 
+for node in $nodes
+do
+    # if [ "$node" != "$SPARK_BOSS" ]
+    # then
+    echo $node
+    ssh -t $node start_spark_worker.sh $SPARK_MASTER_HOST
+   # fi				
+done
+