changes for archiving.

[cdsc_reddit.git] / start_spark_cluster.sh

diff --git a/start_spark_cluster.sh b/start_spark_cluster.sh
new file mode 100755 (executable)
index 0000000..c6c0ea4
--- /dev/null
+++ b/start_spark_cluster.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+nodes="$(scontrol show hostnames)"
+
+export SPARK_MASTER_HOST=$(hostname)
+echo $SPARK_MASTER_HOST
+# singularity instance stop spark-boss
+# rm -r $HOME/.singularity/instances/sing/$(hostname)/nathante/spark-boss
+ 
+# for node in $nodes
+# dol
+#     echo $node
+#     ssh $node "singularity instance stop --all -F"
+# done
+
+# singularity instance start /gscratch/comdata/users/nathante/cdsc_base.sif spark-boss
+#apptainer exec /gscratch/comdata/users/nathante/containers/nathante.sif
+start-master.sh 
+for node in $nodes
+do
+    # if [ "$node" != "$SPARK_BOSS" ]
+    # then
+    echo $node
+    ssh -t $node start_spark_worker.sh $SPARK_MASTER_HOST
+   # fi                                
+done
+