added script to start up pyspark in a jupyter server

NIH-HPC · May 19, 2022 · ce3167a · ce3167a
1 parent 4756d68
commit ce3167a
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/pyspark-jupyter b/pyspark-jupyter
@@ -0,0 +1,10 @@
+#! /bin/bash
+
+if [[ "${PORT1:-none}" == "none" ]] ; then
+    PORT1=$(( (RANDOM % 10000) + 2000 ))
+fi
+echo "Running on port ${PORT1} on $(hostname) listening on localhost"
+
+export PYSPARK_DRIVER_PYTHON=jupyter
+export PYSPARK_DRIVER_PYTHON_OPTS="notebook --port=${PORT1} --ip=localhost --no-browser"
+exec pyspark "$@"