Fix shutdown issues (#1117)

* Add kernel_id to message
* Relocate clean up into start_ipython
* Cleanup error handling of listener shutdown
* Set max start attempts to avoid YARN RM auto-restart race condition

Author: kevin-bates

enterprise_gateway/services/processproxies/processproxy.py

@@ -377,7 +377,9 @@ def _post_connection(self, connection_info: dict) -> None:
             self.log.error("No kernel id found in response!  Kernel launch will fail.")
             return
         if kernel_id not in self._response_registry:
-            self.log.error("Kernel id '{}' has not been registered and will not be processed!")
+            self.log.error(
+                f"Kernel id '{kernel_id}' has not been registered and will not be processed!"
+            )
             return
 
         self.log.debug(f"Connection info received for kernel '{kernel_id}': {connection_info}")
@@ -1482,22 +1484,19 @@ def _send_listener_request(self, request, shutdown_socket=False):
                         if isinstance(e2, OSError) and e2.errno == errno.ENOTCONN:
                             # Listener is not connected.  This is probably a follow-on to ECONNREFUSED on connect
                             self.log.debug(
-                                "ERROR: OSError(ENOTCONN) raised on socket shutdown, "
-                                f"listener likely not connected. Cannot send {request}"
+                                f"OSError(ENOTCONN) raised on socket shutdown, listener "
+                                f"has likely already exited. Cannot send '{request}'"
                             )
                         else:
                             self.log.warning(
-                                "Exception occurred attempting to shutdown communication socket to {}:{} "
-                                "for KernelID '{}' (ignored): {}".format(
-                                    self.comm_ip, self.comm_port, self.kernel_id, str(e2)
-                                )
+                                f"Exception occurred attempting to shutdown communication "
+                                f"socket to {self.comm_ip}:{self.comm_port} "
+                                f"for KernelID '{self.kernel_id}' (ignored): {str(e2)}"
                             )
                 sock.close()
         else:
             self.log.debug(
-                "Invalid comm port, not sending request '{}' to comm_port '{}'.",
-                request,
-                self.comm_port,
+                f"Invalid comm port, not sending request '{request}' to comm_port '{self.comm_port}'."
             )
 
     def send_signal(self, signum):

etc/kernel-launchers/python/scripts/launch_ipykernel.py

@@ -435,6 +435,15 @@ def start_ipython(
     app.initialize([])
     app.start()
 
+    # cleanup
+    conn_file = kwargs["connection_file"]
+    try:
+        import os  # re-import os since it's removed during namespace manipulation during startup
+
+        os.remove(conn_file)
+    except Exception as e:
+        print(f"Could not delete connection file '{conn_file}' at exit due to error: {e}")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -598,10 +607,3 @@ def start_ipython(
         ip=ip,
         kernel_class_name=kernel_class_name,
     )
-
-    try:
-        os.remove(connection_file)
-    except Exception as e:
-        logger.warning(
-            f"Could not delete connection file '{connection_file}' at exit due to error: {e}"
-        )

etc/kernelspecs/spark_R_conductor_cluster/kernel.json

@@ -7,7 +7,7 @@
     }
   },
   "env": {
-    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} ${KERNEL_EXTRA_SPARK_OPTS}",
+    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "LAUNCH_OPTS": "--customAppName ${KERNEL_ID}"
   },
   "argv": [

etc/kernelspecs/spark_R_yarn_cluster/kernel.json

@@ -8,7 +8,7 @@
   },
   "env": {
     "SPARK_HOME": "/usr/hdp/current/spark2-client",
-    "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH --conf spark.sparkr.r.command=/opt/conda/lib/R/bin/Rscript ${KERNEL_EXTRA_SPARK_OPTS}",
+    "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH --conf spark.sparkr.r.command=/opt/conda/lib/R/bin/Rscript --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "LAUNCH_OPTS": ""
   },
   "argv": [

etc/kernelspecs/spark_python_conductor_cluster/kernel.json

@@ -8,7 +8,7 @@
     "debugger": true
   },
   "env": {
-    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} ${KERNEL_EXTRA_SPARK_OPTS}",
+    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "LAUNCH_OPTS": ""
   },
   "argv": [

etc/kernelspecs/spark_python_yarn_cluster/kernel.json

@@ -11,7 +11,7 @@
     "SPARK_HOME": "/usr/hdp/current/spark2-client",
     "PYSPARK_PYTHON": "/opt/conda/bin/python",
     "PYTHONPATH": "${HOME}/.local/lib/python3.7/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip",
-    "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.appMasterEnv.PYTHONUSERBASE=/home/${KERNEL_USERNAME}/.local --conf spark.yarn.appMasterEnv.PYTHONPATH=${HOME}/.local/lib/python3.7/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH ${KERNEL_EXTRA_SPARK_OPTS}",
+    "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.appMasterEnv.PYTHONUSERBASE=/home/${KERNEL_USERNAME}/.local --conf spark.yarn.appMasterEnv.PYTHONPATH=${HOME}/.local/lib/python3.7/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "LAUNCH_OPTS": ""
   },
   "argv": [

etc/kernelspecs/spark_scala_conductor_cluster/kernel.json

@@ -7,7 +7,7 @@
     }
   },
   "env": {
-    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} ${KERNEL_EXTRA_SPARK_OPTS}",
+    "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "__TOREE_OPTS__": "--alternate-sigint USR2 --spark-context-initialization-mode eager",
     "LAUNCH_OPTS": "",
     "DEFAULT_INTERPRETER": "Scala"

etc/kernelspecs/spark_scala_yarn_cluster/kernel.json

@@ -8,7 +8,7 @@
   },
   "env": {
     "SPARK_HOME": "/usr/hdp/current/spark2-client",
-    "__TOREE_SPARK_OPTS__": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d ${KERNEL_EXTRA_SPARK_OPTS}",
+    "__TOREE_SPARK_OPTS__": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}",
     "__TOREE_OPTS__": "--alternate-sigint USR2",
     "LAUNCH_OPTS": "",
     "DEFAULT_INTERPRETER": "Scala"