feat(bootstrap): resume gateway from existing state and persist SSH handshake secret

Add a resume code path to gateway start so existing Docker volume state
(k3s, etcd, sandboxes, secrets) is reused instead of requiring a full
destroy/recreate cycle. When the container is gone but the volume remains
(e.g. Docker restart), the CLI automatically creates a new container with
the existing volume and reconciles PKI and secrets.

Move the SSH handshake HMAC secret from ephemeral generation in the
cluster entrypoint (regenerated on every container start) to a Kubernetes
Secret that persists in etcd on the Docker volume. This ensures sandbox
SSH sessions survive container restarts.

Key changes:
- Add DeployOptions.resume flag with resume branch in deploy flow
- Add cleanup_gateway_container for volume-preserving failure cleanup
- Auto-resume in gateway_admin_deploy (stopped/volume-only states)
- Auto-bootstrap tries resume first, falls back to recreate
- Add unless-stopped Docker restart policy to gateway container
- Reconcile SSH handshake secret as K8s Secret alongside TLS PKI
- Update Helm chart to read secret via secretKeyRef
- Add SSH handshake secret to cluster health check

Closes #487

Author: drew

crates/openshell-bootstrap/src/constants.rs

@@ -11,6 +11,8 @@ pub const SERVER_TLS_SECRET_NAME: &str = "openshell-server-tls";
 pub const SERVER_CLIENT_CA_SECRET_NAME: &str = "openshell-server-client-ca";
 /// K8s secret holding the client TLS certificate, key, and CA cert (shared by CLI and sandboxes).
 pub const CLIENT_TLS_SECRET_NAME: &str = "openshell-client-tls";
+/// K8s secret holding the SSH handshake HMAC secret (shared by gateway and sandbox pods).
+pub const SSH_HANDSHAKE_SECRET_NAME: &str = "openshell-ssh-handshake";
 
 pub fn container_name(name: &str) -> String {
     format!("openshell-cluster-{name}")

crates/openshell-bootstrap/src/docker.rs

@@ -9,7 +9,8 @@ use bollard::Docker;
 use bollard::errors::Error as BollardError;
 use bollard::models::{
     ContainerCreateBody, DeviceRequest, HostConfig, HostConfigCgroupnsModeEnum,
-    NetworkCreateRequest, NetworkDisconnectRequest, PortBinding, VolumeCreateRequest,
+    NetworkCreateRequest, NetworkDisconnectRequest, PortBinding, RestartPolicy,
+    RestartPolicyNameEnum, VolumeCreateRequest,
 };
 use bollard::query_parameters::{
     CreateContainerOptions, CreateImageOptions, InspectContainerOptions, InspectNetworkOptions,
@@ -532,6 +533,12 @@ pub async fn ensure_container(
         port_bindings: Some(port_bindings),
         binds: Some(vec![format!("{}:/var/lib/rancher/k3s", volume_name(name))]),
         network_mode: Some(network_name(name)),
+        // Automatically restart the container when Docker restarts, unless the
+        // user explicitly stopped it with `gateway stop`.
+        restart_policy: Some(RestartPolicy {
+            name: Some(RestartPolicyNameEnum::UNLESS_STOPPED),
+            maximum_retry_count: None,
+        }),
         // Add host gateway aliases for DNS resolution.
         // This allows both the entrypoint script and the running gateway
         // process to reach services on the Docker host.
@@ -919,6 +926,48 @@ pub async fn destroy_gateway_resources(docker: &Docker, name: &str) -> Result<()
     Ok(())
 }
 
+/// Clean up the gateway container and network, preserving the persistent volume.
+///
+/// Used when a resume attempt fails — we want to remove the container we may
+/// have just created but keep the volume so the user can retry without losing
+/// their k3s/etcd state and sandbox data.
+pub async fn cleanup_gateway_container(docker: &Docker, name: &str) -> Result<()> {
+    let container_name = container_name(name);
+    let net_name = network_name(name);
+
+    // Disconnect container from network
+    let _ = docker
+        .disconnect_network(
+            &net_name,
+            NetworkDisconnectRequest {
+                container: container_name.clone(),
+                force: Some(true),
+            },
+        )
+        .await;
+
+    let _ = stop_container(docker, &container_name).await;
+
+    let remove_container = docker
+        .remove_container(
+            &container_name,
+            Some(RemoveContainerOptions {
+                force: true,
+                ..Default::default()
+            }),
+        )
+        .await;
+    if let Err(err) = remove_container
+        && !is_not_found(&err)
+    {
+        return Err(err).into_diagnostic();
+    }
+
+    force_remove_network(docker, &net_name).await?;
+
+    Ok(())
+}
+
 /// Forcefully remove a Docker network, disconnecting any remaining
 /// containers first. This ensures that stale Docker network endpoints
 /// cannot prevent port bindings from being released.

crates/openshell-bootstrap/src/lib.rs

@@ -26,12 +26,13 @@ use miette::{IntoDiagnostic, Result};
 use std::sync::{Arc, Mutex};
 
 use crate::constants::{
-    CLIENT_TLS_SECRET_NAME, SERVER_CLIENT_CA_SECRET_NAME, SERVER_TLS_SECRET_NAME, network_name,
-    volume_name,
+    CLIENT_TLS_SECRET_NAME, SERVER_CLIENT_CA_SECRET_NAME, SERVER_TLS_SECRET_NAME,
+    SSH_HANDSHAKE_SECRET_NAME, network_name, volume_name,
 };
 use crate::docker::{
-    check_existing_gateway, check_port_conflicts, destroy_gateway_resources, ensure_container,
-    ensure_image, ensure_network, ensure_volume, start_container, stop_container,
+    check_existing_gateway, check_port_conflicts, cleanup_gateway_container,
+    destroy_gateway_resources, ensure_container, ensure_image, ensure_network, ensure_volume,
+    start_container, stop_container,
 };
 use crate::metadata::{
     create_gateway_metadata, create_gateway_metadata_with_host, local_gateway_host,
@@ -119,6 +120,11 @@ pub struct DeployOptions {
     /// When false, an existing gateway is left as-is and deployment is
     /// skipped (the caller is responsible for prompting the user first).
     pub recreate: bool,
+    /// When true, resume from existing state (volume, stopped container)
+    /// instead of erroring out. The deploy flow reuses the existing volume
+    /// and creates a new container if needed, preserving k3s/etcd state,
+    /// sandbox pods, and secrets.
+    pub resume: bool,
 }
 
 impl DeployOptions {
@@ -135,6 +141,7 @@ impl DeployOptions {
             registry_token: None,
             gpu: false,
             recreate: false,
+            resume: false,
         }
     }
 
@@ -200,6 +207,13 @@ impl DeployOptions {
         self.recreate = recreate;
         self
     }
+
+    /// Set whether to resume from existing state (volume, stopped container).
+    #[must_use]
+    pub fn with_resume(mut self, resume: bool) -> Self {
+        self.resume = resume;
+        self
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -264,6 +278,7 @@ where
     let registry_token = options.registry_token;
     let gpu = options.gpu;
     let recreate = options.recreate;
+    let resume = options.resume;
 
     // Wrap on_log in Arc<Mutex<>> so we can share it with pull_remote_image
     // which needs a 'static callback for the bollard streaming pull.
@@ -288,12 +303,28 @@ where
         (preflight.docker, None)
     };
 
-    // If an existing gateway is found, either tear it down (when recreate is
-    // requested) or bail out so the caller can prompt the user / reuse it.
+    // Guard: recreate takes precedence if both are set (shouldn't happen in practice).
+    if resume && recreate {
+        tracing::warn!("both resume and recreate set; recreate takes precedence");
+    }
+
+    // If an existing gateway is found, decide how to proceed:
+    // - recreate: destroy everything and start fresh
+    // - resume: keep existing state and create/start the container
+    // - neither: error out (caller should prompt the user)
     if let Some(existing) = check_existing_gateway(&target_docker, &name).await? {
         if recreate {
             log("[status] Removing existing gateway".to_string());
             destroy_gateway_resources(&target_docker, &name).await?;
+        } else if resume {
+            if existing.container_running {
+                log("[status] Gateway is already running".to_string());
+            } else {
+                log("[status] Resuming gateway from existing state".to_string());
+            }
+            // Fall through to ensure_* calls — they are idempotent and will
+            // reuse the existing volume, create a container if needed, and
+            // start it.
         } else {
             return Err(miette::miette!(
                 "Gateway '{name}' already exists (container_running={}).\n\
@@ -455,6 +486,11 @@ where
 
         store_pki_bundle(&name, &pki_bundle)?;
 
+        // Reconcile SSH handshake secret: reuse existing K8s secret if present,
+        // generate and persist a new one otherwise. This secret is stored in etcd
+        // (on the persistent volume) so it survives container restarts.
+        reconcile_ssh_handshake_secret(&target_docker, &name, &log).await?;
+
         // Push locally-built component images into the k3s containerd runtime.
         // This is the "push" path for local development — images are exported from
         // the local Docker daemon and streamed into the cluster's containerd so
@@ -524,15 +560,30 @@ where
             docker: target_docker,
         }),
         Err(deploy_err) => {
-            // Automatically clean up Docker resources (volume, container, network,
-            // image) so the environment is left in a retryable state.
-            tracing::info!("deploy failed, cleaning up gateway resources for '{name}'");
-            if let Err(cleanup_err) = destroy_gateway_resources(&target_docker, &name).await {
-                tracing::warn!(
-                    "automatic cleanup after failed deploy also failed: {cleanup_err}. \
-                     Manual cleanup may be required: \
-                     openshell gateway destroy --name {name}"
+            if resume {
+                // When resuming, preserve the volume so the user can retry.
+                // Only clean up the container and network that we may have created.
+                tracing::info!(
+                    "resume failed, cleaning up container for '{name}' (preserving volume)"
                 );
+                if let Err(cleanup_err) = cleanup_gateway_container(&target_docker, &name).await {
+                    tracing::warn!(
+                        "automatic cleanup after failed resume also failed: {cleanup_err}. \
+                         Manual cleanup may be required: \
+                         openshell gateway destroy --name {name}"
+                    );
+                }
+            } else {
+                // Automatically clean up Docker resources (volume, container, network,
+                // image) so the environment is left in a retryable state.
+                tracing::info!("deploy failed, cleaning up gateway resources for '{name}'");
+                if let Err(cleanup_err) = destroy_gateway_resources(&target_docker, &name).await {
+                    tracing::warn!(
+                        "automatic cleanup after failed deploy also failed: {cleanup_err}. \
+                         Manual cleanup may be required: \
+                         openshell gateway destroy --name {name}"
+                    );
+                }
             }
             Err(deploy_err)
         }
@@ -837,6 +888,72 @@ where
     Ok((bundle, true))
 }
 
+/// Reconcile the SSH handshake HMAC secret as a Kubernetes Secret.
+///
+/// If the secret already exists in the cluster, this is a no-op. Otherwise a
+/// fresh 32-byte hex secret is generated and applied. Because the secret lives
+/// in etcd (backed by the persistent Docker volume), it survives container
+/// restarts without regeneration — existing sandbox SSH sessions remain valid.
+async fn reconcile_ssh_handshake_secret<F>(docker: &Docker, name: &str, log: &F) -> Result<()>
+where
+    F: Fn(String) + Sync,
+{
+    use miette::WrapErr;
+
+    let cname = container_name(name);
+    let kubeconfig = constants::KUBECONFIG_PATH;
+
+    // Check if the secret already exists.
+    let (output, exit_code) = exec_capture_with_exit(
+        docker,
+        &cname,
+        vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            format!(
+                "KUBECONFIG={kubeconfig} kubectl -n openshell get secret {SSH_HANDSHAKE_SECRET_NAME} -o jsonpath='{{.data.secret}}' 2>/dev/null"
+            ),
+        ],
+    )
+    .await?;
+
+    if exit_code == 0 && !output.trim().is_empty() {
+        tracing::debug!(
+            "existing SSH handshake secret found ({} bytes encoded)",
+            output.trim().len()
+        );
+        log("[progress] Reusing existing SSH handshake secret".to_string());
+        return Ok(());
+    }
+
+    // Generate a new 32-byte hex secret and create the K8s secret.
+    log("[progress] Generating SSH handshake secret".to_string());
+    let (output, exit_code) = exec_capture_with_exit(
+        docker,
+        &cname,
+        vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            format!(
+                "SECRET=$(head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \\n') && \
+                 KUBECONFIG={kubeconfig} kubectl -n openshell create secret generic {SSH_HANDSHAKE_SECRET_NAME} \
+                 --from-literal=secret=$SECRET --dry-run=client -o yaml | \
+                 KUBECONFIG={kubeconfig} kubectl apply -f -"
+            ),
+        ],
+    )
+    .await?;
+
+    if exit_code != 0 {
+        return Err(miette::miette!(
+            "failed to create SSH handshake secret (exit {exit_code}): {output}"
+        ))
+        .wrap_err("failed to apply SSH handshake secret");
+    }
+
+    Ok(())
+}
+
 /// Load existing TLS secrets from the cluster and reconstruct a [`PkiBundle`].
 ///
 /// Returns an error string describing why secrets couldn't be loaded (for logging).

crates/openshell-cli/src/bootstrap.rs

@@ -144,43 +144,62 @@ pub async fn run_bootstrap(
     );
     eprintln!();
 
-    // Auto-bootstrap always recreates if stale Docker resources are found
-    // (e.g. metadata was deleted but container/volume still exist).
-    let mut options = openshell_bootstrap::DeployOptions::new(&gateway_name).with_recreate(true);
-    if let Some(dest) = remote {
-        let mut remote_opts = openshell_bootstrap::RemoteOptions::new(dest);
-        if let Some(key) = ssh_key {
-            remote_opts = remote_opts.with_ssh_key(key);
+    // Build base deploy options. The auto-bootstrap path tries to resume from
+    // existing state first (preserving sandboxes and secrets), falling back to
+    // a full recreate if the resume fails.
+    let build_options = |resume: bool, recreate: bool| {
+        let mut opts = openshell_bootstrap::DeployOptions::new(&gateway_name)
+            .with_resume(resume)
+            .with_recreate(recreate)
+            .with_gpu(gpu);
+        if let Some(dest) = remote {
+            let mut remote_opts = openshell_bootstrap::RemoteOptions::new(dest);
+            if let Some(key) = ssh_key {
+                remote_opts = remote_opts.with_ssh_key(key);
+            }
+            opts = opts.with_remote(remote_opts);
         }
-        options = options.with_remote(remote_opts);
-    }
-    // Read registry credentials from environment for the auto-bootstrap path.
-    // The explicit `--registry-username` / `--registry-token` flags are only
-    // on `gateway start`; when bootstrapping via `sandbox create`, the env
-    // vars are the mechanism.
-    if let Ok(username) = std::env::var("OPENSHELL_REGISTRY_USERNAME")
-        && !username.trim().is_empty()
-    {
-        options = options.with_registry_username(username);
-    }
-    if let Ok(token) = std::env::var("OPENSHELL_REGISTRY_TOKEN")
-        && !token.trim().is_empty()
-    {
-        options = options.with_registry_token(token);
-    }
-    // Read gateway host override from environment. Needed whenever the
-    // client cannot reach the Docker host at 127.0.0.1 — CI containers,
-    // WSL, remote Docker hosts, etc. The explicit `--gateway-host` flag
-    // is only on `gateway start`; this env var covers the auto-bootstrap
-    // path triggered by `sandbox create`.
-    if let Ok(host) = std::env::var("OPENSHELL_GATEWAY_HOST")
-        && !host.trim().is_empty()
+        // Read registry credentials from environment for the auto-bootstrap path.
+        // The explicit `--registry-username` / `--registry-token` flags are only
+        // on `gateway start`; when bootstrapping via `sandbox create`, the env
+        // vars are the mechanism.
+        if let Ok(username) = std::env::var("OPENSHELL_REGISTRY_USERNAME")
+            && !username.trim().is_empty()
+        {
+            opts = opts.with_registry_username(username);
+        }
+        if let Ok(token) = std::env::var("OPENSHELL_REGISTRY_TOKEN")
+            && !token.trim().is_empty()
+        {
+            opts = opts.with_registry_token(token);
+        }
+        // Read gateway host override from environment. Needed whenever the
+        // client cannot reach the Docker host at 127.0.0.1 — CI containers,
+        // WSL, remote Docker hosts, etc. The explicit `--gateway-host` flag
+        // is only on `gateway start`; this env var covers the auto-bootstrap
+        // path triggered by `sandbox create`.
+        if let Ok(host) = std::env::var("OPENSHELL_GATEWAY_HOST")
+            && !host.trim().is_empty()
+        {
+            opts = opts.with_gateway_host(host);
+        }
+        opts
+    };
+
+    // Try resume first to preserve existing sandboxes and secrets.
+    let handle = match deploy_gateway_with_panel(
+        build_options(true, false),
+        &gateway_name,
+        location,
+    )
+    .await
     {
-        options = options.with_gateway_host(host);
-    }
-    options = options.with_gpu(gpu);
-
-    let handle = deploy_gateway_with_panel(options, &gateway_name, location).await?;
+        Ok(handle) => handle,
+        Err(resume_err) => {
+            tracing::warn!("auto-bootstrap resume failed, falling back to recreate: {resume_err}");
+            deploy_gateway_with_panel(build_options(false, true), &gateway_name, location).await?
+        }
+    };
     let server = handle.gateway_endpoint().to_string();
 
     print_deploy_summary(&gateway_name, &handle);