feat(sandbox): add configurable imagePullPolicy for sandbox pods (#256)

Add OPENSHELL_SANDBOX_IMAGE_PULL_POLICY env/CLI flag that propagates
through Config -> SandboxClient -> pod spec generation. When set, the
policy is applied to both inline and pod-template sandbox containers.

Helm chart exposes server.sandboxImagePullPolicy (default empty = K8s
default behavior). cluster-deploy-fast sets it to Always so dev
clusters always pull fresh sandbox images.

Author: drew

crates/navigator-core/src/config.rs

@@ -32,6 +32,13 @@ pub struct Config {
     #[serde(default)]
     pub sandbox_image: String,
 
+    /// Kubernetes `imagePullPolicy` for sandbox pods (e.g. `Always`,
+    /// `IfNotPresent`, `Never`).  Defaults to empty, which lets Kubernetes
+    /// apply its own default (`:latest` → `Always`, anything else →
+    /// `IfNotPresent`).
+    #[serde(default)]
+    pub sandbox_image_pull_policy: String,
+
     /// gRPC endpoint for sandboxes to connect back to OpenShell.
     /// Used by sandbox pods to fetch their policy at startup.
     #[serde(default)]
@@ -108,6 +115,7 @@ impl Config {
             database_url: String::new(),
             sandbox_namespace: default_sandbox_namespace(),
             sandbox_image: String::new(),
+            sandbox_image_pull_policy: String::new(),
             grpc_endpoint: String::new(),
             ssh_gateway_host: default_ssh_gateway_host(),
             ssh_gateway_port: default_ssh_gateway_port(),
@@ -155,6 +163,13 @@ impl Config {
         self
     }
 
+    /// Create a new configuration with a sandbox image pull policy.
+    #[must_use]
+    pub fn with_sandbox_image_pull_policy(mut self, policy: impl Into<String>) -> Self {
+        self.sandbox_image_pull_policy = policy.into();
+        self
+    }
+
     /// Create a new configuration with a gRPC endpoint for sandbox callback.
     #[must_use]
     pub fn with_grpc_endpoint(mut self, endpoint: impl Into<String>) -> Self {

crates/navigator-server/src/lib.rs

@@ -113,6 +113,7 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul
     let sandbox_client = SandboxClient::new(
         config.sandbox_namespace.clone(),
         config.sandbox_image.clone(),
+        config.sandbox_image_pull_policy.clone(),
         config.grpc_endpoint.clone(),
         format!("0.0.0.0:{}", config.sandbox_ssh_port),
         config.ssh_handshake_secret.clone(),

crates/navigator-server/src/main.rs

@@ -49,6 +49,10 @@ struct Args {
     #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")]
     sandbox_image: Option<String>,
 
+    /// Kubernetes imagePullPolicy for sandbox pods (Always, IfNotPresent, Never).
+    #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")]
+    sandbox_image_pull_policy: Option<String>,
+
     /// gRPC endpoint for sandboxes to callback to `OpenShell`.
     /// This should be reachable from within the Kubernetes cluster.
     #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
@@ -157,6 +161,10 @@ async fn main() -> Result<()> {
         config = config.with_sandbox_image(image);
     }
 
+    if let Some(policy) = args.sandbox_image_pull_policy {
+        config = config.with_sandbox_image_pull_policy(policy);
+    }
+
     if let Some(endpoint) = args.grpc_endpoint {
         config = config.with_grpc_endpoint(endpoint);
     }

crates/navigator-server/src/sandbox/mod.rs

@@ -37,6 +37,9 @@ pub struct SandboxClient {
     client: Client,
     namespace: String,
     default_image: String,
+    /// Kubernetes `imagePullPolicy` for sandbox containers.  When empty the
+    /// field is omitted from the pod spec and Kubernetes applies its default.
+    image_pull_policy: String,
     grpc_endpoint: String,
     ssh_listen_addr: String,
     ssh_handshake_secret: String,
@@ -59,6 +62,7 @@ impl SandboxClient {
     pub async fn new(
         namespace: String,
         default_image: String,
+        image_pull_policy: String,
         grpc_endpoint: String,
         ssh_listen_addr: String,
         ssh_handshake_secret: String,
@@ -79,6 +83,7 @@ impl SandboxClient {
             client,
             namespace,
             default_image,
+            image_pull_policy,
             grpc_endpoint,
             ssh_listen_addr,
             ssh_handshake_secret,
@@ -149,6 +154,7 @@ impl SandboxClient {
         obj.data = sandbox_to_k8s_spec(
             sandbox.spec.as_ref(),
             &self.default_image,
+            &self.image_pull_policy,
             &sandbox.id,
             &sandbox.name,
             &self.grpc_endpoint,
@@ -721,6 +727,7 @@ fn apply_supervisor_bootstrap(pod_template: &mut serde_json::Value, default_imag
 fn sandbox_to_k8s_spec(
     spec: Option<&SandboxSpec>,
     default_image: &str,
+    image_pull_policy: &str,
     sandbox_id: &str,
     sandbox_name: &str,
     grpc_endpoint: &str,
@@ -746,6 +753,7 @@ fn sandbox_to_k8s_spec(
                 sandbox_template_to_k8s(
                     template,
                     default_image,
+                    image_pull_policy,
                     sandbox_id,
                     sandbox_name,
                     grpc_endpoint,
@@ -777,6 +785,7 @@ fn sandbox_to_k8s_spec(
             sandbox_template_to_k8s(
                 &SandboxTemplate::default(),
                 default_image,
+                image_pull_policy,
                 sandbox_id,
                 sandbox_name,
                 grpc_endpoint,
@@ -798,6 +807,7 @@ fn sandbox_to_k8s_spec(
 fn sandbox_template_to_k8s(
     template: &SandboxTemplate,
     default_image: &str,
+    image_pull_policy: &str,
     sandbox_id: &str,
     sandbox_name: &str,
     grpc_endpoint: &str,
@@ -812,6 +822,7 @@ fn sandbox_template_to_k8s(
             pod_template,
             template,
             default_image,
+            image_pull_policy,
             sandbox_id,
             sandbox_name,
             grpc_endpoint,
@@ -861,6 +872,12 @@ fn sandbox_template_to_k8s(
     };
     if !image.is_empty() {
         container.insert("image".to_string(), serde_json::json!(image));
+        if !image_pull_policy.is_empty() {
+            container.insert(
+                "imagePullPolicy".to_string(),
+                serde_json::json!(image_pull_policy),
+            );
+        }
     }
 
     // Build environment variables - start with OpenShell-required vars
@@ -945,6 +962,7 @@ fn inject_pod_template(
     mut pod_template: serde_json::Value,
     template: &SandboxTemplate,
     default_image: &str,
+    image_pull_policy: &str,
     sandbox_id: &str,
     sandbox_name: &str,
     grpc_endpoint: &str,
@@ -1006,6 +1024,16 @@ fn inject_pod_template(
             !client_tls_secret_name.is_empty(),
         );
 
+        // Inject imagePullPolicy on the agent container.
+        if !image_pull_policy.is_empty() {
+            if let Some(container_obj) = container.as_object_mut() {
+                container_obj.insert(
+                    "imagePullPolicy".to_string(),
+                    serde_json::json!(image_pull_policy),
+                );
+            }
+        }
+
         // Inject TLS volumeMount on the agent container.
         if !client_tls_secret_name.is_empty()
             && let Some(container_obj) = container.as_object_mut()

deploy/helm/openshell/templates/statefulset.yaml

@@ -51,6 +51,10 @@ spec:
               value: {{ .Values.server.sandboxNamespace | quote }}
             - name: OPENSHELL_SANDBOX_IMAGE
               value: {{ .Values.server.sandboxImage | quote }}
+            {{- if .Values.server.sandboxImagePullPolicy }}
+            - name: OPENSHELL_SANDBOX_IMAGE_PULL_POLICY
+              value: {{ .Values.server.sandboxImagePullPolicy | quote }}
+            {{- end }}
             - name: OPENSHELL_GRPC_ENDPOINT
               value: {{ if .Values.server.disableTls }}{{ .Values.server.grpcEndpoint | replace "https://" "http://" | quote }}{{ else }}{{ .Values.server.grpcEndpoint | quote }}{{ end }}
             {{- if .Values.server.sshGatewayHost }}

deploy/helm/openshell/values.yaml

@@ -68,6 +68,10 @@ server:
   sandboxNamespace: openshell
   dbUrl: "sqlite:/var/openshell/openshell.db"
   sandboxImage: "ghcr.io/nvidia/openshell/sandbox:latest"
+  # Kubernetes imagePullPolicy for sandbox pods.  Empty = Kubernetes default
+  # (Always for :latest, IfNotPresent otherwise).  Set to "Always" for dev
+  # clusters so new images are picked up without manual eviction.
+  sandboxImagePullPolicy: ""
   # gRPC endpoint for sandboxes to callback to OpenShell (must be reachable from pods)
   grpcEndpoint: "https://openshell.openshell.svc.cluster.local:8080"
   # Public host/port returned to CLI clients for SSH proxy CONNECT requests.

tasks/scripts/cluster-deploy-fast.sh

@@ -411,6 +411,7 @@ if [[ "${needs_helm_upgrade}" == "1" ]]; then
     --set image.pullPolicy=Always \
     --set-string server.grpcEndpoint=https://openshell.openshell.svc.cluster.local:8080 \
     --set server.sandboxImage=${IMAGE_REPO_BASE}/sandbox:${IMAGE_TAG} \
+    --set server.sandboxImagePullPolicy=Always \
     --set server.tls.certSecretName=openshell-server-tls \
     --set server.tls.clientCaSecretName=openshell-server-client-ca \
     --set server.tls.clientTlsSecretName=openshell-client-tls \