fix(customizer): change entrypoint of side car to avoid API spin up for LoRA and unwritable dirs

docs/set-up/config-reference.mdx

@@ -558,12 +558,11 @@ models:
         peft_refresh_interval: 30
         # default: 'nmp-api'
         lora_sidecar_image_name: nmp-api
+        # Container command for the LoRA adapters sidecar. Replaces the image ENTRYPOINT (e.g. `nemo`), so the adapters module is invoked directly and does not write platform instance state under the image $HOME (which is unwritable under the pod's runtime uid).
         lora_sidecar_command:
-        - nemo
-        - services
-        - run
-        - --sidecars
-        - adapters
+        - python
+        - -m
+        - nmp.core.models.sidecars.adapters.main
         lora_sidecar_args: []
         # BusyBox image repository for LoRA cache init containers. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/library/busybox'
         busybox_image: docker.io/library/busybox

k8s/helm/values.yaml

@@ -453,6 +453,12 @@ basePlatformConfig: |
           enabled: true
           k8s_executor: local-k8s
           default_executor: local-k8s
+          # Bypass the image `nemo` ENTRYPOINT; invoke the adapters module directly.
+          # Avoids PermissionError when the sidecar writes instance state under $HOME.
+          lora_sidecar_command:
+            - python
+            - -m
+            - nmp.core.models.sidecars.adapters.main
 
   deployments:
     executors:

plugins/nemo-deployments/tests/unit/backends/docker/docker_helpers.py

@@ -75,7 +75,7 @@ def lora_config(*, restart_policy: RestartPolicy = "Always") -> DeploymentConfig
             Container(
                 name="lora-adapters",
                 image="my-registry/nmp-api:local",
-                command=["nemo", "services", "run", "--sidecars", "adapters"],
+                command=["python", "-m", "nmp.core.models.sidecars.adapters.main"],
                 volumeMounts=[
                     VolumeMount(name="weights", mountPath="/model-store", readOnly=True),
                     VolumeMount(name="scratch", mountPath="/scratch"),

services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py

@@ -69,12 +69,12 @@
 _WEIGHTS_MOUNT = "/model-store"
 _SCRATCH_MOUNT = "/scratch"
 _LORA_MOUNT = "/scratch/loras"
-# The adapters sidecar's `nemo services run` writes state under $XDG_STATE_HOME (default
-# ~/.local/state) and its local data dir under $XDG_DATA_HOME (default ~/.local/share, via
-# nmp_user_data_dir()). The pod runs it as the vLLM uid (2000), which does not own the
-# nmp-api image's $HOME (/home/nvs, uid 1000), so both defaults are unwritable and the
-# sidecar crash-loops. Redirect both to the writable scratch volume, outside the
-# /scratch/loras subtree the adapters controller GCs.
+# The adapters sidecar writes under the XDG base dirs: $XDG_STATE_HOME (default
+# ~/.local/state), $XDG_DATA_HOME (default ~/.local/share, via nmp_user_data_dir()),
+# and $XDG_CONFIG_HOME (default ~/.config). The pod runs it as the vLLM uid (2000),
+# which does not own the nmp-api image's $HOME (/home/nvs, uid 1000), so those defaults
+# are unwritable and the sidecar crash-loops. Redirect all three to the writable
+# scratch volume, outside the /scratch/loras subtree the adapters controller GCs.
 _LORA_SIDECAR_XDG_HOME = f"{_SCRATCH_MOUNT}/.local"
 _SCRATCH_VOLUME_SIZE = "1Gi"
 
@@ -200,6 +200,7 @@ def _lora_sidecar(
         "NMP_BASE_URL": _container_reachable_platform_base_url(runtime=resolved.runtime),
         "XDG_STATE_HOME": _LORA_SIDECAR_XDG_HOME,
         "XDG_DATA_HOME": _LORA_SIDECAR_XDG_HOME,
+        "XDG_CONFIG_HOME": _LORA_SIDECAR_XDG_HOME,
     }
     if engine == ENGINE_VLLM:
         sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] = MODEL_STORE_PATH

services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py

@@ -32,7 +32,13 @@ class DeploymentsPluginConfig(BaseModel):
     peft_refresh_interval: int = 30
     lora_sidecar_image_name: str = "nmp-api"
     lora_sidecar_command: list[str] = Field(
-        default_factory=lambda: ["nemo", "services", "run", "--sidecars", "adapters"]
+        default_factory=lambda: ["python", "-m", "nmp.core.models.sidecars.adapters.main"],
+        description=(
+            "Container command for the LoRA adapters sidecar. Replaces the image ENTRYPOINT "
+            "(e.g. `nemo`), so the adapters module is invoked directly and does not write "
+            "platform instance state under the image $HOME (which is unwritable under the "
+            "pod's runtime uid)."
+        ),
     )
     lora_sidecar_args: list[str] = Field(default_factory=list)
     busybox_image: str = Field(

services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py

@@ -359,14 +359,17 @@ def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None:
     sidecar = k8s.server_config.init_containers[-1]
     assert sidecar.restart_policy == "Always"
     assert sidecar.image == "registry/nmp-api:tag"
+    assert sidecar.command == ["python", "-m", "nmp.core.models.sidecars.adapters.main"]
     env = {item.name: item.value for item in sidecar.env}
     assert env["NIM_PEFT_SOURCE"] == "/scratch/loras"
-    # Sidecar `nemo services run` writes both its instance state ($XDG_STATE_HOME) and its
-    # local data dir ($XDG_DATA_HOME, via nmp_user_data_dir) -- both must land on the
-    # writable scratch volume, not the nmp-api image's $HOME (unwritable under the pod's
-    # vLLM uid, so the sidecar crash-loops on PermissionError). See _lora_sidecar.
+    # Sidecar writes under $XDG_STATE_HOME, $XDG_DATA_HOME (via nmp_user_data_dir), and
+    # $XDG_CONFIG_HOME. All three must land on the writable scratch volume, not the image
+    # $HOME (unwritable under the pod's vLLM uid). Invoking the adapters module directly
+    # (not `nemo services run`) avoids the platform runner's home-dir writes that caused
+    # PermissionError / READY→PENDING regressions (NVBug 6573168). See _lora_sidecar.
     assert env["XDG_STATE_HOME"] == "/scratch/.local"
     assert env["XDG_DATA_HOME"] == "/scratch/.local"
+    assert env["XDG_CONFIG_HOME"] == "/scratch/.local"
     assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store"
     assert env["NMP_BASE_URL"] == "http://platform.example:8080"
     assert env["VLLM_ENDPOINT"] == "http://127.0.0.1:8000"