diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index 195eaca783..94bb3245d8 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -558,12 +558,11 @@ models: peft_refresh_interval: 30 # default: 'nmp-api' lora_sidecar_image_name: nmp-api + # Container command for the LoRA adapters sidecar. Replaces the image ENTRYPOINT (e.g. `nemo`), so the adapters module is invoked directly and does not write platform instance state under the image $HOME (which is unwritable under the pod's runtime uid). lora_sidecar_command: - - nemo - - services - - run - - --sidecars - - adapters + - python + - -m + - nmp.core.models.sidecars.adapters.main lora_sidecar_args: [] # BusyBox image repository for LoRA cache init containers. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/library/busybox' busybox_image: docker.io/library/busybox diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 3f3a6cd495..957311b6fc 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -453,6 +453,12 @@ basePlatformConfig: | enabled: true k8s_executor: local-k8s default_executor: local-k8s + # Bypass the image `nemo` ENTRYPOINT; invoke the adapters module directly. + # Avoids PermissionError when the sidecar writes instance state under $HOME. + lora_sidecar_command: + - python + - -m + - nmp.core.models.sidecars.adapters.main deployments: executors: diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/docker_helpers.py b/plugins/nemo-deployments/tests/unit/backends/docker/docker_helpers.py index 5f3c887516..3ff1631c81 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/docker_helpers.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/docker_helpers.py @@ -75,7 +75,7 @@ def lora_config(*, restart_policy: RestartPolicy = "Always") -> DeploymentConfig Container( name="lora-adapters", image="my-registry/nmp-api:local", - command=["nemo", "services", "run", "--sidecars", "adapters"], + command=["python", "-m", "nmp.core.models.sidecars.adapters.main"], volumeMounts=[ VolumeMount(name="weights", mountPath="/model-store", readOnly=True), VolumeMount(name="scratch", mountPath="/scratch"), diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py index 027fa8bda0..ce60701830 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py @@ -69,12 +69,12 @@ _WEIGHTS_MOUNT = "/model-store" _SCRATCH_MOUNT = "/scratch" _LORA_MOUNT = "/scratch/loras" -# The adapters sidecar's `nemo services run` writes state under $XDG_STATE_HOME (default -# ~/.local/state) and its local data dir under $XDG_DATA_HOME (default ~/.local/share, via -# nmp_user_data_dir()). The pod runs it as the vLLM uid (2000), which does not own the -# nmp-api image's $HOME (/home/nvs, uid 1000), so both defaults are unwritable and the -# sidecar crash-loops. Redirect both to the writable scratch volume, outside the -# /scratch/loras subtree the adapters controller GCs. +# The adapters sidecar writes under the XDG base dirs: $XDG_STATE_HOME (default +# ~/.local/state), $XDG_DATA_HOME (default ~/.local/share, via nmp_user_data_dir()), +# and $XDG_CONFIG_HOME (default ~/.config). The pod runs it as the vLLM uid (2000), +# which does not own the nmp-api image's $HOME (/home/nvs, uid 1000), so those defaults +# are unwritable and the sidecar crash-loops. Redirect all three to the writable +# scratch volume, outside the /scratch/loras subtree the adapters controller GCs. _LORA_SIDECAR_XDG_HOME = f"{_SCRATCH_MOUNT}/.local" _SCRATCH_VOLUME_SIZE = "1Gi" @@ -200,6 +200,7 @@ def _lora_sidecar( "NMP_BASE_URL": _container_reachable_platform_base_url(runtime=resolved.runtime), "XDG_STATE_HOME": _LORA_SIDECAR_XDG_HOME, "XDG_DATA_HOME": _LORA_SIDECAR_XDG_HOME, + "XDG_CONFIG_HOME": _LORA_SIDECAR_XDG_HOME, } if engine == ENGINE_VLLM: sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] = MODEL_STORE_PATH diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py index 19dca500c4..092f7c893f 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py @@ -32,7 +32,13 @@ class DeploymentsPluginConfig(BaseModel): peft_refresh_interval: int = 30 lora_sidecar_image_name: str = "nmp-api" lora_sidecar_command: list[str] = Field( - default_factory=lambda: ["nemo", "services", "run", "--sidecars", "adapters"] + default_factory=lambda: ["python", "-m", "nmp.core.models.sidecars.adapters.main"], + description=( + "Container command for the LoRA adapters sidecar. Replaces the image ENTRYPOINT " + "(e.g. `nemo`), so the adapters module is invoked directly and does not write " + "platform instance state under the image $HOME (which is unwritable under the " + "pod's runtime uid)." + ), ) lora_sidecar_args: list[str] = Field(default_factory=list) busybox_image: str = Field( diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py index 9635977e3e..c2d4b45a72 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py @@ -359,14 +359,17 @@ def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None: sidecar = k8s.server_config.init_containers[-1] assert sidecar.restart_policy == "Always" assert sidecar.image == "registry/nmp-api:tag" + assert sidecar.command == ["python", "-m", "nmp.core.models.sidecars.adapters.main"] env = {item.name: item.value for item in sidecar.env} assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" - # Sidecar `nemo services run` writes both its instance state ($XDG_STATE_HOME) and its - # local data dir ($XDG_DATA_HOME, via nmp_user_data_dir) -- both must land on the - # writable scratch volume, not the nmp-api image's $HOME (unwritable under the pod's - # vLLM uid, so the sidecar crash-loops on PermissionError). See _lora_sidecar. + # Sidecar writes under $XDG_STATE_HOME, $XDG_DATA_HOME (via nmp_user_data_dir), and + # $XDG_CONFIG_HOME. All three must land on the writable scratch volume, not the image + # $HOME (unwritable under the pod's vLLM uid). Invoking the adapters module directly + # (not `nemo services run`) avoids the platform runner's home-dir writes that caused + # PermissionError / READY→PENDING regressions (NVBug 6573168). See _lora_sidecar. assert env["XDG_STATE_HOME"] == "/scratch/.local" assert env["XDG_DATA_HOME"] == "/scratch/.local" + assert env["XDG_CONFIG_HOME"] == "/scratch/.local" assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" assert env["NMP_BASE_URL"] == "http://platform.example:8080" assert env["VLLM_ENDPOINT"] == "http://127.0.0.1:8000"