diff --git a/docker/customizer/removals/files/nmp-automodel-training.txt b/docker/customizer/removals/files/nmp-automodel-training.txt index fdc1f97e01..cb5f0223d9 100644 --- a/docker/customizer/removals/files/nmp-automodel-training.txt +++ b/docker/customizer/removals/files/nmp-automodel-training.txt @@ -12,3 +12,7 @@ /usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libopus-*.so* /usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libsndfile-*.so* /usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libswscale-*.so* +# The soundfile shim goes with its payload above. transformers gates `import soundfile` on +# find_spec("soundfile"), so leaving the module behind makes that probe report an audio +# backend that can no longer dlopen its library. +/opt/venv/lib/python3.*/site-packages/soundfile.py diff --git a/docker/customizer/removals/files/nmp-rl-training.txt b/docker/customizer/removals/files/nmp-rl-training.txt index 215d0aff8e..a2d2eebd83 100644 --- a/docker/customizer/removals/files/nmp-rl-training.txt +++ b/docker/customizer/removals/files/nmp-rl-training.txt @@ -5,3 +5,6 @@ /opt/nemo_rl_venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so /opt/ray_venvs/*/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so /opt/uv_cache/archive-v0/*/_soundfile_data/libsndfile_*.so +/opt/nemo_rl_venv/lib/python3.*/site-packages/soundfile.py +/opt/ray_venvs/*/lib/python3.*/site-packages/soundfile.py +/opt/uv_cache/archive-v0/*/soundfile.py diff --git a/docker/rl/Dockerfile.nmp-rl-training b/docker/rl/Dockerfile.nmp-rl-training index d2675669d8..7e2b98ad04 100644 --- a/docker/rl/Dockerfile.nmp-rl-training +++ b/docker/rl/Dockerfile.nmp-rl-training @@ -87,7 +87,13 @@ RUN --mount=type=bind,source=docker/scripts/remove-listed-files.sh,target=/tmp/r # Point runtime uv writes at the per-user cache instead, which is already owned by USER_UID above. # The prefetched venvs are unaffected: their symlinks are absolute /opt/uv_cache paths resolved at # build time, so changing UV_CACHE_DIR only redirects FUTURE uv operations. +# HOME must follow the runtime user. The base sets HOME=/home/nmp-build for its build user +# (uid 2000) and that value survives into the published image, but this stage runs as +# ${USER_UID}, which cannot write there. Anything resolving `~` at import time then dies: +# swanlab, which nemo_rl.utils.logger imports unconditionally, calls os.mkdir("~/.swanlab") +# from its module body, so the GRPO driver fails before reading a config. ENV PATH="/opt/nemo_rl_venv/bin:${PATH}" \ + HOME=/home/${USERNAME} \ UV_CACHE_DIR=/home/${USERNAME}/.cache/uv ENTRYPOINT ["/opt/venv/bin/python"] CMD ["-m", "nmp.rl.tasks.training", "--help"] diff --git a/tests/smoke_gpu/test_customizer_automodel.py b/tests/smoke_gpu/test_customizer_automodel.py index 1ab6d86815..1dd5377b9e 100644 --- a/tests/smoke_gpu/test_customizer_automodel.py +++ b/tests/smoke_gpu/test_customizer_automodel.py @@ -21,7 +21,12 @@ import pytest from python_package_versions import assert_python_package_min_versions -SOUNDFILE_LIBSNDFILE_PATTERN = "/opt/venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so" +SOUNDFILE_PATTERNS = ( + "/opt/venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so", + # The shim is removed with its payload; see the removals file for why keeping it is + # worse than not shipping soundfile at all. + "/opt/venv/lib/python3.*/site-packages/soundfile.py", +) MINIMUM_PYTHON_PACKAGE_VERSIONS = { "bitsandbytes": "0.49.2", "mamba-ssm": "2.3.0", @@ -68,10 +73,24 @@ def test_nmp_automodel_training_importable(): @pytest.mark.smoke_nmp_automodel_training def test_soundfile_libsndfile_removed(): - remaining = sorted(glob(SOUNDFILE_LIBSNDFILE_PATTERN)) + remaining = sorted(path for pattern in SOUNDFILE_PATTERNS for path in glob(pattern)) assert remaining == [], f"file cleanup left scanner-visible libsndfile files: {remaining}" +@pytest.mark.smoke_nmp_automodel_training +def test_transformers_audio_backend_probe_is_off(): + """The payload and its shim must be removed together. + + ``transformers.audio_utils`` runs ``if is_soundfile_available(): import soundfile``, and + that probe is ``find_spec("soundfile")`` -- file presence, not loadability. Removing only + the codec leaves the probe answering yes for a backend that then fails to dlopen, which + breaks ``from transformers import AutoProcessor`` and everything importing through it. + """ + from transformers.utils.import_utils import is_soundfile_available + + assert not is_soundfile_available() + + @pytest.mark.smoke_nmp_automodel_training def test_dali_still_importable_after_file_cleanup(): import importlib.metadata diff --git a/tests/smoke_gpu/test_rl_training.py b/tests/smoke_gpu/test_rl_training.py index 3f7159bbe6..756a6364dc 100644 --- a/tests/smoke_gpu/test_rl_training.py +++ b/tests/smoke_gpu/test_rl_training.py @@ -37,6 +37,11 @@ "/opt/nemo_rl_venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so", "/opt/ray_venvs/*/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so", "/opt/uv_cache/archive-v0/*/_soundfile_data/libsndfile_*.so", + # The shim is removed with the codec; see docker/rl/codec-file-removals.txt for why + # keeping it is worse than not shipping soundfile at all. + "/opt/nemo_rl_venv/lib/python3.*/site-packages/soundfile.py", + "/opt/ray_venvs/*/lib/python3.*/site-packages/soundfile.py", + "/opt/uv_cache/archive-v0/*/soundfile.py", ) BASE_VENV_MINIMUM_PYTHON_PACKAGE_VERSIONS = { "wandb": "0.28.2", @@ -239,6 +244,27 @@ def test_soundfile_libsndfile_removed(): assert remaining == [], f"file cleanup left scanner-visible libsndfile files: {remaining}" +@pytest.mark.smoke_nmp_rl_training +def test_transformers_audio_backend_probe_is_off(): + """Removing the codec must also switch off the probe that guards its import. + + ``transformers.audio_utils`` does ``if is_soundfile_available(): import soundfile``, + and that probe is ``find_spec("soundfile")`` -- file presence, not loadability. Delete + the codec but keep the module and the probe says yes to a backend that then fails to + dlopen, so ``from transformers import AutoProcessor`` raises. That import is at module + scope in ``nemo_rl.algorithms.grpo``, so the GRPO driver dies before it reads a config. + """ + from transformers.utils.import_utils import is_soundfile_available + + assert not is_soundfile_available() + + +@pytest.mark.smoke_nmp_rl_training +def test_grpo_driver_module_imports(): + """The exact import the GRPO driver performs first, and the one the codec strip broke.""" + from nemo_rl.algorithms.grpo import MasterConfig # noqa: F401 + + @pytest.mark.smoke_nmp_rl_training def test_no_git_directories_shipped(): """The published image must not carry repository history.