diff --git a/nemo_rl/data/datasets/eval_datasets/mmau.py b/nemo_rl/data/datasets/eval_datasets/mmau.py index 9c810d2310..4cb4630320 100644 --- a/nemo_rl/data/datasets/eval_datasets/mmau.py +++ b/nemo_rl/data/datasets/eval_datasets/mmau.py @@ -18,10 +18,10 @@ from typing import Any import numpy as np -import soundfile as sf from datasets import Audio, load_dataset from nemo_rl.data.datasets.response_datasets.avqa import _resample_audio +from nemo_rl.data.datasets.utils import read_audio from nemo_rl.data.interfaces import TaskDataSpec from nemo_rl.data.processors import vlm_hf_data_processor @@ -58,7 +58,7 @@ def __init__( def format_data(self, data: dict[str, Any]) -> dict[str, Any]: """Convert a raw MMAU item into messages format for vlm_hf_data_processor.""" audio_raw = data["audio"] - audio_array, orig_sr = sf.read(io.BytesIO(audio_raw["bytes"])) + audio_array, orig_sr = read_audio(io.BytesIO(audio_raw["bytes"])) # Convert to mono if stereo if audio_array.ndim > 1: diff --git a/nemo_rl/data/datasets/response_datasets/audiomcq.py b/nemo_rl/data/datasets/response_datasets/audiomcq.py index ae60242dec..5912a00289 100644 --- a/nemo_rl/data/datasets/response_datasets/audiomcq.py +++ b/nemo_rl/data/datasets/response_datasets/audiomcq.py @@ -17,13 +17,12 @@ from typing import Any import numpy as np -import soundfile as sf from datasets import Dataset, load_dataset from huggingface_hub import snapshot_download from scipy.signal import resample_poly from nemo_rl.data.datasets.raw_dataset import RawDataset -from nemo_rl.data.datasets.utils import get_huggingface_cache_path +from nemo_rl.data.datasets.utils import get_huggingface_cache_path, read_audio DEFAULT_TEMPLATE = ( "{question} Please choose the answer from the following options: {choices}. " @@ -165,7 +164,7 @@ def format_data(self, data: dict[str, Any]) -> dict[str, Any]: f"(source_dataset={source!r}, audio_path={audio_path!r})." ) - audio_array, orig_sr = sf.read(absolute_path) + audio_array, orig_sr = read_audio(absolute_path) # Mono downmix for multi-channel waveforms before resampling. if audio_array.ndim > 1: diff --git a/nemo_rl/data/datasets/response_datasets/avqa.py b/nemo_rl/data/datasets/response_datasets/avqa.py index b17da9300d..df1b142bcb 100644 --- a/nemo_rl/data/datasets/response_datasets/avqa.py +++ b/nemo_rl/data/datasets/response_datasets/avqa.py @@ -18,11 +18,11 @@ from typing import Any import numpy as np -import soundfile as sf from datasets import Audio, Dataset, load_dataset from scipy.signal import resample_poly from nemo_rl.data.datasets.raw_dataset import RawDataset +from nemo_rl.data.datasets.utils import read_audio DEFAULT_TEMPLATE = ( "{question} Please choose the answer from the following options: {choices}. " @@ -109,7 +109,7 @@ def __init__( def format_data(self, data: dict[str, Any]) -> dict[str, Any]: audio_raw = data["audio"] - audio_array, orig_sr = sf.read(io.BytesIO(audio_raw["bytes"])) + audio_array, orig_sr = read_audio(io.BytesIO(audio_raw["bytes"])) # Resample to 16kHz if needed if orig_sr != 16000: diff --git a/nemo_rl/data/datasets/utils.py b/nemo_rl/data/datasets/utils.py index 4e924d1ffb..e1777d5d5f 100644 --- a/nemo_rl/data/datasets/utils.py +++ b/nemo_rl/data/datasets/utils.py @@ -51,6 +51,25 @@ def load_audio_from_file(path: str, sampling_rate: int = 16000) -> np.ndarray: return waveform.squeeze(0).numpy().astype(np.float32) +def read_audio(source: Union[str, io.BytesIO]) -> tuple[np.ndarray, int]: + """Read an audio file or buffer as ``(samples, sample_rate)``. + + ``soundfile`` is imported lazily because it dlopens ``libsndfile`` at import + time, so images that strip the bundled codecs can still import the audio + dataset modules and run text-only training. + """ + try: + import soundfile as sf + except (ImportError, OSError) as e: + raise RuntimeError( + "Reading audio requires the 'soundfile' package and its bundled " + "libsndfile library, which are unavailable in this environment. " + "Install them with `uv pip install --reinstall soundfile`." + ) from e + + return sf.read(source) + + def assert_no_double_bos(token_ids: torch.Tensor, tokenizer: TokenizerType) -> None: """Assert that there are no double starting BOS tokens in the message.