Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions nemo_rl/data/datasets/eval_datasets/mmau.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
from typing import Any

import numpy as np
import soundfile as sf
from datasets import Audio, load_dataset

from nemo_rl.data.datasets.response_datasets.avqa import _resample_audio
from nemo_rl.data.datasets.utils import read_audio
from nemo_rl.data.interfaces import TaskDataSpec
from nemo_rl.data.processors import vlm_hf_data_processor

Expand Down Expand Up @@ -58,7 +58,7 @@ def __init__(
def format_data(self, data: dict[str, Any]) -> dict[str, Any]:
"""Convert a raw MMAU item into messages format for vlm_hf_data_processor."""
audio_raw = data["audio"]
audio_array, orig_sr = sf.read(io.BytesIO(audio_raw["bytes"]))
audio_array, orig_sr = read_audio(io.BytesIO(audio_raw["bytes"]))

# Convert to mono if stereo
if audio_array.ndim > 1:
Expand Down
5 changes: 2 additions & 3 deletions nemo_rl/data/datasets/response_datasets/audiomcq.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,12 @@
from typing import Any

import numpy as np
import soundfile as sf
from datasets import Dataset, load_dataset
from huggingface_hub import snapshot_download
from scipy.signal import resample_poly

from nemo_rl.data.datasets.raw_dataset import RawDataset
from nemo_rl.data.datasets.utils import get_huggingface_cache_path
from nemo_rl.data.datasets.utils import get_huggingface_cache_path, read_audio

DEFAULT_TEMPLATE = (
"{question} Please choose the answer from the following options: {choices}. "
Expand Down Expand Up @@ -165,7 +164,7 @@ def format_data(self, data: dict[str, Any]) -> dict[str, Any]:
f"(source_dataset={source!r}, audio_path={audio_path!r})."
)

audio_array, orig_sr = sf.read(absolute_path)
audio_array, orig_sr = read_audio(absolute_path)

# Mono downmix for multi-channel waveforms before resampling.
if audio_array.ndim > 1:
Expand Down
4 changes: 2 additions & 2 deletions nemo_rl/data/datasets/response_datasets/avqa.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
from typing import Any

import numpy as np
import soundfile as sf
from datasets import Audio, Dataset, load_dataset
from scipy.signal import resample_poly

from nemo_rl.data.datasets.raw_dataset import RawDataset
from nemo_rl.data.datasets.utils import read_audio

DEFAULT_TEMPLATE = (
"{question} Please choose the answer from the following options: {choices}. "
Expand Down Expand Up @@ -109,7 +109,7 @@ def __init__(

def format_data(self, data: dict[str, Any]) -> dict[str, Any]:
audio_raw = data["audio"]
audio_array, orig_sr = sf.read(io.BytesIO(audio_raw["bytes"]))
audio_array, orig_sr = read_audio(io.BytesIO(audio_raw["bytes"]))

# Resample to 16kHz if needed
if orig_sr != 16000:
Expand Down
19 changes: 19 additions & 0 deletions nemo_rl/data/datasets/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,25 @@ def load_audio_from_file(path: str, sampling_rate: int = 16000) -> np.ndarray:
return waveform.squeeze(0).numpy().astype(np.float32)


def read_audio(source: Union[str, io.BytesIO]) -> tuple[np.ndarray, int]:
"""Read an audio file or buffer as ``(samples, sample_rate)``.

``soundfile`` is imported lazily because it dlopens ``libsndfile`` at import
time, so images that strip the bundled codecs can still import the audio
dataset modules and run text-only training.
"""
try:
import soundfile as sf
except (ImportError, OSError) as e:
raise RuntimeError(
"Reading audio requires the 'soundfile' package and its bundled "
"libsndfile library, which are unavailable in this environment. "
"Install them with `uv pip install --reinstall soundfile`."
) from e

return sf.read(source)


def assert_no_double_bos(token_ids: torch.Tensor, tokenizer: TokenizerType) -> None:
"""Assert that there are no double starting BOS tokens in the message.

Expand Down
Loading