add suggestions

Signed-off-by: Sarah Yurick <sarahyurick@gmail.com>

Author: sarahyurick

nemo_curator/backends/base.py

@@ -122,11 +122,12 @@ def process_batch(self, tasks: list[Task]) -> list[Task]:
             raise ValueError(msg)
 
         # Record failed tasks for later inspection or retry bookkeeping.
-        record_failed_tasks(failed_tasks)
+        if failed_tasks:
+            record_failed_tasks()
 
         # Filter tasks based on the Slurm array configuration.
         slurm_array = resolve_slurm_array_config(is_source_stage=is_source_stage)
-        if slurm_array is not None:
+        if slurm_array is not None and is_source_stage:
             results = filter_slurm_array_source_tasks(results, slurm_array, self.stage.name)
 
         # Opt-in resumability: fire per-source deltas (no-op when no actor registered).

nemo_curator/backends/failed_task_markers.py

@@ -16,7 +16,6 @@
 import uuid
 from pathlib import Path
 
-from nemo_curator.tasks.sentinels import FailedTask
 from nemo_curator.utils.retry_manifest import METADATA_DIRNAME
 
 FAILED_TASKS_DIR_ENV_VAR = "NEMO_CURATOR_FAILED_TASKS_DIR"
@@ -61,10 +60,8 @@ def configure_slurm_array_failed_task_manifest_dir(checkpoint_path: str | Path,
     return _configure_failed_task_manifest_dir(manifest_dir)
 
 
-def record_failed_tasks(failed_tasks: list[FailedTask]) -> None:
+def record_failed_tasks() -> None:
     """Write one attempt-scoped manifest after any FailedTask is detected."""
-    if not failed_tasks:
-        return
     manifest_dir = os.environ.get(FAILED_TASKS_DIR_ENV_VAR)
     if not manifest_dir:
         return

tests/backends/test_base_stage_adapter.py

@@ -104,7 +104,7 @@ def filter_tasks(
             calls["filter_tasks"] += 1
             return tasks
 
-        def record_failed_tasks(_failed_tasks: list[FailedTask]) -> None:
+        def record_failed_tasks() -> None:
             calls["record_failed_tasks"] += 1
 
         monkeypatch.setattr(base_module, "resolve_slurm_array_config", resolve_config)

tests/backends/test_failed_task_markers.py

@@ -26,13 +26,6 @@
     failed_task_manifest_exists,
     record_failed_tasks,
 )
-from nemo_curator.tasks.sentinels import FailedTask
-
-
-def _failed_task(task_id: str = "0_7_0") -> FailedTask:
-    task = FailedTask()
-    task.task_id = task_id
-    return task
 
 
 class TestFailedTaskManifest:
@@ -82,7 +75,7 @@ def test_record_failed_tasks_writes_single_manifest(
         manifest_dir = tmp_path / "failed-tasks"
         monkeypatch.setenv(FAILED_TASKS_DIR_ENV_VAR, str(manifest_dir))
 
-        record_failed_tasks([_failed_task("0_7_0"), _failed_task("0_8_0")])
+        record_failed_tasks()
 
         manifest_files = list(manifest_dir.glob("*.json"))
         assert manifest_files == [manifest_dir / FAILED_TASK_MANIFEST_FILENAME]
@@ -93,55 +86,42 @@ def test_additional_failed_tasks_leave_existing_manifest_unchanged(
     ) -> None:
         manifest_dir = tmp_path / "failed-tasks"
         monkeypatch.setenv(FAILED_TASKS_DIR_ENV_VAR, str(manifest_dir))
-        record_failed_tasks([_failed_task("0_7_0")])
+        record_failed_tasks()
         manifest_file = manifest_dir / FAILED_TASK_MANIFEST_FILENAME
-        original_manifest = manifest_file.read_text()
 
-        record_failed_tasks([_failed_task("0_8_0")])
+        record_failed_tasks()
 
         assert list(manifest_dir.glob("*.json")) == [manifest_file]
-        assert manifest_file.read_text() == original_manifest
 
     def test_record_failed_tasks_without_configured_attempt_is_noop(
         self, tmp_path: Path, monkeypatch: MonkeyPatch
     ) -> None:
         monkeypatch.delenv(FAILED_TASKS_DIR_ENV_VAR, raising=False)
         monkeypatch.chdir(tmp_path)
 
-        record_failed_tasks([_failed_task()])
+        record_failed_tasks()
 
         assert not (tmp_path / ".nemo_curator_metadata").exists()
 
-    def test_record_failed_tasks_does_not_write_manifest_for_empty_list(
-        self, tmp_path: Path, monkeypatch: MonkeyPatch
-    ) -> None:
-        manifest_dir = tmp_path / "failed-tasks"
-        monkeypatch.setenv(FAILED_TASKS_DIR_ENV_VAR, str(manifest_dir))
-
-        record_failed_tasks([])
-
-        assert not manifest_dir.exists()
-        assert not failed_task_manifest_exists()
-
     def test_record_failed_tasks_propagates_manifest_write_failure(
         self, tmp_path: Path, monkeypatch: MonkeyPatch
     ) -> None:
         manifest_dir = tmp_path / "failed-tasks"
         monkeypatch.setenv(FAILED_TASKS_DIR_ENV_VAR, str(manifest_dir))
 
-        def fail_touch(self: Path, mode: int = 0o666, exist_ok: bool = True) -> None:
+        def fail_touch(_self: Path, _mode: int = 0o666, _exist_ok: bool = True) -> None:
             msg = "storage unavailable"
             raise OSError(msg)
 
         monkeypatch.setattr(Path, "touch", fail_touch)
 
         with pytest.raises(OSError, match="storage unavailable"):
-            record_failed_tasks([_failed_task()])
+            record_failed_tasks()
 
     def test_failed_task_manifest_exists_accepts_explicit_directory(self, tmp_path: Path) -> None:
         manifest_dir = tmp_path / "failed-tasks"
         manifest_dir.mkdir()
-        (manifest_dir / FAILED_TASK_MANIFEST_FILENAME).write_text('{"status":"failed_tasks"}\n')
+        (manifest_dir / FAILED_TASK_MANIFEST_FILENAME).touch()
 
         assert failed_task_manifest_exists(manifest_dir)
         assert not failed_task_manifest_exists(tmp_path / "missing")