improve embedding persistence

Author: clemsgrs

slide2vec/distributed/pipeline_worker.py

@@ -19,8 +19,8 @@ def main(argv=None) -> int:
     import slide2vec.distributed as distributed
     from slide2vec.api import Model
     from slide2vec.inference import (
+        _build_incremental_persist_callback,
         _compute_embedded_slides,
-        _persist_embedded_slide,
         load_successful_tiled_slides,
     )
     from slide2vec.progress import JsonlProgressReporter, activate_progress_reporter
@@ -70,21 +70,20 @@ def main(argv=None) -> int:
         )
         context = activate_progress_reporter(reporter) if reporter is not None else nullcontext()
         with context:
-            embedded_slides = _compute_embedded_slides(
+            persist_callback, _, _ = _build_incremental_persist_callback(
+                model=model,
+                preprocessing=preprocessing,
+                execution=execution,
+                process_list_path=None,
+            )
+            _compute_embedded_slides(
                 model,
                 assigned_slides,
                 assigned_tiling_results,
                 preprocessing=preprocessing,
                 execution=execution,
+                on_embedded_slide=persist_callback,
             )
-            for embedded_slide, tiling_result in zip(embedded_slides, assigned_tiling_results):
-                _persist_embedded_slide(
-                    model,
-                    embedded_slide,
-                    tiling_result,
-                    preprocessing=preprocessing,
-                    execution=execution,
-                )
         return 0
     finally:
         if dist.is_available() and dist.is_initialized():

slide2vec/encoders/validation.py

@@ -63,14 +63,18 @@ def validate_encoder_config(
     if not mismatches:
         return
 
-    message = (
-        f"Model '{encoder_name}' is configured with "
-        f"{'; '.join(mismatches)}. "
-        "Set `model.allow_non_recommended_settings=true` in YAML/CLI or "
-        "`allow_non_recommended_settings=True` in `Model.from_preset(...)` "
-        "to continue with a warning."
-    )
     if allow_non_recommended:
-        logger.warning(message)
+        logger.warning(
+            f"Model '{encoder_name}' is configured with "
+            f"{'; '.join(mismatches)}. "
+            "Warning-only mode is enabled because "
+            "`allow_non_recommended_settings=True`."
+        )
     else:
-        raise ValueError(message)
+        raise ValueError(
+            f"Model '{encoder_name}' is configured with "
+            f"{'; '.join(mismatches)}. "
+            "Set `model.allow_non_recommended_settings=true` in YAML/CLI or "
+            "`allow_non_recommended_settings=True` in `Model.from_preset(...)` "
+            "to continue."
+        )

slide2vec/inference.py

@@ -1021,24 +1021,31 @@ def run_pipeline_with_coordinates(
                 slide_artifacts=slide_artifacts,
                 process_list_path=process_list_path,
             )
-        embedded_slides = _compute_embedded_slides(
-            model,
-            embeddable_slides,
-            embeddable_tiling_results,
+        local_persist_callback, tile_or_hier_artifacts, slide_artifacts = _build_incremental_persist_callback(
+            model=model,
             preprocessing=resolved_preprocessing,
             execution=execution,
+            process_list_path=process_list_path,
         )
-        tile_artifacts, hierarchical_artifacts, slide_artifacts = _collect_local_pipeline_artifacts(
-            model=model,
-            embedded_slides=embedded_slides,
-            tiling_results=embeddable_tiling_results,
+        _compute_embedded_slides(
+            model,
+            embeddable_slides,
+            embeddable_tiling_results,
             preprocessing=resolved_preprocessing,
             execution=execution,
+            on_embedded_slide=local_persist_callback,
         )
+        tile_artifacts: list[TileEmbeddingArtifact] = []
+        hierarchical_artifacts: list[HierarchicalEmbeddingArtifact] = []
+        for artifact in tile_or_hier_artifacts:
+            if isinstance(artifact, HierarchicalEmbeddingArtifact):
+                hierarchical_artifacts.append(artifact)
+            elif artifact is not None:
+                tile_artifacts.append(artifact)
         return RunResult(
             tile_artifacts=tile_artifacts,
             hierarchical_artifacts=hierarchical_artifacts,
-            slide_artifacts=slide_artifacts,
+            slide_artifacts=list(slide_artifacts),
             process_list_path=process_list_path,
         )
     except Exception as exc: