Migrate ClearEx ingestion and analysis stores to Zarr v3

environment.yml

@@ -3,13 +3,13 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.11
+  - python>=3.12,<3.13
   - pip
   - cython
-  - dask=2025.1.0
+  - dask=2026.1.1
   - dask-image
   - dask-jobqueue
-  - distributed=2025.1.0
+  - distributed=2026.1.1
   - h5py
   - jupyterlab
   - matplotlib
@@ -23,10 +23,10 @@ dependencies:
   - scikit-image
   - scipy
   - simpleitk
-  - tifffile
-  - zarr<3.0
+  - tifffile=2025.10.16
+  - zarr=3.1.5
   - pip:
       - antspyx
       - neuroglancer (>=2.40.1,<3.0.0)
       - PyQt6>=6.7
-      - -e .
+      - -e .[viewer]

pyproject.toml

@@ -18,10 +18,10 @@ dependencies = [
     "antspyx",
     "basicpy",
     "cython>=3.1.4",
-    "dask==2025.1.0",
+    "dask==2026.1.1",
     "dask-image",
     "dask-jobqueue",
-    "distributed==2025.1.0",
+    "distributed==2026.1.1",
     "h5py",
     # u-Segment3D currently requires imagecodecs<2025.
     "imagecodecs>=2024.9.22,<2025",
@@ -36,8 +36,8 @@ dependencies = [
     "scikit-image",
     "scipy<1.13",
     "seaborn",
-    "tifffile==2025.1.10",
-    "zarr<3.0",
+    "tifffile==2025.10.16",
+    "zarr>=3,<4",
 ]
 
 [project.optional-dependencies]
@@ -52,6 +52,11 @@ usegment3d = [
     "cellpose<3",
 ]
 
+viewer = [
+    "ome-zarr>=0.14.0",
+    "napari-ome-zarr>=0.7.2",
+]
+
 dev = [
     "black>=25.11.0",
     "pre-commit",

src/clearex/CODEX.md

@@ -130,6 +130,30 @@ This directory contains the runtime orchestration surface for ClearEx.
 - Detailed operational guidance for this area now lives in
   `src/clearex/visualization/README.md`.
 
+## Recent Runtime Updates (2026-03-20)
+
+- Added store-level spatial calibration for Navigate multiposition datasets:
+  - `WorkflowConfig` now carries `SpatialCalibrationConfig`,
+  - canonical text form is `z=...,y=...,x=...`,
+  - allowed bindings are `+/-x`, `+/-y`, `+/-z`, `+/-f`, and `none`,
+  - the root store attr `spatial_calibration` persists schema, mapping, and
+    `theta_mode`,
+  - missing attrs resolve to identity instead of requiring backfilled config.
+- Setup flow now exposes a lightweight `Spatial Calibration` control per
+  experiment:
+  - one draft is kept per experiment while setup is open,
+  - existing stores prefill the current mapping,
+  - `Next` writes the resolved mapping to every reused or newly prepared store.
+- Headless workflows now accept `--stage-axis-map` for Navigate
+  `experiment.yml` inputs and existing Zarr/N5 stores.
+- Visualization position affines now derive world `z/y/x` translations from
+  the stored calibration:
+  - Navigate `F` is available as a placement source,
+  - `none` zeroes a world axis translation,
+  - sign inversion is supported,
+  - `THETA` remains rotation of the `z/y` plane about world `x`.
+- Provenance now records the effective spatial calibration used by the run.
+
 ## Sequencing and Inputs
 
 - Operation order is driven by `analysis_parameters[<op>]["execution_order"]`.

src/clearex/codex.md

@@ -130,6 +130,30 @@ This directory contains the runtime orchestration surface for ClearEx.
 - Detailed operational guidance for this area now lives in
   `src/clearex/visualization/README.md`.
 
+## Recent Runtime Updates (2026-03-20)
+
+- Added store-level spatial calibration for Navigate multiposition datasets:
+  - `WorkflowConfig` now carries `SpatialCalibrationConfig`,
+  - canonical text form is `z=...,y=...,x=...`,
+  - allowed bindings are `+/-x`, `+/-y`, `+/-z`, `+/-f`, and `none`,
+  - the root store attr `spatial_calibration` persists schema, mapping, and
+    `theta_mode`,
+  - missing attrs resolve to identity instead of requiring backfilled config.
+- Setup flow now exposes a lightweight `Spatial Calibration` control per
+  experiment:
+  - one draft is kept per experiment while setup is open,
+  - existing stores prefill the current mapping,
+  - `Next` writes the resolved mapping to every reused or newly prepared store.
+- Headless workflows now accept `--stage-axis-map` for Navigate
+  `experiment.yml` inputs and existing Zarr/N5 stores.
+- Visualization position affines now derive world `z/y/x` translations from
+  the stored calibration:
+  - Navigate `F` is available as a placement source,
+  - `none` zeroes a world axis translation,
+  - sign inversion is supported,
+  - `THETA` remains rotation of the `z/y` plane about world `x`.
+- Provenance now records the effective spatial calibration used by the run.
+
 ## Sequencing and Inputs
 
 - Operation order is driven by `analysis_parameters[<op>]["execution_order"]`.

src/clearex/deconvolution/pipeline.py

@@ -45,6 +45,7 @@
 # Local Imports
 from clearex.deconvolution.petakit import run_petakit_deconvolution
 from clearex.io.provenance import register_latest_output_reference
+from clearex.io.zarr_storage import create_or_overwrite_array
 
 if TYPE_CHECKING:
     from dask.distributed import Client
@@ -1069,25 +1070,37 @@ def _persist_synthetic_psf_assets(
             voxel_z_um=float(voxel_z_um),
         )
         channel_group = synthetic_group.create_group(f"ch{int(channel_index):02d}")
-        channel_group.create_dataset(
+        create_or_overwrite_array(
+            root=channel_group,
             name="combined_psf_zyx",
+            shape=tuple(int(v) for v in artifacts.combined_psf_zyx.shape),
+            dtype=np.float32,
             data=np.asarray(artifacts.combined_psf_zyx, dtype=np.float32),
             overwrite=True,
         )
-        channel_group.create_dataset(
+        create_or_overwrite_array(
+            root=channel_group,
             name="detection_psf_zyx",
+            shape=tuple(int(v) for v in artifacts.detection_psf_zyx.shape),
+            dtype=np.float32,
             data=np.asarray(artifacts.detection_psf_zyx, dtype=np.float32),
             overwrite=True,
         )
         if artifacts.illumination_psf_zyx is not None:
-            channel_group.create_dataset(
+            create_or_overwrite_array(
+                root=channel_group,
                 name="illumination_psf_zyx",
+                shape=tuple(int(v) for v in artifacts.illumination_psf_zyx.shape),
+                dtype=np.float32,
                 data=np.asarray(artifacts.illumination_psf_zyx, dtype=np.float32),
                 overwrite=True,
             )
         preview_bytes = np.frombuffer(artifacts.preview_png_bytes, dtype=np.uint8)
-        channel_group.create_dataset(
+        create_or_overwrite_array(
+            root=channel_group,
             name="preview_png",
+            shape=tuple(int(v) for v in preview_bytes.shape),
+            dtype=preview_bytes.dtype,
             data=preview_bytes,
             overwrite=True,
         )

src/clearex/detect/pipeline.py

@@ -48,6 +48,7 @@
     remove_close_blobs,
 )
 from clearex.io.provenance import register_latest_output_reference
+from clearex.io.zarr_storage import create_or_overwrite_array
 
 if TYPE_CHECKING:
     from dask.distributed import Client
@@ -553,8 +554,11 @@ def save_particle_detections_to_store(
 
     detection_array = np.asarray(detections, dtype=np.float32)
     row_chunks = int(min(max(1, detection_array.shape[0]), 16384))
-    latest_group.create_dataset(
+    create_or_overwrite_array(
+        root=latest_group,
         name="detections",
+        shape=tuple(int(v) for v in detection_array.shape),
+        dtype=detection_array.dtype,
         data=detection_array,
         chunks=(row_chunks, len(_PARTICLE_COLUMNS)),
         overwrite=True,
@@ -572,8 +576,11 @@ def save_particle_detections_to_store(
         else np.empty((0, 4), dtype=np.float32)
     )
     points_chunks = int(min(max(1, napari_points.shape[0]), 16384))
-    latest_group.create_dataset(
+    create_or_overwrite_array(
+        root=latest_group,
         name="points_tzyx",
+        shape=tuple(int(v) for v in napari_points.shape),
+        dtype=napari_points.dtype,
         data=napari_points,
         chunks=(points_chunks, 4),
         overwrite=True,

src/clearex/flatfield/pipeline.py

@@ -46,6 +46,7 @@
 import zarr
 
 from clearex.io.provenance import register_latest_output_reference
+from clearex.io.zarr_storage import write_dask_array
 
 if TYPE_CHECKING:
     from dask.distributed import Client
@@ -1368,10 +1369,11 @@ def _emit(percent: int, message: str) -> None:
             downsampled = downsampled.rechunk(level_chunks)
 
         level_component = f"{base_parent}/data_pyramid/level_{level_index}"
-        write_task = da.to_zarr(
-            downsampled,
-            url=str(zarr_path),
+        write_task = write_dask_array(
+            zarr_path=zarr_path,
             component=level_component,
+            array=downsampled,
+            chunks=level_chunks,
             overwrite=True,
             compute=False,
         )