Add ManifestStore for loading data from ManifestArrays (#490)

* Draft ManifestStore implementation

---------

Co-authored-by: Tom Nicholas <[email protected]>
Co-authored-by: Kyle Barron <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 4 people

docs/conf.py

@@ -40,6 +40,7 @@
     "numpy": ("https://numpy.org/doc/stable/", None),
     "zarr": ("https://zarr.readthedocs.io/en/stable/", None),
     "xarray": ("https://docs.xarray.dev/en/stable/", None),
+    "obstore": ("https://developmentseed.org/obstore/latest/", None),
 }
 
 # Add any paths that contain templates here, relative to this directory.

docs/releases.rst

@@ -9,6 +9,8 @@ v1.3.3 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- Added experimental ManifestStore (:pull:`490`).
+
 Breaking changes
 ~~~~~~~~~~~~~~~~
 

pyproject.toml

@@ -38,7 +38,9 @@ remote = [
     "aiohttp",
     "s3fs",
 ]
-
+obstore = [
+    "obstore>=0.5.1",
+]
 # non-kerchunk-based readers
 hdf = [
     "virtualizarr[remote]",
@@ -172,11 +174,11 @@ run-tests-html-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov
 [tool.pixi.environments]
 min-deps = ["dev", "hdf", "hdf5", "hdf5-lib"] # VirtualiZarr/conftest.py using h5py, so the minimum set of dependencies for testing still includes hdf libs
 # Inherit from min-deps to get all the test commands, along with optional dependencies
-test = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib"]
-test-py311 = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "py311"] # test against python 3.11
-test-py312 = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "py312"] # test against python 3.12
+test = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "obstore"]
+test-py311 = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "obstore", "py311"] # test against python 3.11
+test-py312 = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "obstore", "py312"] # test against python 3.12
 upstream = ["dev", "hdf", "hdf5", "hdf5-lib", "netcdf3", "upstream", "icechunk-dev"]
-all = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "all_readers", "all_writers"]
+all = ["dev", "remote", "hdf", "hdf5", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "obstore", "all_readers", "all_writers"]
 docs = ["docs"]
 
 

virtualizarr/manifests/__init__.py

@@ -2,4 +2,6 @@
 # This is just to avoid conflicting with some type of file called manifest that .gitignore recommends ignoring.
 
 from .array import ManifestArray  # type: ignore # noqa
+from .group import ManifestGroup  # type: ignore # noqa
 from .manifest import ChunkEntry, ChunkManifest  # type: ignore # noqa
+from .store import ManifestStore  # type: ignore # noqa

virtualizarr/manifests/group.py

@@ -0,0 +1,37 @@
+from typing import Mapping, TypeAlias
+
+from zarr.core.group import GroupMetadata
+
+from virtualizarr.manifests import ManifestArray
+
+ManifestArrayVariableMapping: TypeAlias = dict[str, ManifestArray]
+
+
+class ManifestGroup:
+    """
+    Virtualized representation of multiple ManifestArrays as a Zarr Group.
+    """
+
+    # TODO: Consider refactoring according to https://github.com/zarr-developers/VirtualiZarr/pull/490#discussion_r2007805272
+    _manifest_arrays: Mapping[str, ManifestArray]
+    _metadata: GroupMetadata
+
+    def __init__(
+        self,
+        manifest_arrays: ManifestArrayVariableMapping,
+        attributes: dict,
+    ) -> None:
+        """
+        Create a ManifestGroup from the dictionary of ManifestArrays and the group / dataset level metadata
+
+        Parameters
+        ----------
+        attributes : attributes to include in Group metadata
+        manifest_dict : ManifestArrayVariableMapping
+        """
+
+        self._metadata = GroupMetadata(attributes=attributes)
+        self._manifest_arrays = manifest_arrays
+
+    def __str__(self) -> str:
+        return f"ManifestGroup(manifest_arrays={self._manifest_arrays}, metadata={self._metadata})"