Test failures with Zarr 3 #1777
Comments
|
I am aware of problems and am looking into fixing everything alongside fsspec/kerchunk#516 . It may take some time. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I've run a test build with Zarr 3
This update has caused some tests to fail due to some issue in
LazyReferenceMapper:____________________ ERROR at setup of test_append_parquet _____________________ m = <fsspec.implementations.memory.MemoryFileSystem object at 0x7fc8beb41fd0> @pytest.fixture() def lazy_refs(m): zarr = pytest.importorskip("zarr") l = LazyReferenceMapper.create("memory://refs", fs=m) > g = zarr.open(l, mode="w") fsspec/implementations/tests/test_reference.py:758: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.13/site-packages/zarr/_compat.py:43: in inner_f return f(*args, **kwargs) /usr/lib/python3.13/site-packages/zarr/api/synchronous.py:190: in open obj = sync( /usr/lib/python3.13/site-packages/zarr/core/sync.py:142: in sync raise return_result /usr/lib/python3.13/site-packages/zarr/core/sync.py:98: in _runner return await coro /usr/lib/python3.13/site-packages/zarr/api/asynchronous.py:309: in open store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ store_like = <fsspec.implementations.reference.LazyReferenceMapper object at 0x7fc8bd84b770> async def make_store_path( store_like: StoreLike | None, *, path: str | None = "", mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> StorePath: """ Convert a `StoreLike` object into a StorePath object. This function takes a `StoreLike` object and returns a `StorePath` object. The `StoreLike` object can be a `Store`, `StorePath`, `Path`, `str`, or `dict[str, Buffer]`. If the `StoreLike` object is a Store or `StorePath`, it is converted to a `StorePath` object. If the `StoreLike` object is a Path or str, it is converted to a LocalStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer], it is converted to a `MemoryStore` object and then to a `StorePath` object. If the `StoreLike` object is None, a `MemoryStore` object is created and converted to a `StorePath` object. If the `StoreLike` object is a str and starts with a protocol, it is converted to a RemoteStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer] and the mode is not None, the `MemoryStore` object is created with the given mode. If the `StoreLike` object is a str and starts with a protocol, the RemoteStore object is created with the given mode and storage options. Parameters ---------- store_like : StoreLike | None The object to convert to a `StorePath` object. path : str | None, optional The path to use when creating the `StorePath` object. If None, the default path is the empty string. mode : StoreAccessMode | None, optional The mode to use when creating the `StorePath` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- StorePath The converted StorePath object. Raises ------ TypeError If the StoreLike object is not one of the supported types. """ from zarr.storage._fsspec import FsspecStore # circular import used_storage_options = False path_normalized = normalize_path(path) if isinstance(store_like, StorePath): result = store_like / path_normalized else: assert mode in (None, "r", "r+", "a", "w", "w-") # if mode 'r' was provided, we'll open any new stores as read-only _read_only = mode == "r" if isinstance(store_like, Store): store = store_like elif store_like is None: store = await MemoryStore.open(read_only=_read_only) elif isinstance(store_like, Path): store = await LocalStore.open(root=store_like, read_only=_read_only) elif isinstance(store_like, str): storage_options = storage_options or {} if _is_fsspec_uri(store_like): used_storage_options = True store = FsspecStore.from_url( store_like, storage_options=storage_options, read_only=_read_only ) else: store = await LocalStore.open(root=Path(store_like), read_only=_read_only) elif isinstance(store_like, dict): # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) else: msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] > raise TypeError(msg) E TypeError: Unsupported type for store_like: 'LazyReferenceMapper' /usr/lib/python3.13/site-packages/zarr/storage/_common.py:316: TypeError =================================== FAILURES =================================== ____________________________ test_missing_nonasync _____________________________ m = <fsspec.mapping.FSMap object at 0x7fc8bdb72f10> def test_missing_nonasync(m): zarr = pytest.importorskip("zarr") zarray = { "chunks": [1], "compressor": None, "dtype": "<f8", "fill_value": "NaN", "filters": [], "order": "C", "shape": [10], "zarr_format": 2, } refs = {".zarray": json.dumps(zarray)} m = fsspec.get_mapper("reference://", fo=refs, remote_protocol="memory") > a = zarr.open_array(m) fsspec/implementations/tests/test_reference.py:448: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.13/site-packages/zarr/api/synchronous.py:1051: in open_array sync( /usr/lib/python3.13/site-packages/zarr/core/sync.py:142: in sync raise return_result /usr/lib/python3.13/site-packages/zarr/core/sync.py:98: in _runner return await coro /usr/lib/python3.13/site-packages/zarr/api/asynchronous.py:1232: in open_array store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ store_like = <fsspec.mapping.FSMap object at 0x7fc8bdb72f10> async def make_store_path( store_like: StoreLike | None, *, path: str | None = "", mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> StorePath: """ Convert a `StoreLike` object into a StorePath object. This function takes a `StoreLike` object and returns a `StorePath` object. The `StoreLike` object can be a `Store`, `StorePath`, `Path`, `str`, or `dict[str, Buffer]`. If the `StoreLike` object is a Store or `StorePath`, it is converted to a `StorePath` object. If the `StoreLike` object is a Path or str, it is converted to a LocalStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer], it is converted to a `MemoryStore` object and then to a `StorePath` object. If the `StoreLike` object is None, a `MemoryStore` object is created and converted to a `StorePath` object. If the `StoreLike` object is a str and starts with a protocol, it is converted to a RemoteStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer] and the mode is not None, the `MemoryStore` object is created with the given mode. If the `StoreLike` object is a str and starts with a protocol, the RemoteStore object is created with the given mode and storage options. Parameters ---------- store_like : StoreLike | None The object to convert to a `StorePath` object. path : str | None, optional The path to use when creating the `StorePath` object. If None, the default path is the empty string. mode : StoreAccessMode | None, optional The mode to use when creating the `StorePath` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- StorePath The converted StorePath object. Raises ------ TypeError If the StoreLike object is not one of the supported types. """ from zarr.storage._fsspec import FsspecStore # circular import used_storage_options = False path_normalized = normalize_path(path) if isinstance(store_like, StorePath): result = store_like / path_normalized else: assert mode in (None, "r", "r+", "a", "w", "w-") # if mode 'r' was provided, we'll open any new stores as read-only _read_only = mode == "r" if isinstance(store_like, Store): store = store_like elif store_like is None: store = await MemoryStore.open(read_only=_read_only) elif isinstance(store_like, Path): store = await LocalStore.open(root=store_like, read_only=_read_only) elif isinstance(store_like, str): storage_options = storage_options or {} if _is_fsspec_uri(store_like): used_storage_options = True store = FsspecStore.from_url( store_like, storage_options=storage_options, read_only=_read_only ) else: store = await LocalStore.open(root=Path(store_like), read_only=_read_only) elif isinstance(store_like, dict): # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) else: msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] > raise TypeError(msg) E TypeError: Unsupported type for store_like: 'FSMap' /usr/lib/python3.13/site-packages/zarr/storage/_common.py:316: TypeError _____________________________ test_parquet_no_data _____________________________ m = <fsspec.implementations.memory.MemoryFileSystem object at 0x7fc8beb41fd0> def test_parquet_no_data(m): zarr = pytest.importorskip("zarr") lz = fsspec.implementations.reference.LazyReferenceMapper.create( "memory://out.parq", fs=m ) > g = zarr.open_group(lz, mode="w") fsspec/implementations/tests/test_reference.py:833: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.13/site-packages/zarr/_compat.py:43: in inner_f return f(*args, **kwargs) /usr/lib/python3.13/site-packages/zarr/api/synchronous.py:524: in open_group sync( /usr/lib/python3.13/site-packages/zarr/core/sync.py:142: in sync raise return_result /usr/lib/python3.13/site-packages/zarr/core/sync.py:98: in _runner return await coro /usr/lib/python3.13/site-packages/zarr/api/asynchronous.py:800: in open_group store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ store_like = <fsspec.implementations.reference.LazyReferenceMapper object at 0x7fc8bcc00050> async def make_store_path( store_like: StoreLike | None, *, path: str | None = "", mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> StorePath: """ Convert a `StoreLike` object into a StorePath object. This function takes a `StoreLike` object and returns a `StorePath` object. The `StoreLike` object can be a `Store`, `StorePath`, `Path`, `str`, or `dict[str, Buffer]`. If the `StoreLike` object is a Store or `StorePath`, it is converted to a `StorePath` object. If the `StoreLike` object is a Path or str, it is converted to a LocalStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer], it is converted to a `MemoryStore` object and then to a `StorePath` object. If the `StoreLike` object is None, a `MemoryStore` object is created and converted to a `StorePath` object. If the `StoreLike` object is a str and starts with a protocol, it is converted to a RemoteStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer] and the mode is not None, the `MemoryStore` object is created with the given mode. If the `StoreLike` object is a str and starts with a protocol, the RemoteStore object is created with the given mode and storage options. Parameters ---------- store_like : StoreLike | None The object to convert to a `StorePath` object. path : str | None, optional The path to use when creating the `StorePath` object. If None, the default path is the empty string. mode : StoreAccessMode | None, optional The mode to use when creating the `StorePath` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- StorePath The converted StorePath object. Raises ------ TypeError If the StoreLike object is not one of the supported types. """ from zarr.storage._fsspec import FsspecStore # circular import used_storage_options = False path_normalized = normalize_path(path) if isinstance(store_like, StorePath): result = store_like / path_normalized else: assert mode in (None, "r", "r+", "a", "w", "w-") # if mode 'r' was provided, we'll open any new stores as read-only _read_only = mode == "r" if isinstance(store_like, Store): store = store_like elif store_like is None: store = await MemoryStore.open(read_only=_read_only) elif isinstance(store_like, Path): store = await LocalStore.open(root=store_like, read_only=_read_only) elif isinstance(store_like, str): storage_options = storage_options or {} if _is_fsspec_uri(store_like): used_storage_options = True store = FsspecStore.from_url( store_like, storage_options=storage_options, read_only=_read_only ) else: store = await LocalStore.open(root=Path(store_like), read_only=_read_only) elif isinstance(store_like, dict): # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) else: msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] > raise TypeError(msg) E TypeError: Unsupported type for store_like: 'LazyReferenceMapper' /usr/lib/python3.13/site-packages/zarr/storage/_common.py:316: TypeError __________________________ test_parquet_no_references __________________________ m = <fsspec.implementations.memory.MemoryFileSystem object at 0x7fc8beb41fd0> def test_parquet_no_references(m): zarr = pytest.importorskip("zarr") lz = fsspec.implementations.reference.LazyReferenceMapper.create( "memory://out.parq", fs=m ) > g = zarr.open_group(lz, mode="w") fsspec/implementations/tests/test_reference.py:853: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.13/site-packages/zarr/_compat.py:43: in inner_f return f(*args, **kwargs) /usr/lib/python3.13/site-packages/zarr/api/synchronous.py:524: in open_group sync( /usr/lib/python3.13/site-packages/zarr/core/sync.py:142: in sync raise return_result /usr/lib/python3.13/site-packages/zarr/core/sync.py:98: in _runner return await coro /usr/lib/python3.13/site-packages/zarr/api/asynchronous.py:800: in open_group store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ store_like = <fsspec.implementations.reference.LazyReferenceMapper object at 0x7fc8bd7dbd90> async def make_store_path( store_like: StoreLike | None, *, path: str | None = "", mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> StorePath: """ Convert a `StoreLike` object into a StorePath object. This function takes a `StoreLike` object and returns a `StorePath` object. The `StoreLike` object can be a `Store`, `StorePath`, `Path`, `str`, or `dict[str, Buffer]`. If the `StoreLike` object is a Store or `StorePath`, it is converted to a `StorePath` object. If the `StoreLike` object is a Path or str, it is converted to a LocalStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer], it is converted to a `MemoryStore` object and then to a `StorePath` object. If the `StoreLike` object is None, a `MemoryStore` object is created and converted to a `StorePath` object. If the `StoreLike` object is a str and starts with a protocol, it is converted to a RemoteStore object and then to a `StorePath` object. If the `StoreLike` object is a dict[str, Buffer] and the mode is not None, the `MemoryStore` object is created with the given mode. If the `StoreLike` object is a str and starts with a protocol, the RemoteStore object is created with the given mode and storage options. Parameters ---------- store_like : StoreLike | None The object to convert to a `StorePath` object. path : str | None, optional The path to use when creating the `StorePath` object. If None, the default path is the empty string. mode : StoreAccessMode | None, optional The mode to use when creating the `StorePath` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- StorePath The converted StorePath object. Raises ------ TypeError If the StoreLike object is not one of the supported types. """ from zarr.storage._fsspec import FsspecStore # circular import used_storage_options = False path_normalized = normalize_path(path) if isinstance(store_like, StorePath): result = store_like / path_normalized else: assert mode in (None, "r", "r+", "a", "w", "w-") # if mode 'r' was provided, we'll open any new stores as read-only _read_only = mode == "r" if isinstance(store_like, Store): store = store_like elif store_like is None: store = await MemoryStore.open(read_only=_read_only) elif isinstance(store_like, Path): store = await LocalStore.open(root=store_like, read_only=_read_only) elif isinstance(store_like, str): storage_options = storage_options or {} if _is_fsspec_uri(store_like): used_storage_options = True store = FsspecStore.from_url( store_like, storage_options=storage_options, read_only=_read_only ) else: store = await LocalStore.open(root=Path(store_like), read_only=_read_only) elif isinstance(store_like, dict): # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) else: msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] > raise TypeError(msg) E TypeError: Unsupported type for store_like: 'LazyReferenceMapper' /usr/lib/python3.13/site-packages/zarr/storage/_common.py:316: TypeErrorThe text was updated successfully, but these errors were encountered: