From 563e99d0711bf8ddc2bd3131a0fb02ca440b9755 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 4 Dec 2024 12:17:44 -0600 Subject: [PATCH] Wrap sync fs for xarray.to_zarr --- src/zarr/storage/fsspec.py | 4 ++++ tests/test_store/test_fsspec.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/zarr/storage/fsspec.py b/src/zarr/storage/fsspec.py index c9edd8f8ac..b588c763a9 100644 --- a/src/zarr/storage/fsspec.py +++ b/src/zarr/storage/fsspec.py @@ -3,6 +3,8 @@ import warnings from typing import TYPE_CHECKING, Any +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + from zarr.abc.store import ByteRangeRequest, Store from zarr.storage.common import _dereference_path @@ -166,6 +168,8 @@ def from_url( opts = {"asynchronous": True, **opts} fs, path = url_to_fs(url, **opts) + if not fs.async_impl: + fs = AsyncFileSystemWrapper(fs) # fsspec is not consistent about removing the scheme from the path, so check and strip it here # https://github.com/fsspec/filesystem_spec/issues/1722 diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index b307f2cdf4..c8953f6027 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -6,6 +6,7 @@ import pytest from botocore.session import Session +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper import zarr.api.asynchronous from zarr.core.buffer import Buffer, cpu, default_buffer_prototype @@ -214,3 +215,19 @@ async def test_empty_nonexistent_path(self, store_kwargs) -> None: store_kwargs["path"] += "/abc" store = await self.store_cls.open(**store_kwargs) assert await store.is_empty("") + + +def test_wrap_sync_filesystem(): + """The local fs is not async so we should expect it to be wrapped automatically""" + store = FsspecStore.from_url("local://test/path") + + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + + +def test_no_wrap_async_filesystem(): + """An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs""" + store = FsspecStore.from_url("https://test/path") + + assert not isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl