Skip to content

Commit

Permalink
Wrap sync fs for xarray.to_zarr
Browse files Browse the repository at this point in the history
  • Loading branch information
moradology committed Jan 8, 2025
1 parent f360fc6 commit 563e99d
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 0 deletions.
4 changes: 4 additions & 0 deletions src/zarr/storage/fsspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import warnings
from typing import TYPE_CHECKING, Any

from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper

from zarr.abc.store import ByteRangeRequest, Store
from zarr.storage.common import _dereference_path

Expand Down Expand Up @@ -166,6 +168,8 @@ def from_url(
opts = {"asynchronous": True, **opts}

fs, path = url_to_fs(url, **opts)
if not fs.async_impl:
fs = AsyncFileSystemWrapper(fs)

# fsspec is not consistent about removing the scheme from the path, so check and strip it here
# https://github.com/fsspec/filesystem_spec/issues/1722
Expand Down
17 changes: 17 additions & 0 deletions tests/test_store/test_fsspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import pytest
from botocore.session import Session
from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper

import zarr.api.asynchronous
from zarr.core.buffer import Buffer, cpu, default_buffer_prototype
Expand Down Expand Up @@ -214,3 +215,19 @@ async def test_empty_nonexistent_path(self, store_kwargs) -> None:
store_kwargs["path"] += "/abc"
store = await self.store_cls.open(**store_kwargs)
assert await store.is_empty("")


def test_wrap_sync_filesystem():
"""The local fs is not async so we should expect it to be wrapped automatically"""
store = FsspecStore.from_url("local://test/path")

assert isinstance(store.fs, AsyncFileSystemWrapper)
assert store.fs.async_impl


def test_no_wrap_async_filesystem():
"""An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs"""
store = FsspecStore.from_url("https://test/path")

assert not isinstance(store.fs, AsyncFileSystemWrapper)
assert store.fs.async_impl

0 comments on commit 563e99d

Please sign in to comment.