Simplify mypy config for tests (#2156)

* Simplify mypy config for tests

* Add a list of test modules to expliclty ignore typing

* More test mypy fixes

* Remove old test_metadata file

* Fix ignoring v2 tests

* Fix name test

Author: dstansby

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     rev: v1.11.2
     hooks:
       - id: mypy
-        files: src|tests/v3/test_(api|array|buffer).py
+        files: src|tests
         additional_dependencies:
           # Package dependencies
           - asciitree

pyproject.toml

@@ -225,6 +225,7 @@ python_version = "3.10"
 ignore_missing_imports = true
 namespace_packages = false
 
+
 strict = true
 warn_unreachable = true
 
@@ -236,6 +237,22 @@ module = [
 ]
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = [
+    "tests.v2.*",
+    "tests.v3.package_with_entrypoint.*",
+    "tests.v3.test_codecs.*",
+    "tests.v3.test_metadata.*",
+    "tests.v3.test_store.*",
+    "tests.v3.test_config",
+    "tests.v3.test_group",
+    "tests.v3.test_indexing",
+    "tests.v3.test_properties",
+    "tests.v3.test_sync",
+    "tests.v3.test_v2",
+]
+ignore_errors = true
+
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]
@@ -262,6 +279,6 @@ markers = [
 
 [tool.repo-review]
 ignore = [
-	"PC111",  # fix Python code in documentation - enable later 
+	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]

tests/v3/test_metadata.py tests/__init__.py

@@ -1,27 +1,27 @@
 from __future__ import annotations
 
+import pathlib
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
+import numpy as np
+import numpy.typing as npt
+import pytest
+from hypothesis import HealthCheck, Verbosity, settings
+
 from zarr import AsyncGroup, config
+from zarr.store import LocalStore, MemoryStore, StorePath
+from zarr.store.remote import RemoteStore
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Generator, Iterator
     from types import ModuleType
     from typing import Any, Literal
 
     from _pytest.compat import LEGACY_PATH
 
     from zarr.abc.store import Store
     from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat
-import pathlib
-from dataclasses import dataclass, field
-
-import numpy as np
-import pytest
-from hypothesis import HealthCheck, Verbosity, settings
-
-from zarr.store import LocalStore, MemoryStore, StorePath
-from zarr.store.remote import RemoteStore
 
 
 async def parse_store(
@@ -102,7 +102,7 @@ def xp(request: pytest.FixtureRequest) -> Iterator[ModuleType]:
 
 
 @pytest.fixture(autouse=True)
-def reset_config():
+def reset_config() -> Generator[None, None, None]:
     config.reset()
     yield
     config.reset()
@@ -116,7 +116,7 @@ class ArrayRequest:
 
 
 @pytest.fixture
-def array_fixture(request: pytest.FixtureRequest) -> np.ndarray:
+def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]:
     array_request: ArrayRequest = request.param
     return (
         np.arange(np.prod(array_request.shape))

tests/v3/conftest.py

@@ -1,5 +1,6 @@
 import os.path
 import sys
+from collections.abc import Generator
 
 import pytest
 
@@ -10,7 +11,7 @@
 
 
 @pytest.fixture()
-def set_path():
+def set_path() -> Generator[None, None, None]:
     sys.path.append(here)
     zarr.registry._collect_entrypoints()
     yield
@@ -23,22 +24,22 @@ def set_path():
 
 @pytest.mark.usefixtures("set_path")
 @pytest.mark.parametrize("codec_name", ["TestEntrypointCodec", "TestEntrypointGroup.Codec"])
-def test_entrypoint_codec(codec_name):
+def test_entrypoint_codec(codec_name: str) -> None:
     config.set({"codecs.test": "package_with_entrypoint." + codec_name})
     cls_test = zarr.registry.get_codec_class("test")
     assert cls_test.__qualname__ == codec_name
 
 
 @pytest.mark.usefixtures("set_path")
-def test_entrypoint_pipeline():
+def test_entrypoint_pipeline() -> None:
     config.set({"codec_pipeline.path": "package_with_entrypoint.TestEntrypointCodecPipeline"})
     cls = zarr.registry.get_pipeline_class()
     assert cls.__name__ == "TestEntrypointCodecPipeline"
 
 
 @pytest.mark.usefixtures("set_path")
 @pytest.mark.parametrize("buffer_name", ["TestEntrypointBuffer", "TestEntrypointGroup.Buffer"])
-def test_entrypoint_buffer(buffer_name):
+def test_entrypoint_buffer(buffer_name: str) -> None:
     config.set(
         {
             "buffer": "package_with_entrypoint." + buffer_name,

tests/v3/test_codec_entrypoints.py

@@ -109,7 +109,7 @@ def test_parse_shapelike_valid(data: Iterable[int]) -> None:
 
 # todo: more dtypes
 @pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)])
-def parse_dtype(data: tuple[str, np.dtype]) -> None:
+def parse_dtype(data: tuple[str, np.dtype[Any]]) -> None:
     unparsed, parsed = data
     assert parse_dtype(unparsed) == parsed
 

tests/v3/test_common.py

@@ -10,7 +10,7 @@
 import zarr
 from zarr import Array, zeros
 from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline
-from zarr.abc.store import ByteSetter
+from zarr.abc.store import ByteSetter, Store
 from zarr.codecs import BatchedCodecPipeline, BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec
 from zarr.core.array_spec import ArraySpec
 from zarr.core.buffer import NDBuffer
@@ -77,17 +77,18 @@ def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any)
         assert config.get(key) == new_val
 
 
-def test_fully_qualified_name():
+def test_fully_qualified_name() -> None:
     class MockClass:
         pass
 
-    assert "v3.test_config.test_fully_qualified_name.<locals>.MockClass" == fully_qualified_name(
-        MockClass
+    assert (
+        fully_qualified_name(MockClass)
+        == "tests.v3.test_config.test_fully_qualified_name.<locals>.MockClass"
     )
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_codec_pipeline_class(store):
+def test_config_codec_pipeline_class(store: Store) -> None:
     # has default value
     assert get_pipeline_class().__name__ != ""
 
@@ -138,7 +139,7 @@ class MockEnvCodecPipeline(CodecPipeline):
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_codec_implementation(store):
+def test_config_codec_implementation(store: Store) -> None:
     # has default value
     assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"]
 
@@ -171,7 +172,7 @@ async def _encode_single(
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_ndbuffer_implementation(store):
+def test_config_ndbuffer_implementation(store: Store) -> None:
     # has default value
     assert fully_qualified_name(get_ndbuffer_class()) == config.defaults[0]["ndbuffer"]
 
@@ -191,7 +192,7 @@ def test_config_ndbuffer_implementation(store):
     assert isinstance(got, TestNDArrayLike)
 
 
-def test_config_buffer_implementation():
+def test_config_buffer_implementation() -> None:
     # has default value
     assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"]
 

tests/v3/test_config.py

@@ -681,7 +681,7 @@ async def test_asyncgroup_update_attributes(
     assert agroup_new_attributes.attrs == attributes_new
 
 
-async def test_group_members_async(store: LocalStore | MemoryStore):
+async def test_group_members_async(store: LocalStore | MemoryStore) -> None:
     group = AsyncGroup(
         GroupMetadata(),
         store_path=StorePath(store=store, path="root"),

tests/v3/test_group.py

@@ -25,12 +25,11 @@
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from zarr.abc.store import Store
     from zarr.core.common import ChunkCoords
 
 
 @pytest.fixture
-async def store() -> Iterator[Store]:
+async def store() -> Iterator[StorePath]:
     yield StorePath(await MemoryStore.open(mode="w"))
 
 
@@ -52,7 +51,7 @@ def zarr_array_from_numpy_array(
 
 class CountingDict(MemoryStore):
     @classmethod
-    async def open(cls):
+    async def open(cls) -> CountingDict:
         store = await super().open(mode="w")
         store.counter = Counter()
         return store
@@ -68,7 +67,7 @@ async def set(self, key, value, byte_range=None):
         return await super().set(key, value, byte_range)
 
 
-def test_normalize_integer_selection():
+def test_normalize_integer_selection() -> None:
     assert 1 == normalize_integer_selection(1, 100)
     assert 99 == normalize_integer_selection(-1, 100)
     with pytest.raises(IndexError):
@@ -79,7 +78,7 @@ def test_normalize_integer_selection():
         normalize_integer_selection(-1000, 100)
 
 
-def test_replace_ellipsis():
+def test_replace_ellipsis() -> None:
     # 1D, single item
     assert (0,) == replace_ellipsis(0, (100,))
 
@@ -258,7 +257,7 @@ def _test_get_basic_selection(a, z, selection):
 
 
 # noinspection PyStatementEffect
-def test_get_basic_selection_1d(store: StorePath):
+def test_get_basic_selection_1d(store: StorePath) -> None:
     # setup
     a = np.arange(1050, dtype=int)
     z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,))
@@ -328,7 +327,7 @@ def test_get_basic_selection_1d(store: StorePath):
 
 
 # noinspection PyStatementEffect
-def test_get_basic_selection_2d(store: StorePath):
+def test_get_basic_selection_2d(store: StorePath) -> None:
     # setup
     a = np.arange(10000, dtype=int).reshape(1000, 10)
     z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3))
@@ -349,7 +348,7 @@ def test_get_basic_selection_2d(store: StorePath):
     np.testing.assert_array_equal(z[fancy_selection], [0, 11])
 
 
-def test_fancy_indexing_fallback_on_get_setitem(store: StorePath):
+def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None:
     z = zarr_array_from_numpy_array(store, np.zeros((20, 20)))
     z[[1, 2, 3], [1, 2, 3]] = 1
     np.testing.assert_array_equal(

tests/v3/test_indexing.py

@@ -237,7 +237,7 @@ def test_metadata_to_dict(
 
 @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897])
 @pytest.mark.parametrize("precision", ["ns", "D"])
-async def test_datetime_metadata(fill_value: int, precision: str):
+async def test_datetime_metadata(fill_value: int, precision: str) -> None:
     metadata_dict = {
         "zarr_format": 3,
         "node_type": "array",

tests/v3/test_metadata/__init__.py

@@ -11,7 +11,7 @@
 
 
 @given(st.data())
-def test_roundtrip(data):
+def test_roundtrip(data: st.DataObject) -> None:
     nparray = data.draw(np_arrays)
     zarray = data.draw(arrays(arrays=st.just(nparray)))
     assert_array_equal(nparray, zarray[:])
@@ -23,7 +23,7 @@ def test_roundtrip(data):
 # Uncomment the next line to reproduce the original failure.
 # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/ndR2z7nkDZEDADWpBL4=')
 @pytest.mark.filterwarnings("ignore::RuntimeWarning")
-def test_basic_indexing(data):
+def test_basic_indexing(data: st.DataObject) -> None:
     zarray = data.draw(arrays())
     nparray = zarray[:]
     indexer = data.draw(basic_indices(shape=nparray.shape))
@@ -42,7 +42,7 @@ def test_basic_indexing(data):
 # Uncomment the next line to reproduce the original failure.
 # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/eLmF7qr/C5EDADZUBRM=')
 @pytest.mark.filterwarnings("ignore::RuntimeWarning")
-def test_vindex(data):
+def test_vindex(data: st.DataObject) -> None:
     zarray = data.draw(arrays())
     nparray = zarray[:]
 

tests/v3/test_metadata/test_v3.py

@@ -7,7 +7,7 @@
 from zarr.store.memory import MemoryStore
 
 
-async def test_make_store_path(tmpdir) -> None:
+async def test_make_store_path(tmpdir: str) -> None:
     # None
     store_path = await make_store_path(None)
     assert isinstance(store_path.store, MemoryStore)
@@ -33,4 +33,4 @@ async def test_make_store_path(tmpdir) -> None:
     assert Path(store_path.store.root) == Path(tmpdir)
 
     with pytest.raises(TypeError):
-        await make_store_path(1)
+        await make_store_path(1)  # type: ignore[arg-type]

tests/v3/test_properties.py

@@ -1,5 +1,7 @@
 import os
+from collections.abc import Generator
 
+import botocore.client
 import fsspec
 import pytest
 from upath import UPath
@@ -22,7 +24,7 @@
 
 
 @pytest.fixture(scope="module")
-def s3_base():
+def s3_base() -> Generator[None, None, None]:
     # writable local S3 system
 
     # This fixture is module-scoped, meaning that we can reuse the MotoServer across all tests
@@ -37,7 +39,7 @@ def s3_base():
     server.stop()
 
 
-def get_boto3_client():
+def get_boto3_client() -> botocore.client.BaseClient:
     from botocore.session import Session
 
     # NB: we use the sync botocore client for setup
@@ -46,7 +48,7 @@ def get_boto3_client():
 
 
 @pytest.fixture(autouse=True, scope="function")
-def s3(s3_base):
+def s3(s3_base) -> Generator[s3fs.S3FileSystem, None, None]:
     """
     Quoting Martin Durant:
     pytest-asyncio creates a new event loop for each async test.
@@ -81,7 +83,7 @@ async def alist(it):
     return out
 
 
-async def test_basic():
+async def test_basic() -> None:
     store = await RemoteStore.open(
         f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_url, anon=False
     )