feat(codecs): deprecate the host-byte-order default of BytesCodec on big-endian hosts

`BytesCodec()` without an `endian` argument means `sys.byteorder`, so the
same code writes big-endian chunks on s390x and little-endian chunks
elsewhere. The default will become "little" on every host, which is what
the codec originally defaulted to before #1660 and what the default
serializer uses. On big-endian hosts, omitting `endian` now raises a
ZarrFutureWarning; nothing changes on little-endian hosts.

Callers inside zarr (`zarr.testing` strategies, docstrings) and the tests
now pass `endian` explicitly, so they do not depend on the default.

Refs #3438

Assisted-by: ClaudeCode:claude-opus-5-5

Author: d-v-b

changes/PR_NUMBER.removal.md

@@ -0,0 +1 @@
+On big-endian hosts, constructing `BytesCodec()` without an `endian` argument is deprecated and raises a `ZarrFutureWarning`. The omitted `endian` currently means the host byte order. A future release will change it to `"little"` on every host, so that stored bytes do not depend on the machine that wrote them. To keep the current behavior, pass `endian="big"`; to adopt the new default now, pass `endian="little"`. Nothing changes on little-endian hosts, where the default is already `"little"`. `BytesCodec.from_dict` and zarr's default serializer are unaffected.

src/zarr/api/asynchronous.py

@@ -1024,7 +1024,7 @@ async def create(
         Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead.
 
         If no codecs are provided, default codecs will be used based on the data type of the array.
-        For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`;
+        For most data types, the default codecs are the tuple `(BytesCodec(endian="little"), ZstdCodec())`;
         data types that require a special [`zarr.abc.codec.ArrayBytesCodec`][], like variable-length strings or bytes,
         will use the [`zarr.abc.codec.ArrayBytesCodec`][] required for the data type instead of [`zarr.codecs.BytesCodec`][].
     dimension_names : Iterable[str | None] | None = None

src/zarr/api/synchronous.py

@@ -762,7 +762,7 @@ def create(
         Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead.
 
         If no codecs are provided, default codecs will be used based on the data type of the array.
-        For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`;
+        For most data types, the default codecs are the tuple `(BytesCodec(endian="little"), ZstdCodec())`;
         data types that require a special [`zarr.abc.codec.ArrayBytesCodec`][], like variable-length strings or bytes,
         will use the [`zarr.abc.codec.ArrayBytesCodec`][] required for the data type instead of [`zarr.codecs.BytesCodec`][].
     dimension_names : Iterable[str | None] | None = None

src/zarr/codecs/bytes.py

@@ -3,13 +3,15 @@
 import sys
 import warnings
 from dataclasses import dataclass, replace
+from enum import Enum
 from typing import TYPE_CHECKING, ClassVar, Final, Literal
 
 from zarr.abc.codec import ArrayBytesCodec
 from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta
 from zarr.core.common import JSON, parse_named_configuration
 from zarr.core.dtype.common import HasEndianness
 from zarr.core.dtype.npy.structured import Struct
+from zarr.errors import ZarrFutureWarning
 
 if TYPE_CHECKING:
     from typing import Self
@@ -33,6 +35,30 @@ class Endian(metaclass=_DeprecatedStrEnumMeta):
     _members: ClassVar[dict[str, str]] = {"little": "little", "big": "big"}
 
 
+class _HostEndian(Enum):
+    """Marks an omitted `endian` argument, which currently means the host byte order."""
+
+    token = 0
+
+
+def _resolve_host_endian() -> EndianLiteral:
+    """The byte order an omitted `endian` argument stands for, warning where it will change.
+
+    The default will become `"little"` on every host, so only big-endian hosts are affected.
+    """
+    if sys.byteorder == "big":
+        warnings.warn(
+            "BytesCodec() without an `endian` argument stores chunks in the byte order of the "
+            "host, which is big-endian here. A future version of Zarr Python will default to "
+            "endian='little' on every host, so that stored bytes do not depend on the machine "
+            "that wrote them. Pass endian='big' to keep the current behavior, or "
+            "endian='little' to adopt the new default now.",
+            ZarrFutureWarning,
+            stacklevel=3,
+        )
+    return sys.byteorder
+
+
 def _parse_endian(data: object) -> EndianLiteral:
     if isinstance(data, str) and data in ENDIAN:
         return data  # type: ignore[return-value]
@@ -41,13 +67,24 @@ def _parse_endian(data: object) -> EndianLiteral:
 
 @dataclass(frozen=True)
 class BytesCodec(ArrayBytesCodec):
-    """bytes codec"""
+    """bytes codec
+
+    When `endian` is omitted it is the byte order of the host. That default is deprecated on
+    big-endian hosts: it will become `"little"` on every host, so that stored bytes do not depend
+    on the machine that wrote them.
+    """
 
     is_fixed_size = True
 
     endian: EndianLiteral | None
 
-    def __init__(self, *, endian: Endian | EndianLiteral | None = sys.byteorder) -> None:
+    def __init__(
+        self,
+        *,
+        endian: Endian | EndianLiteral | Literal[_HostEndian.token] | None = _HostEndian.token,
+    ) -> None:
+        if endian is _HostEndian.token:
+            endian = _resolve_host_endian()
         if endian is None:
             endian_parsed: EndianLiteral | None = None
         else:

src/zarr/testing/stateful.py

@@ -145,7 +145,7 @@ def add_array(self, data: DataObject, name: str) -> None:
                 paths=st.just(parent),
                 array_names=st.just(name),
                 zarr_formats=st.just(3),
-                compressors=st.just(BytesCodec()),
+                compressors=st.just(BytesCodec(endian="little")),
                 open_mode="a",
             ),
             label="generated array",

src/zarr/testing/strategies.py

@@ -259,8 +259,8 @@ def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[str | N
 # silently disables the fast path under every property test.
 sharding_inner_codecs: st.SearchStrategy[list[BytesCodec | ZstdCodec]] = st.sampled_from(
     [
-        [BytesCodec()],
-        [BytesCodec(), ZstdCodec()],
+        [BytesCodec(endian="little")],
+        [BytesCodec(endian="little"), ZstdCodec()],
     ]
 )
 
@@ -303,7 +303,7 @@ def array_metadata(
             attributes=draw(attributes),  # type: ignore[arg-type]
             dimension_names=draw(dimension_names(ndim=ndim)),
             chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"),  # FIXME
-            codecs=[BytesCodec()],
+            codecs=[BytesCodec(endian="little")],
             storage_transformers=(),
         )
 
@@ -386,7 +386,7 @@ def _sharding_codecs(
     return ShardingCodec(
         subchunk_write_order=subchunk_write_order,
         codecs=inner_codecs,
-        index_codecs=[BytesCodec(), Crc32cCodec()],
+        index_codecs=[BytesCodec(endian="little"), Crc32cCodec()],
         chunk_shape=chunk_shape,
     )
 

tests/test_array.py

@@ -342,7 +342,7 @@ def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str)
             "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}},
             "data_type": "uint8",
             "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}},
-            "codecs": (BytesCodec().to_dict(),),
+            "codecs": (BytesCodec(endian="little").to_dict(),),
             "fill_value": 0,
             "storage_transformers": ({"test": "should_raise"}),
         }
@@ -353,7 +353,7 @@ def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str)
             "chunks": (1,),
             "dtype": "|u1",
             "dimension_separator": ".",
-            "codecs": (BytesCodec().to_dict(),),
+            "codecs": (BytesCodec(endian="little").to_dict(),),
             "fill_value": 0,
             "order": "C",
             "storage_transformers": ({"test": "should_raise"}),

tests/test_chunk_transform.py

@@ -60,9 +60,9 @@ def _make_nd_buffer(arr: np.ndarray[Any, np.dtype[Any]]) -> NDBuffer:
 @pytest.mark.parametrize(
     ("shape", "codecs"),
     [
-        ((100,), (BytesCodec(),)),
-        ((100,), (BytesCodec(), GzipCodec())),
-        ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec())),
+        ((100,), (BytesCodec(endian="little"),)),
+        ((100,), (BytesCodec(endian="little"), GzipCodec())),
+        ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec(endian="little"), ZstdCodec())),
     ],
     ids=["bytes-only", "with-compression", "full-chain"],
 )
@@ -90,14 +90,14 @@ def test_construction_rejects_non_sync(shape: tuple[int, ...], codecs: tuple[Cod
 @pytest.mark.parametrize(
     ("arr", "codecs"),
     [
-        (np.arange(100, dtype="float64"), (BytesCodec(),)),
-        (np.arange(100, dtype="float64"), (BytesCodec(), GzipCodec(level=1))),
+        (np.arange(100, dtype="float64"), (BytesCodec(endian="little"),)),
+        (np.arange(100, dtype="float64"), (BytesCodec(endian="little"), GzipCodec(level=1))),
         (
             np.arange(12, dtype="float64").reshape(3, 4),
-            (TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec(level=1)),
+            (TransposeCodec(order=(1, 0)), BytesCodec(endian="little"), ZstdCodec(level=1)),
         ),
-        (np.arange(100, dtype="float64"), (BytesCodec(), Crc32cCodec())),
-        (np.arange(50, dtype="int32"), (BytesCodec(), ZstdCodec(level=1))),
+        (np.arange(100, dtype="float64"), (BytesCodec(endian="little"), Crc32cCodec())),
+        (np.arange(50, dtype="int32"), (BytesCodec(endian="little"), ZstdCodec(level=1))),
     ],
     ids=["bytes-only", "gzip", "transpose+zstd", "crc32c", "int32"],
 )
@@ -118,9 +118,9 @@ def test_encode_decode_roundtrip(
 @pytest.mark.parametrize(
     ("shape", "codecs", "input_size", "expected_size"),
     [
-        ((100,), (BytesCodec(),), 800, 800),
-        ((100,), (BytesCodec(), Crc32cCodec()), 800, 804),
-        ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec()), 96, 96),
+        ((100,), (BytesCodec(endian="little"),), 800, 800),
+        ((100,), (BytesCodec(endian="little"), Crc32cCodec()), 800, 804),
+        ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec(endian="little")), 96, 96),
     ],
     ids=["bytes-only", "crc32c", "transpose"],
 )
@@ -147,7 +147,7 @@ def _encode_sync(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer
 
     spec = _make_array_spec((3, 4), np.dtype("float64"))
     chain = ChunkTransform(
-        codecs=(NoneReturningAACodec(order=(1, 0)), BytesCodec()),
+        codecs=(NoneReturningAACodec(order=(1, 0)), BytesCodec(endian="little")),
     )
     arr = np.arange(12, dtype="float64").reshape(3, 4)
     nd_buf = _make_nd_buffer(arr)

tests/test_codec_pipeline.py

@@ -275,7 +275,7 @@ async def _encode_single(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any:
 
 _CODEC_FACTORY: dict[str, Callable[[], Codec]] = {
     _AA: lambda: TransposeCodec(order=(0, 1)),
-    _AB: BytesCodec,
+    _AB: lambda: BytesCodec(endian="little"),
     _BB: GzipCodec,
 }
 

tests/test_codec_pipeline_suite.py

@@ -285,7 +285,7 @@ def _val(n: int, dtype: str, offset: int = 1) -> np.ndarray:
             "chunks": (2, 4),
             "shards": None,
             "filters": [TransposeCodec(order=(1, 0))],
-            "serializer": BytesCodec(),
+            "serializer": BytesCodec(endian="little"),
             **_I32,
         },
         writes=((slice(None), np.arange(96, dtype="int32").reshape(8, 12)),),
@@ -298,7 +298,7 @@ def _val(n: int, dtype: str, offset: int = 1) -> np.ndarray:
             "chunks": (2, 4),
             "shards": None,
             "filters": [TransposeCodec(order=(1, 0))],
-            "serializer": BytesCodec(),
+            "serializer": BytesCodec(endian="little"),
             "compressors": GzipCodec(level=1),
             **_I32,
         },
@@ -517,7 +517,9 @@ def test_partial_write_after_reopen_is_correct(
             compressors=None,
             config={"write_empty_chunks": True},
             serializer=ShardingCodec(
-                chunk_shape=inner, codecs=[BytesCodec()], subchunk_write_order=subchunk_write_order
+                chunk_shape=inner,
+                codecs=[BytesCodec(endian="little")],
+                subchunk_write_order=subchunk_write_order,
             ),
         )
         ref = np.arange(24, dtype="int32").reshape(shape)