diff --git a/docs/about/releases.md b/docs/about/releases.md index a5dc07368..3d4e902d4 100644 --- a/docs/about/releases.md +++ b/docs/about/releases.md @@ -16,6 +16,8 @@ Adds a `nrefs` accessor for counting virtual chunk references, a `mode` paramete ### Bug fixes - Fix parsing kerchunk references that use a **structured (record) dtype**, as produced for a FITS `BinTableHDU` (e.g. an SDSS spectrum). Such references round-trip the dtype through JSON as a list of `[name, format]` lists and encode the `fill_value` as base64 raw bytes; `from_kerchunk_refs` now coerces the field specs back to tuples before `np.dtype` and decodes the base64 `fill_value` into a structured scalar, instead of raising `TypeError`. By [David Stuebe](https://github.com/emfdavid). +- Read the bytes codec's `endian` robustly across zarr versions when converting v3 metadata to v2. zarr's `BytesCodec.endian` was an `Endian` enum through 3.2.x and is a plain `str` on zarr's `main` (following zarr's enum-to-string-literal deprecation, zarr-developers/zarr-python#3968); `convert_v3_to_v2_metadata` read `endian.value`, which raises `AttributeError` on the plain string, so writing kerchunk references for big-endian data would break against newer zarr. The endianness is now normalized to a string either way. + By [David Stuebe](https://github.com/emfdavid). - Writing a virtual `DataTree` with `region` or `append_dim` (forwarded to each node) previously always failed with `ContainsGroupError`, because every group was unconditionally created; the existing groups are now opened instead. By [Aaron Spring](https://github.com/aaronspring). diff --git a/virtualizarr/tests/test_writers/test_kerchunk.py b/virtualizarr/tests/test_writers/test_kerchunk.py index 705853f28..7a6434821 100644 --- a/virtualizarr/tests/test_writers/test_kerchunk.py +++ b/virtualizarr/tests/test_writers/test_kerchunk.py @@ -351,6 +351,26 @@ def test_convert_v3_to_v2_metadata( assert v2_metadata.attributes == {} +def test_codec_endian_normalizes_enum_and_str(): + # zarr's BytesCodec.endian was an Endian enum through 3.2.x and became a plain str on main + # (zarr-developers/zarr-python#3968). _codec_endian must return the string either way, so + # the str shape (which released zarr can't construct) is fed via a stand-in codec. + from types import SimpleNamespace + from typing import cast + + from zarr.abc.codec import ArrayBytesCodec + from zarr.codecs import BytesCodec + + from virtualizarr.utils import _codec_endian + + assert _codec_endian(BytesCodec(endian="big")) == "big" + assert _codec_endian(BytesCodec(endian="little")) == "little" + assert _codec_endian(BytesCodec(endian=None)) is None + # zarr-main shape: endian is already a plain string + assert _codec_endian(cast(ArrayBytesCodec, SimpleNamespace(endian="big"))) == "big" + assert _codec_endian(cast(ArrayBytesCodec, SimpleNamespace(endian=None))) is None + + def test_warn_if_no_virtual_vars(): non_virtual_ds = xr.Dataset({"foo": ("x", [10, 20, 30]), "x": ("x", [1, 2, 3])}) with pytest.warns(UserWarning, match="non-virtual"): diff --git a/virtualizarr/utils.py b/virtualizarr/utils.py index 87af9094b..a83a840c2 100644 --- a/virtualizarr/utils.py +++ b/virtualizarr/utils.py @@ -96,6 +96,20 @@ def determine_chunk_grid_shape( return tuple(ceildiv(length, chunksize) for length, chunksize in zip(shape, chunks)) +def _codec_endian(codec: ArrayBytesCodec) -> str | None: + """The bytes codec's endianness as a plain string, or None if it has none. + + ``BytesCodec.endian`` was a ``zarr.codecs.bytes.Endian`` enum through zarr 3.2.x and became a + plain ``str`` on zarr's ``main`` branch, following zarr's enum-to-string-literal deprecation + (zarr-developers/zarr-python#3968, #3457). Normalize both so the endianness check works across + zarr versions without touching the deprecated ``Endian`` enum. + """ + endian = getattr(codec, "endian", None) + if endian is None: + return None + return endian if isinstance(endian, str) else endian.value + + def convert_v3_to_v2_metadata( v3_metadata: ArrayV3Metadata, fill_value: Any = None ) -> ArrayV2Metadata: @@ -125,10 +139,9 @@ def convert_v3_to_v2_metadata( # This logic is based on the (default) Bytes codec's endian property, # but other codec pipelines could store endianness elsewhere. big_endian = any( - isinstance(codec, ArrayBytesCodec) - and getattr(codec, "endian", None) is not None - and codec.endian.value == "big" # type: ignore[attr-defined] + _codec_endian(codec) == "big" for codec in v3_metadata.codecs + if isinstance(codec, ArrayBytesCodec) ) if big_endian: na_dtype = v3_metadata.data_type.to_native_dtype().newbyteorder(">")