diff --git a/pixi.lock b/pixi.lock index 1e39cef..fa7f3f9 100644 --- a/pixi.lock +++ b/pixi.lock @@ -9874,8 +9874,8 @@ packages: requires_python: '>=3.6' - pypi: ./ name: virtual-tiff - version: 0.2.2.dev12+g9d9b47fdb.d20260212 - sha256: 3856c231f61aa517de9cb27a702bae9b43fac3514dcf1d192ae9d8c705ecb16a + version: 0.2.2.dev11+g49b851b3f.d20260225 + sha256: 84f29f6df980bebe9b0977c265974d5faf72ca82acfa3fbbb95b1ce5b8341ca8 requires_dist: - async-tiff>=0.4.0 - imagecodecs diff --git a/pyproject.toml b/pyproject.toml index 9d7f2d7..32c8978 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,12 @@ docs = ["dev", "docs"] test-min-versions = ["dev", "test", "minimum-versions", "py311"] # test minimum versions with python 3.11 test = ["dev", "test", "latest-versions", "py314"] # test latest versions with python 3.14 +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:Imagecodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning", + "ignore:Dataset has no geotransform, gcps, or rpcs. The identity matrix will be returned.:rasterio.errors.NotGeoreferencedWarning", +] + [tool.coverage.run] source_pkgs = [""] branch = true diff --git a/src/virtual_tiff/codecs.py b/src/virtual_tiff/codecs.py index 4f3c556..d468c24 100644 --- a/src/virtual_tiff/codecs.py +++ b/src/virtual_tiff/codecs.py @@ -1,23 +1,46 @@ -# Copied and modified from https://github.com/zarr-developers/zarr-python/blob/bb55f0c58320a6d27be3a0ba918feee398a53db4/src/zarr/codecs/bytes.py - +# Adapted from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/codecs/bytes.py and https://github.com/zarr-developers/zarr-python/pull/3332 from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass, replace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal, Self, cast, overload import numpy as np -from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec +from zarr.abc.codec import ( + ArrayArrayCodec, + ArrayBytesCodec, + CodecJSON, + CodecJSON_V2, + CodecJSON_V3, +) from zarr.codecs.bytes import Endian from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer -from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.common import JSON from zarr.registry import register_codec if TYPE_CHECKING: - from typing import Self - from zarr.core.array_spec import ArraySpec +def check_codecjson_v2(data: object) -> bool: + return isinstance(data, Mapping) and "id" in data and isinstance(data["id"], str) + + +ZarrFormat = Literal[2, 3] + + +def _parse_endian(data: object) -> Endian | None: + if data is None: + return None + if isinstance(data, Endian): + return data + if isinstance(data, str) and data in ("little", "big"): + return Endian(data) + raise ValueError( + f"Invalid endian value: {data!r}. Expected 'little', 'big', or None." + ) + + @dataclass(frozen=True) class ChunkyCodec(ArrayBytesCodec): is_fixed_size = True @@ -25,24 +48,56 @@ class ChunkyCodec(ArrayBytesCodec): endian: Endian | None def __init__(self, *, endian: Endian | str | None = "little") -> None: - endian_parsed = None if endian is None else parse_enum(endian, Endian) - object.__setattr__(self, "endian", endian_parsed) + object.__setattr__(self, "endian", _parse_endian(endian)) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: - _, configuration_parsed = parse_named_configuration( - data, "ChunkyCodec", require_configuration=False - ) - configuration_parsed = configuration_parsed or {} - return cls(**configuration_parsed) # type: ignore[arg-type] + return cls.from_json(data) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: - if self.endian is not None: - return { - "name": "ChunkyCodec", - "configuration": {"endian": self.endian.value}, - } - return {"name": "ChunkyCodec"} + return cast(dict[str, JSON], self.to_json(zarr_format=3)) + + @classmethod + def _from_json_v2(cls, data: CodecJSON) -> Self: + if isinstance(data, Mapping): + return cls(endian=data.get("endian")) + raise ValueError(f"Invalid JSON: {data}") + + @classmethod + def _from_json_v3(cls, data: CodecJSON) -> Self: + if isinstance(data, str): + return cls() + if isinstance(data, Mapping): + config = data.get("configuration", {}) + return cls(**config) + raise ValueError(f"Invalid JSON: {data}") + + @classmethod + def from_json(cls, data: CodecJSON) -> Self: + if check_codecjson_v2(data): + return cls._from_json_v2(data) + return cls._from_json_v3(data) + + @overload + def to_json(self, zarr_format: Literal[2]) -> CodecJSON_V2: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> CodecJSON_V3: ... + + def to_json(self, zarr_format: ZarrFormat) -> CodecJSON_V2 | CodecJSON_V3: + if zarr_format == 2: + if self.endian is not None: + return {"id": "ChunkyCodec", "endian": self.endian.value} # type: ignore[return-value, typeddict-item] + return {"id": "ChunkyCodec"} # type: ignore[return-value] + elif zarr_format == 3: + if self.endian is not None: + return { + "name": "ChunkyCodec", + "configuration": {"endian": self.endian.value}, + } + return {"name": "ChunkyCodec"} + raise ValueError( + f"Unsupported Zarr format {zarr_format}. Expected 2 or 3." + ) # pragma: no cover def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: if array_spec.dtype.item_size == 0: @@ -123,14 +178,38 @@ def __init__(self) -> None: @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: - _, configuration_parsed = parse_named_configuration( - data, "HorizontalDeltaCodec", require_configuration=False - ) - configuration_parsed = configuration_parsed or {} - return cls(**configuration_parsed) # type: ignore[arg-type] + return cls.from_json(data) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: - return {"name": "HorizontalDeltaCodec"} + return cast(dict[str, JSON], self.to_json(zarr_format=3)) + + @classmethod + def _from_json_v2(cls, data: CodecJSON) -> Self: + return cls() + + @classmethod + def _from_json_v3(cls, data: CodecJSON) -> Self: + return cls() + + @classmethod + def from_json(cls, data: CodecJSON) -> Self: + if check_codecjson_v2(data): + return cls._from_json_v2(data) + return cls._from_json_v3(data) + + @overload + def to_json(self, zarr_format: Literal[2]) -> CodecJSON_V2: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> CodecJSON_V3: ... + + def to_json(self, zarr_format: ZarrFormat) -> CodecJSON_V2 | CodecJSON_V3: + if zarr_format == 2: + return {"id": "HorizontalDeltaCodec"} # type: ignore[return-value] + elif zarr_format == 3: + return {"name": "HorizontalDeltaCodec"} + raise ValueError( + f"Unsupported Zarr format {zarr_format}. Expected 2 or 3." + ) # pragma: no cover def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return self diff --git a/src/virtual_tiff/constants.py b/src/virtual_tiff/constants.py index dd6665e..c399e5c 100644 --- a/src/virtual_tiff/constants.py +++ b/src/virtual_tiff/constants.py @@ -1,4 +1,4 @@ -from numcodecs.zarr3 import LZMA, Zlib +from zarr.codecs.numcodecs import LZMA, Zlib from virtual_tiff.imagecodecs import ( DeflateCodec, diff --git a/src/virtual_tiff/imagecodecs.py b/src/virtual_tiff/imagecodecs.py index 0736eba..2a371f3 100644 --- a/src/virtual_tiff/imagecodecs.py +++ b/src/virtual_tiff/imagecodecs.py @@ -1,100 +1,103 @@ -# Copied and slightly adapted from https://github.com/zarr-developers/numcodecs/blob/main/numcodecs/zarr3.py +# Adapted from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/codecs/numcodecs/_codecs.py from __future__ import annotations import asyncio +from collections.abc import Mapping from dataclasses import dataclass, replace -from functools import cached_property -from typing import Self +from typing import ( + TYPE_CHECKING, + Any, + Literal, + Self, + cast, + overload, +) from warnings import warn -import numcodecs import numpy as np -from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec +from zarr.abc.codec import ( + ArrayArrayCodec, + BytesBytesCodec, + CodecJSON, + CodecJSON_V2, + CodecJSON_V3, +) from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON +from zarr.registry import get_numcodec -CODEC_PREFIX = "imagecodecs_" +from virtual_tiff.codecs import check_codecjson_v2 +if TYPE_CHECKING: + from zarr.abc.numcodec import Numcodec -def _expect_name_prefix(codec_name: str) -> str: - if not codec_name.startswith(CODEC_PREFIX): - raise ValueError( - f"Expected name to start with '{CODEC_PREFIX}'. Got {codec_name} instead." - ) # pragma: no cover - return codec_name.removeprefix(CODEC_PREFIX) + +ZarrFormat = Literal[2, 3] @dataclass(frozen=True) class _ImageCodecsCodec: codec_name: str - codec_config: JSON - - def __init_subclass__(cls, *, codec_name: str | None = None, **kwargs): - """To be used only when creating the actual public-facing codec class.""" - super().__init_subclass__(**kwargs) - if codec_name is not None: - namespace = codec_name - - cls_name = f"{CODEC_PREFIX}{namespace}.{cls.__name__}" - cls.codec_name = f"{CODEC_PREFIX}{namespace}" - cls.__doc__ = f""" - See :class:`{cls_name}` for more details and parameters. - """ - - def __init__(self, **codec_config: JSON) -> None: - if not self.codec_name: - raise ValueError( - "The codec name needs to be supplied through the `codec_name` attribute." - ) # pragma: no cover - _expect_name_prefix(self.codec_name) - - if "id" not in codec_config: - codec_config = { - "id": self.codec_name, # type: ignore - **codec_config, - } - elif codec_config["id"] != self.codec_name: - raise ValueError( - f"Codec id does not match {self.codec_name}. Got: {codec_config['id']}." - ) # pragma: no cover - - object.__setattr__(self, "codec_config", codec_config) + _codec: Numcodec + codec_config: Mapping[str, Any] + + def __init__(self, **codec_config: Any) -> None: + codec = get_numcodec( + { + "id": self.codec_name, + **{k: v for k, v in codec_config.items() if k != "id"}, + } # type: ignore[typeddict-item] + ) + object.__setattr__(self, "_codec", codec) + object.__setattr__(self, "codec_config", codec.get_config()) warn( "Imagecodecs codecs are not in the Zarr version 3 specification and " "may not be supported by other zarr implementations.", category=UserWarning, + stacklevel=2, ) - @cached_property - def _codec(self) -> numcodecs.abc.Codec: - return numcodecs.get_codec(dict(self.codec_config)) + def to_dict(self) -> dict[str, JSON]: + return cast(dict[str, JSON], self.to_json(zarr_format=3)) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: - configuration = data.get("configuration", {}) - return cls(**configuration) + return cls.from_json(data) # type: ignore[arg-type] - def to_dict(self) -> JSON: - codec_config = {k: v for k, v in self.codec_config.items() if k != "id"} - if codec_config: - return { - "name": self.codec_name, - "configuration": codec_config, - } - return {"name": self.codec_name} + @classmethod + def _from_json_v2(cls, data: CodecJSON_V2) -> Self: + return cls(**data) - def compute_encoded_size( - self, input_byte_length: int, chunk_spec: ArraySpec - ) -> int: - raise NotImplementedError # pragma: no cover + @classmethod + def _from_json_v3(cls, data: CodecJSON_V3) -> Self: + if isinstance(data, str): + return cls() + return cls(**data.get("configuration", {})) + @classmethod + def from_json(cls, data: CodecJSON) -> Self: + if check_codecjson_v2(data): + return cls._from_json_v2(data) # type: ignore[arg-type] + return cls._from_json_v3(data) # type: ignore[arg-type] -class _ImageCodecsBytesBytesCodec(_ImageCodecsCodec, BytesBytesCodec): - def __init__(self, **codec_config: JSON) -> None: - super().__init__(**codec_config) + @overload + def to_json(self, zarr_format: Literal[2]) -> CodecJSON_V2: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> CodecJSON_V3: ... + + def to_json(self, zarr_format: ZarrFormat) -> CodecJSON_V2 | CodecJSON_V3: + codec_config = {k: v for k, v in self.codec_config.items() if k != "id"} + if zarr_format == 2: + return {"id": self.codec_name, **codec_config} # type: ignore[return-value, typeddict-item] + else: + if codec_config: + return {"name": self.codec_name, "configuration": codec_config} + return {"name": self.codec_name} + +class _ImageCodecsBytesBytesCodec(_ImageCodecsCodec, BytesBytesCodec): async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec ) -> Buffer: @@ -116,11 +119,13 @@ async def _encode_single( ) -> Buffer: return await asyncio.to_thread(self._encode, chunk_bytes, chunk_spec.prototype) + def compute_encoded_size( + self, input_byte_length: int, chunk_spec: ArraySpec + ) -> int: + raise NotImplementedError -class _ImageCodecsArrayArrayCodec(_ImageCodecsCodec, ArrayArrayCodec): - def __init__(self, **codec_config: JSON) -> None: - super().__init__(**codec_config) +class _ImageCodecsArrayArrayCodec(_ImageCodecsCodec, ArrayArrayCodec): async def _decode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec ) -> NDBuffer: @@ -137,30 +142,25 @@ async def _encode_single( out = await asyncio.to_thread(self._codec.encode, chunk_ndarray) return chunk_spec.prototype.nd_buffer.from_ndarray_like(out) - -class _ImageCodecsArrayBytesCodec(_ImageCodecsCodec, ArrayBytesCodec): - def __init__(self, **codec_config: JSON) -> None: - super().__init__(**codec_config) - - async def _decode_single( - self, chunk_buffer: Buffer, chunk_spec: ArraySpec - ) -> NDBuffer: - chunk_bytes = chunk_buffer.to_bytes() - out = await asyncio.to_thread(self._codec.decode, chunk_bytes) - return chunk_spec.prototype.nd_buffer.from_ndarray_like( - out.reshape(chunk_spec.shape) - ) + def compute_encoded_size( + self, input_byte_length: int, chunk_spec: ArraySpec + ) -> int: + raise NotImplementedError # array-to-array codecs ("filters") -class DeltaCodec(_ImageCodecsArrayArrayCodec, codec_name="delta"): +class DeltaCodec(_ImageCodecsArrayArrayCodec): + codec_name = "imagecodecs_delta" + def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: if astype := self.codec_config.get("astype"): return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[call-overload] return chunk_spec -class FloatPredCodec(_ImageCodecsArrayArrayCodec, codec_name="floatpred"): +class FloatPredCodec(_ImageCodecsArrayArrayCodec): + codec_name = "imagecodecs_floatpred" + def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: if astype := self.codec_config.get("astype"): return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[call-overload] @@ -170,56 +170,56 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: # bytes-to-bytes codecs -class DeflateCodec(_ImageCodecsBytesBytesCodec, codec_name="deflate"): - pass +class DeflateCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_deflate" -class JetRawCodec(_ImageCodecsBytesBytesCodec, codec_name="jetraw"): - pass +class JetRawCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jetraw" -class JpegCodec(_ImageCodecsBytesBytesCodec, codec_name="jpeg"): - pass +class JpegCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jpeg" -class Jpeg8Codec(_ImageCodecsBytesBytesCodec, codec_name="jpeg8"): - pass +class Jpeg8Codec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jpeg8" -class Jpeg2KCodec(_ImageCodecsBytesBytesCodec, codec_name="jpeg2k"): - pass +class Jpeg2KCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jpeg2k" -class JpegXRCodec(_ImageCodecsBytesBytesCodec, codec_name="jpegxr"): - pass +class JpegXRCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jpegxr" -class JpegXLCodec(_ImageCodecsBytesBytesCodec, codec_name="jpegxl"): - pass +class JpegXLCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_jpegxl" -class LercCodec(_ImageCodecsBytesBytesCodec, codec_name="lerc"): - pass +class LercCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_lerc" -class LZWCodec(_ImageCodecsBytesBytesCodec, codec_name="lzw"): - pass +class LZWCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_lzw" -class PackBitsCodec(_ImageCodecsBytesBytesCodec, codec_name="packbits"): - pass +class PackBitsCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_packbits" -class PngCodec(_ImageCodecsBytesBytesCodec, codec_name="png"): - pass +class PngCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_png" -class WebpCodec(_ImageCodecsBytesBytesCodec, codec_name="webp"): - pass +class WebpCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_webp" -class ZstdCodec(_ImageCodecsBytesBytesCodec, codec_name="zstd"): - pass +class ZstdCodec(_ImageCodecsBytesBytesCodec): + codec_name = "imagecodecs_zstd" __all__ = [ diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 90f769f..8c16d30 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -1,7 +1,98 @@ +import warnings + +import numpy as np import pytest +from zarr.codecs.bytes import Endian +from zarr.core.array_spec import ArrayConfig, ArraySpec +from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer.cpu import NDBuffer +from zarr.core.dtype import Float32, UInt8, UInt16 + +from virtual_tiff.codecs import ( + ChunkyCodec, + HorizontalDeltaCodec, + _parse_endian, + check_codecjson_v2, +) +from virtual_tiff.imagecodecs import ( + DeflateCodec, + DeltaCodec, + FloatPredCodec, + LZWCodec, + ZstdCodec, +) + +_DEFAULT_CONFIG = ArrayConfig(order="C", write_empty_chunks=True) + + +def _make_spec(shape, dtype, fill_value=0): + return ArraySpec( + shape=shape, + dtype=dtype, + fill_value=fill_value, + config=_DEFAULT_CONFIG, + prototype=default_buffer_prototype(), + ) + + +# --- check_codecjson_v2 tests --- + + +def test_check_codecjson_v2_with_id(): + assert check_codecjson_v2({"id": "foo"}) is True + + +def test_check_codecjson_v2_with_name(): + assert check_codecjson_v2({"name": "foo"}) is False + + +def test_check_codecjson_v2_with_string(): + assert check_codecjson_v2("foo") is False + + +def test_check_codecjson_v2_with_non_string_id(): + assert check_codecjson_v2({"id": 123}) is False + + +def test_check_codecjson_v2_with_empty_dict(): + assert check_codecjson_v2({}) is False + + +# --- _parse_endian tests --- + + +def test_parse_endian_none(): + assert _parse_endian(None) is None + + +def test_parse_endian_string_little(): + assert _parse_endian("little") == Endian.little + + +def test_parse_endian_string_big(): + assert _parse_endian("big") == Endian.big + + +def test_parse_endian_enum_passthrough(): + assert _parse_endian(Endian.big) is Endian.big + + +def test_parse_endian_invalid(): + with pytest.raises(ValueError, match="Invalid endian value"): + _parse_endian("middle") + + +def test_parse_endian_invalid_type(): + with pytest.raises(ValueError, match="Invalid endian value"): + _parse_endian(42) + -from virtual_tiff.codecs import ChunkyCodec -from virtual_tiff.imagecodecs import DeflateCodec, LZWCodec +# --- ChunkyCodec tests --- + + +def test_chunky_codec_default_endian(): + codec = ChunkyCodec() + assert codec.endian == Endian.little @pytest.mark.parametrize("endian", ["big", "little"]) @@ -14,17 +105,227 @@ def test_chunky_codec_roundtrip_preserves_endian(endian): assert restored.endian == codec.endian +@pytest.mark.parametrize("endian", ["big", "little"]) +def test_chunky_codec_to_json_v3(endian): + codec = ChunkyCodec(endian=endian) + v3 = codec.to_json(zarr_format=3) + assert v3["name"] == "ChunkyCodec" + assert v3["configuration"]["endian"] == endian + restored = ChunkyCodec.from_json(v3) + assert restored.endian == codec.endian + + +@pytest.mark.parametrize("endian", ["big", "little"]) +def test_chunky_codec_to_json_v2(endian): + codec = ChunkyCodec(endian=endian) + v2 = codec.to_json(zarr_format=2) + assert v2["id"] == "ChunkyCodec" + assert v2["endian"] == endian + restored = ChunkyCodec.from_json(v2) + assert restored.endian == codec.endian + + +def test_chunky_codec_from_json_auto_detects_format(): + codec = ChunkyCodec(endian="big") + v2 = codec.to_json(zarr_format=2) + v3 = codec.to_json(zarr_format=3) + assert ChunkyCodec.from_json(v2).endian == codec.endian + assert ChunkyCodec.from_json(v3).endian == codec.endian + + +def test_chunky_codec_none_endian_to_json_v3(): + codec = ChunkyCodec(endian=None) + v3 = codec.to_json(zarr_format=3) + assert v3 == {"name": "ChunkyCodec"} + assert "configuration" not in v3 + + +def test_chunky_codec_none_endian_to_json_v2(): + codec = ChunkyCodec(endian=None) + v2 = codec.to_json(zarr_format=2) + assert v2 == {"id": "ChunkyCodec"} + assert "endian" not in v2 + + +def test_chunky_codec_from_json_v3_string_only(): + """A v3 codec can be just a name string with no configuration.""" + restored = ChunkyCodec._from_json_v3("ChunkyCodec") + assert restored.endian == Endian.little # default + + +def test_chunky_codec_from_json_v2_invalid(): + with pytest.raises(ValueError, match="Invalid JSON"): + ChunkyCodec._from_json_v2(42) + + +def test_chunky_codec_from_json_v3_invalid(): + with pytest.raises(ValueError, match="Invalid JSON"): + ChunkyCodec._from_json_v3(42) + + +def test_chunky_codec_compute_encoded_size(): + codec = ChunkyCodec() + assert codec.compute_encoded_size(100, _make_spec((10,), UInt8())) == 100 + + +@pytest.mark.asyncio +async def test_chunky_codec_decode_single_little_endian(): + codec = ChunkyCodec(endian="little") + data = np.array([1, 2, 3], dtype=" 0).""" + codec = ChunkyCodec(endian="little") + evolved = codec.evolve_from_array_spec(_make_spec((10,), UInt8())) + assert evolved.endian == Endian.little + + +def test_chunky_codec_evolve_from_array_spec_multi_byte(): + """endian should be preserved for multi-byte dtypes.""" + codec = ChunkyCodec(endian="big") + evolved = codec.evolve_from_array_spec(_make_spec((10,), UInt16())) + assert evolved.endian == Endian.big + + +def test_chunky_codec_evolve_from_array_spec_none_endian_multi_byte(): + """endian=None with multi-byte dtype should raise.""" + codec = ChunkyCodec(endian=None) + with pytest.raises(ValueError, match="endian"): + codec.evolve_from_array_spec(_make_spec((10,), UInt16())) + + +# --- HorizontalDeltaCodec tests --- + + +def test_horizontal_delta_to_json_v3(): + codec = HorizontalDeltaCodec() + v3 = codec.to_json(zarr_format=3) + assert v3 == {"name": "HorizontalDeltaCodec"} + restored = HorizontalDeltaCodec.from_json(v3) + assert isinstance(restored, HorizontalDeltaCodec) + + +def test_horizontal_delta_to_json_v2(): + codec = HorizontalDeltaCodec() + v2 = codec.to_json(zarr_format=2) + assert v2["id"] == "HorizontalDeltaCodec" + restored = HorizontalDeltaCodec.from_json(v2) + assert isinstance(restored, HorizontalDeltaCodec) + + +def test_horizontal_delta_roundtrip(): + codec = HorizontalDeltaCodec() + restored = HorizontalDeltaCodec.from_dict(codec.to_dict()) + assert isinstance(restored, HorizontalDeltaCodec) + + +def test_horizontal_delta_from_json_auto_detects_format(): + codec = HorizontalDeltaCodec() + v2 = codec.to_json(zarr_format=2) + v3 = codec.to_json(zarr_format=3) + assert isinstance(HorizontalDeltaCodec.from_json(v2), HorizontalDeltaCodec) + assert isinstance(HorizontalDeltaCodec.from_json(v3), HorizontalDeltaCodec) + + +@pytest.mark.asyncio +async def test_horizontal_delta_decode_cumsum(): + """HorizontalDeltaCodec decodes via cumulative sum along last axis.""" + codec = HorizontalDeltaCodec() + # Input: differences [1, 2, 3] -> cumsum -> [1, 3, 6] + diffs = np.array([[1, 2, 3]], dtype=np.int32) + nd_buf = NDBuffer.from_ndarray_like(diffs) + spec = _make_spec((1, 3), Float32()) + result = await codec._decode_single(nd_buf, spec) + expected = np.array([[1, 3, 6]], dtype=np.int32) + np.testing.assert_array_equal(result.as_ndarray_like(), expected) + + +@pytest.mark.asyncio +async def test_horizontal_delta_encode_raises(): + codec = HorizontalDeltaCodec() + data = np.array([[1, 2, 3]], dtype=np.int32) + nd_buf = NDBuffer.from_ndarray_like(data) + spec = _make_spec((1, 3), Float32()) + with pytest.raises(NotImplementedError): + await codec._encode_single(nd_buf, spec) + + +def test_horizontal_delta_compute_encoded_size(): + codec = HorizontalDeltaCodec() + assert codec.compute_encoded_size(100, _make_spec((10,), UInt8())) == 100 + + +def test_horizontal_delta_evolve_from_array_spec(): + codec = HorizontalDeltaCodec() + spec = _make_spec((10,), UInt8()) + assert codec.evolve_from_array_spec(spec) is codec + + +# --- ImageCodecs tests --- + + def test_imagecodecs_roundtrip(): - """Codec metadata must from_dict/to_dict cycles.""" + """Codec metadata must survive from_dict/to_dict cycles.""" codec = LZWCodec() output = codec.to_dict() roundtripped = LZWCodec.from_dict(output) - assert codec == roundtripped - assert "name" not in output.get("configuration", {}) + assert codec.codec_config == roundtripped.codec_config output = roundtripped.to_dict() roundtripped = LZWCodec.from_dict(output) - assert codec == roundtripped - assert "name" not in output.get("configuration", {}) + assert codec.codec_config == roundtripped.codec_config + + +@pytest.mark.parametrize("codec_cls", [LZWCodec, DeflateCodec]) +def test_imagecodecs_to_json_v3(codec_cls): + codec = codec_cls() + v3 = codec.to_json(zarr_format=3) + assert v3["name"] == codec.codec_name + restored = codec_cls.from_json(v3) + assert restored.codec_config["id"] == codec.codec_config["id"] + + +@pytest.mark.parametrize("codec_cls", [LZWCodec, DeflateCodec]) +def test_imagecodecs_to_json_v2(codec_cls): + codec = codec_cls() + v2 = codec.to_json(zarr_format=2) + assert v2["id"] == codec.codec_name + restored = codec_cls.from_json(v2) + assert restored.codec_config["id"] == codec.codec_config["id"] + + +@pytest.mark.parametrize("codec_cls", [LZWCodec, DeflateCodec]) +def test_imagecodecs_from_json_auto_detects_format(codec_cls): + codec = codec_cls() + v2 = codec.to_json(zarr_format=2) + v3 = codec.to_json(zarr_format=3) + assert codec_cls.from_json(v2).codec_config["id"] == codec.codec_config["id"] + assert codec_cls.from_json(v3).codec_config["id"] == codec.codec_config["id"] @pytest.mark.parametrize("codec_cls", [LZWCodec, DeflateCodec]) @@ -37,3 +338,94 @@ def test_imagecodecs_from_dict_ignores_wrapper_keys(codec_cls): assert "name" not in restored.codec_config assert "configuration" not in restored.codec_config assert "id" in restored.codec_config + + +def test_imagecodecs_from_json_v3_string_only(): + """A v3 codec specified as just a name string should work.""" + restored = LZWCodec._from_json_v3("imagecodecs_lzw") + assert restored.codec_config["id"] == "imagecodecs_lzw" + + +def test_imagecodecs_with_configuration(): + """Codecs with extra config should preserve it through roundtrips.""" + codec = DeflateCodec(level=6) + assert codec.codec_config["level"] == 6 + v3 = codec.to_json(zarr_format=3) + assert "configuration" in v3 + assert v3["configuration"]["level"] == 6 + restored = DeflateCodec.from_json(v3) + assert restored.codec_config["level"] == 6 + + +def test_imagecodecs_v2_with_configuration(): + """v2 format should inline config alongside the id.""" + codec = DeflateCodec(level=6) + v2 = codec.to_json(zarr_format=2) + assert v2["id"] == "imagecodecs_deflate" + assert v2["level"] == 6 + restored = DeflateCodec.from_json(v2) + assert restored.codec_config["level"] == 6 + + +def test_imagecodecs_v3_no_configuration(): + """Codecs with only the 'id' in config should omit 'configuration' key in v3.""" + codec = LZWCodec() + v3 = codec.to_json(zarr_format=3) + assert "configuration" not in v3 + assert v3 == {"name": "imagecodecs_lzw"} + + +def test_imagecodecs_emits_warning(): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + LZWCodec() + assert len(w) == 1 + assert "not in the Zarr version 3 specification" in str(w[0].message) + + +def test_imagecodecs_compute_encoded_size_raises(): + codec = LZWCodec() + with pytest.raises(NotImplementedError): + codec.compute_encoded_size(100, _make_spec((10,), UInt8())) + + +@pytest.mark.asyncio +async def test_imagecodecs_bytes_bytes_encode_decode_roundtrip(): + """LZW encode -> decode should recover the original bytes.""" + codec = LZWCodec() + original = np.arange(256, dtype=np.uint8) + buf = default_buffer_prototype().buffer.from_bytes(original.tobytes()) + spec = _make_spec((256,), UInt8()) + encoded = await codec._encode_single(buf, spec) + assert encoded.to_bytes() != original.tobytes() # compressed + decoded = await codec._decode_single(encoded, spec) + np.testing.assert_array_equal( + np.frombuffer(decoded.to_bytes(), dtype=np.uint8), original + ) + + +@pytest.mark.asyncio +async def test_imagecodecs_zstd_encode_decode_roundtrip(): + codec = ZstdCodec() + data = np.arange(100, dtype=np.float32) + buf = default_buffer_prototype().buffer.from_bytes(data.tobytes()) + spec = _make_spec((100,), Float32(), fill_value=0.0) + encoded = await codec._encode_single(buf, spec) + decoded = await codec._decode_single(encoded, spec) + np.testing.assert_array_equal( + np.frombuffer(decoded.to_bytes(), dtype=np.float32), data + ) + + +def test_imagecodecs_delta_resolve_metadata_no_astype(): + codec = DeltaCodec() + spec = _make_spec((10,), UInt8()) + result = codec.resolve_metadata(spec) + assert result is spec + + +def test_imagecodecs_floatpred_resolve_metadata_no_astype(): + codec = FloatPredCodec(shape=(10,), dtype="f4") + spec = _make_spec((10,), Float32(), fill_value=0.0) + result = codec.resolve_metadata(spec) + assert result is spec