From a046ef03fe53d6d7fcdbcac79d6af57b3aa44bfd Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:11:08 +0100 Subject: [PATCH 01/12] scaffold: create ZarrCore package at lib/ZarrCore/ Minimal core package with Project.toml (UUID, deps on JSON, DiskArrays, OffsetArrays, DateTimes64, Dates) and module entry point defining ZarrFormat{V}, DV constant, and AbstractCodecPipeline. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/ZarrCore/Project.toml | 17 +++++++++++++++++ lib/ZarrCore/src/ZarrCore.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 lib/ZarrCore/Project.toml create mode 100644 lib/ZarrCore/src/ZarrCore.jl diff --git a/lib/ZarrCore/Project.toml b/lib/ZarrCore/Project.toml new file mode 100644 index 00000000..a155517c --- /dev/null +++ b/lib/ZarrCore/Project.toml @@ -0,0 +1,17 @@ +name = "ZarrCore" +uuid = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" +version = "0.1.0" + +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[compat] +JSON = "0.21, 1" +DiskArrays = "0.4.2" +OffsetArrays = "0.11, 1.0" +DateTimes64 = "1" +julia = "1.10" diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl new file mode 100644 index 00000000..c63b9aae --- /dev/null +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -0,0 +1,29 @@ +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +const DV = ZarrFormat(Val(2)) + +abstract type AbstractCodecPipeline end + +include("chunkkeyencoding.jl") +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +include("metadata3.jl") +include("pipeline.jl") +include("Storage/Storage.jl") +include("ZArray.jl") +include("ZGroup.jl") + +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +end # module From 3d80bcee9248407830ac151ea849089651e49ee0 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:11:23 +0100 Subject: [PATCH 02/12] core: add types, registries, codecs, and filters to ZarrCore - chunkkeyencoding.jl: AbstractChunkKeyEncoding, ChunkKeyEncoding - MaxLengthStrings.jl: MaxLengthString type - Compressors: abstract Compressor, compressortypes registry, NoCompressor, default_compressor(), zcompress/zuncompress fallbacks - Codecs: V3Codecs module with V3Codec{In,Out} abstract type, v3_codec_parsers registry, codec_to_dict dispatch, BytesCodec, TransposeCodec - Filters: all pure-Julia filters (Fletcher32, Shuffle, Delta, Quantize, FixedScaleOffset, VLen) Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/ZarrCore/src/Codecs/Codecs.jl | 49 +++++++ lib/ZarrCore/src/Codecs/V3/V3.jl | 109 +++++++++++++++ lib/ZarrCore/src/Compressors/Compressors.jl | 113 +++++++++++++++ lib/ZarrCore/src/Filters/Filters.jl | 95 +++++++++++++ lib/ZarrCore/src/Filters/delta.jl | 45 ++++++ lib/ZarrCore/src/Filters/fixedscaleoffset.jl | 52 +++++++ lib/ZarrCore/src/Filters/fletcher32.jl | 85 ++++++++++++ lib/ZarrCore/src/Filters/quantize.jl | 52 +++++++ lib/ZarrCore/src/Filters/shuffle.jl | 70 ++++++++++ lib/ZarrCore/src/Filters/vlenfilters.jl | 82 +++++++++++ lib/ZarrCore/src/MaxLengthStrings.jl | 74 ++++++++++ lib/ZarrCore/src/chunkkeyencoding.jl | 137 +++++++++++++++++++ 12 files changed, 963 insertions(+) create mode 100644 lib/ZarrCore/src/Codecs/Codecs.jl create mode 100644 lib/ZarrCore/src/Codecs/V3/V3.jl create mode 100644 lib/ZarrCore/src/Compressors/Compressors.jl create mode 100644 lib/ZarrCore/src/Filters/Filters.jl create mode 100644 lib/ZarrCore/src/Filters/delta.jl create mode 100644 lib/ZarrCore/src/Filters/fixedscaleoffset.jl create mode 100644 lib/ZarrCore/src/Filters/fletcher32.jl create mode 100644 lib/ZarrCore/src/Filters/quantize.jl create mode 100644 lib/ZarrCore/src/Filters/shuffle.jl create mode 100644 lib/ZarrCore/src/Filters/vlenfilters.jl create mode 100644 lib/ZarrCore/src/MaxLengthStrings.jl create mode 100644 lib/ZarrCore/src/chunkkeyencoding.jl diff --git a/lib/ZarrCore/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl new file mode 100644 index 00000000..9f5e775f --- /dev/null +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -0,0 +1,49 @@ +module Codecs + +using JSON: JSON + +""" + abstract type Codec + +The abstract supertype for all Zarr codecs + +## Interface + +All subtypes of `Codec` SHALL implement the following methods: + +- `zencode(a, c::Codec)`: compress the array `a` using the codec `c`. +- `zdecode(a, c::Codec, T)`: decode the array `a` using the codec `c` + and return an array of type `T`. +- `JSON.lower(c::Codec)`: return a JSON representation of the codec `c`, which + follows the Zarr specification for that codec. +- `getCodec(::Type{<:Codec}, d::Dict)`: return a codec object from a given + dictionary `d` which contains the codec's parameters according to the Zarr spec. + +Subtypes of `Codec` MAY also implement the following methods: + +- `zencode!(encoded, data, c::Codec)`: encode the array `data` using the + codec `c` and store the result in the array `encoded`. +- `zdecode!(data, encoded, c::Codec)`: decode the array `encoded` + using the codec `c` and store the result in the array `data`. + +Finally, an entry MUST be added to the `VN.codectypes` dictionary for each codec type where N is the +Zarr format version. +This must also follow the Zarr specification's name for that compressor. The name of the compressor +is the key, and the value is the compressor type (e.g. `BloscCodec` or `NoCodec`). + +For example, the Blosc codec is named "blosc" in the Zarr spec, so the entry for [`BloscCodec`](@ref) +must be added to `codectypes` as `codectypes["blosc"] = BloscCodec`. +""" + +abstract type Codec end + +zencode(a, c::Codec) = error("Unimplemented") +zencode!(encoded, data, c::Codec) = error("Unimplemented") +zdecode(a, c::Codec, T::Type) = error("Unimplemented") +zdecode!(data, encoded, c::Codec) = error("Unimplemented") +JSON.lower(c::Codec) = error("Unimplemented") +getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") + +include("V3/V3.jl") + +end diff --git a/lib/ZarrCore/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl new file mode 100644 index 00000000..e49e1cc8 --- /dev/null +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -0,0 +1,109 @@ +module V3Codecs + +import ..Codecs: zencode, zdecode, zencode!, zdecode! +using JSON: JSON + +abstract type V3Codec{In,Out} end + +""" +Registry mapping V3 codec names (strings) to parser functions. +Each parser function takes a config dict and returns a V3Codec instance. +""" +const v3_codec_parsers = Dict{String, Function}() + +""" + codec_to_dict(c::V3Codec) -> Dict{String,Any} + +Serialize a V3Codec to a JSON-compatible dictionary. +All V3Codec subtypes should implement this method. +""" +function codec_to_dict end + +""" + codec_category(::V3Codec{In,Out}) -> Tuple{Symbol,Symbol} + +Return the (input, output) category of a codec, e.g. (:array, :bytes). +""" +codec_category(::V3Codec{In,Out}) where {In,Out} = (In, Out) + +""" + encoded_shape(codec::V3Codec, sz::NTuple{N,Int}) -> NTuple{N,Int} + +Return the shape of the output of `codec_encode(codec, data)` given the input shape. +Default implementation returns the input shape unchanged. +""" +encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz + +# --- BytesCodec (array -> bytes) --- + +struct BytesCodec <: V3Codec{:array, :bytes} + endian::Symbol # :little or :big + function BytesCodec(endian::Symbol) + endian in (:little, :big) || + throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) + new(endian) + end +end +BytesCodec() = BytesCodec(:little) + +const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 +_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN + +function codec_encode(c::BytesCodec, data::AbstractArray) + if _needs_bswap(c.endian) + return reinterpret(UInt8, bswap.(vec(data))) |> collect + else + return reinterpret(UInt8, vec(data)) |> collect + end +end + +function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} + arr = collect(reinterpret(T, encoded)) + if _needs_bswap(c.endian) + arr = bswap.(arr) + end + return reshape(arr, shape) +end + +function codec_to_dict(c::BytesCodec) + Dict{String,Any}( + "name" => "bytes", + "configuration" => Dict{String,Any}("endian" => string(c.endian)) + ) +end + +# Register BytesCodec parser +v3_codec_parsers["bytes"] = function(config) + endian = Symbol(get(config, "endian", "little")) + BytesCodec(endian) +end + +# --- TransposeCodec (array -> array) --- + +struct TransposeCodec{N} <: V3Codec{:array, :array} + order::NTuple{N, Int} # permutation (1-based Julia indexing) +end + +encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) + +function codec_encode(c::TransposeCodec, data::AbstractArray) + return permutedims(data, c.order) +end + +function codec_decode(c::TransposeCodec, encoded::AbstractArray) + inv_order = Tuple(invperm(collect(c.order))) + return permutedims(encoded, inv_order) +end + +function codec_to_dict(c::TransposeCodec) + # Zarr v3 spec uses 0-based C-order for the transpose order + Dict{String,Any}( + "name" => "transpose", + "configuration" => Dict{String,Any}("order" => collect(c.order .- 1)) + ) +end + +# Note: TransposeCodec is NOT registered in v3_codec_parsers here because +# parsing requires shape context from metadata3.jl + +end diff --git a/lib/ZarrCore/src/Compressors/Compressors.jl b/lib/ZarrCore/src/Compressors/Compressors.jl new file mode 100644 index 00000000..59accd55 --- /dev/null +++ b/lib/ZarrCore/src/Compressors/Compressors.jl @@ -0,0 +1,113 @@ +import JSON # for JSON.lower + +_reinterpret(::Type{T}, x::AbstractArray{S, 0}) where {T, S} = reinterpret(T, reshape(x, 1)) +_reinterpret(::Type{T}, x::AbstractArray) where T = reinterpret(T, x) + +""" + abstract type Compressor + +The abstract supertype for all Zarr compressors. + +## Interface + +All subtypes of `Compressor` SHALL implement the following methods: + +- `zcompress(a, c::Compressor)`: compress the array `a` using the compressor `c`. +- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` + and return an array of type `T`. +- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which + follows the Zarr specification for that compressor. +- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given + dictionary `d` which contains the compressor's parameters according to the Zarr spec. + +Subtypes of `Compressor` MAY also implement the following methods: + +- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the + compressor `c` and store the result in the array `compressed`. +- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` + using the compressor `c` and store the result in the array `data`. + +Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. +This must also follow the Zarr specification's name for that compressor. The name of the compressor +is the key, and the value is the compressor type (e.g. `BloscCompressor` or `NoCompressor`). + +For example, the Blosc compressor is named "blosc" in the Zarr spec, so the entry for [`BloscCompressor`](@ref) +must be added to `compressortypes` as `compressortypes["blosc"] = BloscCompressor`. +""" +abstract type Compressor end + +const compressortypes = Dict{Union{String,Nothing}, Type{<: Compressor}}() + +# ## Fallback definitions for the compressor interface +# Define fallbacks and generic methods for the compressor interface +getCompressor(compdict::Dict) = haskey(compdict, "id") ? + getCompressor(compressortypes[compdict["id"]], compdict) : + getCompressor(compressortypes[compdict["name"]], compdict["configuration"]) +getCompressor(::Nothing) = NoCompressor() + +# Compression when no filter is given +zcompress!(compressed,data,c,::Nothing) = zcompress!(compressed,data,c) +zuncompress!(data,compressed,c,::Nothing) = zuncompress!(data,compressed,c) + +# Fallback definition of mutating form of compress and uncompress +function zcompress!(compressed, data, c) + empty!(compressed) + append!(compressed,zcompress(data, c)) +end +zuncompress!(data, compressed, c) = copyto!(data, zuncompress(compressed, c, eltype(data))) + + +# Function given a filter stack +function zcompress!(compressed, data, c, f) + a2 = foldl(f, init=data) do anow, fnow + zencode(anow,fnow) + end + zcompress!(compressed, a2, c) +end + +function zuncompress!(data, compressed, c, f) + data2 = zuncompress(compressed, c, desttype(last(f))) + a2 = foldr(f, init = data2) do fnow, anow + zdecode(anow, fnow) + end + copyto!(data, a2) +end + +# ## `NoCompressor` +# The default and most minimal implementation of a compressor follows here, which does +# no actual compression. This is a good reference implementation for other compressors. + +""" + NoCompressor() + +Creates an object that can be passed to ZArray constructors without compression. +""" +struct NoCompressor <: Compressor end + +function zuncompress(a, ::NoCompressor, T) + _reinterpret(T,a) +end + +function zcompress(a, ::NoCompressor) + _reinterpret(UInt8,a) +end + +JSON.lower(::NoCompressor) = nothing + +compressortypes[nothing] = NoCompressor + +""" + DEFAULT_COMPRESSOR_FACTORY + +A Ref holding a zero-argument function that returns the default compressor. +Override by setting `ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> MyCompressor()`. +""" +const DEFAULT_COMPRESSOR_FACTORY = Ref{Function}(() -> NoCompressor()) + +""" + default_compressor() + +Return the default compressor. By default returns `NoCompressor()`. +Override by setting `ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> MyCompressor()`. +""" +default_compressor() = DEFAULT_COMPRESSOR_FACTORY[]() diff --git a/lib/ZarrCore/src/Filters/Filters.jl b/lib/ZarrCore/src/Filters/Filters.jl new file mode 100644 index 00000000..829f31ff --- /dev/null +++ b/lib/ZarrCore/src/Filters/Filters.jl @@ -0,0 +1,95 @@ +import JSON + +""" + abstract type Filter{T,TENC} + +The supertype for all Zarr filters. + +## Interface + +All subtypes MUST implement the following methods: + +- [`zencode(ain, filter::Filter)`](@ref zencode): Encodes data `ain` using the filter, and returns a vector of bytes. +- [`zdecode(ain, filter::Filter)`](@ref zdecode): Decodes data `ain`, a vector of bytes, using the filter, and returns the original data. +- [`JSON.lower`](@ref): Returns a JSON-serializable dictionary representing the filter, according to the Zarr specification. +- [`getfilter(::Type{<: Filter}, filterdict)`](@ref getfilter): Returns the filter type read from a given filter dictionary. + +If the filter has type parameters, it MUST also implement: +- [`sourcetype(::Filter)::T`](@ref sourcetype): equivalent to `dtype` in the Python Zarr implementation. +- [`desttype(::Filter)::T`](@ref desttype): equivalent to `atype` in the Python Zarr implementation. + +Finally, an entry MUST be added to the `filterdict` dictionary for each filter type. +This must also follow the Zarr specification's name for that filter. The name of the filter +is the key, and the value is the filter type (e.g. `VLenUInt8Filter` or `Fletcher32Filter`). + + +Subtypes include: [`VLenArrayFilter`](@ref), [`VLenUTF8Filter`](@ref), [`Fletcher32Filter`](@ref). +""" +abstract type Filter{T,TENC} end + +""" + zencode(ain, filter::Filter) + +Encodes data `ain` using the filter, and returns a vector of bytes. +""" +function zencode end + +""" + zdecode(ain, filter::Filter) + +Decodes data `ain`, a vector of bytes, using the filter, and returns the original data. +""" +function zdecode end + +""" + getfilter(::Type{<: Filter}, filterdict) + +Returns the filter type read from a given specification dictionary, which must follow the Zarr specification. +""" +function getfilter end + +""" + sourcetype(::Filter)::T + +Returns the source type of the filter. +""" +function sourcetype end + +""" + desttype(::Filter)::T + +Returns the destination type of the filter. +""" +function desttype end + +filterdict = Dict{String,Type{<:Filter}}() + +function getfilters(d::Dict) + if !haskey(d,"filters") + return nothing + else + if d["filters"] === nothing || isempty(d["filters"]) + return nothing + end + f = map(d["filters"]) do f + try + getfilter(filterdict[f["id"]], f) + catch e + @show f + rethrow(e) + end + end + return (f...,) + end +end +sourcetype(::Filter{T}) where T = T +desttype(::Filter{<:Any,T}) where T = T + +zencode(ain,::Nothing) = ain + +include("vlenfilters.jl") +include("fletcher32.jl") +include("fixedscaleoffset.jl") +include("shuffle.jl") +include("quantize.jl") +include("delta.jl") diff --git a/lib/ZarrCore/src/Filters/delta.jl b/lib/ZarrCore/src/Filters/delta.jl new file mode 100644 index 00000000..9d1de048 --- /dev/null +++ b/lib/ZarrCore/src/Filters/delta.jl @@ -0,0 +1,45 @@ +#= +# Delta compression + + +=# + +""" + DeltaFilter(; DecodingType, [EncodingType = DecodingType]) + +Delta-based compression for Zarr arrays. (Delta encoding is Julia `diff`, decoding is Julia `cumsum`). +""" +struct DeltaFilter{T, TENC} <: Filter{T, TENC} +end + +function DeltaFilter(; DecodingType = Float16, EncodingType = DecodingType) + return DeltaFilter{DecodingType, EncodingType}() +end + +DeltaFilter{T}() where T = DeltaFilter{T, T}() + +function zencode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} + arr = reinterpret(DecodingType, vec(data)) + + enc = similar(arr, EncodingType) + # perform the delta operation + enc[begin] = arr[begin] + enc[begin+1:end] .= diff(arr) + return enc +end + +function zdecode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} + encoded = reinterpret(EncodingType, vec(data)) + decoded = DecodingType.(cumsum(encoded)) + return decoded +end + +function JSON.lower(filter::DeltaFilter{T, Tenc}) where {T, Tenc} + return Dict("id" => "delta", "dtype" => typestr(T), "astype" => typestr(Tenc)) +end + +function getfilter(::Type{<: DeltaFilter}, d) + return DeltaFilter{typestr(d["dtype"], haskey(d, "astype") ? typestr(d["astype"]) : d["dtype"])}() +end + +filterdict["delta"] = DeltaFilter \ No newline at end of file diff --git a/lib/ZarrCore/src/Filters/fixedscaleoffset.jl b/lib/ZarrCore/src/Filters/fixedscaleoffset.jl new file mode 100644 index 00000000..9e12c52d --- /dev/null +++ b/lib/ZarrCore/src/Filters/fixedscaleoffset.jl @@ -0,0 +1,52 @@ + +""" + FixedScaleOffsetFilter{T,TENC}(scale, offset) + +A compressor that scales and offsets the data. + +!!! note + The geographic CF standards define scale/offset decoding as `x * scale + offset`, + but this filter defines it as `x / scale + offset`. Constructing a `FixedScaleOffsetFilter` + from CF data means `FixedScaleOffsetFilter(1/cf_scale_factor, cf_add_offset)`. +""" +struct FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc} <: Filter{T, Tenc} + scale::ScaleOffsetType + offset::ScaleOffsetType +end + +FixedScaleOffsetFilter{T}(scale::ScaleOffsetType, offset::ScaleOffsetType) where {T, ScaleOffsetType} = FixedScaleOffsetFilter{T, ScaleOffsetType}(scale, offset) +FixedScaleOffsetFilter(scale::ScaleOffsetType, offset::ScaleOffsetType) where {ScaleOffsetType} = FixedScaleOffsetFilter{ScaleOffsetType, ScaleOffsetType}(scale, offset) + +function FixedScaleOffsetFilter(; scale::ScaleOffsetType, offset::ScaleOffsetType, T, Tenc = T) where ScaleOffsetType + return FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}(scale, offset) +end + +function zencode(a::AbstractArray, c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {T, Tenc, ScaleOffsetType} + if Tenc <: Integer + return [round(Tenc, (a - c.offset) * c.scale) for a in a] # apply scale and offset, and round to nearest integer + else + return [convert(Tenc, (a - c.offset) * c.scale) for a in a] # apply scale and offset + end +end + +function zdecode(a::AbstractArray, c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {T, Tenc, ScaleOffsetType} + return [convert(Base.nonmissingtype(T), (a / c.scale) + c.offset) for a in a] +end + + +function getfilter(::Type{<: FixedScaleOffsetFilter}, d::Dict) + scale = d["scale"] + offset = d["offset"] + # Types must be converted from strings to the actual Julia types they represent. + string_T = d["dtype"] + string_Tenc = get(d, "astype", string_T) + T = typestr(string_T) + Tenc = typestr(string_Tenc) + return FixedScaleOffsetFilter{Tenc, T, Tenc}(scale, offset) +end + +function JSON.lower(c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {ScaleOffsetType, T, Tenc} + return Dict("id" => "fixedscaleoffset", "scale" => c.scale, "offset" => c.offset, "dtype" => typestr(T), "astype" => typestr(Tenc)) +end + +filterdict["fixedscaleoffset"] = FixedScaleOffsetFilter diff --git a/lib/ZarrCore/src/Filters/fletcher32.jl b/lib/ZarrCore/src/Filters/fletcher32.jl new file mode 100644 index 00000000..d854cb90 --- /dev/null +++ b/lib/ZarrCore/src/Filters/fletcher32.jl @@ -0,0 +1,85 @@ +#= +# Fletcher32 filter + +This "filter" basically injects a 4-byte checksum at the end of the data, to ensure data integrity. + +The implementation is based on the [numcodecs implementation here](https://github.com/zarr-developers/numcodecs/blob/79d1a8d4f9c89d3513836aba0758e0d2a2a1cfaf/numcodecs/fletcher32.pyx) +and the [original C implementation for NetCDF](https://github.com/Unidata/netcdf-c/blob/main/plugins/H5checksum.c#L109) linked therein. + +=# + +""" + Fletcher32Filter() + +A compressor that uses the Fletcher32 checksum algorithm to compress and uncompress data. + +Note that this goes from UInt8 to UInt8, and is effectively only checking +the checksum and cropping the last 4 bytes of the data during decoding. +""" +struct Fletcher32Filter <: Filter{UInt8, UInt8} +end + +getfilter(::Type{<: Fletcher32Filter}, d::Dict) = Fletcher32Filter() +JSON.lower(::Fletcher32Filter) = Dict("id" => "fletcher32") +filterdict["fletcher32"] = Fletcher32Filter + +function _checksum_fletcher32(data::AbstractArray{UInt8}) + len = length(data) ÷ 2 # length in 16-bit words + sum1::UInt32 = 0 + sum2::UInt32 = 0 + data_idx = 1 + + #= + Compute the checksum for pairs of bytes. + The magic `360` value is the largest number of sums that can be performed without overflow in UInt32. + =# + while len > 0 + tlen = len > 360 ? 360 : len + len -= tlen + while tlen > 0 + sum1 += begin # create a 16 bit word from two bytes, the first one shifted to the end of the word + (UInt16(data[data_idx]) << 8) | UInt16(data[data_idx + 1]) + end + sum2 += sum1 + data_idx += 2 + tlen -= 1 + if tlen < 1 + break + end + end + sum1 = (sum1 & 0xffff) + (sum1 >> 16) + sum2 = (sum2 & 0xffff) + (sum2 >> 16) + end + + # if the length of the data is odd, add the first byte to the checksum again (?!) + if length(data) % 2 == 1 + sum1 += UInt16(data[1]) << 8 + sum2 += sum1 + sum1 = (sum1 & 0xffff) + (sum1 >> 16) + sum2 = (sum2 & 0xffff) + (sum2 >> 16) + end + return (sum2 << 16) | sum1 +end + +function zencode(data, ::Fletcher32Filter) + bytes = reinterpret(UInt8, vec(data)) + checksum = _checksum_fletcher32(bytes) + result = copy(bytes) + append!(result, reinterpret(UInt8, [checksum])) # TODO: decompose this without the extra allocation of wrapping in Array + return result +end + +function zdecode(data, ::Fletcher32Filter) + bytes = reinterpret(UInt8, data) + checksum = _checksum_fletcher32(view(bytes, 1:length(bytes) - 4)) + stored_checksum = only(reinterpret(UInt32, view(bytes, (length(bytes) - 3):length(bytes)))) + if checksum != stored_checksum + throw(ErrorException(""" + Checksum mismatch in Fletcher32 decoding. + + The computed value is $(checksum) and the stored value is $(stored_checksum). + This might be a sign that the data is corrupted. + """)) # TODO: make this a custom error type + end + return view(bytes, 1:length(bytes) - 4) +end diff --git a/lib/ZarrCore/src/Filters/quantize.jl b/lib/ZarrCore/src/Filters/quantize.jl new file mode 100644 index 00000000..c5d7c9a4 --- /dev/null +++ b/lib/ZarrCore/src/Filters/quantize.jl @@ -0,0 +1,52 @@ +#= +# Quantize compression + + +=# + +""" + QuantizeFilter(; digits, DecodingType, [EncodingType = DecodingType]) + +Quantization based compression for Zarr arrays. +""" +struct QuantizeFilter{T, TENC} <: Filter{T, TENC} + digits::Int32 +end + +function QuantizeFilter(; digits = 10, T = Float16, Tenc = T) + return QuantizeFilter{T, Tenc}(digits) +end + +QuantizeFilter{T, Tenc}(; digits = 10) where {T, Tenc} = QuantizeFilter{T, Tenc}(digits) +QuantizeFilter{T}(; digits = 10) where T = QuantizeFilter{T, T}(digits) + +function zencode(data::AbstractArray, filter::QuantizeFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} + arr = reinterpret(DecodingType, vec(data)) + + precision = 10.0^(-filter.digits) + + _exponent = log(10, precision) # log 10 in base `precision` + exponent = _exponent < 0 ? floor(Int, _exponent) : ceil(Int, _exponent) + + bits = ceil(log(2, 10.0^(-exponent))) + scale = 2.0^bits + + enc = @. convert(EncodingType, round(scale * arr) / scale) + + return enc +end + +# Decoding is a no-op; quantization is a lossy filter but data is encoded directly. +function zdecode(data::AbstractArray, filter::QuantizeFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} + return data +end + +function JSON.lower(filter::QuantizeFilter{T, Tenc}) where {T, Tenc} + return Dict("id" => "quantize", "digits" => filter.digits, "dtype" => typestr(T), "astype" => typestr(Tenc)) +end + +function getfilter(::Type{<: QuantizeFilter}, d) + return QuantizeFilter{typestr(d["dtype"], typestr(d["astype"]))}(; digits = d["digits"]) +end + +filterdict["quantize"] = QuantizeFilter \ No newline at end of file diff --git a/lib/ZarrCore/src/Filters/shuffle.jl b/lib/ZarrCore/src/Filters/shuffle.jl new file mode 100644 index 00000000..6a01f5d4 --- /dev/null +++ b/lib/ZarrCore/src/Filters/shuffle.jl @@ -0,0 +1,70 @@ +#= +# Shuffle compression + +This file implements the shuffle compressor. +=# + +struct ShuffleFilter <: Filter{UInt8, UInt8} + elementsize::Csize_t +end + +ShuffleFilter(; elementsize = 4) = ShuffleFilter(elementsize) + +function _do_shuffle!(dest::AbstractVector{UInt8}, source::AbstractVector{UInt8}, elementsize::Csize_t) + count = fld(length(source), elementsize) # elementsize is in bytes, so this works + for i in 0:(count-1) + offset = i * elementsize + for byte_index in 0:(elementsize-1) + j = byte_index * count + i + dest[j+1] = source[offset + byte_index+1] + end + end +end + +function _do_unshuffle!(dest::AbstractVector{UInt8}, source::AbstractVector{UInt8}, elementsize::Csize_t) + count = fld(length(source), elementsize) # elementsize is in bytes, so this works + for i in 0:(elementsize-1) + offset = i * count + for byte_index in 0:(count-1) + j = byte_index * elementsize + i + dest[j+1] = source[offset + byte_index+1] + end + end +end + +function zencode(a::AbstractArray, c::ShuffleFilter) + if c.elementsize <= 1 # no shuffling needed if elementsize is 1 + return a + end + source = reinterpret(UInt8, vec(a)) + dest = Vector{UInt8}(undef, length(source)) + _do_shuffle!(dest, source, c.elementsize) + return dest +end + +function zdecode(a::AbstractArray, c::ShuffleFilter) + if c.elementsize <= 1 # no shuffling needed if elementsize is 1 + return a + end + source = reinterpret(UInt8, vec(a)) + dest = Vector{UInt8}(undef, length(source)) + _do_unshuffle!(dest, source, c.elementsize) + return dest +end + +function getfilter(::Type{ShuffleFilter}, d::Dict) + return ShuffleFilter(d["elementsize"]) +end + +function JSON.lower(c::ShuffleFilter) + return Dict("id" => "shuffle", "elementsize" => Int64(c.elementsize)) +end + +filterdict["shuffle"] = ShuffleFilter +#= + +# Tests + + + +=# \ No newline at end of file diff --git a/lib/ZarrCore/src/Filters/vlenfilters.jl b/lib/ZarrCore/src/Filters/vlenfilters.jl new file mode 100644 index 00000000..dad91dfa --- /dev/null +++ b/lib/ZarrCore/src/Filters/vlenfilters.jl @@ -0,0 +1,82 @@ +#= +# Variable-length filters + +This file implements variable-length filters for Zarr, i.e., filters that write arrays of variable-length arrays ("ragged arrays"). + +Specifically, it implements the `VLenArrayFilter` and `VLenUTF8Filter` types, which are used to encode and decode variable-length arrays and UTF-8 strings, respectively. +=# + +# ## VLenArrayFilter + +""" + VLenArrayFilter(T) + +Encodes and decodes variable-length arrays of arbitrary data type `T`. +""" +struct VLenArrayFilter{T} <: Filter{T,UInt8} end +# We don't need to define `sourcetype` and `desttype` for this filter, since the generic implementations are sufficient. + +JSON.lower(::VLenArrayFilter{T}) where T = Dict("id"=>"vlen-array","dtype"=> typestr(T) ) +getfilter(::Type{<:VLenArrayFilter}, f) = VLenArrayFilter{typestr(f["dtype"])}() +filterdict["vlen-array"] = VLenArrayFilter + +function zdecode(ain, ::VLenArrayFilter{T}) where T + f = IOBuffer(ain) + nitems = read(f, UInt32) + out = Array{Vector{T}}(undef,nitems) + for i=1:nitems + len1 = read(f,UInt32) + out[i] = read!(f,Array{T}(undef,len1 ÷ sizeof(T))) + end + close(f) + out +end + +#Encodes Array of Vectors `ain` into bytes +function zencode(ain,::VLenArrayFilter) + b = IOBuffer() + nitems = length(ain) + write(b,UInt32(nitems)) + for a in ain + write(b, UInt32(length(a) * sizeof(eltype(a)))) + write(b, a) + end + take!(b) +end + +# ## VLenUTF8Filter + +""" + VLenUTF8Filter + +Encodes and decodes variable-length unicode strings +""" +struct VLenUTF8Filter <: Filter{String, UInt8} end + +JSON.lower(::VLenUTF8Filter) = Dict("id"=>"vlen-utf8") +getfilter(::Type{<:VLenUTF8Filter}, f) = VLenUTF8Filter() +filterdict["vlen-utf8"] = VLenUTF8Filter + +function zdecode(ain, ::VLenUTF8Filter) + f = IOBuffer(ain) + nitems = read(f, UInt32) + out = Array{String}(undef, nitems) + for i in 1:nitems + clen = read(f, UInt32) + out[i] = String(read(f, clen)) + end + close(f) + out +end + +function zencode(ain, ::VLenUTF8Filter) + b = IOBuffer() + nitems = length(ain) + write(b, UInt32(nitems)) + for a in ain + utf8encoded = transcode(String, a) + write(b, UInt32(ncodeunits(utf8encoded))) + write(b, utf8encoded) + end + take!(b) +end diff --git a/lib/ZarrCore/src/MaxLengthStrings.jl b/lib/ZarrCore/src/MaxLengthStrings.jl new file mode 100644 index 00000000..1937f9ac --- /dev/null +++ b/lib/ZarrCore/src/MaxLengthStrings.jl @@ -0,0 +1,74 @@ +#This is a modified version of FixedSizeStrings.jl +#adapted so that appended zero Chars are omitted from +#the string. So most credits got to authors of +#https://github.com/JuliaComputing/FixedSizeStrings.jl +# Maybe this should be moved to FizedSizeStrings.jl, +#but until then let's keep it here... +module MaxLengthStrings +import Base: iterate, lastindex, getindex, sizeof, length, ncodeunits, codeunit, isvalid, read, write +export MaxLengthString + +struct MaxLengthString{N,T} <: AbstractString + data::NTuple{N,T} + function MaxLengthString{N,T}(itr) where {N,T} + new(totuple_appendzero(NTuple{N,T},itr)) + end +end +import Base: tuple_type_head, tuple_type_tail +#totuple_appendzero(::Type{Tuple{}}, itr, s...) = () +tuple_append(::Type{NTuple{N,T}}) where {N,T} = (zero(T), tuple_append(NTuple{N-1,T})...) +tuple_append(::Type{Tuple{}}) = () +function totuple_appendzero(::Type{Tuple{}},itr,s...) + iterate(itr, s...)===nothing || error("String is too long to fit into MaxLengthString") + () +end +function totuple_appendzero(::Type{NTuple{N,T}}, itr, s...)::Tuple{Vararg{T,N}} where {N,T} + y = iterate(itr, s...) + if y === nothing + tuple_append(NTuple{N,T}) + else + (convert(T, y[1]), totuple_appendzero(NTuple{N-1,T}, itr, y[2])...) + end +end + +function MaxLengthString(s::AbstractString,N=length(s),T=UInt8) + MaxLengthString{N,T}(rpad(s,N,'\0')) +end + +function iterate(s::MaxLengthString{N}, i::Int = 1) where N + i > N && return nothing + c = s.data[i] + iszero(c) && return nothing + return (Char(c), i+1) +end + +lastindex(s::MaxLengthString{N}) where {N} = findlast(!iszero,s.data) + +function getindex(s::MaxLengthString, i::Int) + checkbounds(s,i) + Char(s.data[i]) +end + +sizeof(s::MaxLengthString) = sizeof(s.data) + +length(s::MaxLengthString) = findlast(!iszero,s.data) + +ncodeunits(s::MaxLengthString) = length(s) + +codeunit(::MaxLengthString{<:Any,T}) where T = T +function codeunit(s::MaxLengthString, i::Integer) + checkbounds(s,i) + s.data[i] +end + +isvalid(s::MaxLengthString{<:Any,UInt8}, i::Int) = checkbounds(Bool, s, i) +isvalid(s::MaxLengthString, i::Int) = checkbounds(Bool, s, i) && isvalid(Char,s.data[i]) + +function read(io::IO, T::Type{<:MaxLengthString{N}}) where N + return read!(io, Ref{T}())[]::T +end + +function write(io::IO, s::MaxLengthString{N}) where N + return write(io, Ref(s)) +end +end diff --git a/lib/ZarrCore/src/chunkkeyencoding.jl b/lib/ZarrCore/src/chunkkeyencoding.jl new file mode 100644 index 00000000..9a9be200 --- /dev/null +++ b/lib/ZarrCore/src/chunkkeyencoding.jl @@ -0,0 +1,137 @@ +abstract type AbstractChunkKeyEncoding end + +struct ChunkKeyEncoding <: AbstractChunkKeyEncoding + sep::Char + prefix::Bool +end + +# Default Zarr v2 separator +const DS2 = '.' +# Default Zarr v3 separator +const DS3 = '/' + +default_sep(::ZarrFormat{2}) = DS2 +default_sep(::ZarrFormat{3}) = DS3 +default_sep(v::Int) = default_sep(ZarrFormat(v)) +default_prefix(::ZarrFormat{2}) = false +default_prefix(::ZarrFormat{3}) = true +const DS = default_sep(DV) + +@inline function citostring(e::ChunkKeyEncoding, i::CartesianIndex) + if e.prefix + "c$(e.sep)" * join(reverse((i - oneunit(i)).I), e.sep) + else + join(reverse((i - oneunit(i)).I), e.sep) + end +end +@inline citostring(e::ChunkKeyEncoding, ::CartesianIndex{0}) = e.prefix ? "c$(e.sep)0" : "0" + +""" + SuffixChunkKeyEncoding{E<:AbstractChunkKeyEncoding} + +Chunk key encoding that appends a user-defined suffix string to the key +produced by a base chunk key encoding. The primary use case is adding file +extensions (e.g. `.tiff`, `.shard.zip`) so that individual chunk files are +directly usable by other software without Zarr-specific tooling. + +Per the zarr-extensions `suffix` chunk-key-encoding proposal. +""" +struct SuffixChunkKeyEncoding{E<:AbstractChunkKeyEncoding} <: AbstractChunkKeyEncoding + suffix::String + base_encoding::E +end + +SuffixChunkKeyEncoding(suffix::String; sep::Char='/', prefix::Bool=true) = + SuffixChunkKeyEncoding(suffix, ChunkKeyEncoding(sep, prefix)) + +@inline citostring(e::SuffixChunkKeyEncoding, i::CartesianIndex) = + citostring(e.base_encoding, i) * e.suffix + +"""Serialize an `AbstractChunkKeyEncoding` to a JSON-compatible dict.""" +function lower_chunk_key_encoding(e::ChunkKeyEncoding) + Dict{String,Any}( + "name" => e.prefix ? "default" : "v2", + "configuration" => Dict{String,Any}("separator" => string(e.sep)) + ) +end + +function lower_chunk_key_encoding(e::SuffixChunkKeyEncoding) + Dict{String,Any}( + "name" => "suffix", + "configuration" => Dict{String,Any}( + "suffix" => e.suffix, + "base_encoding" => lower_chunk_key_encoding(e.base_encoding) + ) + ) +end + +"""Stores a registered chunk key encoding parser together with its expected return type.""" +struct ChunkKeyEncodingEntry + return_type::Type{<:AbstractChunkKeyEncoding} + parser::Function +end + +""" +Registry mapping chunk key encoding names to `ChunkKeyEncodingEntry` values +(return type + parser function). Use `register_chunk_key_encoding` to add new entries. +""" +const chunk_key_encoding_parsers = Dict{String, ChunkKeyEncodingEntry}() + +""" + register_chunk_key_encoding(parser::Function, name::String[, ::Type{T}]) + +Register a chunk key encoding parser under `name`. The parser must accept a +`Dict{String,Any}` configuration and return an `AbstractChunkKeyEncoding`. + +The optional trailing `Type{T}` argument narrows the declared return type stored +in the registry (defaults to `AbstractChunkKeyEncoding`). Specifying it enables +a runtime assertion in `parse_chunk_key_encoding` and makes the registry +self-documenting. + +Supports do-block syntax: + + register_chunk_key_encoding("myenc") do config + MyEncoding(config["param"]) + end + + register_chunk_key_encoding("myenc", MyEncoding) do config + MyEncoding(config["param"]) + end +""" +function register_chunk_key_encoding(parser::Function, name::String, ::Type{T}) where {T<:AbstractChunkKeyEncoding} + chunk_key_encoding_parsers[name] = ChunkKeyEncodingEntry(T, parser) +end +register_chunk_key_encoding(parser::Function, name::String) = + register_chunk_key_encoding(parser, name, AbstractChunkKeyEncoding) + +""" + parse_chunk_key_encoding(d::AbstractDict) -> AbstractChunkKeyEncoding + +Parse a chunk key encoding dict (as found in `zarr.json`) into an +`AbstractChunkKeyEncoding` by looking up the registered parser for the encoding name. +""" +function parse_chunk_key_encoding(d::AbstractDict)::AbstractChunkKeyEncoding + name = d["name"] + config = get(d, "configuration", Dict{String,Any}())::Dict{String,Any} + haskey(chunk_key_encoding_parsers, name) || + throw(ArgumentError("Unknown chunk_key_encoding of name, $name")) + entry = chunk_key_encoding_parsers[name] + return entry.parser(config)::entry.return_type +end + +# Register built-in encodings +register_chunk_key_encoding("default", ChunkKeyEncoding) do config + ChunkKeyEncoding(only(get(config, "separator", '/')), true) +end + +register_chunk_key_encoding("v2", ChunkKeyEncoding) do config + ChunkKeyEncoding(only(get(config, "separator", '.')), false) +end + +register_chunk_key_encoding("suffix", SuffixChunkKeyEncoding) do config + suffix_str = config["suffix"] + base = parse_chunk_key_encoding(config["base_encoding"]) + SuffixChunkKeyEncoding(suffix_str, base) +end + +_concatpath(p,s) = isempty(p) ? s : rstrip(p,'/') * '/' * s From c73e3cfe1b55d5126b1f446a66b6e353b4e89b34 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:11:33 +0100 Subject: [PATCH 03/12] core: add metadata layer to ZarrCore - metadata.jl: MetadataV2, type string system, fill value encoding/decoding, default_compressor() in constructors - metadata3.jl: MetadataV3 with registry-based codec parsing via parse_v3_codec/v3_codec_parsers (replaces hardcoded if/elseif), codec_to_dict dispatch for serialization (replaces isa checks), compressor_to_v3_bytes_codecs dispatch - pipeline.jl: V2Pipeline, V3Pipeline, pipeline_encode/pipeline_decode! Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/ZarrCore/src/metadata.jl | 293 +++++++++++++++++++++++++ lib/ZarrCore/src/metadata3.jl | 390 ++++++++++++++++++++++++++++++++++ lib/ZarrCore/src/pipeline.jl | 77 +++++++ 3 files changed, 760 insertions(+) create mode 100644 lib/ZarrCore/src/metadata.jl create mode 100644 lib/ZarrCore/src/metadata3.jl create mode 100644 lib/ZarrCore/src/pipeline.jl diff --git a/lib/ZarrCore/src/metadata.jl b/lib/ZarrCore/src/metadata.jl new file mode 100644 index 00000000..09859610 --- /dev/null +++ b/lib/ZarrCore/src/metadata.jl @@ -0,0 +1,293 @@ +import Dates: Date, DateTime +using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring + +"""NumPy array protocol type string (typestr) format + +A string providing the basic type of the homogeneous array. The basic string format +consists of 3 parts: a character describing the byteorder of the data +(<: little-endian, >: big-endian, |: not-relevant), a character code giving the basic +type of the array, and an integer providing the number of bytes the type uses. + +https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding +""" + +include("MaxLengthStrings.jl") +using .MaxLengthStrings: MaxLengthString + +primitive type ASCIIChar <: AbstractChar 8 end +ASCIIChar(x::UInt8) = reinterpret(ASCIIChar, x) +ASCIIChar(x::Integer) = ASCIIChar(UInt8(x)) +Base.UInt8(x::ASCIIChar) = reinterpret(UInt8, x) +Base.codepoint(x::ASCIIChar) = UInt8(x) +Base.show(io::IO, x::ASCIIChar) = print(io, Char(x)) +Base.zero(::Union{ASCIIChar,Type{ASCIIChar}}) = ASCIIChar(Base.zero(UInt8)) + +Base.zero(t::Union{String, Type{String}}) = "" + +typestr(t::Type) = string('<', 'V', sizeof(t)) +typestr(t::Type{>:Missing}) = typestr(Base.nonmissingtype(t)) +typestr(t::Type{Bool}) = string('|', 'b', sizeof(t)) +typestr(t::Type{<:Int8}) = string("|i1") +typestr(t::Type{<:Signed}) = string('<', 'i', sizeof(t)) +typestr(t::Type{<:UInt8}) = string("|u1") +typestr(t::Type{<:Unsigned}) = string('<', 'u', sizeof(t)) +typestr(t::Type{Complex{T}} where T<:AbstractFloat) = string('<', 'c', sizeof(t)) +typestr(t::Type{<:AbstractFloat}) = string('<', 'f', sizeof(t)) +typestr(::Type{MaxLengthString{N,UInt32}}) where N = string('<', 'U', N) +typestr(::Type{MaxLengthString{N,UInt8}}) where N = string('<', 'S', N) +typestr(::Type{<:Array}) = "|O" +typestr(t::Type{<:DateTime64}) = pydatetime_string(t) +typestr(::Type{<:AbstractString}) = "|O" + +const typestr_regex = r"^([<|>])([tbiufcmMOSUV])(\d*)(\[\w+\])?$" +const typemap = Dict{Tuple{Char, Int}, DataType}( + ('b', 1) => Bool, + ('S', 1) => ASCIIChar, + ('U', 1) => Char, +) +sizemapf(x::Type{<:Number}) = sizeof(x) +typecharf(::Type{<:Signed}) = 'i' +typecharf(::Type{<:Unsigned}) = 'u' +typecharf(::Type{<:AbstractFloat}) = 'f' +typecharf(::Type{<:Complex}) = 'c' +foreach([Float16,Float32,Float64,Int8,Int16,Int32,Int64,Int128, + UInt8,UInt16,UInt32,UInt64,UInt128, + Complex{Float16},Complex{Float32},Complex{Float64}]) do t + typemap[(typecharf(t),sizemapf(t))] = t +end + +function typestr(s::AbstractString, filterlist=nothing) + m = match(typestr_regex, s) + if m === nothing + throw(ArgumentError("$s is not a valid numpy typestr")) + else + + byteorder, typecode, typesize, typespec = m.captures + if typecode == "O" + if filterlist === nothing + throw(ArgumentError("Object array can only be parsed when an appropriate filter is defined")) + end + return sourcetype(first(filterlist)) + end + isempty(typesize) && throw((ArgumentError("$s is not a valid numpy typestr"))) + tc, ts = first(typecode), parse(Int, typesize) + if (tc in ('U','S')) && ts > 1 + return MaxLengthString{ts,tc=='U' ? UInt32 : UInt8} + end + if tc == 'M' && ts == 8 + #We have a datetime64 value + return datetime_from_pystring(s) + end + # convert typecode to Char and typesize to Int + typemap[(tc,ts)] + end +end + + +"""Metadata configuration of the stored array + +Each array requires essential configuration metadata to be stored, enabling correct +interpretation of the stored data. This metadata is encoded using JSON and stored as the +value of the ".zarray" key within an array store. + +# Type Parameters +* T - element type of the array +* N - dimensionality of the array +* C - compressor +* F - filters + +# See Also + +https://zarr.readthedocs.io/en/stable/spec/v2.html#metadata +""" +abstract type AbstractMetadata{T,N,E <: AbstractChunkKeyEncoding} end +Base.ndims(::AbstractMetadata{<:Any,N}) where N = N + + +"""Metadata for Zarr version 2 arrays""" +struct MetadataV2{T,N,C,F} <: AbstractMetadata{T,N,ChunkKeyEncoding} + zarr_format::Int + node_type::String + shape::Base.RefValue{NTuple{N, Int}} + chunks::NTuple{N, Int} + dtype::String # structured data types not yet supported + compressor::C + fill_value::Union{T, Nothing} + order::Char + filters::F # not yet supported + chunk_key_encoding::ChunkKeyEncoding + function MetadataV2{T2,N,C,F}(zarr_format, node_type, shape, chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F} + zarr_format == 2 || throw(ArgumentError("MetadataV2 only functions if zarr_format == 2")) + #Do some sanity checks to make sure we have a sane array + any(<(0), shape) && throw(ArgumentError("Size must be positive")) + any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + order === 'C' || throw(ArgumentError("Currently only 'C' storage order is supported")) + new{T2,N,C,F}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) + end +end +zarr_format(::MetadataV2) = ZarrFormat(Val(2)) + +# Type alias for backward compatibility +const Metadata = AbstractMetadata + +#To make unit tests pass with ref shape +function Base.:(==)(m1::MetadataV2, m2::MetadataV2) + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.compressor == m2.compressor && + m1.fill_value == m2.fill_value && + m1.order == m2.order && + m1.filters == m2.filters && + m1.chunk_key_encoding == m2.chunk_key_encoding +end + + +"Construct Metadata based on your data" +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; + node_type::String="array", + compressor::C=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + filters=nothing, + fill_as_missing = false, + dimension_separator::Char = '.' + ) where {T, N, C} + return Metadata(A, chunks, ZarrFormat(zarr_format); + node_type=node_type, + compressor=compressor, + fill_value=fill_value, + order=order, + filters=filters, + fill_as_missing=fill_as_missing, + chunk_key_encoding=ChunkKeyEncoding(dimension_separator, default_prefix(ZarrFormat(zarr_format))) + ) +end + +# V2 constructor +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; + node_type::String="array", + compressor::C=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + filters::F=nothing, + fill_as_missing = false, + chunk_key_encoding=ChunkKeyEncoding('.', false) + ) where {T, N, C, F} + T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} + MetadataV2{T2,N,C,typeof(filters)}( + 2, + node_type, + size(A), + chunks, + typestr(eltype(A)), + compressor, + fill_value, + order, + filters, + chunk_key_encoding, + ) +end + +Metadata(s::Union{AbstractString, IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) + +"Construct Metadata from Dict" +function Metadata(d::AbstractDict, fill_as_missing) + zarr_format = d["zarr_format"]::Int + zarr_format ∉ (2, 3) && throw(ArgumentError("Zarr.jl currently only supports v2 or v3 of the specification")) + return Metadata(d, fill_as_missing, ZarrFormat(zarr_format)) +end + +# V2 constructor from Dict +function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{2}) + # Zarr v2 metadata is only for arrays + node_type = "array" + + compdict = d["compressor"] + if isnothing(compdict) + # try the last filter, for Kerchunk compat + if !isnothing(d["filters"]) && haskey(compressortypes, d["filters"][end]["id"]) + compdict = pop!(d["filters"]) # TODO: this will not work with JSON3! + end + end + compressor = getCompressor(compdict) + + filters = getfilters(d) + + T = typestr(d["dtype"], filters) + N = length(d["shape"]) + C = typeof(compressor) + F = typeof(filters) + + fv = fill_value_decoding(d["fill_value"], T) + + TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} + + dim_sep = only(get(d, "dimension_separator", '.')) + + MetadataV2{TU,N,C,F}( + d["zarr_format"], + node_type, + NTuple{N, Int}(d["shape"]) |> reverse, + NTuple{N, Int}(d["chunks"]) |> reverse, + d["dtype"], + compressor, + fv, + first(d["order"]), + filters, + ChunkKeyEncoding(dim_sep, false), + ) +end + + +"Describes how to lower Metadata to JSON, used in json(::Metadata)" +function JSON.lower(md::MetadataV2) + Dict{String, Any}( + "zarr_format" => Int(md.zarr_format), + "node_type" => md.node_type, + "shape" => md.shape[] |> reverse, + "chunks" => md.chunks |> reverse, + "dtype" => md.dtype, + "compressor" => md.compressor, + "fill_value" => fill_value_encoding(md.fill_value), + "order" => md.order, + "filters" => md.filters, + "dimension_separator" => md.chunk_key_encoding.sep + ) +end + + +# Fill value encoding and decoding as described in +# https://zarr.readthedocs.io/en/stable/spec/v2.html#fill-value-encoding + +fill_value_encoding(v) = v +fill_value_encoding(::Nothing)=nothing +function fill_value_encoding(v::AbstractFloat) + if isnan(v) + "NaN" + elseif isinf(v) + v>0 ? "Infinity" : "-Infinity" + else + v + end +end + +Base.eltype(::AbstractMetadata{T}) where T = T + +# this correctly parses "NaN" and "Infinity" +fill_value_decoding(v::AbstractString, T::Type{<:Number}) = parse(T, v) +fill_value_decoding(v::Nothing, ::Any) = v +fill_value_decoding(v, T) = T(v) +fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) +fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v +fill_value_decoding(v::Nothing, ::Type{ZarrCore.ASCIIChar}) = v +# Sometimes when translating between CF (climate and forecast) convention data +# and Zarr groups, fill values are left as "negative integers" to encode unsigned +# integers. So, we have to convert to the signed type with the same number of bytes +# as the unsigned integer, then reinterpret as unsigned. That's how a fill value +# of -1 can have a realistic meaning with an unsigned dtype. +# However, we have to apply this correction only if the integer is negative. +# If it's positive, then the value might be out of range of the signed integer type. +fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = sign(v) < 0 ? reinterpret(T, signed(T)(v)) : T(v) diff --git a/lib/ZarrCore/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl new file mode 100644 index 00000000..555ee772 --- /dev/null +++ b/lib/ZarrCore/src/metadata3.jl @@ -0,0 +1,390 @@ +""" +Prototype Zarr version 3 support +""" + +const typemap3 = Dict{String, DataType}() +foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t + typemap3[lowercase(string(t))] = t +end +typemap3["complex64"] = ComplexF32 +typemap3["complex128"] = ComplexF64 + +function typestr3(t::Type) + return lowercase(string(t)) +end +# TODO: Check raw types +function typestr3(::Type{NTuple{N,UInt8}}) where {N} + return "r$(N*8)" +end + +function typestr3(s::AbstractString, codecs=nothing) + if !haskey(typemap3, s) + if startswith(s, "r") + num_bits = tryparse(Int, s[2:end]) + if isnothing(num_bits) + throw(ArgumentError("$s is not a known type")) + end + if mod(num_bits, 8) == 0 + return NTuple{num_bits÷8,UInt8} + else + throw(ArgumentError("$s must describe a raw type with bit size that is a multiple of 8 bits")) + end + end + end + return typemap3[s] +end + +function check_keys(d::AbstractDict, keys) + for key in keys + if !haskey(d, key) + throw(ArgumentError("Zarr v3 metadata must have a key called $key")) + end + end +end + +"""Metadata for Zarr version 3 arrays""" +struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} + zarr_format::Int + node_type::String + shape::Base.RefValue{NTuple{N, Int}} + chunks::NTuple{N, Int} + dtype::String # data_type in v3 + pipeline::P + fill_value::Union{T, Nothing} + chunk_key_encoding::E + function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} + zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) + #Do some sanity checks to make sure we have a sane array + any(<(0), shape) && throw(ArgumentError("Size must be positive")) + any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + end +end +MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) +zarr_format(::MetadataV3) = ZarrFormat(Val(3)) + +function Base.:(==)(m1::MetadataV3, m2::MetadataV3) + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.fill_value == m2.fill_value && + m1.pipeline == m2.pipeline && + m1.chunk_key_encoding == m2.chunk_key_encoding +end + +""" +Derive the storage order ('C' or 'F') from the codec pipeline of a MetadataV3. + +Throws `ArgumentError` if the order cannot be unambiguously determined, which +occurs when: +- the pipeline contains more than one array->array codec, +- an array->array codec is not a `TransposeCodec` (unknown effect on order), or +- the `TransposeCodec` permutation is neither the identity (C order) nor the + full reversal (F order). +""" +function get_order(md::MetadataV3) + array_array = md.pipeline.array_array + if length(array_array) == 0 + return 'C' + end + if length(array_array) > 1 + throw(ArgumentError( + "Cannot determine storage order: pipeline has $(length(array_array)) " * + "array->array codecs; composed permutations yield an indeterminate order" + )) + end + codec = only(array_array) + if !(codec isa Codecs.V3Codecs.TransposeCodec) + throw(ArgumentError( + "Cannot determine storage order: unrecognized array->array codec $(typeof(codec))" + )) + end + N = ndims(md) + c_perm = ntuple(identity, N) + f_perm = ntuple(i -> N - i + 1, N) + if codec.order == c_perm + return 'C' + elseif codec.order == f_perm + return 'F' + else + throw(ArgumentError( + "Cannot determine storage order: TransposeCodec permutation $(codec.order) " * + "is neither C order $c_perm nor F order $f_perm" + )) + end +end +get_order(md::MetadataV2) = md.order + +# --- Registry-based codec parsing --- + +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + if codec_name == "transpose" + return _parse_transpose_codec(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end + +function _parse_transpose_codec(config::Dict, shape_length::Int) + _order = config["order"] + if _order isa AbstractString + n = shape_length + if _order == "C" + @warn "Transpose codec dimension order of C is deprecated" + perm = ntuple(identity, n) + elseif _order == "F" + @warn "Transpose codec dimension order of F is deprecated" + perm = ntuple(i -> n - i + 1, n) + else + throw(ArgumentError("Unknown transpose order string: $_order")) + end + else + perm = Tuple(Int.(_order) .+ 1) + end + return Codecs.V3Codecs.TransposeCodec(perm) +end + +# --- Metadata3 from Dict --- + +function Metadata3(d::AbstractDict, fill_as_missing) + check_keys(d, ("zarr_format", "node_type")) + + zarr_format = d["zarr_format"]::Int + + node_type = d["node_type"]::String + if node_type ∉ ("group", "array") + throw(ArgumentError("Unknown node_type of $node_type")) + end + + zarr_format == 3 || throw(ArgumentError("Metadata3 only functions if zarr_format == 3")) + + # Groups + if node_type == "group" + # Groups only need zarr_format and node_type + # Optionally they can have attributes + for key in keys(d) + if key ∉ ("zarr_format", "node_type", "attributes") + throw(ArgumentError("Zarr v3 group metadata cannot have a key called $key")) + end + end + + group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) + return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) + end + + # Array keys + mandatory_keys = [ + "zarr_format", + "node_type", + "shape", + "data_type", + "chunk_grid", + "chunk_key_encoding", + "fill_value", + "codecs", + ] + optional_keys = [ + "attributes", + "storage_transformers", + "dimension_names", + ] + + check_keys(d, mandatory_keys) + for key in keys(d) + if key ∉ mandatory_keys && key ∉ optional_keys + throw(ArgumentError("Zarr v3 metadata cannot have a key called $key")) + end + end + + # Shape + shape = Int.(d["shape"]) + + # Datatype + data_type = d["data_type"]::String + + # Chunk Grid + chunk_grid = d["chunk_grid"] + if chunk_grid["name"] == "regular" + chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) + if length(shape) != length(chunks) + throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) + end + else + throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) + end + + # Chunk Key Encoding + chunk_key_encoding = d["chunk_key_encoding"] + + # Build V3Pipeline from codec chain using registry-based parsing + array_array_codecs = [] + array_bytes_codec = nothing + bytes_bytes_codecs = [] + + for codec in d["codecs"] + codec_name = codec["name"] + config = get(codec, "configuration", Dict{String,Any}()) + parsed = parse_v3_codec(codec_name, config, length(shape)) + cat = Codecs.V3Codecs.codec_category(parsed) + if cat == (:array, :array) + push!(array_array_codecs, parsed) + elseif cat == (:array, :bytes) + array_bytes_codec = parsed + elseif cat == (:bytes, :bytes) + push!(bytes_bytes_codecs, parsed) + end + end + + isnothing(array_bytes_codec) && throw(ArgumentError("V3 codec chain must contain a 'bytes' codec")) + pipeline = V3Pipeline(Tuple(array_array_codecs), array_bytes_codec, Tuple(bytes_bytes_codecs)) + + # Type Parameters + T = typestr3(data_type) + N = length(shape) + + fv = fill_value_decoding(d["fill_value"], T)::T + + TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} + + chunk_key_encoding = parse_chunk_key_encoding(chunk_key_encoding) + E = typeof(chunk_key_encoding) + + MetadataV3{TU, N, typeof(pipeline), E}( + zarr_format, + node_type, + NTuple{N, Int}(shape) |> reverse, + NTuple{N, Int}(chunks) |> reverse, + data_type, + pipeline, + fv, + chunk_key_encoding, + ) +end + +# --- Metadata3 from Array (takes bytes_bytes_codecs directly) --- + +"Construct MetadataV3 based on your data" +function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; + node_type::String="array", + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + compressor=nothing, + bytes_bytes_codecs::Tuple=(compressor === nothing ? () : compressor_to_v3_bytes_codecs(compressor)), + fill_as_missing = false, + dimension_separator::Char = '/' + ) where {T, N} + @warn("Zarr v3 support is experimental") + T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} + if fill_value === nothing + fill_value = zero(T) + end + T_base = Base.nonmissingtype(T2) + array_array_codecs = if order == 'F' + (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) + else + () + end + array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) + pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) + chunk_key_encoding = ChunkKeyEncoding(dimension_separator, true) + return MetadataV3{T2, N, typeof(pipeline), typeof(chunk_key_encoding)}( + 3, + node_type, + size(A), + chunks, + typestr3(eltype(A)), + pipeline, + fill_value, + chunk_key_encoding, + ) +end + +# --- Compressor to V3 bytes codecs dispatch --- + +""" + compressor_to_v3_bytes_codecs(compressor) -> Tuple + +Convert a compressor to a tuple of V3 bytes->bytes codecs. +Subtypes should add methods for their specific compressor types. +""" +compressor_to_v3_bytes_codecs(::NoCompressor) = () +compressor_to_v3_bytes_codecs(c) = throw(ArgumentError("Unsupported compressor type for v3: $(typeof(c)). Register a method for compressor_to_v3_bytes_codecs.")) + +# --- Metadata(A, chunks, ::ZarrFormat{3}) using compressor_to_v3_bytes_codecs --- + +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; + node_type::String="array", + compressor=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + filters=nothing, + fill_as_missing = false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) + ) where {T, N, E} + bb_codecs = compressor_to_v3_bytes_codecs(compressor) + return Metadata3(A, chunks; + node_type=node_type, + fill_value=fill_value, + order=order, + endian=endian, + bytes_bytes_codecs=bb_codecs, + fill_as_missing=fill_as_missing, + dimension_separator=chunk_key_encoding.sep + ) +end + +# --- lower3: serialize MetadataV3 using codec_to_dict dispatch --- + +function lower3(md::MetadataV3{T}) where T + chunk_grid = Dict{String,Any}( + "name" => "regular", + "configuration" => Dict{String,Any}( + "chunk_shape" => md.chunks |> reverse + ) + ) + + # chunk_key_encoding + chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) + + # Build codecs from pipeline using codec_to_dict dispatch + codecs = Dict{String,Any}[] + p = md.pipeline + + # array->array codecs + for codec in p.array_array + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + + # array->bytes codec + push!(codecs, Codecs.V3Codecs.codec_to_dict(p.array_bytes)) + + # bytes->bytes codecs + for codec in p.bytes_bytes + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + + Dict{String, Any}( + "zarr_format" => Int(md.zarr_format), + "node_type" => md.node_type, + "shape" => md.shape[] |> reverse, + "data_type" => typestr3(T), + "chunk_grid" => chunk_grid, + "chunk_key_encoding" => chunk_key_encoding, + "fill_value" => fill_value_encoding(md.fill_value), + "codecs" => codecs + ) +end + +# V3 constructor from Dict - delegate to Metadata3 +function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{3}) + return Metadata3(d, fill_as_missing) +end + +function JSON.lower(md::MetadataV3) + return lower3(md) +end diff --git a/lib/ZarrCore/src/pipeline.jl b/lib/ZarrCore/src/pipeline.jl new file mode 100644 index 00000000..c7eb9229 --- /dev/null +++ b/lib/ZarrCore/src/pipeline.jl @@ -0,0 +1,77 @@ +""" +V2Pipeline wraps the existing v2 compressor + filter pair. +Delegates to zcompress!/zuncompress! with zero behavior change. +""" +struct V2Pipeline{C<:Compressor, F} <: AbstractCodecPipeline + compressor::C + filters::F +end + +function pipeline_encode(p::V2Pipeline, data::AbstractArray, fill_value) + if fill_value !== nothing && all(isequal(fill_value), data) + return nothing + end + dtemp = UInt8[] + zcompress!(dtemp, data, p.compressor, p.filters) + return dtemp +end + +function pipeline_decode!(p::V2Pipeline, output::AbstractArray, compressed::Vector{UInt8}) + zuncompress!(output, compressed, p.compressor, p.filters) + return output +end + +""" +V3Pipeline holds a three-phase v3 codec chain: +- array_array: tuple of array->array codecs (e.g. transpose) +- array_bytes: single array->bytes codec (e.g. bytes, sharding) +- bytes_bytes: tuple of bytes->bytes codecs (e.g. gzip, blosc, crc32c) +""" +struct V3Pipeline{AA, AB, BB} <: AbstractCodecPipeline + array_array::AA + array_bytes::AB + bytes_bytes::BB +end + +function pipeline_encode(p::V3Pipeline, data::AbstractArray, fill_value) + if fill_value !== nothing && all(isequal(fill_value), data) + return nothing + end + # Phase 1: array->array codecs (forward order) + result = data + for codec in p.array_array + result = Codecs.V3Codecs.codec_encode(codec, result) + end + # Phase 2: array->bytes codec + bytes = Codecs.V3Codecs.codec_encode(p.array_bytes, result) + # Phase 3: bytes->bytes codecs (forward order) + for codec in p.bytes_bytes + bytes = Codecs.V3Codecs.codec_encode(codec, bytes) + end + return bytes +end + +function pipeline_decode!(p::V3Pipeline, output::AbstractArray, compressed::Vector{UInt8}) + # Phase 3 reverse: bytes->bytes codecs (reverse order) + bytes = compressed + for codec in reverse(collect(p.bytes_bytes)) + bytes = Codecs.V3Codecs.codec_decode(codec, bytes) + end + # Phase 2 reverse: bytes->array codec + # Compute the intermediate shape — the shape data has after array_array encoding + intermediate_shape = foldl( + (sz, codec) -> Codecs.V3Codecs.encoded_shape(codec, sz), + p.array_array; init=size(output) + ) + arr = Codecs.V3Codecs.codec_decode(p.array_bytes, bytes, eltype(output), intermediate_shape) + # Phase 1 reverse: array->array codecs (reverse order) + for codec in reverse(collect(p.array_array)) + arr = Codecs.V3Codecs.codec_decode(codec, arr) + end + copyto!(output, arr) + return output +end + +# Convenience: extract pipeline from metadata +get_pipeline(m::MetadataV2) = V2Pipeline(m.compressor, m.filters) +get_pipeline(m::MetadataV3) = m.pipeline From f272c736bbd20bece8fa4cf56f8c76d61341228a Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:11:45 +0100 Subject: [PATCH 04/12] core: add storage backends and ZArray/ZGroup to ZarrCore - Storage.jl: AbstractStore, storageregexlist, metadata I/O, SequentialRead/ConcurrentRead strategies - DirectoryStore, DictStore, ConsolidatedStore (core stores) - ZArray.jl: ZArray type, readblock!/writeblock!, zcreate/zzeros/zopen with default_compressor() - ZGroup.jl: ZGroup, zgroup, zopen, consolidate_metadata (without HTTP.serve/writezip) Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/ZarrCore/src/Storage/Storage.jl | 273 +++++++++++ lib/ZarrCore/src/Storage/consolidated.jl | 101 ++++ lib/ZarrCore/src/Storage/dictstore.jl | 54 +++ lib/ZarrCore/src/Storage/directorystore.jl | 59 +++ lib/ZarrCore/src/ZArray.jl | 507 +++++++++++++++++++++ lib/ZarrCore/src/ZGroup.jl | 204 +++++++++ 6 files changed, 1198 insertions(+) create mode 100644 lib/ZarrCore/src/Storage/Storage.jl create mode 100644 lib/ZarrCore/src/Storage/consolidated.jl create mode 100644 lib/ZarrCore/src/Storage/dictstore.jl create mode 100644 lib/ZarrCore/src/Storage/directorystore.jl create mode 100644 lib/ZarrCore/src/ZArray.jl create mode 100644 lib/ZarrCore/src/ZGroup.jl diff --git a/lib/ZarrCore/src/Storage/Storage.jl b/lib/ZarrCore/src/Storage/Storage.jl new file mode 100644 index 00000000..918c6db6 --- /dev/null +++ b/lib/ZarrCore/src/Storage/Storage.jl @@ -0,0 +1,273 @@ + +# Defines different storages for zarr arrays. Currently only regular files (DirectoryStore) +# and Dictionaries are supported + +""" + abstract type AbstractStore + +This the abstract supertype for all Zarr store implementations. Currently only regular files ([`DirectoryStore`](@ref)) +and Dictionaries are supported. + +## Interface + +All subtypes of `AbstractStore` must implement the following methods: + +- [`storagesize(d::AbstractStore, p::AbstractString)`](@ref storagesize) +- [`subdirs(d::AbstractStore, p::AbstractString)`](@ref subdirs) +- [`subkeys(d::AbstractStore, p::AbstractString)`](@ref subkeys) +- [`isinitialized(d::AbstractStore, p::AbstractString)`](@ref isinitialized) +- [`storefromstring(::Type{<: AbstractStore}, s, _)`](@ref storefromstring) +- `Base.getindex(d::AbstractStore, i::AbstractString)`: return the data stored in key `i` as a Vector{UInt8} +- `Base.setindex!(d::AbstractStore, v, i::AbstractString)`: write the values in `v` to the key `i` of the given store `d` + +They may optionally implement the following methods: + +- [`store_read_strategy(s::AbstractStore)`](@ref store_read_strategy): return the read strategy for the given store. See [`SequentialRead`](@ref) and [`ConcurrentRead`](@ref). +""" +abstract type AbstractStore end + +# Define the interface + +""" + storagesize(d::AbstractStore, p::AbstractString) + +This function shall return the size of all data files in a store at path `p`. +""" +function storagesize end + + +""" + Base.getindex(d::AbstractStore,i::String) + +Returns the data stored in the given key as a Vector{UInt8} +""" +Base.getindex(d::AbstractStore,i::AbstractString) = error("getindex not implemented for store $(typeof(d))") + +""" + Base.setindex!(d::AbstractStore,v,i::String) + +Writes the values in v to the given store and key. +""" +Base.setindex!(d::AbstractStore,v,i::AbstractString) = error("setindex not implemented for store $(typeof(d))") + +""" + subdirs(d::AbstractStore, p) + +Returns a list of keys for children stores in the given store at path p. +""" +function subdirs end + +""" + subkeys(d::AbstractStore, p) + +Returns the keys of files in the given store. +""" +function subkeys end + +# Function to construct the full path to a chunk given the base path, Cartesian Index i, and the chunk ecoding +store_readchunk(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = s[p, citostring(e, i)] +store_deletechunk(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = delete!(s, p, citostring(e, i)) +store_writechunk(s::AbstractStore, v, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = s[p, citostring(e, i)] = v +store_isinitialized(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = isinitialized(s, p, citostring(e, i)) + + +#Functions to concat path and key +Base.getindex(s::AbstractStore, p, i::AbstractString) = s[_concatpath(p, i)] +Base.delete!(s::AbstractStore, p, i::AbstractString) = delete!(s, _concatpath(p, i)) +Base.haskey(s::AbstractStore, k::AbstractString) = isinitialized(s, k) +Base.setindex!(s::AbstractStore, v, p, i::AbstractString) = setindex!(s, v, _concatpath(p, i)) + + +maybecopy(x) = copy(x) +maybecopy(x::String) = x + + +function getattrs(::ZarrFormat{2}, s::AbstractStore, p) + atts = s[p,".zattrs"] + if atts === nothing + Dict() + else + JSON.parse(replace(String(maybecopy(atts)),": NaN,"=>": \"NaN\","); dicttype = Dict{String,Any}) + end +end + +function getattrs(::ZarrFormat{3}, s::AbstractStore, p) + md = s[p, "zarr.json"] + if md === nothing + error("zarr.json not found") + else + md = JSON.parse(replace(String(maybecopy(md)), ": NaN," => ": \"NaN\","); dicttype=Dict{String,Any}) + return get(md, "attributes", Dict{String,Any}()) + end +end + +function writeattrs(::ZarrFormat{2}, s::AbstractStore, p, att::Dict; indent_json::Bool=false) + b = IOBuffer() + + if indent_json + JSON.print(b,att,4) + else + JSON.print(b,att) + end + + s[p,".zattrs"] = take!(b) + att +end + +function writeattrs(::ZarrFormat{3}, s::AbstractStore, p, att::Dict; indent_json::Bool=false) + # This is messy, we need to open zarr.json and replace the attributes section + md = s[p, "zarr.json"] + if md === nothing + error("zarr.json not found") + else + md = JSON.parse(replace(String(maybecopy(md)), ": NaN," => ": \"NaN\",")) + end + md = Dict(md) + md["attributes"] = att + + b = IOBuffer() + + if indent_json + JSON.print(b, md, 4) + else + JSON.print(b, md) + end + + s[p, "zarr.json"] = take!(b) + att +end + +is_zarr3(s::AbstractStore, p) = isinitialized(s,_concatpath(p,"zarr.json")) +is_zarr2(s::AbstractStore, p) = is_zarray(ZarrFormat(Val(2)), s, p) || is_zgroup(ZarrFormat((Val(2))), s, p) + +is_zgroup(::ZarrFormat{2}, s::AbstractStore, p) = isinitialized(s, _concatpath(p, ".zgroup")) +is_zarray(::ZarrFormat{2}, s::AbstractStore, p) = isinitialized(s, _concatpath(p, ".zarray")) +function is_zgroup(::ZarrFormat{3}, s::AbstractStore, p) + isinitialized(s, _concatpath(p, "zarr.json")) || return false + try + metadata = getmetadata(ZarrFormat(Val(3)), s, p, false) + metadata.node_type == "group" + catch e + e isa ArgumentError || rethrow() + @warn "Skipping $p: $(e.msg)" + false + end +end +function is_zarray(::ZarrFormat{3}, s::AbstractStore, p) + isinitialized(s, _concatpath(p, "zarr.json")) || return false + try + metadata = getmetadata(ZarrFormat(Val(3)), s, p, false) + metadata.node_type == "array" + catch e + e isa ArgumentError || rethrow() + @warn "Skipping $p: $(e.msg)" + false + end +end + + +isinitialized(s::AbstractStore, p, i::AbstractString) = isinitialized(s, _concatpath(p, i)) +isinitialized(s::AbstractStore, i::AbstractString) = s[i] !== nothing + +getmetadata(::ZarrFormat{2}, s::AbstractStore, p, fill_as_missing) = Metadata(String(maybecopy(s[p, ".zarray"])), fill_as_missing) + +getmetadata(::ZarrFormat{3}, s::AbstractStore, p, fill_as_missing) = Metadata(String(maybecopy(s[p, "zarr.json"])), fill_as_missing) + +function writemetadata(::ZarrFormat{2}, s::AbstractStore, p, m::AbstractMetadata; indent_json::Bool=false) + met = IOBuffer() + + if indent_json + JSON.print(met,m,4) + else + JSON.print(met,m) + end + + s[p,".zarray"] = take!(met) + m +end +function writemetadata(::ZarrFormat{3}, s::AbstractStore, p, m::AbstractMetadata; indent_json::Bool=false) + met = IOBuffer() + + if indent_json + JSON.print(met, m, 4) + else + JSON.print(met, m) + end + + s[p, "zarr.json"] = take!(met) + m +end + + + +## Handling sequential vs parallel IO +struct SequentialRead end +struct ConcurrentRead + ntasks::Int +end +store_read_strategy(::AbstractStore) = SequentialRead() + +channelsize(s) = channelsize(store_read_strategy(s)) +channelsize(::SequentialRead) = 0 +channelsize(c::ConcurrentRead) = c.ntasks + +read_items!(s::AbstractStore, c::AbstractChannel, e::AbstractChunkKeyEncoding, p, i) = read_items!(s, c, store_read_strategy(s), e, p, i) +function read_items!(s::AbstractStore, c::AbstractChannel, ::SequentialRead, e::AbstractChunkKeyEncoding, p, i) + for ii in i + res = store_readchunk(s, p, ii, e) + put!(c,(ii=>res)) + end +end +function read_items!(s::AbstractStore, c::AbstractChannel, r::ConcurrentRead, e::AbstractChunkKeyEncoding, p, i) + ntasks = r.ntasks + #@show ntasks + asyncmap(i,ntasks = ntasks) do ii + #@show ii,objectid(current_task),p + res = store_readchunk(s, p, ii, e) + #@show ii,length(res) + put!(c,(ii=>res)) + nothing + end +end + +write_items!(s::AbstractStore, c::AbstractChannel, e::AbstractChunkKeyEncoding, p, i) = write_items!(s, c, store_read_strategy(s), e, p, i) +function write_items!(s::AbstractStore, c::AbstractChannel, ::SequentialRead, e::AbstractChunkKeyEncoding, p, i) + for _ in 1:length(i) + ii,data = take!(c) + if data === nothing + if store_isinitialized(s, p, ii, e) + store_deletechunk(s, p, ii, e) + end + else + store_writechunk(s, data, p, ii, e) + end + end + close(c) +end + +function write_items!(s::AbstractStore, c::AbstractChannel, r::ConcurrentRead, e::AbstractChunkKeyEncoding, p, i) + ntasks = r.ntasks + asyncmap(i,ntasks = ntasks) do _ + ii,data = take!(c) + if data === nothing + if store_isinitialized(s, p, ii, e) + store_deletechunk(s, p, ii, e) + end + else + store_writechunk(s, data, p, ii, e) = data + end + nothing + end + close(c) +end + +isemptysub(s::AbstractStore, p) = isempty(subkeys(s,p)) && isempty(subdirs(s,p)) + +#Here different storage backends can register regexes that are checked against +#during auto-check of storage format when doing zopen +storageregexlist = Pair[] + +#include("formattedstore.jl") +include("directorystore.jl") +include("dictstore.jl") +include("consolidated.jl") diff --git a/lib/ZarrCore/src/Storage/consolidated.jl b/lib/ZarrCore/src/Storage/consolidated.jl new file mode 100644 index 00000000..de1cc291 --- /dev/null +++ b/lib/ZarrCore/src/Storage/consolidated.jl @@ -0,0 +1,101 @@ +""" +A store that wraps any other AbstractStore but has access to the consolidated metadata +stored in the .zmetadata key. Whenever data attributes or metadata are accessed, the +data will be read from the dictionary instead. +""" +struct ConsolidatedStore{P} <: AbstractStore + parent::P + path::String + cons::Dict{String,Any} +end +function ConsolidatedStore(s::AbstractStore, p) + d = s[p, ".zmetadata"] + if d === nothing + throw(ArgumentError("Could not find consolidated metadata for store $s")) + end + meta = JSON.parse(String(copy(d)); dicttype = Dict{String,Any}) + ConsolidatedStore(s, p, meta["metadata"]) +end + +function Base.show(io::IO,d::ConsolidatedStore) + b = IOBuffer() + show(b,d.parent) + print(io, "Consolidated ", String(take!(b))) +end + +storagesize(d::ConsolidatedStore,p) = storagesize(d.parent,p) +function Base.getindex(d::ConsolidatedStore,i::String) + d.parent[i] +end +function getmetadata(::ZarrFormat{2}, d::ConsolidatedStore, p, fill_as_missing) + return Metadata(d.cons[_unconcpath(d, p, ".zarray")], fill_as_missing) +end +function getmetadata(::ZarrFormat{3}, d::ConsolidatedStore, p, fill_as_missing) + return Metadata(d.cons[_unconcpath(d, p, "zarr.json")], fill_as_missing) +end +function getattrs(::ZarrFormat{2}, d::ConsolidatedStore, p) + return get(d.cons, _unconcpath(d, p, ".zattrs"), Dict{String,Any}()) +end + +function getattrs(::ZarrFormat{3}, d::ConsolidatedStore, p) + return get(d.cons, _unconcpath(d, p, ".zattrs"), Dict{String,Any}()) +end +function _unconcpath(d,p) + startswith(p,d.path) || error("Requested key is not in consolidated path") + lstrip(replace(p,d.path=>"", count=1),'/') +end +_unconcpath(d,p,s) = _concatpath(_unconcpath(d,p),s) +is_zarray(::ZarrFormat{2}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, ".zarray")) +is_zgroup(::ZarrFormat{2}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, ".zgroup")) +is_zarray(::ZarrFormat{3}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, "zarr.json")) && get(d.cons[_unconcpath(d, p, "zarr.json")], "node_type", "") == "array" +is_zgroup(::ZarrFormat{3}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, "zarr.json")) && get(d.cons[_unconcpath(d, p, "zarr.json")], "node_type", "") == "group" +ZarrFormat(d::ConsolidatedStore, path) = ZarrFormat(d.parent, path) # detect format from parent, not cons +check_consolidated_write(i::String) = split(i,'/')[end] in (".zattrs",".zarray",".zgroup") && + throw(ArgumentError("Can not modify consolidated metadata, please re-open the dataset with `consolidated=false`")) + +_pdict(d::ConsolidatedStore,p) = filter(((k,v),)->startswith(k,p),d.cons) +function subdirs(d::ConsolidatedStore,p) + p2 = _unconcpath(d,p) + d2 = _pdict(d,p2) + _searchsubdict(d2, p2, (sp, lp) -> length(sp) > lp + 1) +end + +function subkeys(d::ConsolidatedStore,p) + subkeys(d.parent,p) +end + + + +function Base.setindex!(d::ConsolidatedStore,v,i::String) + #Here we check that we don't overwrite consolidated information + check_consolidated_write(i) + d.parent[i] = v +end +function Base.delete!(d::ConsolidatedStore,i::String) + check_consolidated_write(i) + delete!(d.parent,i) +end + +store_read_strategy(s::ConsolidatedStore) = store_read_strategy(s.parent) + +function consolidate_metadata(s::AbstractStore,d,prefix) + for k in (".zattrs",".zarray",".zgroup") + v = s[prefix,k] + if v !== nothing + d[_concatpath(prefix,k)] = JSON.parse(String(copy(v)); dicttype = Dict{String,Any}) + end + end + foreach(subdirs(s,prefix)) do subname + consolidate_metadata(s,d,string(prefix,subname,"/")) + end + d +end +function consolidate_metadata(s::AbstractStore,p) + d = consolidate_metadata(s,Dict{String,Any}(),p) + buf = IOBuffer() + JSON.print(buf,Dict("metadata"=>d,"zarr_consolidated_format"=>1),4) + s[p,".zmetadata"] = take!(buf) + ConsolidatedStore(s,p,d) +end + +consolidate_metadata(s) = consolidate_metadata(zopen(s,"w")) diff --git a/lib/ZarrCore/src/Storage/dictstore.jl b/lib/ZarrCore/src/Storage/dictstore.jl new file mode 100644 index 00000000..d21bcb61 --- /dev/null +++ b/lib/ZarrCore/src/Storage/dictstore.jl @@ -0,0 +1,54 @@ +# Stores data in a simple dict in memory +struct DictStore <: AbstractStore + a::Dict{String,Vector{UInt8}} +end +DictStore() = DictStore(Dict{String,Vector{UInt8}}()) + +Base.show(io::IO,d::DictStore) = print(io,"Dictionary Storage") +function _pdict(d::DictStore,p) + p = (isempty(p) || endswith(p,'/')) ? p : p*'/' + filter(((k,v),)->startswith(k,p),d.a) +end +function storagesize(d::DictStore,p) + sum(i -> (last(split(i[1],'/')) ∉ (".zattrs",".zarray") ? sizeof(i[2]) : zero(sizeof(i[2]))), _pdict(d,p); init=0) +end + +function Base.getindex(d::DictStore,i::AbstractString) + get(d.a,i,nothing) +end +function Base.setindex!(d::DictStore,v,i::AbstractString) + d.a[i] = v +end +Base.delete!(d::DictStore, i::AbstractString) = delete!(d.a,i) + +function subdirs(d::DictStore,p) + d2 = _pdict(d,p) + _searchsubdict(d2,p,(sp,lp)->length(sp) > lp+1) +end + +function subkeys(d::DictStore,p) + d2 = _pdict(d,p) + _searchsubdict(d2,p,(sp,lp)->length(sp) == lp+1) +end + +function _searchsubdict(d2,p,condition) + o = Set{String}() + pspl = split(rstrip(p,'/'),'/') + lp = if length(pspl) == 1 && isempty(pspl[1]) + 0 + else + length(pspl) + end + for k in keys(d2) + sp = split(k,'/') + if condition(sp,lp) + push!(o,sp[lp+1]) + end + end + collect(o) +end + + +#getsub(d::DictStore, p, n) = _substore(d,p).subdirs[n] + +#path(d::DictStore) = "" diff --git a/lib/ZarrCore/src/Storage/directorystore.jl b/lib/ZarrCore/src/Storage/directorystore.jl new file mode 100644 index 00000000..2913176e --- /dev/null +++ b/lib/ZarrCore/src/Storage/directorystore.jl @@ -0,0 +1,59 @@ +"Normalize logical storage path" +function normalize_path(p::AbstractString) + # \ to / since normpath on linux won't handle it + p = replace(p, '\\'=>'/') + p = normpath(p) + # \ to / again since normpath on windows creates \ + p = replace(p, '\\'=>'/') + p == "/" ? p : rstrip(p, '/') +end + +# Stores files in a regular file system +struct DirectoryStore <: AbstractStore + folder::String + function DirectoryStore(p) + mkpath(normalize_path(p)) + new(normalize_path(p)) + end +end + +function Base.getindex(d::DirectoryStore, i::String) + fname=joinpath(d.folder,i) + if isfile(fname) + read(fname) + else + nothing + end +end + +function Base.setindex!(d::DirectoryStore,v,i::String) + fname=d.folder * "/" * i + folder = dirname(fname) + isdir(folder) || mkpath(folder) + write(fname,v) + v +end + + +function storagesize(d::DirectoryStore,p) + sum(f -> filesize(d.folder * "/" * p * "/" * f), filter(i->i ∉ (".zattrs",".zarray"),readdir(d.folder * "/" * p)); init=0) +end + +function subdirs(s::DirectoryStore,p) + pbase = joinpath(s.folder,p) + if !isdir(pbase) + return String[] + else + return filter(i -> isdir(joinpath(s.folder,p, i)), readdir(pbase)) + end +end +function subkeys(s::DirectoryStore,p) + pbase = joinpath(s.folder,p) + if !isdir(pbase) + return String[] + else + return filter(i -> isfile(joinpath(s.folder,p, i)), readdir(pbase)) + end +end +Base.delete!(s::DirectoryStore, k::String) = isfile(joinpath(s.folder, k)) && rm(joinpath(s.folder, k)) + diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl new file mode 100644 index 00000000..72219492 --- /dev/null +++ b/lib/ZarrCore/src/ZArray.jl @@ -0,0 +1,507 @@ +import JSON +import OffsetArrays: OffsetArray +import DiskArrays: AbstractDiskArray +import DiskArrays +using DateTimes64: DateTime64 +using Dates: Day, Millisecond + +""" +Number of tasks to use for async reading of chunks. Warning: setting this to very high values can lead to a large memory footprint. +""" +const concurrent_io_tasks = Ref(50) + +getfillval(::Type{T}, t::String) where {T <: Number} = parse(T, t) +getfillval(::Type{T}, t::Union{T,Nothing}) where {T} = t + +struct SenMissArray{T,N} <: AbstractArray{Union{T,Missing},N} + x::Array{T,N} + senval::T +end +SenMissArray(x::Array{T,N},v) where {T,N} = SenMissArray{T,N}(x,convert(T,v)) +Base.size(x::SenMissArray) = size(x.x) +senval(x::SenMissArray) = x.senval +function Base.getindex(x::SenMissArray,i::Int) + v = x.x[i] + isequal(v,senval(x)) ? missing : v +end +Base.setindex!(x::SenMissArray,v,i::Int) = x.x[i] = v +Base.setindex!(x::SenMissArray,::Missing,i::Int) = x.x[i] = senval(x) +Base.IndexStyle(::Type{<:SenMissArray})=Base.IndexLinear() + +# Struct representing a Zarr Array in Julia, note that +# chunks(chunk size) and size are always in Julia column-major order +struct ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata{T,N}} <: AbstractDiskArray{T,N} + metadata::M + storage::S + path::String + attrs::Dict + writeable::Bool +end + +Base.eltype(::ZArray{T}) where {T} = T +Base.ndims(::ZArray{<:Any,N}) where {N} = N +Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N, Int} +function Base.size(z::ZArray{<:Any,N}, i::Integer) where {N} + len = length(z.metadata.shape[]) + if 0 < i <= len + z.metadata.shape[][i]::Int + elseif i > len + 1 + else + error("arraysize: dimension out of range") + end +end +Base.length(z::ZArray) = prod(z.metadata.shape[])::Int +Base.lastindex(z::ZArray{<:Any,N}, n::Integer) where {N} = size(z, n)::Int +Base.lastindex(z::ZArray{<:Any,1}) = size(z, 1)::Int + +function Base.show(io::IO,z::ZArray) + print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +end +function Base.show(io::IO,::MIME"text/plain",z::ZArray) + print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +end + +zname(z::ZArray) = zname(z.path) + +function zname(s::String) + spl = split(rstrip(s,'/'),'/') + isempty(last(spl)) ? "root" : last(spl) +end + + +""" + storagesize(z::ZArray) + +Returns the size of the compressed data stored in the ZArray `z` in bytes +""" +storagesize(z::ZArray) = storagesize(z.storage,z.path) + +""" + storageratio(z::ZArray) + +Returns the ratio of the size of the uncompressed data in `z` and the size of the compressed data. +""" +storageratio(z::ZArray) = length(z)*sizeof(eltype(z))/storagesize(z) + +storageratio(z::ZArray{<:Vector}) = "unknown" + +nobytes(z::ZArray) = length(z)*sizeof(eltype(z)) +nobytes(z::ZArray{<:Vector}) = "unknown" +nobytes(z::ZArray{<:String}) = "unknown" + +zinfo(z::ZArray) = zinfo(stdout,z) +function zinfo(io::IO,z::ZArray) + ninit = sum(chunkindices(z)) do i + store_isinitialized(z.storage, z.path, i, z.metadata.chunk_key_encoding) + end + allinfos = [ + "Type" => "ZArray", + "Data type" => eltype(z), + "Shape" => size(z), + "Chunk Shape" => z.metadata.chunks, + "Order" => try get_order(z.metadata) catch e "unknown ($(e.msg))" end, + "Read-Only" => !z.writeable, + "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), + "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, + "Store type" => z.storage, + "No. bytes" => nobytes(z), + "No. bytes stored" => storagesize(z), + "Storage ratio" => storageratio(z), + "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" + ] + foreach(allinfos) do ii + println(io,rpad(ii[1],20),": ",ii[2]) + end +end + +function ZArray(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=false) where T<:AbstractStore + zv = if zarr_format == :auto + ZarrFormat(s, path) + else + ZarrFormat(zarr_format) + end + metadata = getmetadata(zv, s, path, fill_as_missing) + attrs = getattrs(zv, s, path) + writeable = mode == "w" + startswith(path,"/") && error("Paths should never start with a leading '/'") + ZArray(metadata, s, string(path), attrs, writeable) +end + +zarr_format(z::ZArray) = zarr_format(z.metadata) +dimension_separator(z::ZArray) = dimension_separator(z.metadata) + + +""" + trans_ind(r, bs) + +For a given index and blocksize determines which chunks of the Zarray will have to +be accessed. +""" +trans_ind(r::AbstractUnitRange, bs) = fld1(first(r),bs):fld1(last(r),bs) +trans_ind(r::Integer, bs) = fld1(r,bs) + +function boundint(r1, s2, o2) + r2 = range(o2+1,length=s2) + f1, f2 = first(r1), first(r2) + l1, l2 = last(r1),last(r2) + UnitRange(f1 > f2 ? f1 : f2, l1 < l2 ? l1 : l2) +end + +function getchunkarray(z::ZArray{>:Missing}) + # temporary workaround to use strings as data values + inner = fill(z.metadata.fill_value, z.metadata.chunks) + a = SenMissArray(inner,z.metadata.fill_value) +end +_zero(T) = zero(T) +_zero(T::Type{<:MaxLengthString}) = T("") +_zero(T::Type{ASCIIChar}) = ASCIIChar(0) +_zero(::Type{<:Vector{T}}) where T = T[] +_zero(::Type{Char}) = Char(0) +getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) + +maybeinner(a::Array) = a +maybeinner(a::SenMissArray) = a.x +resetbuffer!(fv,a::Array) = fv === nothing || fill!(a,fv) +resetbuffer!(_,a::SenMissArray) = fill!(a,missing) + +# Function to read or write from a zarr array. Could be refactored +# using type system to get rid of the `if readmode` statements. +function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} + + output_base_offsets = map(i->first(i)-1,r.indices) + # Determines which chunks are affected + blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + # Allocate array of the size of a chunks where uncompressed data can be held + #bufferdict = IdDict((current_task()=>getchunkarray(z),)) + a = getchunkarray(z) + # Now loop through the chunks + c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) + + task = @async begin + read_items!($(z.storage), c, $(z.metadata.chunk_key_encoding), $(z.path), $(blockr)) + end + bind(c,task) + + try + for i in 1:length(blockr) + + bI,chunk_compressed = take!(c) + + current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) + + indranges = map(boundint,r.indices,size(a),current_chunk_offsets) + + uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) + nothing + end + finally + close(c) + end + + aout +end + +function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} + + z.writeable || error("Can not write to read-only ZArray") + input_base_offsets = map(i->first(i)-1,r.indices) + # Determines which chunks are affected + blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + # Allocate array of the size of a chunks where uncompressed data can be held + #bufferdict = IdDict((current_task()=>getchunkarray(z),)) + a = getchunkarray(z) + # Now loop through the chunks + readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) + + readtask = @async begin + read_items!(z.storage, readchannel, z.metadata.chunk_key_encoding, z.path, blockr) + end + bind(readchannel,readtask) + + writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) + + writetask = @async begin + write_items!(z.storage, writechannel, z.metadata.chunk_key_encoding, z.path, blockr) + end + bind(writechannel,writetask) + + try + for i in 1:length(blockr) + + bI,chunk_compressed = take!(readchannel) + + current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) + + indranges = map(boundint,r.indices,size(a),current_chunk_offsets) + + if isnothing(chunk_compressed) || (length.(indranges) != size(a)) + resetbuffer!(z.metadata.fill_value,a) + end + + curchunk = if length.(indranges) != size(a) + view(a,dotminus.(indranges,current_chunk_offsets)...) + else + a + end + + if chunk_compressed !== nothing + uncompress_raw!(a,z,chunk_compressed) + end + + curchunk .= view(ain,dotminus.(indranges,input_base_offsets)...) + + put!(writechannel,bI=>compress_raw(maybeinner(a),z)) + nothing + end + finally + close(readchannel) + close(writechannel) + end + ain +end + +DiskArrays.readblock!(a::ZArray,aout,i::AbstractUnitRange...) = readblock!(aout,a,CartesianIndices(i)) +DiskArrays.writeblock!(a::ZArray,v,i::AbstractUnitRange...) = writeblock!(v,a,CartesianIndices(i)) +DiskArrays.haschunks(::ZArray) = DiskArrays.Chunked() +DiskArrays.eachchunk(a::ZArray) = DiskArrays.GridChunks(a,a.metadata.chunks) + +""" + uncompress_raw!(a::DenseArray{T},z::ZArray{T,N},i::CartesianIndex{N}) + +Read the chunk specified by `i` from the Zarray `z` and write its content to `a` +""" +function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N + if curchunk === nothing + if isnothing(z.metadata.fill_value) + throw(ArgumentError("The array $z got missing chunks and no fill_value")) + end + fill!(a, z.metadata.fill_value) + else + pipeline_decode!(get_pipeline(z.metadata), a, curchunk) + end + a +end + +dotminus(x,y) = x.-y + +function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) + + uncompress_raw!(a,z,chunk_compressed) + + if length.(indranges) == size(a) + aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a + else + curchunk = a[dotminus.(indranges,current_chunk_offsets)...] + aout[dotminus.(indranges, output_base_offsets)...] = curchunk + end +end + +function compress_raw(a,z) + length(a) == prod(z.metadata.chunks) || throw(DimensionMismatch("Array size does not equal chunk size")) + pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) +end + + + +""" + zcreate(T, dims...;kwargs) + +Creates a new empty zarr array with element type `T` and array dimensions `dims`. The following keyword arguments are accepted: + +* `path=""` directory name to store a persistent array. If left empty, an in-memory array will be created +* `name=""` name of the zarr array, defaults to the directory name +* `zarr_format`=$(DV) Zarr format version (2 or 3) +* `storagetype` determines the storage to use, current options are `DirectoryStore` or `DictStore` +* `chunks=dims` size of the individual array chunks, must be a tuple of length `length(dims)` +* `fill_value=nothing` value to represent missing values +* `fill_as_missing=false` set to `true` shall fillvalue s be converted to `missing`s +* `filters`=filters to be applied +* `compressor=default_compressor()` compressor type and properties +* `attrs=Dict()` a dict containing key-value pairs with metadata attributes associated to the array +* `writeable=true` determines if the array is opened in read-only or write mode +* `indent_json=false` determines if indents are added to format the json files `.zarray` and `.zattrs`. This makes them more readable, but increases file size. +* `dimension_separator='.'` sets how chunks are encoded. The Zarr v2 default is '.' such that the first 3D chunk would be `0.0.0`. The Zarr v3 default is `/`. +""" +function zcreate(::Type{T}, dims::Integer...; + name="", + path=nothing, + zarr_format=DV, + dimension_separator=default_sep(zarr_format), + kwargs... + ) where T + + if path===nothing + store = DictStore() + else + store = DirectoryStore(joinpath(path, name)) + end + zcreate(T, store, dims...; zarr_format, dimension_separator, kwargs...) +end + +function zcreate(::Type{T},storage::AbstractStore, + dims...; + path = "", + zarr_format = DV, + chunks=dims, + fill_value=nothing, + fill_as_missing=false, + compressor=default_compressor(), + filters = filterfromtype(T), + attrs=Dict(), + writeable=true, + indent_json=false, + dimension_separator=nothing + ) where {T} + + v = ZarrFormat(zarr_format) + if isnothing(dimension_separator) + dimension_separator = default_sep(v) + end + if dimension_separator isa AbstractString + # Convert AbstractString to Char + dimension_separator = only(dimension_separator) + end + chunk_key_encoding = ChunkKeyEncoding(dimension_separator, default_prefix(v)) + + length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) + N = length(dims) + C = typeof(compressor) + + # Create a dummy array to use with Metadata constructor + # This allows us to leverage the multiple dispatch in Metadata constructors + dummy_array = Array{T,N}(undef, dims...) + metadata = Metadata(dummy_array, chunks, v; + compressor=compressor, + fill_value=fill_value, + filters=filters, + fill_as_missing=fill_as_missing, + chunk_key_encoding=chunk_key_encoding + ) + + # Extract the element type from the metadata (handles T2 calculation) + T2 = eltype(metadata) + + isemptysub(storage,path) || error("$storage $path is not empty") + + writemetadata(v, storage, path, metadata, indent_json=indent_json) + + writeattrs(v, storage, path, attrs, indent_json=indent_json) + + ZArray(metadata, storage, path, attrs, writeable) +end + +filterfromtype(::Type{<:Any}) = nothing + +function filterfromtype(::Type{<:AbstractArray{T}}) where T + #Here we have to apply the vlenarray filter + (VLenArrayFilter{T}(),) +end + +filterfromtype(::Type{<:Union{<:AbstractString, Union{<:AbstractString, Missing}}}) = (VLenUTF8Filter(),) +filterfromtype(::Type{<:Union{MaxLengthString, Union{MaxLengthString, Missing}}}) = nothing + +#Not all Array types can be mapped directly to a valid ZArray encoding. +#Here we try to determine the correct element type +to_zarrtype(::AbstractArray{T}) where T = T +to_zarrtype(a::AbstractArray{<:Date}) = DateTime64{Day} +to_zarrtype(a::AbstractArray{<:DateTime}) = DateTime64{Millisecond} + +function ZArray(a::AbstractArray, args...; kwargs...) + z = zcreate(to_zarrtype(a), args..., size(a)...; kwargs...) + z[:]=a + z +end + +""" + chunkindices(z::ZArray) + +Returns the Cartesian Indices of the chunks of a given ZArray +""" +chunkindices(z::ZArray) = CartesianIndices(map((s, c) -> 1:ceil(Int, s/c), z.metadata.shape[], z.metadata.chunks)) + +""" + zzeros(T, dims...; kwargs... ) + +Creates a zarr array and initializes all values with zero. Accepts the same keyword arguments as `zcreate` +""" +function zzeros(T,dims...;kwargs...) + z = zcreate(T,dims...;kwargs...) + as = zeros(T, z.metadata.chunks...) + data_encoded = compress_raw(as,z) + p = z.path + if data_encoded !== nothing + for i in chunkindices(z) + store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) + end + end + z +end + +#Resizing Zarr arrays +""" + resize!(z::ZArray{T,N}, newsize::NTuple{N}) + +Resizes a `ZArray` to the new specified size. If the size along any of the +axes is decreased, unused chunks will be deleted from the store. +""" +function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N} + any(<(0), newsize) && throw(ArgumentError("Size must be positive")) + oldsize = z.metadata.shape[] + z.metadata.shape[] = newsize + #Check if array was shrunk + if any(map(<,newsize, oldsize)) + prune_oob_chunks(z.storage, z.path, oldsize, newsize, z.metadata.chunks, z.metadata.chunk_key_encoding) + end + writemetadata(zarr_format(z), z.storage, z.path, z.metadata) + nothing +end +Base.resize!(z::ZArray, newsize::Integer...) = resize!(z,newsize) + +""" + append!(z::ZArray{<:Any, N},a;dims = N) + +Appends an AbstractArray to an existinng `ZArray` along the dimension dims. The +size of the `ZArray` is increased accordingly and data appended. + +Example: + +````julia +z=zzeros(Int,5,3) +append!(z,[1,2,3],dims=1) #Add a new row +append!(z,ones(Int,6,2)) #Add two new columns +z[:,:] +```` +""" +function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N + #Determine how many entries to add to axis + otherdims = sort!(setdiff(1:N,dims)) + othersize = size(z)[otherdims] + if ndims(a)==N + nadd = size(a,dims) + size(a)[otherdims]==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) + elseif ndims(a)==N-1 + size(a)==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) + nadd = 1 + else + throw(DimensionMismatch("Number of dimensions of array must be either $N or $(N-1)")) + end + oldsize = size(z) + newsize = ntuple(i->i==dims ? oldsize[i]+nadd : oldsize[i], N) + resize!(z,newsize) + appendinds = ntuple(i->i==dims ? (oldsize[i]+1:newsize[i]) : Colon(),N) + z[appendinds...] = a + nothing +end + +function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks, chunk_key_encoding) + dimstoshorten = findall(map(<,newsize, oldsize)) + for idim in dimstoshorten + delrange = (fld1(newsize[idim],chunks[idim])+1):(fld1(oldsize[idim],chunks[idim])) + allchunkranges = map(i->1:fld1(oldsize[i],chunks[i]),1:length(oldsize)) + r = (allchunkranges[1:idim-1]..., delrange, allchunkranges[idim+1:end]...) + for cI in CartesianIndices(r) + store_deletechunk(s, path, cI, chunk_key_encoding) + end + end +end diff --git a/lib/ZarrCore/src/ZGroup.jl b/lib/ZarrCore/src/ZGroup.jl new file mode 100644 index 00000000..90c05247 --- /dev/null +++ b/lib/ZarrCore/src/ZGroup.jl @@ -0,0 +1,204 @@ +struct ZGroup{S<:AbstractStore} + storage::S + path::String + arrays::Dict{String, ZArray} + groups::Dict{String, ZGroup} + attrs::Dict + writeable::Bool +end + +# path can also be a SubString{String} +ZGroup(storage, path::AbstractString, arrays, groups, attrs, writeable) = + ZGroup(storage, String(path), arrays, groups, attrs, writeable) + +zname(g::ZGroup) = zname(g.path) + + + +#Open an existing ZGroup +function ZGroup(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=false) where T<:AbstractStore + arrays = Dict{String, ZArray}() + groups = Dict{String, ZGroup}() + zv = if zarr_format == :auto + ZarrFormat(s, path) + else + ZarrFormat(zarr_format) + end + for d in subdirs(s,path) + dshort = split(d,'/')[end] + subpath = _concatpath(path,dshort) + if is_zarray(zv, s, subpath) + m = zopen_noerr(s, mode, zv, path=_concatpath(path, dshort), fill_as_missing=fill_as_missing) + arrays[dshort] = m + elseif is_zgroup(zv, s, subpath) + m = zopen_noerr(s, mode, zv, path=_concatpath(path, dshort), fill_as_missing=fill_as_missing) + groups[dshort] = m + end + end + attrs = getattrs(zv, s, path) + startswith(path,"/") && error("Paths should never start with a leading '/'") + ZGroup(s, path, arrays, groups, attrs,mode=="w") +end + +#Function to guess a Zarr format from a store and a path, useful for guessing format when trying to open a group/array +ZarrFormat(s::AbstractStore, path) = is_zarr2(s, path) ? ZarrFormat(2) : + is_zarr3(s, path) ? ZarrFormat(3) : + throw(ArgumentError("Specified store $s in path $(path) is neither a ZArray nor a ZGroup in a recognized zarr format.")) + + +""" + zopen_noerr(AbstractStore, mode = "r"; consolidated = false) + +Works like `zopen` with the single difference that no error is thrown when +the path or store does not point to a valid zarr array or group, but nothing +is returned instead. +""" +function zopen_noerr(s::AbstractStore, mode, zv::ZarrFormat; + consolidated = false, + path="", + lru = 0, + fill_as_missing=false) + + consolidated && isinitialized(s, ".zmetadata") && return zopen(ConsolidatedStore(s, path), mode, path=path, lru=lru, fill_as_missing=fill_as_missing) + if lru !== 0 + error("LRU caches are not supported anymore by the current Zarr version. Please use an earlier version of Zarr for now and open an issue at Zarr.jl if you need this functionality") + end + if is_zarray(zv, s, path) + return ZArray(s, mode, path, zv; fill_as_missing=fill_as_missing) + elseif is_zgroup(zv, s, path) + return ZGroup(s, mode, path, zv; fill_as_missing=fill_as_missing) + else + return nothing + end +end + +function Base.show(io::IO, g::ZGroup) + print(io, "ZarrGroup at ", g.storage, " and path ", g.path) + !isempty(g.arrays) && print(io, "\nVariables: ", map(i -> string(zname(i), " "), values(g.arrays))...) + !isempty(g.groups) && print(io, "\nGroups: ", map(i -> string(zname(i), " "), values(g.groups))...) + nothing +end +Base.haskey(g::ZGroup,k)= haskey(g.groups,k) || haskey(g.arrays,k) + + +function Base.getindex(g::ZGroup, k) + if haskey(g.groups, k) + return g.groups[k] + elseif haskey(g.arrays, k) + return g.arrays[k] + else + throw(KeyError("Zarr Dataset does not contain $k")) + end +end + + +""" + zopen(s::AbstractStore, mode="r"; consolidated = false, path = "", lru = 0) + +Opens a zarr Array or Group at Store `s`. If `consolidated` is set to "true", +Zarr will search for a consolidated metadata field as created by the python zarr +`consolidate_metadata` function. This can substantially speed up metadata parsing +of large zarr groups. Setting `lru` to a value > 0 means that chunks that have been +accessed before will be cached and consecutive reads will happen from the cache. +Here, `lru` denotes the number of chunks that remain in memory. The expected zarr version +can be supplied through `zarr_format` and defaults to `:auto` which tries to detect +if the zarr version is v2 or v3. +""" +function zopen(s::AbstractStore, mode="r"; + zarr_format=:auto, + consolidated = false, + path = "", + lru = 0, + fill_as_missing = false) + + zarr_format = if zarr_format == :auto + ZarrFormat(s, path) + else + ZarrFormat(zarr_format) + end + # add interfaces to Stores later + r = zopen_noerr(s, mode, zarr_format; consolidated=consolidated, path=path, lru=lru, fill_as_missing=fill_as_missing) + if r === nothing + throw(ArgumentError("Specified store $s in path $(path) is neither a ZArray nor a ZGroup")) + else + return r + end +end + +""" + zopen(p::String, mode="r") + +Open a zarr Array or group at disc path p. +""" +function zopen(s::String, mode="r"; kwargs...) + store, path = storefromstring(s,false) + zopen(store, mode; path=path, kwargs...) +end + +function storefromstring(s, create=true) + for (r,t) in storageregexlist + if match(r,s) !== nothing + return storefromstring(t,s,create) + end + end + if create || isdir(s) + return DirectoryStore(s), "" + else + throw(ArgumentError("Path $s is not a directory.")) + end +end + +""" + zgroup(s::AbstractStore; attrs=Dict()) + +Create a new zgroup in the store `s` +""" +function zgroup(s::AbstractStore, path::String="", zarr_format=ZarrFormat(2); attrs=Dict(), indent_json::Bool=false) + zv = zarr_format isa ZarrFormat ? zarr_format : ZarrFormat(zarr_format) + isemptysub(s, path) || error("Store is not empty") + write_group_metadata(zv, s, path, attrs; indent_json=indent_json) + ZGroup(s, path, Dict{String,ZArray}(), Dict{String,ZGroup}(), attrs, true) +end + +function write_group_metadata(::ZarrFormat{2}, s::AbstractStore, path, attrs; indent_json::Bool=false) + d = Dict("zarr_format" => 2) + b = IOBuffer() + indent_json ? JSON.print(b, d, 4) : JSON.print(b, d) + s[path, ".zgroup"] = take!(b) + writeattrs(ZarrFormat(Val(2)), s, path, attrs, indent_json=indent_json) +end + +function write_group_metadata(::ZarrFormat{3}, s::AbstractStore, path, attrs; indent_json::Bool=false) + d = Dict{String,Any}("zarr_format" => 3, "node_type" => "group") + if !isempty(attrs) + d["attributes"] = attrs + end + b = IOBuffer() + indent_json ? JSON.print(b, d, 4) : JSON.print(b, d) + s[path, "zarr.json"] = take!(b) +end + +zgroup(s::String;kwargs...)=zgroup(storefromstring(s, true)...;kwargs...) + +"Create a subgroup of the group g" +function zgroup(g::ZGroup, name; attrs=Dict()) + g.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) + subpath = _concatpath(g.path, name) + # Detect format from parent + zv = is_zarr3(g.storage, g.path) ? ZarrFormat(3) : ZarrFormat(2) + g.groups[name] = zgroup(g.storage, subpath, zv; attrs=attrs) +end + +"Create a new subarray of the group g" +function zcreate(::Type{T},g::ZGroup, name::AbstractString, addargs...; kwargs...) where T + g.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) + name = string(name) + z = zcreate(T, g.storage, addargs...; path = _concatpath(g.path,name), kwargs...) + g.arrays[name] = z + return z +end + +function consolidate_metadata(z::Union{ZArray,ZGroup}) + z.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) + consolidate_metadata(z.storage,z.path) +end From 7ecf6f10557070bd837c3c8f963b97150530fc48 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:11:52 +0100 Subject: [PATCH 05/12] core: add ZarrCore exports and smoke tests - Exports: ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup - Smoke tests: NoCompressor roundtrip, DirectoryStore, DictStore, V3 uncompressed roundtrip, default_compressor verification Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/ZarrCore/test/Project.toml | 7 ++++++ lib/ZarrCore/test/runtests.jl | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 lib/ZarrCore/test/Project.toml create mode 100644 lib/ZarrCore/test/runtests.jl diff --git a/lib/ZarrCore/test/Project.toml b/lib/ZarrCore/test/Project.toml new file mode 100644 index 00000000..86e1c9d3 --- /dev/null +++ b/lib/ZarrCore/test/Project.toml @@ -0,0 +1,7 @@ +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" + +[sources.ZarrCore] +path = ".." diff --git a/lib/ZarrCore/test/runtests.jl b/lib/ZarrCore/test/runtests.jl new file mode 100644 index 00000000..db6051c7 --- /dev/null +++ b/lib/ZarrCore/test/runtests.jl @@ -0,0 +1,43 @@ +using Test +using ZarrCore +using JSON + +@testset "ZarrCore" begin + @testset "NoCompressor roundtrip" begin + z = zzeros(Int64, 4, 4) + @test z isa ZArray + @test size(z) == (4, 4) + @test z[1,1] == 0 + z[2,3] = 42 + @test z[2,3] == 42 + end + + @testset "DirectoryStore" begin + p = mktempdir() + z = zcreate(Float64, 10, 10; path=p, chunks=(5,5)) + z[:] = reshape(1.0:100.0, 10, 10) + z2 = zopen(p) + @test z2[1,1] == 1.0 + @test z2[10,10] == 100.0 + end + + @testset "DictStore" begin + z = zcreate(Int32, 6, 6; chunks=(3,3)) + z[:] = ones(Int32, 6, 6) + @test z[1,1] == 1 + end + + @testset "V3 uncompressed roundtrip" begin + z = zcreate(Float32, 8, 8; + zarr_format=3, + chunks=(4,4), + compressor=ZarrCore.NoCompressor()) + z[:] = reshape(Float32.(1:64), 8, 8) + @test z[1,1] == 1.0f0 + @test z[8,8] == 64.0f0 + end + + @testset "default_compressor is NoCompressor" begin + @test ZarrCore.default_compressor() isa ZarrCore.NoCompressor + end +end From 11d6ed5e1918dba7fd90a3114c386ccf8d575ace Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:12:04 +0100 Subject: [PATCH 06/12] wrapper: rewrite Zarr.jl to depend on ZarrCore - src/Zarr.jl: imports from ZarrCore, re-exports public API, includes compressor/codec/store files, overrides default_compressor() to BloscCompressor via __init__, adds Codecs.V3Codecs wrapper module for backward-compatible module paths - src/Codecs/V3/compression_codecs.jl: GzipV3Codec, BloscV3Codec, ZstdV3Codec, CRC32cV3Codec, ShardingCodec with full infrastructure, all registering into ZarrCore.Codecs.V3Codecs.v3_codec_parsers - Compressor files: registration moved to Zarr.__init__ - Storage files: minor fixes for ZarrCore compatibility Co-Authored-By: Claude Opus 4.6 (1M context) --- src/Codecs/V3/compression_codecs.jl | 323 ++++++++++++++++++++++++++++ src/Compressors/blosc.jl | 2 +- src/Compressors/zlib.jl | 2 +- src/Compressors/zstd.jl | 2 +- src/Storage/gcstore.jl | 4 +- src/Storage/http.jl | 7 +- src/Zarr.jl | 163 ++++++++++++-- 7 files changed, 470 insertions(+), 33 deletions(-) create mode 100644 src/Codecs/V3/compression_codecs.jl diff --git a/src/Codecs/V3/compression_codecs.jl b/src/Codecs/V3/compression_codecs.jl new file mode 100644 index 00000000..40938f42 --- /dev/null +++ b/src/Codecs/V3/compression_codecs.jl @@ -0,0 +1,323 @@ +# V3 compression codecs that depend on Blosc, ChunkCodecLibZlib, ChunkCodecLibZstd, CRC32c +# These register into ZarrCore's V3 codec registry. + +using CRC32c: CRC32c +using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions +using ChunkCodecCore: encode as cc_encode, decode as cc_decode + +import ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode + +# --- CRC32cCodec (internal helper) --- + +struct CRC32cCodec +end + +function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) + hash::UInt32 = 0x00000000 + while(bytesavailable(input) > 0) + sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) + read!(input, sized_buffer) + write(output, sized_buffer) + hash = CRC32c.crc32c(sized_buffer, hash) + end + return hash +end + +function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=false) + zencode!(output, input, c) + return take!(output) +end +function zencode!(output::IO, input::IO, c::CRC32cCodec) + hash = crc32c_stream!(output, input) + write(output, hash) + return output +end +function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=true) + zdecode!(output, input, c) + return take!(output) +end +function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) + input_vec = take!(input) + truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) + hash = crc32c_stream!(output, truncated_input) + if input_vec[end-3:end] != reinterpret(UInt8, [hash]) + throw(IOError("CRC32c hash does not match")) + end + return output +end + +# --- GzipV3Codec --- + +struct GzipV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +GzipV3Codec() = GzipV3Codec(6) + +function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) + opts = GzipEncodeOptions(; level=c.level) + return cc_encode(opts, data) +end +function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) + return cc_decode(LibZGzipCodec(), encoded) +end +codec_to_dict(c::GzipV3Codec) = Dict{String,Any}( + "name" => "gzip", + "configuration" => Dict{String,Any}("level" => c.level) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- BloscV3Codec --- + +struct BloscV3Codec <: V3Codec{:bytes, :bytes} + cname::String + clevel::Int + shuffle::Int + blocksize::Int + typesize::Int +end +BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) + +function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) + comp = BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return zcompress(data, comp) +end +function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) + comp = BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return collect(zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::BloscV3Codec) = Dict{String,Any}( + "name" => "blosc", + "configuration" => Dict{String,Any}( + "cname" => c.cname, + "clevel" => c.clevel, + "shuffle" => c.shuffle == 0 ? "noshuffle" : + c.shuffle == 1 ? "shuffle" : + c.shuffle == 2 ? "bitshuffle" : + throw(ArgumentError("Unknown shuffle integer: $(c.shuffle)")), + "blocksize" => c.blocksize, + "typesize" => c.typesize + ) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- ZstdV3Codec --- + +struct ZstdV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +ZstdV3Codec() = ZstdV3Codec(3) + +function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) + comp = ZstdCompressor(level=c.level) + return zcompress(data, comp) +end +function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) + comp = ZstdCompressor(level=c.level) + return collect(zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::ZstdV3Codec) = Dict{String,Any}( + "name" => "zstd", + "configuration" => Dict{String,Any}("level" => c.level) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- CRC32cV3Codec --- + +struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} +end + +function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) + out = UInt8[] + return zencode!(out, data, CRC32cCodec()) +end +function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) + out = UInt8[] + return zdecode!(out, encoded, CRC32cCodec()) +end +codec_to_dict(::CRC32cV3Codec) = Dict{String,Any}("name" => "crc32c") + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- ShardingCodec --- + +""" + ShardingCodec{N} + +Sharding codec for Zarr v3. Sharding splits chunks into smaller "shards" and stores them +in a single file with an index mapping chunk coordinates to shard locations. + +# Fields +- `chunk_shape`: Shape of each shard (NTuple{N,Int}) +- `codecs`: Vector of codecs to apply to shard data (e.g., [BytesCodec(), GzipCodec()]) +- `index_codecs`: Vector of codecs to apply to the index (e.g., [BytesCodec()]) +- `index_location`: Location of index in shard file, either `:start` or `:end` +""" +struct ShardingCodec{N} <: V3Codec{:array, :bytes} + chunk_shape::NTuple{N,Int} + codecs::Vector{V3Codec} + index_codecs::Vector{V3Codec} + index_location::Symbol +end + +const MAX_UINT64 = typemax(UInt64) + +""" + ChunkShardInfo + +Information about a chunk's location within a shard. +""" +struct ChunkShardInfo + offset::UInt64 + nbytes::UInt64 +end + +ChunkShardInfo() = ChunkShardInfo(MAX_UINT64, MAX_UINT64) + +""" + ShardIndex{N} + +Internal structure representing the shard index. +Stores chunk location info for an N-dimensional grid of chunks. +Empty chunks are marked with ChunkShardInfo(MAX_UINT64, MAX_UINT64) +""" +struct ShardIndex{N} + chunks::Array{ChunkShardInfo, N} +end + +function ShardIndex(chunks_per_shard::NTuple{N,Int}) where N + chunks = fill(ChunkShardInfo(), chunks_per_shard) + return ShardIndex{N}(chunks) +end + +function get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N + info = idx.chunks[chunk_coords...] + if info.offset == MAX_UINT64 && info.nbytes == MAX_UINT64 + return nothing + end + return (Int(info.offset), Int(info.offset + info.nbytes)) +end + +function set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) where N + idx.chunks[chunk_coords...] = ChunkShardInfo(UInt64(offset), UInt64(nbytes)) +end + +function set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N + idx.chunks[chunk_coords...] = ChunkShardInfo() +end + +function calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) where N + return ntuple(i -> div(shard_shape[i], chunk_shape[i]), N) +end + +function get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) where N + return ntuple(N) do i + start_idx = (chunk_coords[i] - 1) * chunk_shape[i] + 1 + end_idx = min(chunk_coords[i] * chunk_shape[i], shard_shape[i]) + start_idx:end_idx + end +end + +function apply_codec_chain(data, codecs::Vector{V3Codec}) + result = data + for codec in codecs + result = zencode(result, codec) + end + return result +end + +function reverse_codec_chain(data, codecs::Vector{V3Codec}) + result = data + for codec in reverse(codecs) + result = zdecode(result, codec) + end + return result +end + +function encode_shard_index(index::ShardIndex{N}, index_codecs::Vector{V3Codec}) where N + n_chunks = length(index.chunks) + index_data = Vector{UInt64}(undef, 2 * n_chunks) + idx = 1 + for cart_idx in CartesianIndices(index.chunks) + info = index.chunks[cart_idx] + index_data[idx] = info.offset + index_data[idx + 1] = info.nbytes + idx += 2 + end + index_bytes = reinterpret(UInt8, index_data) + encoded = apply_codec_chain(index_bytes, index_codecs) + return encoded +end + +function decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N + decoded_bytes = reverse_codec_chain(index_bytes, index_codecs) + n_chunks = prod(chunks_per_shard) + expected_length = n_chunks * 2 * sizeof(UInt64) + if length(decoded_bytes) != expected_length + throw(DimensionMismatch("Index size mismatch: expected $expected_length, got $(length(decoded_bytes))")) + end + index_data = reinterpret(UInt64, decoded_bytes) + chunks = Array{ChunkShardInfo, N}(undef, chunks_per_shard) + idx = 1 + for cart_idx in CartesianIndices(chunks) + offset = index_data[idx] + nbytes = index_data[idx + 1] + chunks[cart_idx] = ChunkShardInfo(offset, nbytes) + idx += 2 + end + return ShardIndex{N}(chunks) +end + +function compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N + index = ShardIndex(chunks_per_shard) + encoded = encode_shard_index(index, index_codecs) + return length(encoded) +end + +# Sharding codec parser registration moved to _register_v3_codec_parsers!() + +""" + _register_v3_codec_parsers!() + +Register all V3 codec parsers into ZarrCore's registry. Called from Zarr.__init__(). +""" +function _register_v3_codec_parsers!() + v3_codec_parsers["gzip"] = function(config) + level = get(config, "level", 6) + GzipV3Codec(level) + end + + v3_codec_parsers["blosc"] = function(config) + cname = get(config, "cname", "lz4") + clevel = get(config, "clevel", 5) + shuffle_val = get(config, "shuffle", "noshuffle") + shuffle_int = shuffle_val isa Integer ? shuffle_val : + shuffle_val == "noshuffle" ? 0 : + shuffle_val == "shuffle" ? 1 : + shuffle_val == "bitshuffle" ? 2 : + throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) + blocksize = get(config, "blocksize", 0) + typesize = get(config, "typesize", 4) + BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize) + end + + v3_codec_parsers["zstd"] = function(config) + level = get(config, "level", 3) + ZstdV3Codec(level) + end + + v3_codec_parsers["crc32c"] = function(config) + CRC32cV3Codec() + end + + v3_codec_parsers["sharding_indexed"] = function(config) + throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) + end +end diff --git a/src/Compressors/blosc.jl b/src/Compressors/blosc.jl index 789a298f..efe3c770 100644 --- a/src/Compressors/blosc.jl +++ b/src/Compressors/blosc.jl @@ -67,4 +67,4 @@ end JSON.lower(c::BloscCompressor) = Dict("id"=>"blosc", "cname"=>c.cname, "clevel"=>c.clevel, "shuffle"=>c.shuffle, "blocksize"=>c.blocksize) -Zarr.compressortypes["blosc"] = BloscCompressor \ No newline at end of file +# Registration moved to Zarr.__init__ \ No newline at end of file diff --git a/src/Compressors/zlib.jl b/src/Compressors/zlib.jl index 6b82feef..5977ff48 100644 --- a/src/Compressors/zlib.jl +++ b/src/Compressors/zlib.jl @@ -42,4 +42,4 @@ end JSON.lower(z::ZlibCompressor) = Dict("id"=>"zlib", "level" => z.config.level) -Zarr.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file +# Registration moved to Zarr.__init__ \ No newline at end of file diff --git a/src/Compressors/zstd.jl b/src/Compressors/zstd.jl index 6cd80a08..7b6e7c9d 100644 --- a/src/Compressors/zstd.jl +++ b/src/Compressors/zstd.jl @@ -52,4 +52,4 @@ function JSON.lower(z::ZstdCompressor) end end -Zarr.compressortypes["zstd"] = ZstdCompressor +# Registration moved to Zarr.__init__ diff --git a/src/Storage/gcstore.jl b/src/Storage/gcstore.jl index 02a2051c..cf2d320d 100644 --- a/src/Storage/gcstore.jl +++ b/src/Storage/gcstore.jl @@ -130,9 +130,7 @@ function subdirs(s::GCStore, p) return dirs end -pushfirst!(storageregexlist,r"^https://storage.googleapis.com"=>GCStore) -pushfirst!(storageregexlist,r"^http://storage.googleapis.com"=>GCStore) -push!(storageregexlist,r"^gs://"=>GCStore) +# Storage regex registration moved to Zarr.__init__ function storefromstring(::Type{<:GCStore}, url,_) uri = URI(url) diff --git a/src/Storage/http.jl b/src/Storage/http.jl index 980284f2..7286531b 100644 --- a/src/Storage/http.jl +++ b/src/Storage/http.jl @@ -37,18 +37,13 @@ end end -push!(storageregexlist,r"^https://"=>HTTPStore) -push!(storageregexlist,r"^http://"=>HTTPStore) +# Storage regex registration moved to Zarr.__init__ function storefromstring(::Type{<:HTTPStore}, s,_) http_store = HTTPStore(s) try if http_store["", ".zmetadata"] !== nothing http_store = ConsolidatedStore(http_store,"") end - if is_zarray(http_store, "") - meta = getmetadata(http_store, "", false) - http_store = FormattedStore{meta.zarr_format, meta.dimension_separator}(http_store) - end catch err @warn exception=err "Additional metadata was not available for HTTPStore." end diff --git a/src/Zarr.jl b/src/Zarr.jl index 76fc25bf..8639d114 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -1,30 +1,151 @@ module Zarr +using ZarrCore import JSON import Blosc -struct ZarrFormat{V} - version::Val{V} -end -Base.Int(v::ZarrFormat{V}) where V = V -@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) -ZarrFormat(v::ZarrFormat) = v -#Default Zarr Version -const DV = ZarrFormat(Val(2)) - -include("chunkkeyencoding.jl") -abstract type AbstractCodecPipeline end -include("metadata.jl") -include("metadata3.jl") -include("Compressors/Compressors.jl") -include("Codecs/Codecs.jl") -include("Storage/Storage.jl") -include("Filters/Filters.jl") -include("ZArray.jl") -include("pipeline.jl") -include("ZGroup.jl") +# Import names that will be extended by included files (import allows method extension) +import ZarrCore: getCompressor, zuncompress, zuncompress!, zcompress, zcompress!, + storagesize, storefromstring, store_read_strategy, + subdirs, subkeys, isinitialized, + consolidate_metadata, compressor_to_v3_bytes_codecs +# Import types, non-extended functions, and constants via using +using ZarrCore: ZarrCore, + # Types + ZArray, ZGroup, AbstractStore, DirectoryStore, DictStore, ConsolidatedStore, + AbstractMetadata, MetadataV2, MetadataV3, + AbstractCodecPipeline, V2Pipeline, V3Pipeline, + Compressor, NoCompressor, + Filter, + ZarrFormat, + AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + ASCIIChar, + # Functions (not extended) + zcreate, zopen, zzeros, zgroup, + storageratio, zinfo, zname, + pipeline_encode, pipeline_decode!, get_pipeline, get_order, + Metadata, Metadata3, + typestr, typestr3, + fill_value_encoding, fill_value_decoding, + compressortypes, default_compressor, + getfilters, filterdict, + zencode, zdecode, + citostring, + default_sep, default_prefix, + storageregexlist, + chunkindices, + _reinterpret, + # Storage functions (not extended) + SequentialRead, ConcurrentRead, + concurrent_io_tasks, + is_zarray, is_zgroup, is_zarr2, is_zarr3, + getmetadata, writemetadata, getattrs, writeattrs, + isemptysub, _concatpath, + store_readchunk, store_writechunk, store_deletechunk, store_isinitialized, + read_items!, write_items!, channelsize, maybecopy, + # Storage utility functions + normalize_path, + # Filter types + VLenArrayFilter, VLenUTF8Filter, + Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, + QuantizeFilter, DeltaFilter, + # DateTime support + DateTime64, + # Constants + DV, DS, DS2, DS3 + +# Re-export the same symbols as before export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, - zinfo, DirectoryStore, S3Store, GCStore, zgroup + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +# Bring in the MaxLengthStrings submodule +using ZarrCore: MaxLengthStrings +using .MaxLengthStrings: MaxLengthString + +# Include compressor implementations (they register into ZarrCore.compressortypes) +include("Compressors/blosc.jl") +include("Compressors/zlib.jl") +include("Compressors/zstd.jl") + +# Override default compressor and register compressors/codecs/stores at module init time +# (Dict/Array mutations during precompile don't persist, so they must happen in __init__) +function __init__() + ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> BloscCompressor() + + # Register compressor types + compressortypes["blosc"] = BloscCompressor + compressortypes["zlib"] = ZlibCompressor + compressortypes["zstd"] = ZstdCompressor + + # Register V3 codec parsers + _register_v3_codec_parsers!() + + # Register storage URL resolvers + _register_storage_regexes!() +end + +function _register_storage_regexes!() + # GCStore regexes (must be first to match before generic HTTP) + pushfirst!(storageregexlist, r"^https://storage.googleapis.com" => GCStore) + pushfirst!(storageregexlist, r"^http://storage.googleapis.com" => GCStore) + push!(storageregexlist, r"^gs://" => GCStore) + # HTTPStore regexes + push!(storageregexlist, r"^https://" => HTTPStore) + push!(storageregexlist, r"^http://" => HTTPStore) + # S3Store regex + push!(storageregexlist, r"^s3://" => S3Store) +end + +# Include V3 compression codec implementations +include("Codecs/V3/compression_codecs.jl") + +# Now define the compressor-to-v3 codec mappings +function compressor_to_v3_bytes_codecs(c::BloscCompressor) + (BloscV3Codec(c.cname, c.clevel, c.shuffle, c.blocksize, sizeof(UInt8)),) +end +function compressor_to_v3_bytes_codecs(c::ZlibCompressor) + level = c.config.level == -1 ? 6 : c.config.level + (GzipV3Codec(level),) +end +function compressor_to_v3_bytes_codecs(c::ZstdCompressor) + (ZstdV3Codec(c.config.compressionLevel),) +end + +# Create a Codecs module wrapper so that Zarr.Codecs.V3Codecs.XXX paths work for tests. +# This re-exports everything from ZarrCore.Codecs.V3Codecs plus the compression codecs +# defined in Zarr. +module Codecs + module V3Codecs + # Re-export everything from ZarrCore.Codecs.V3Codecs + using ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode, codec_category, encoded_shape, + BytesCodec, TransposeCodec + # Re-export the compression codecs defined in Zarr + using ...Zarr: BloscV3Codec, GzipV3Codec, ZstdV3Codec, CRC32cV3Codec, ShardingCodec + end +end + +# Include network/archive storage backends +include("Storage/gcstore.jl") +include("Storage/http.jl") +include("Storage/zipstore.jl") + +# Register S3Store stub +struct S3Store <: AbstractStore + bucket::String + aws::Any +end +function S3Store(args...) + error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") +end + +# Store URL resolvers registered in __init__ + +# HTTP.serve and writezip for ZArray/ZGroup +HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) +writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) + +export S3Store, GCStore end # module From d822a7af8729953ad2746654b3d96e6b652c088e Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:12:24 +0100 Subject: [PATCH 07/12] deps: add ZarrCore dependency, remove unused DataStructures - Add ZarrCore as path dependency via [sources.ZarrCore] - Remove DataStructures from [deps] and [compat] (unused) - Add ZarrCore to test/Project.toml with source path Co-Authored-By: Claude Opus 4.6 (1M context) --- Project.toml | 6 ++++-- test/Project.toml | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index cfe36f77..4a1554a9 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" @@ -19,8 +18,12 @@ OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +[sources.ZarrCore] +path = "lib/ZarrCore" + [weakdeps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -34,7 +37,6 @@ ChunkCodecCore = "1" ChunkCodecLibZlib = "1" ChunkCodecLibZstd = "1" CRC32c = "1.10, 1.11" -DataStructures = "0.17, 0.18, 0.19" DateTimes64 = "1" DiskArrays = "0.4.2" HTTP = "^1.3" diff --git a/test/Project.toml b/test/Project.toml index ff8f8c15..51cf9837 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -11,7 +11,11 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" [sources] CondaPkg = {rev = "manifest-check", url = "https://github.com/JamesWrigley/CondaPkg.jl"} Zarr = {path = ".."} + +[sources.ZarrCore] +path = "../lib/ZarrCore" From ec63c2d36b96157d3f61179a491fac5a807a009f Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:12:32 +0100 Subject: [PATCH 08/12] cleanup: remove files migrated to ZarrCore Remove 21 source files from src/ that now live in lib/ZarrCore/src/. Zarr.jl no longer includes these directly. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/Codecs/Codecs.jl | 49 --- src/Codecs/V3/V3.jl | 650 -------------------------------- src/Compressors/Compressors.jl | 109 ------ src/Filters/Filters.jl | 95 ----- src/Filters/delta.jl | 45 --- src/Filters/fixedscaleoffset.jl | 52 --- src/Filters/fletcher32.jl | 85 ----- src/Filters/quantize.jl | 52 --- src/Filters/shuffle.jl | 70 ---- src/Filters/vlenfilters.jl | 82 ---- src/MaxLengthStrings.jl | 74 ---- src/Storage/Storage.jl | 291 -------------- src/Storage/consolidated.jl | 101 ----- src/Storage/dictstore.jl | 54 --- src/Storage/directorystore.jl | 59 --- src/ZArray.jl | 507 ------------------------- src/ZGroup.jl | 206 ---------- src/chunkkeyencoding.jl | 137 ------- src/metadata.jl | 293 -------------- src/metadata3.jl | 456 ---------------------- src/pipeline.jl | 77 ---- 21 files changed, 3544 deletions(-) delete mode 100644 src/Codecs/Codecs.jl delete mode 100644 src/Codecs/V3/V3.jl delete mode 100644 src/Compressors/Compressors.jl delete mode 100644 src/Filters/Filters.jl delete mode 100644 src/Filters/delta.jl delete mode 100644 src/Filters/fixedscaleoffset.jl delete mode 100644 src/Filters/fletcher32.jl delete mode 100644 src/Filters/quantize.jl delete mode 100644 src/Filters/shuffle.jl delete mode 100644 src/Filters/vlenfilters.jl delete mode 100644 src/MaxLengthStrings.jl delete mode 100644 src/Storage/Storage.jl delete mode 100644 src/Storage/consolidated.jl delete mode 100644 src/Storage/dictstore.jl delete mode 100644 src/Storage/directorystore.jl delete mode 100644 src/ZArray.jl delete mode 100644 src/ZGroup.jl delete mode 100644 src/chunkkeyencoding.jl delete mode 100644 src/metadata.jl delete mode 100644 src/metadata3.jl delete mode 100644 src/pipeline.jl diff --git a/src/Codecs/Codecs.jl b/src/Codecs/Codecs.jl deleted file mode 100644 index ec6e6205..00000000 --- a/src/Codecs/Codecs.jl +++ /dev/null @@ -1,49 +0,0 @@ -module Codecs - -using JSON: JSON - -""" - abstract type Codec - -The abstract supertype for all Zarr codecs - -## Interface - -All subtypes of `Codec` SHALL implement the following methods: - -- `zencode(a, c::Codec)`: compress the array `a` using the codec `c`. -- `zdecode(a, c::Codec, T)`: decode the array `a` using the codec `c` - and return an array of type `T`. -- `JSON.lower(c::Codec)`: return a JSON representation of the codec `c`, which - follows the Zarr specification for that codec. -- `getCodec(::Type{<:Codec}, d::Dict)`: return a codec object from a given - dictionary `d` which contains the codec's parameters according to the Zarr spec. - -Subtypes of `Codec` MAY also implement the following methods: - -- `zencode!(encoded, data, c::Codec)`: encode the array `data` using the - codec `c` and store the result in the array `encoded`. -- `zdecode!(data, encoded, c::Codec)`: decode the array `encoded` - using the codec `c` and store the result in the array `data`. - -Finally, an entry MUST be added to the `VN.codectypes` dictionary for each codec type where N is the -Zarr format version. -This must also follow the Zarr specification's name for that compressor. The name of the compressor -is the key, and the value is the compressor type (e.g. `BloscCodec` or `NoCodec`). - -For example, the Blosc codec is named "blosc" in the Zarr spec, so the entry for [`BloscCodec`](@ref) -must be added to `codectypes` as `codectypes["blosc"] = BloscCodec`. -""" - -abstract type Codec end - -zencode(a, c::Codec) = error("Unimplemented") -zencode!(encoded, data, c::Codec) = error("Unimplemented") -zdecode(a, c::Codec, T::Type) = error("Unimplemented") -zdecode!(data, encoded, c::Codec) = error("Unimplemented") -JSON.lower(c::Codec) = error("Unimplemented") -getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") - -include("V3/V3.jl") - -end diff --git a/src/Codecs/V3/V3.jl b/src/Codecs/V3/V3.jl deleted file mode 100644 index 5c27690d..00000000 --- a/src/Codecs/V3/V3.jl +++ /dev/null @@ -1,650 +0,0 @@ -module V3Codecs - -import ..Codecs: zencode, zdecode, zencode!, zdecode! -# Import compressor types and functions from Zarr (grandparent module) -import ...Zarr: ZlibCompressor, ZstdCompressor, zcompress, zuncompress -import ...Zarr: BloscCompressor as ZarrBloscCompressor -using CRC32c: CRC32c -using JSON: JSON -using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions -using ChunkCodecCore: encode as cc_encode, decode as cc_decode - -abstract type V3Codec{In,Out} end -const codectypes = Dict{String, V3Codec}() - -@enum BloscCompressor begin - lz4 - lz4hc - blosclz - zstd - snappy - zlib -end - -@enum BloscShuffle begin - noshuffle - shuffle - bitshuffle -end - -struct BloscCodec <: V3Codec{:bytes, :bytes} - cname::BloscCompressor - clevel::Int64 - shuffle::BloscShuffle - typesize::UInt8 - blocksize::UInt -end -name(::BloscCodec) = "blosc" - -struct BytesCodec <: V3Codec{:array, :bytes} - endian::Symbol # :little or :big - function BytesCodec(endian::Symbol) - endian ∈ (:little, :big) || - throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) - new(endian) - end -end -BytesCodec() = BytesCodec(:little) -name(::BytesCodec) = "bytes" - -const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 -_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN - -struct CRC32cCodec <: V3Codec{:bytes, :bytes} -end -name(::CRC32cCodec) = "crc32c" - -struct GzipCodec <: V3Codec{:bytes, :bytes} -end -name(::GzipCodec) = "gzip" - - -#= -zencode(a, c::Codec) = error("Unimplemented") -zencode!(encoded, data, c::Codec) = error("Unimplemented") -zdecode(a, c::Codec, T::Type) = error("Unimplemented") -zdecode!(data, encoded, c::Codec) = error("Unimplemented") -=# - -function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) - hash::UInt32 = 0x00000000 - while(bytesavailable(input) > 0) - sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) - read!(input, sized_buffer) - write(output, sized_buffer) - hash = CRC32c.crc32c(sized_buffer, hash) - end - return hash -end -function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) - output = IOBuffer(encoded, read=false, write=true) - input = IOBuffer(data, read=true, write=false) - zencode!(output, input, c) - return take!(output) -end -function zencode!(output::IO, input::IO, c::CRC32cCodec) - hash = crc32c_stream!(output, input) - write(output, hash) - return output -end -function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) - output = IOBuffer(encoded, read=false, write=true) - input = IOBuffer(data, read=true, write=true) - zdecode!(output, input, c) - return take!(output) -end -function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) - input_vec = take!(input) - truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) - hash = crc32c_stream!(output, truncated_input) - if input_vec[end-3:end] != reinterpret(UInt8, [hash]) - throw(IOError("CRC32c hash does not match")) - end - return output -end - -""" - ShardingCodec{N} - -Sharding codec for Zarr v3. Sharding splits chunks into smaller "shards" and stores them -in a single file with an index mapping chunk coordinates to shard locations. - -# Fields -- `chunk_shape`: Shape of each shard (NTuple{N,Int}) -- `codecs`: Vector of codecs to apply to shard data (e.g., [BytesCodec(), GzipCodec()]) -- `index_codecs`: Vector of codecs to apply to the index (e.g., [BytesCodec()]) -- `index_location`: Location of index in shard file, either `:start` or `:end` - -# Implementation Notes -Sharding works by: -1. Taking a chunk of data and splitting it into shards based on `chunk_shape` -2. Encoding each shard using the `codecs` pipeline -3. Creating an index that maps (chunk_coords, shard_coords) -> (offset, size) in the shard file -4. Encoding the index using `index_codecs` -5. Writing the shard file with index at `index_location` (start or end) - -""" -struct ShardingCodec{N} <: V3Codec{:array, :bytes} - chunk_shape::NTuple{N,Int} # Shape of each shard - codecs::Vector{V3Codec} # Codecs to apply to shard data - index_codecs::Vector{V3Codec} # Codecs to apply to the index - index_location::Symbol # :start or :end -end -name(::ShardingCodec) = "sharding_indexed" - -""" - JSON.lower(c::ShardingCodec) - -Serialize ShardingCodec to JSON format for Zarr v3 metadata. -""" -function JSON.lower(c::ShardingCodec) - return Dict( - "name" => "sharding_indexed", - "configuration" => Dict( - "chunk_shape" => collect(c.chunk_shape), - "codecs" => [JSON.lower(codec) for codec in c.codecs], - "index_codecs" => [JSON.lower(codec) for codec in c.index_codecs], - "index_location" => string(c.index_location) - ) - ) -end - -""" - getCodec(::Type{ShardingCodec}, d::Dict) - -Deserialize ShardingCodec from JSON configuration dict. -""" -function getCodec(::Type{ShardingCodec}, d::Dict) - config = d["configuration"] - N = length(config["chunk_shape"]) - chunk_shape = NTuple{N,Int}(config["chunk_shape"]) - codecs = [getCodec(codec_dict) for codec_dict in config["codecs"]] - index_codecs = [getCodec(codec_dict) for codec_dict in config["index_codecs"]] - index_location = Symbol(get(config, "index_location", "end")) - return ShardingCodec{N}(chunk_shape, codecs, index_codecs, index_location) -end - -const MAX_UINT64 = typemax(UInt64) - -""" - ChunkShardInfo - -Information about a chunk's location within a shard. -""" -struct ChunkShardInfo - offset::UInt64 # Byte offset within shard where chunk begins - nbytes::UInt64 # Number of bytes the chunk occupies -end - -ChunkShardInfo() = ChunkShardInfo(MAX_UINT64, MAX_UINT64) # Empty chunk marker - -""" - ShardIndex{N} - -Internal structure representing the shard index. -Stores chunk location info for an N-dimensional grid of chunks. -Empty chunks are marked with ChunkShardInfo(MAX_UINT64, MAX_UINT64) -""" -struct ShardIndex{N} - chunks::Array{ChunkShardInfo, N} # N-dimensional array of chunk info -end - -""" - ShardIndex(chunks_per_shard::NTuple{N,Int}) - -Create an empty shard index with all chunks marked as empty. -""" -function ShardIndex(chunks_per_shard::NTuple{N,Int}) where N - chunks = fill(ChunkShardInfo(), chunks_per_shard) - return ShardIndex{N}(chunks) -end - -""" - get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) - -Get the byte range (offset, offset+nbytes) for a chunk, or nothing if empty. -""" -function get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N - info = idx.chunks[chunk_coords...] - - if info.offset == MAX_UINT64 && info.nbytes == MAX_UINT64 - return nothing - end - - return (Int(info.offset), Int(info.offset + info.nbytes)) -end - -""" - set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) - -Set the byte range for a chunk in the index. -""" -function set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) where N - idx.chunks[chunk_coords...] = ChunkShardInfo(UInt64(offset), UInt64(nbytes)) -end - -""" - set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) - -Mark a chunk as empty in the index. -""" -function set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N - idx.chunks[chunk_coords...] = ChunkShardInfo() -end - -""" - calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) - -Calculate how many chunks fit in each shard dimension. -""" -function calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) where N - return ntuple(i -> div(shard_shape[i], chunk_shape[i]), N) -end - -""" - get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) - -Get the array slice ranges for a chunk within a shard. -chunk_coords are 1-based indices. -""" -function get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) where N - return ntuple(N) do i - start_idx = (chunk_coords[i] - 1) * chunk_shape[i] + 1 - end_idx = min(chunk_coords[i] * chunk_shape[i], shard_shape[i]) - start_idx:end_idx - end -end - -""" - apply_codec_chain(data, codecs::Vector{V3Codec}) - -Apply codec pipeline in forward order (encoding). -""" -function apply_codec_chain(data, codecs::Vector{V3Codec}) - result = data - for codec in codecs - result = zencode(result, codec) - end - return result -end - -""" - reverse_codec_chain(data, codecs::Vector{V3Codec}) - -Apply codec pipeline in reverse order (decoding). -""" -function reverse_codec_chain(data, codecs::Vector{V3Codec}) - result = data - for codec in reverse(codecs) - result = zdecode(result, codec) - end - return result -end - -""" - encode_shard_index(index::ShardIndex, index_codecs::Vector{V3Codec}) - -Encode the shard index using the index codec pipeline. - -Per Zarr v3 spec, the index is linearized in C-order (row-major) with alternating -offset/nbytes values: [chunk_0_offset, chunk_0_nbytes, chunk_1_offset, chunk_1_nbytes, ...] -``` -""" -function encode_shard_index(index::ShardIndex{N}, index_codecs::Vector{V3Codec}) where N - # Pre-allocate buffer for index data - n_chunks = length(index.chunks) - index_data = Vector{UInt64}(undef, 2 * n_chunks) - - # Iterate in C-order (row-major) and interleave offset/nbytes - idx = 1 - for cart_idx in CartesianIndices(index.chunks) - info = index.chunks[cart_idx] - index_data[idx] = info.offset - index_data[idx + 1] = info.nbytes - idx += 2 - end - - # Convert to bytes - index_bytes = reinterpret(UInt8, index_data) - - # Apply index codecs - encoded = apply_codec_chain(index_bytes, index_codecs) - - return encoded -end - -""" - decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) - -Decode the shard index from bytes. - -The bytes are in C-order with alternating offset/nbytes: -[offset0, nbytes0, offset1, nbytes1, ...] -""" -function decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N - # Decode using index codecs (in reverse order) - decoded_bytes = reverse_codec_chain(index_bytes, index_codecs) - - # Expected size: 16 bytes (2 * UInt64) per chunk - n_chunks = prod(chunks_per_shard) - expected_length = n_chunks * 2 * sizeof(UInt64) - - if length(decoded_bytes) != expected_length - throw(DimensionMismatch("Index size mismatch: expected $expected_length, got $(length(decoded_bytes))")) - end - - # Reinterpret as UInt64 array: [offset1, nbytes1, offset1, nbytes1, ...] - index_data = reinterpret(UInt64, decoded_bytes) - - # Reconstruct the N-dimensional array of ChunkShardInfo - chunks = Array{ChunkShardInfo, N}(undef, chunks_per_shard) - - idx = 1 - for cart_idx in CartesianIndices(chunks) - offset = index_data[idx] - nbytes = index_data[idx + 1] - chunks[cart_idx] = ChunkShardInfo(offset, nbytes) - idx += 2 - end - - return ShardIndex{N}(chunks) -end - -""" - compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) - -Compute the byte size of the encoded shard index. -Per spec: "The size of the index can be determined by applying c.compute_encoded_size -for each index codec recursively. The initial size is the byte size of the index array, -i.e. 16 * chunks per shard." -""" -function compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N - # Initial size: 16 bytes per chunk (2 * UInt64) - n_chunks = prod(chunks_per_shard) - size = n_chunks * 16 - - # Apply each codec's size transformation - # For most codecs, we need to actually encode to know the size - # For simplicity, we encode an empty index - index = ShardIndex(chunks_per_shard) - encoded = encode_shard_index(index, index_codecs) - - return length(encoded) -end - -""" - zencode!(encoded::Vector{UInt8}, data::AbstractArray, c::ShardingCodec) - -Encode array data using sharding codec following Zarr v3 spec. - -Per spec: "In the sharding_indexed binary format, inner chunks are written successively -in a shard, where unused space between them is allowed, followed by an index referencing them." -""" -function zencode!(encoded::Vector{UInt8}, data::AbstractArray, c::ShardingCodec{N}) where N - shard_shape = size(data) - chunks_per_shard = calculate_chunks_per_shard(shard_shape, c.chunk_shape) - - # Create empty index - index = ShardIndex(chunks_per_shard) - - # Buffers for encoded chunks - chunk_buffers = Vector{UInt8}[] - current_offset = 0 - - # Process chunks in C order (row-major) - # Per spec: "The actual order of the chunk content is not fixed" - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - - # Extract chunk data from shard - slice_ranges = get_chunk_slice_in_shard(chunk_coords, c.chunk_shape, shard_shape) - chunk_data = data[slice_ranges...] - - # Encode chunk using codec pipeline - encoded_chunk = apply_codec_chain(chunk_data, c.codecs) - - # Skip if chunk is empty (no bytes) - if isempty(encoded_chunk) - set_chunk_empty!(index, chunk_coords) - continue - end - - nbytes = length(encoded_chunk) - - # Record offset and length in index - set_chunk_slice!(index, chunk_coords, current_offset, nbytes) - - push!(chunk_buffers, encoded_chunk) - current_offset += nbytes - end - - # Encode the index - encoded_index = encode_shard_index(index, c.index_codecs) - index_size = length(encoded_index) - - # If index is at start, adjust all offsets to account for index size - if c.index_location == :start - # Add index_size to all non-empty chunk offsets - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - info = index.chunks[cart_idx] - if info.offset != MAX_UINT64 - index.chunks[cart_idx] = ChunkShardInfo(info.offset + index_size, info.nbytes) - end - end - # Re-encode index with corrected offsets - encoded_index = encode_shard_index(index, c.index_codecs) - end - - # If all chunks are empty, return empty buffer (no shard) - if isempty(chunk_buffers) - resize!(encoded, 0) - return encoded - end - - # Assemble final shard: [index] + chunks or chunks + [index] - total_size = (c.index_location == :start ? index_size : 0) + - current_offset + - (c.index_location == :end ? index_size : 0) - - resize!(encoded, total_size) - output = IOBuffer(encoded, write=true) - - if c.index_location == :start - write(output, encoded_index) - for buf in chunk_buffers - write(output, buf) - end - else # :end - for buf in chunk_buffers - write(output, buf) - end - write(output, encoded_index) - end - - return encoded -end - -""" - zdecode!(data::AbstractArray, encoded::Vector{UInt8}, c::ShardingCodec) - -Decode sharded data back to array following Zarr v3 spec. - -Per spec: "A simple implementation to decode inner chunks in a shard would -(a) read the entire value from the store into a byte buffer, -(b) parse the shard index from the beginning or end of the buffer and -(c) cut out the relevant bytes that belong to the requested chunk." -""" -function zdecode!(data::AbstractArray, encoded::Vector{UInt8}, c::ShardingCodec{N}) where N - # Handle empty shard (no data) - if isempty(encoded) - fill!(data, zero(eltype(data))) # Fill with zeros (or should use fill_value from spec) - return data - end - - shard_shape = size(data) - chunks_per_shard = calculate_chunks_per_shard(shard_shape, c.chunk_shape) - - # Compute encoded index size - index_size = compute_encoded_index_size(chunks_per_shard, c.index_codecs) - - # Extract index bytes based on location - if c.index_location == :start - index_bytes = encoded[1:index_size] - chunk_data_offset = index_size - else # :end - index_bytes = encoded[end-index_size+1:end] - chunk_data_offset = 0 - end - - # Decode the index - index = decode_shard_index(index_bytes, chunks_per_shard, c.index_codecs) - - # Decode each chunk and place into output array - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - - # Get chunk byte range from index - chunk_slice = get_chunk_slice(index, chunk_coords) - - # Get array slice for this chunk - array_slice = get_chunk_slice_in_shard(chunk_coords, c.chunk_shape, shard_shape) - - if chunk_slice === nothing - # Empty chunk - fill with zeros (or fill_value) - # Per spec: "Empty inner chunks are interpreted as being filled with the fill value" - data[array_slice...] .= zero(eltype(data)) - continue - end - - # Extract chunk bytes - # Offsets in index are relative to start of chunk data - offset_start, offset_end = chunk_slice - - # Adjust for where chunk data begins in the shard - byte_start = chunk_data_offset + offset_start + 1 # Julia 1-based indexing - byte_end = chunk_data_offset + offset_end - - encoded_chunk = encoded[byte_start:byte_end] - - # Decode chunk using codec pipeline (in reverse) - decoded_chunk = reverse_codec_chain(encoded_chunk, c.codecs) - - # Place decoded chunk into output array - expected_shape = length.(array_slice) - data[array_slice...] = reshape(decoded_chunk, expected_shape) - end - - return data -end - -struct TransposeCodec{N} <: V3Codec{:array, :array} - order::NTuple{N, Int} # permutation (1-based Julia indexing) -end -name(::TransposeCodec) = "transpose" - -# codec_encode / codec_decode methods for V3 codecs - -# --- BytesCodec (array -> bytes) --- - -function codec_encode(c::BytesCodec, data::AbstractArray) - if _needs_bswap(c.endian) - return reinterpret(UInt8, bswap.(vec(data))) |> collect - else - return reinterpret(UInt8, vec(data)) |> collect - end -end - -function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} - arr = collect(reinterpret(T, encoded)) - if _needs_bswap(c.endian) - arr = bswap.(arr) - end - return reshape(arr, shape) -end - -# --- TransposeCodec (array -> array) --- - -"""Return the shape of the output of `codec_encode(codec, data)` given the input shape.""" -encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz -encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) - -function codec_encode(c::TransposeCodec, data::AbstractArray) - return permutedims(data, c.order) -end - -function codec_decode(c::TransposeCodec, encoded::AbstractArray) - inv_order = Tuple(invperm(collect(c.order))) - return permutedims(encoded, inv_order) -end - -# --- Wrapper codecs (bytes -> bytes) that delegate to Zarr compressors --- -# These use fully-qualified names to avoid conflicts with the BloscCompressor -# enum and other names already defined in this module. - -struct GzipV3Codec <: V3Codec{:bytes, :bytes} - level::Int -end -GzipV3Codec() = GzipV3Codec(6) -name(::GzipV3Codec) = "gzip" - -function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) - opts = GzipEncodeOptions(; level=c.level) - return cc_encode(opts, data) -end - -function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) - return cc_decode(LibZGzipCodec(), encoded) -end - -struct BloscV3Codec <: V3Codec{:bytes, :bytes} - cname::String - clevel::Int - shuffle::Int - blocksize::Int - typesize::Int -end -BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) -name(::BloscV3Codec) = "blosc" - -function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) - comp = ZarrBloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) - return zcompress(data, comp) -end - -function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) - comp = ZarrBloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) - return collect(zuncompress(encoded, comp, UInt8)) -end - -struct ZstdV3Codec <: V3Codec{:bytes, :bytes} - level::Int -end -ZstdV3Codec() = ZstdV3Codec(3) -name(::ZstdV3Codec) = "zstd" - -function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) - comp = ZstdCompressor(level=c.level) - return zcompress(data, comp) -end - -function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) - comp = ZstdCompressor(level=c.level) - return collect(zuncompress(encoded, comp, UInt8)) -end - -struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} -end -name(::CRC32cV3Codec) = "crc32c" - -function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) - out = UInt8[] - return zencode!(out, data, CRC32cCodec()) -end - -function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) - out = UInt8[] - return zdecode!(out, encoded, CRC32cCodec()) -end - -end diff --git a/src/Compressors/Compressors.jl b/src/Compressors/Compressors.jl deleted file mode 100644 index 58a80109..00000000 --- a/src/Compressors/Compressors.jl +++ /dev/null @@ -1,109 +0,0 @@ -import JSON # for JSON.lower - -_reinterpret(::Type{T}, x::AbstractArray{S, 0}) where {T, S} = reinterpret(T, reshape(x, 1)) -_reinterpret(::Type{T}, x::AbstractArray) where T = reinterpret(T, x) - -""" - abstract type Compressor - -The abstract supertype for all Zarr compressors. - -## Interface - -All subtypes of `Compressor` SHALL implement the following methods: - -- `zcompress(a, c::Compressor)`: compress the array `a` using the compressor `c`. -- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` - and return an array of type `T`. -- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which - follows the Zarr specification for that compressor. -- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given - dictionary `d` which contains the compressor's parameters according to the Zarr spec. - -Subtypes of `Compressor` MAY also implement the following methods: - -- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the - compressor `c` and store the result in the array `compressed`. -- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` - using the compressor `c` and store the result in the array `data`. - -Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. -This must also follow the Zarr specification's name for that compressor. The name of the compressor -is the key, and the value is the compressor type (e.g. `BloscCompressor` or `NoCompressor`). - -For example, the Blosc compressor is named "blosc" in the Zarr spec, so the entry for [`BloscCompressor`](@ref) -must be added to `compressortypes` as `compressortypes["blosc"] = BloscCompressor`. -""" -abstract type Compressor end - -const compressortypes = Dict{Union{String,Nothing}, Type{<: Compressor}}() - -# function getCompressor end -# function zcompress end -# function zuncompress end -# function zcompress! end -# function zuncompress! end -# JSON.lower is neither defined nor documented here, since that would be documentation piracy :yarr: - -# Include the compressor implementations -include("blosc.jl") -include("zlib.jl") -include("zstd.jl") - -# ## Fallback definitions for the compressor interface -# Define fallbacks and generic methods for the compressor interface -getCompressor(compdict::Dict) = haskey(compdict, "id") ? - getCompressor(compressortypes[compdict["id"]], compdict) : - getCompressor(compressortypes[compdict["name"]], compdict["configuration"]) -getCompressor(::Nothing) = NoCompressor() - -# Compression when no filter is given -zcompress!(compressed,data,c,::Nothing) = zcompress!(compressed,data,c) -zuncompress!(data,compressed,c,::Nothing) = zuncompress!(data,compressed,c) - -# Fallback definition of mutating form of compress and uncompress -function zcompress!(compressed, data, c) - empty!(compressed) - append!(compressed,zcompress(data, c)) -end -zuncompress!(data, compressed, c) = copyto!(data, zuncompress(compressed, c, eltype(data))) - - -# Function given a filter stack -function zcompress!(compressed, data, c, f) - a2 = foldl(f, init=data) do anow, fnow - zencode(anow,fnow) - end - zcompress!(compressed, a2, c) -end - -function zuncompress!(data, compressed, c, f) - data2 = zuncompress(compressed, c, desttype(last(f))) - a2 = foldr(f, init = data2) do fnow, anow - zdecode(anow, fnow) - end - copyto!(data, a2) -end - -# ## `NoCompressor` -# The default and most minimal implementation of a compressor follows here, which does -# no actual compression. This is a good reference implementation for other compressors. - -""" - NoCompressor() - -Creates an object that can be passed to ZArray constructors without compression. -""" -struct NoCompressor <: Compressor end - -function zuncompress(a, ::NoCompressor, T) - _reinterpret(T,a) -end - -function zcompress(a, ::NoCompressor) - _reinterpret(UInt8,a) -end - -JSON.lower(::NoCompressor) = nothing - -compressortypes[nothing] = NoCompressor diff --git a/src/Filters/Filters.jl b/src/Filters/Filters.jl deleted file mode 100644 index 829f31ff..00000000 --- a/src/Filters/Filters.jl +++ /dev/null @@ -1,95 +0,0 @@ -import JSON - -""" - abstract type Filter{T,TENC} - -The supertype for all Zarr filters. - -## Interface - -All subtypes MUST implement the following methods: - -- [`zencode(ain, filter::Filter)`](@ref zencode): Encodes data `ain` using the filter, and returns a vector of bytes. -- [`zdecode(ain, filter::Filter)`](@ref zdecode): Decodes data `ain`, a vector of bytes, using the filter, and returns the original data. -- [`JSON.lower`](@ref): Returns a JSON-serializable dictionary representing the filter, according to the Zarr specification. -- [`getfilter(::Type{<: Filter}, filterdict)`](@ref getfilter): Returns the filter type read from a given filter dictionary. - -If the filter has type parameters, it MUST also implement: -- [`sourcetype(::Filter)::T`](@ref sourcetype): equivalent to `dtype` in the Python Zarr implementation. -- [`desttype(::Filter)::T`](@ref desttype): equivalent to `atype` in the Python Zarr implementation. - -Finally, an entry MUST be added to the `filterdict` dictionary for each filter type. -This must also follow the Zarr specification's name for that filter. The name of the filter -is the key, and the value is the filter type (e.g. `VLenUInt8Filter` or `Fletcher32Filter`). - - -Subtypes include: [`VLenArrayFilter`](@ref), [`VLenUTF8Filter`](@ref), [`Fletcher32Filter`](@ref). -""" -abstract type Filter{T,TENC} end - -""" - zencode(ain, filter::Filter) - -Encodes data `ain` using the filter, and returns a vector of bytes. -""" -function zencode end - -""" - zdecode(ain, filter::Filter) - -Decodes data `ain`, a vector of bytes, using the filter, and returns the original data. -""" -function zdecode end - -""" - getfilter(::Type{<: Filter}, filterdict) - -Returns the filter type read from a given specification dictionary, which must follow the Zarr specification. -""" -function getfilter end - -""" - sourcetype(::Filter)::T - -Returns the source type of the filter. -""" -function sourcetype end - -""" - desttype(::Filter)::T - -Returns the destination type of the filter. -""" -function desttype end - -filterdict = Dict{String,Type{<:Filter}}() - -function getfilters(d::Dict) - if !haskey(d,"filters") - return nothing - else - if d["filters"] === nothing || isempty(d["filters"]) - return nothing - end - f = map(d["filters"]) do f - try - getfilter(filterdict[f["id"]], f) - catch e - @show f - rethrow(e) - end - end - return (f...,) - end -end -sourcetype(::Filter{T}) where T = T -desttype(::Filter{<:Any,T}) where T = T - -zencode(ain,::Nothing) = ain - -include("vlenfilters.jl") -include("fletcher32.jl") -include("fixedscaleoffset.jl") -include("shuffle.jl") -include("quantize.jl") -include("delta.jl") diff --git a/src/Filters/delta.jl b/src/Filters/delta.jl deleted file mode 100644 index 9d1de048..00000000 --- a/src/Filters/delta.jl +++ /dev/null @@ -1,45 +0,0 @@ -#= -# Delta compression - - -=# - -""" - DeltaFilter(; DecodingType, [EncodingType = DecodingType]) - -Delta-based compression for Zarr arrays. (Delta encoding is Julia `diff`, decoding is Julia `cumsum`). -""" -struct DeltaFilter{T, TENC} <: Filter{T, TENC} -end - -function DeltaFilter(; DecodingType = Float16, EncodingType = DecodingType) - return DeltaFilter{DecodingType, EncodingType}() -end - -DeltaFilter{T}() where T = DeltaFilter{T, T}() - -function zencode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} - arr = reinterpret(DecodingType, vec(data)) - - enc = similar(arr, EncodingType) - # perform the delta operation - enc[begin] = arr[begin] - enc[begin+1:end] .= diff(arr) - return enc -end - -function zdecode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} - encoded = reinterpret(EncodingType, vec(data)) - decoded = DecodingType.(cumsum(encoded)) - return decoded -end - -function JSON.lower(filter::DeltaFilter{T, Tenc}) where {T, Tenc} - return Dict("id" => "delta", "dtype" => typestr(T), "astype" => typestr(Tenc)) -end - -function getfilter(::Type{<: DeltaFilter}, d) - return DeltaFilter{typestr(d["dtype"], haskey(d, "astype") ? typestr(d["astype"]) : d["dtype"])}() -end - -filterdict["delta"] = DeltaFilter \ No newline at end of file diff --git a/src/Filters/fixedscaleoffset.jl b/src/Filters/fixedscaleoffset.jl deleted file mode 100644 index 9e12c52d..00000000 --- a/src/Filters/fixedscaleoffset.jl +++ /dev/null @@ -1,52 +0,0 @@ - -""" - FixedScaleOffsetFilter{T,TENC}(scale, offset) - -A compressor that scales and offsets the data. - -!!! note - The geographic CF standards define scale/offset decoding as `x * scale + offset`, - but this filter defines it as `x / scale + offset`. Constructing a `FixedScaleOffsetFilter` - from CF data means `FixedScaleOffsetFilter(1/cf_scale_factor, cf_add_offset)`. -""" -struct FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc} <: Filter{T, Tenc} - scale::ScaleOffsetType - offset::ScaleOffsetType -end - -FixedScaleOffsetFilter{T}(scale::ScaleOffsetType, offset::ScaleOffsetType) where {T, ScaleOffsetType} = FixedScaleOffsetFilter{T, ScaleOffsetType}(scale, offset) -FixedScaleOffsetFilter(scale::ScaleOffsetType, offset::ScaleOffsetType) where {ScaleOffsetType} = FixedScaleOffsetFilter{ScaleOffsetType, ScaleOffsetType}(scale, offset) - -function FixedScaleOffsetFilter(; scale::ScaleOffsetType, offset::ScaleOffsetType, T, Tenc = T) where ScaleOffsetType - return FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}(scale, offset) -end - -function zencode(a::AbstractArray, c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {T, Tenc, ScaleOffsetType} - if Tenc <: Integer - return [round(Tenc, (a - c.offset) * c.scale) for a in a] # apply scale and offset, and round to nearest integer - else - return [convert(Tenc, (a - c.offset) * c.scale) for a in a] # apply scale and offset - end -end - -function zdecode(a::AbstractArray, c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {T, Tenc, ScaleOffsetType} - return [convert(Base.nonmissingtype(T), (a / c.scale) + c.offset) for a in a] -end - - -function getfilter(::Type{<: FixedScaleOffsetFilter}, d::Dict) - scale = d["scale"] - offset = d["offset"] - # Types must be converted from strings to the actual Julia types they represent. - string_T = d["dtype"] - string_Tenc = get(d, "astype", string_T) - T = typestr(string_T) - Tenc = typestr(string_Tenc) - return FixedScaleOffsetFilter{Tenc, T, Tenc}(scale, offset) -end - -function JSON.lower(c::FixedScaleOffsetFilter{ScaleOffsetType, T, Tenc}) where {ScaleOffsetType, T, Tenc} - return Dict("id" => "fixedscaleoffset", "scale" => c.scale, "offset" => c.offset, "dtype" => typestr(T), "astype" => typestr(Tenc)) -end - -filterdict["fixedscaleoffset"] = FixedScaleOffsetFilter diff --git a/src/Filters/fletcher32.jl b/src/Filters/fletcher32.jl deleted file mode 100644 index d854cb90..00000000 --- a/src/Filters/fletcher32.jl +++ /dev/null @@ -1,85 +0,0 @@ -#= -# Fletcher32 filter - -This "filter" basically injects a 4-byte checksum at the end of the data, to ensure data integrity. - -The implementation is based on the [numcodecs implementation here](https://github.com/zarr-developers/numcodecs/blob/79d1a8d4f9c89d3513836aba0758e0d2a2a1cfaf/numcodecs/fletcher32.pyx) -and the [original C implementation for NetCDF](https://github.com/Unidata/netcdf-c/blob/main/plugins/H5checksum.c#L109) linked therein. - -=# - -""" - Fletcher32Filter() - -A compressor that uses the Fletcher32 checksum algorithm to compress and uncompress data. - -Note that this goes from UInt8 to UInt8, and is effectively only checking -the checksum and cropping the last 4 bytes of the data during decoding. -""" -struct Fletcher32Filter <: Filter{UInt8, UInt8} -end - -getfilter(::Type{<: Fletcher32Filter}, d::Dict) = Fletcher32Filter() -JSON.lower(::Fletcher32Filter) = Dict("id" => "fletcher32") -filterdict["fletcher32"] = Fletcher32Filter - -function _checksum_fletcher32(data::AbstractArray{UInt8}) - len = length(data) ÷ 2 # length in 16-bit words - sum1::UInt32 = 0 - sum2::UInt32 = 0 - data_idx = 1 - - #= - Compute the checksum for pairs of bytes. - The magic `360` value is the largest number of sums that can be performed without overflow in UInt32. - =# - while len > 0 - tlen = len > 360 ? 360 : len - len -= tlen - while tlen > 0 - sum1 += begin # create a 16 bit word from two bytes, the first one shifted to the end of the word - (UInt16(data[data_idx]) << 8) | UInt16(data[data_idx + 1]) - end - sum2 += sum1 - data_idx += 2 - tlen -= 1 - if tlen < 1 - break - end - end - sum1 = (sum1 & 0xffff) + (sum1 >> 16) - sum2 = (sum2 & 0xffff) + (sum2 >> 16) - end - - # if the length of the data is odd, add the first byte to the checksum again (?!) - if length(data) % 2 == 1 - sum1 += UInt16(data[1]) << 8 - sum2 += sum1 - sum1 = (sum1 & 0xffff) + (sum1 >> 16) - sum2 = (sum2 & 0xffff) + (sum2 >> 16) - end - return (sum2 << 16) | sum1 -end - -function zencode(data, ::Fletcher32Filter) - bytes = reinterpret(UInt8, vec(data)) - checksum = _checksum_fletcher32(bytes) - result = copy(bytes) - append!(result, reinterpret(UInt8, [checksum])) # TODO: decompose this without the extra allocation of wrapping in Array - return result -end - -function zdecode(data, ::Fletcher32Filter) - bytes = reinterpret(UInt8, data) - checksum = _checksum_fletcher32(view(bytes, 1:length(bytes) - 4)) - stored_checksum = only(reinterpret(UInt32, view(bytes, (length(bytes) - 3):length(bytes)))) - if checksum != stored_checksum - throw(ErrorException(""" - Checksum mismatch in Fletcher32 decoding. - - The computed value is $(checksum) and the stored value is $(stored_checksum). - This might be a sign that the data is corrupted. - """)) # TODO: make this a custom error type - end - return view(bytes, 1:length(bytes) - 4) -end diff --git a/src/Filters/quantize.jl b/src/Filters/quantize.jl deleted file mode 100644 index c5d7c9a4..00000000 --- a/src/Filters/quantize.jl +++ /dev/null @@ -1,52 +0,0 @@ -#= -# Quantize compression - - -=# - -""" - QuantizeFilter(; digits, DecodingType, [EncodingType = DecodingType]) - -Quantization based compression for Zarr arrays. -""" -struct QuantizeFilter{T, TENC} <: Filter{T, TENC} - digits::Int32 -end - -function QuantizeFilter(; digits = 10, T = Float16, Tenc = T) - return QuantizeFilter{T, Tenc}(digits) -end - -QuantizeFilter{T, Tenc}(; digits = 10) where {T, Tenc} = QuantizeFilter{T, Tenc}(digits) -QuantizeFilter{T}(; digits = 10) where T = QuantizeFilter{T, T}(digits) - -function zencode(data::AbstractArray, filter::QuantizeFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} - arr = reinterpret(DecodingType, vec(data)) - - precision = 10.0^(-filter.digits) - - _exponent = log(10, precision) # log 10 in base `precision` - exponent = _exponent < 0 ? floor(Int, _exponent) : ceil(Int, _exponent) - - bits = ceil(log(2, 10.0^(-exponent))) - scale = 2.0^bits - - enc = @. convert(EncodingType, round(scale * arr) / scale) - - return enc -end - -# Decoding is a no-op; quantization is a lossy filter but data is encoded directly. -function zdecode(data::AbstractArray, filter::QuantizeFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType} - return data -end - -function JSON.lower(filter::QuantizeFilter{T, Tenc}) where {T, Tenc} - return Dict("id" => "quantize", "digits" => filter.digits, "dtype" => typestr(T), "astype" => typestr(Tenc)) -end - -function getfilter(::Type{<: QuantizeFilter}, d) - return QuantizeFilter{typestr(d["dtype"], typestr(d["astype"]))}(; digits = d["digits"]) -end - -filterdict["quantize"] = QuantizeFilter \ No newline at end of file diff --git a/src/Filters/shuffle.jl b/src/Filters/shuffle.jl deleted file mode 100644 index 6a01f5d4..00000000 --- a/src/Filters/shuffle.jl +++ /dev/null @@ -1,70 +0,0 @@ -#= -# Shuffle compression - -This file implements the shuffle compressor. -=# - -struct ShuffleFilter <: Filter{UInt8, UInt8} - elementsize::Csize_t -end - -ShuffleFilter(; elementsize = 4) = ShuffleFilter(elementsize) - -function _do_shuffle!(dest::AbstractVector{UInt8}, source::AbstractVector{UInt8}, elementsize::Csize_t) - count = fld(length(source), elementsize) # elementsize is in bytes, so this works - for i in 0:(count-1) - offset = i * elementsize - for byte_index in 0:(elementsize-1) - j = byte_index * count + i - dest[j+1] = source[offset + byte_index+1] - end - end -end - -function _do_unshuffle!(dest::AbstractVector{UInt8}, source::AbstractVector{UInt8}, elementsize::Csize_t) - count = fld(length(source), elementsize) # elementsize is in bytes, so this works - for i in 0:(elementsize-1) - offset = i * count - for byte_index in 0:(count-1) - j = byte_index * elementsize + i - dest[j+1] = source[offset + byte_index+1] - end - end -end - -function zencode(a::AbstractArray, c::ShuffleFilter) - if c.elementsize <= 1 # no shuffling needed if elementsize is 1 - return a - end - source = reinterpret(UInt8, vec(a)) - dest = Vector{UInt8}(undef, length(source)) - _do_shuffle!(dest, source, c.elementsize) - return dest -end - -function zdecode(a::AbstractArray, c::ShuffleFilter) - if c.elementsize <= 1 # no shuffling needed if elementsize is 1 - return a - end - source = reinterpret(UInt8, vec(a)) - dest = Vector{UInt8}(undef, length(source)) - _do_unshuffle!(dest, source, c.elementsize) - return dest -end - -function getfilter(::Type{ShuffleFilter}, d::Dict) - return ShuffleFilter(d["elementsize"]) -end - -function JSON.lower(c::ShuffleFilter) - return Dict("id" => "shuffle", "elementsize" => Int64(c.elementsize)) -end - -filterdict["shuffle"] = ShuffleFilter -#= - -# Tests - - - -=# \ No newline at end of file diff --git a/src/Filters/vlenfilters.jl b/src/Filters/vlenfilters.jl deleted file mode 100644 index dad91dfa..00000000 --- a/src/Filters/vlenfilters.jl +++ /dev/null @@ -1,82 +0,0 @@ -#= -# Variable-length filters - -This file implements variable-length filters for Zarr, i.e., filters that write arrays of variable-length arrays ("ragged arrays"). - -Specifically, it implements the `VLenArrayFilter` and `VLenUTF8Filter` types, which are used to encode and decode variable-length arrays and UTF-8 strings, respectively. -=# - -# ## VLenArrayFilter - -""" - VLenArrayFilter(T) - -Encodes and decodes variable-length arrays of arbitrary data type `T`. -""" -struct VLenArrayFilter{T} <: Filter{T,UInt8} end -# We don't need to define `sourcetype` and `desttype` for this filter, since the generic implementations are sufficient. - -JSON.lower(::VLenArrayFilter{T}) where T = Dict("id"=>"vlen-array","dtype"=> typestr(T) ) -getfilter(::Type{<:VLenArrayFilter}, f) = VLenArrayFilter{typestr(f["dtype"])}() -filterdict["vlen-array"] = VLenArrayFilter - -function zdecode(ain, ::VLenArrayFilter{T}) where T - f = IOBuffer(ain) - nitems = read(f, UInt32) - out = Array{Vector{T}}(undef,nitems) - for i=1:nitems - len1 = read(f,UInt32) - out[i] = read!(f,Array{T}(undef,len1 ÷ sizeof(T))) - end - close(f) - out -end - -#Encodes Array of Vectors `ain` into bytes -function zencode(ain,::VLenArrayFilter) - b = IOBuffer() - nitems = length(ain) - write(b,UInt32(nitems)) - for a in ain - write(b, UInt32(length(a) * sizeof(eltype(a)))) - write(b, a) - end - take!(b) -end - -# ## VLenUTF8Filter - -""" - VLenUTF8Filter - -Encodes and decodes variable-length unicode strings -""" -struct VLenUTF8Filter <: Filter{String, UInt8} end - -JSON.lower(::VLenUTF8Filter) = Dict("id"=>"vlen-utf8") -getfilter(::Type{<:VLenUTF8Filter}, f) = VLenUTF8Filter() -filterdict["vlen-utf8"] = VLenUTF8Filter - -function zdecode(ain, ::VLenUTF8Filter) - f = IOBuffer(ain) - nitems = read(f, UInt32) - out = Array{String}(undef, nitems) - for i in 1:nitems - clen = read(f, UInt32) - out[i] = String(read(f, clen)) - end - close(f) - out -end - -function zencode(ain, ::VLenUTF8Filter) - b = IOBuffer() - nitems = length(ain) - write(b, UInt32(nitems)) - for a in ain - utf8encoded = transcode(String, a) - write(b, UInt32(ncodeunits(utf8encoded))) - write(b, utf8encoded) - end - take!(b) -end diff --git a/src/MaxLengthStrings.jl b/src/MaxLengthStrings.jl deleted file mode 100644 index 1937f9ac..00000000 --- a/src/MaxLengthStrings.jl +++ /dev/null @@ -1,74 +0,0 @@ -#This is a modified version of FixedSizeStrings.jl -#adapted so that appended zero Chars are omitted from -#the string. So most credits got to authors of -#https://github.com/JuliaComputing/FixedSizeStrings.jl -# Maybe this should be moved to FizedSizeStrings.jl, -#but until then let's keep it here... -module MaxLengthStrings -import Base: iterate, lastindex, getindex, sizeof, length, ncodeunits, codeunit, isvalid, read, write -export MaxLengthString - -struct MaxLengthString{N,T} <: AbstractString - data::NTuple{N,T} - function MaxLengthString{N,T}(itr) where {N,T} - new(totuple_appendzero(NTuple{N,T},itr)) - end -end -import Base: tuple_type_head, tuple_type_tail -#totuple_appendzero(::Type{Tuple{}}, itr, s...) = () -tuple_append(::Type{NTuple{N,T}}) where {N,T} = (zero(T), tuple_append(NTuple{N-1,T})...) -tuple_append(::Type{Tuple{}}) = () -function totuple_appendzero(::Type{Tuple{}},itr,s...) - iterate(itr, s...)===nothing || error("String is too long to fit into MaxLengthString") - () -end -function totuple_appendzero(::Type{NTuple{N,T}}, itr, s...)::Tuple{Vararg{T,N}} where {N,T} - y = iterate(itr, s...) - if y === nothing - tuple_append(NTuple{N,T}) - else - (convert(T, y[1]), totuple_appendzero(NTuple{N-1,T}, itr, y[2])...) - end -end - -function MaxLengthString(s::AbstractString,N=length(s),T=UInt8) - MaxLengthString{N,T}(rpad(s,N,'\0')) -end - -function iterate(s::MaxLengthString{N}, i::Int = 1) where N - i > N && return nothing - c = s.data[i] - iszero(c) && return nothing - return (Char(c), i+1) -end - -lastindex(s::MaxLengthString{N}) where {N} = findlast(!iszero,s.data) - -function getindex(s::MaxLengthString, i::Int) - checkbounds(s,i) - Char(s.data[i]) -end - -sizeof(s::MaxLengthString) = sizeof(s.data) - -length(s::MaxLengthString) = findlast(!iszero,s.data) - -ncodeunits(s::MaxLengthString) = length(s) - -codeunit(::MaxLengthString{<:Any,T}) where T = T -function codeunit(s::MaxLengthString, i::Integer) - checkbounds(s,i) - s.data[i] -end - -isvalid(s::MaxLengthString{<:Any,UInt8}, i::Int) = checkbounds(Bool, s, i) -isvalid(s::MaxLengthString, i::Int) = checkbounds(Bool, s, i) && isvalid(Char,s.data[i]) - -function read(io::IO, T::Type{<:MaxLengthString{N}}) where N - return read!(io, Ref{T}())[]::T -end - -function write(io::IO, s::MaxLengthString{N}) where N - return write(io, Ref(s)) -end -end diff --git a/src/Storage/Storage.jl b/src/Storage/Storage.jl deleted file mode 100644 index 0845c604..00000000 --- a/src/Storage/Storage.jl +++ /dev/null @@ -1,291 +0,0 @@ - -# Defines different storages for zarr arrays. Currently only regular files (DirectoryStore) -# and Dictionaries are supported - -""" - abstract type AbstractStore - -This the abstract supertype for all Zarr store implementations. Currently only regular files ([`DirectoryStore`](@ref)) -and Dictionaries are supported. - -## Interface - -All subtypes of `AbstractStore` must implement the following methods: - -- [`storagesize(d::AbstractStore, p::AbstractString)`](@ref storagesize) -- [`subdirs(d::AbstractStore, p::AbstractString)`](@ref subdirs) -- [`subkeys(d::AbstractStore, p::AbstractString)`](@ref subkeys) -- [`isinitialized(d::AbstractStore, p::AbstractString)`](@ref isinitialized) -- [`storefromstring(::Type{<: AbstractStore}, s, _)`](@ref storefromstring) -- `Base.getindex(d::AbstractStore, i::AbstractString)`: return the data stored in key `i` as a Vector{UInt8} -- `Base.setindex!(d::AbstractStore, v, i::AbstractString)`: write the values in `v` to the key `i` of the given store `d` - -They may optionally implement the following methods: - -- [`store_read_strategy(s::AbstractStore)`](@ref store_read_strategy): return the read strategy for the given store. See [`SequentialRead`](@ref) and [`ConcurrentRead`](@ref). -""" -abstract type AbstractStore end - -# Define the interface - -""" - S3Store(bucket::String; aws=nothing) - -An S3-backed Zarr store. Available after loading the `ZarrAWSS3Ext` extension. -""" -struct S3Store <: AbstractStore - bucket::String - aws::Any -end - -function S3Store(args...) - error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") -end - -""" - storagesize(d::AbstractStore, p::AbstractString) - -This function shall return the size of all data files in a store at path `p`. -""" -function storagesize end - - -""" - Base.getindex(d::AbstractStore,i::String) - -Returns the data stored in the given key as a Vector{UInt8} -""" -Base.getindex(d::AbstractStore,i::AbstractString) = error("getindex not implemented for store $(typeof(d))") - -""" - Base.setindex!(d::AbstractStore,v,i::String) - -Writes the values in v to the given store and key. -""" -Base.setindex!(d::AbstractStore,v,i::AbstractString) = error("setindex not implemented for store $(typeof(d))") - -""" - subdirs(d::AbstractStore, p) - -Returns a list of keys for children stores in the given store at path p. -""" -function subdirs end - -""" - subkeys(d::AbstractStore, p) - -Returns the keys of files in the given store. -""" -function subkeys end - -# Function to construct the full path to a chunk given the base path, Cartesian Index i, and the chunk ecoding -store_readchunk(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = s[p, citostring(e, i)] -store_deletechunk(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = delete!(s, p, citostring(e, i)) -store_writechunk(s::AbstractStore, v, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = s[p, citostring(e, i)] = v -store_isinitialized(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = isinitialized(s, p, citostring(e, i)) - - -#Functions to concat path and key -Base.getindex(s::AbstractStore, p, i::AbstractString) = s[_concatpath(p, i)] -Base.delete!(s::AbstractStore, p, i::AbstractString) = delete!(s, _concatpath(p, i)) -Base.haskey(s::AbstractStore, k::AbstractString) = isinitialized(s, k) -Base.setindex!(s::AbstractStore, v, p, i::AbstractString) = setindex!(s, v, _concatpath(p, i)) - - -maybecopy(x) = copy(x) -maybecopy(x::String) = x - - -function getattrs(::ZarrFormat{2}, s::AbstractStore, p) - atts = s[p,".zattrs"] - if atts === nothing - Dict() - else - JSON.parse(replace(String(maybecopy(atts)),": NaN,"=>": \"NaN\","); dicttype = Dict{String,Any}) - end -end - -function getattrs(::ZarrFormat{3}, s::AbstractStore, p) - md = s[p, "zarr.json"] - if md === nothing - error("zarr.json not found") - else - md = JSON.parse(replace(String(maybecopy(md)), ": NaN," => ": \"NaN\","); dicttype=Dict{String,Any}) - return get(md, "attributes", Dict{String,Any}()) - end -end - -function writeattrs(::ZarrFormat{2}, s::AbstractStore, p, att::Dict; indent_json::Bool=false) - b = IOBuffer() - - if indent_json - JSON.print(b,att,4) - else - JSON.print(b,att) - end - - s[p,".zattrs"] = take!(b) - att -end - -function writeattrs(::ZarrFormat{3}, s::AbstractStore, p, att::Dict; indent_json::Bool=false) - # This is messy, we need to open zarr.json and replace the attributes section - md = s[p, "zarr.json"] - if md === nothing - error("zarr.json not found") - else - md = JSON.parse(replace(String(maybecopy(md)), ": NaN," => ": \"NaN\",")) - end - md = Dict(md) - md["attributes"] = att - - b = IOBuffer() - - if indent_json - JSON.print(b, md, 4) - else - JSON.print(b, md) - end - - s[p, "zarr.json"] = take!(b) - att -end - -is_zarr3(s::AbstractStore, p) = isinitialized(s,_concatpath(p,"zarr.json")) -is_zarr2(s::AbstractStore, p) = is_zarray(ZarrFormat(Val(2)), s, p) || is_zgroup(ZarrFormat((Val(2))), s, p) - -is_zgroup(::ZarrFormat{2}, s::AbstractStore, p) = isinitialized(s, _concatpath(p, ".zgroup")) -is_zarray(::ZarrFormat{2}, s::AbstractStore, p) = isinitialized(s, _concatpath(p, ".zarray")) -function is_zgroup(::ZarrFormat{3}, s::AbstractStore, p) - isinitialized(s, _concatpath(p, "zarr.json")) || return false - try - metadata = getmetadata(ZarrFormat(Val(3)), s, p, false) - metadata.node_type == "group" - catch e - e isa ArgumentError || rethrow() - @warn "Skipping $p: $(e.msg)" - false - end -end -function is_zarray(::ZarrFormat{3}, s::AbstractStore, p) - isinitialized(s, _concatpath(p, "zarr.json")) || return false - try - metadata = getmetadata(ZarrFormat(Val(3)), s, p, false) - metadata.node_type == "array" - catch e - e isa ArgumentError || rethrow() - @warn "Skipping $p: $(e.msg)" - false - end -end - - -isinitialized(s::AbstractStore, p, i::AbstractString) = isinitialized(s, _concatpath(p, i)) -isinitialized(s::AbstractStore, i::AbstractString) = s[i] !== nothing - -getmetadata(::ZarrFormat{2}, s::AbstractStore, p, fill_as_missing) = Metadata(String(maybecopy(s[p, ".zarray"])), fill_as_missing) - -getmetadata(::ZarrFormat{3}, s::AbstractStore, p, fill_as_missing) = Metadata(String(maybecopy(s[p, "zarr.json"])), fill_as_missing) - -function writemetadata(::ZarrFormat{2}, s::AbstractStore, p, m::AbstractMetadata; indent_json::Bool=false) - met = IOBuffer() - - if indent_json - JSON.print(met,m,4) - else - JSON.print(met,m) - end - - s[p,".zarray"] = take!(met) - m -end -function writemetadata(::ZarrFormat{3}, s::AbstractStore, p, m::AbstractMetadata; indent_json::Bool=false) - met = IOBuffer() - - if indent_json - JSON.print(met, m, 4) - else - JSON.print(met, m) - end - - s[p, "zarr.json"] = take!(met) - m -end - - - -## Handling sequential vs parallel IO -struct SequentialRead end -struct ConcurrentRead - ntasks::Int -end -store_read_strategy(::AbstractStore) = SequentialRead() - -channelsize(s) = channelsize(store_read_strategy(s)) -channelsize(::SequentialRead) = 0 -channelsize(c::ConcurrentRead) = c.ntasks - -read_items!(s::AbstractStore, c::AbstractChannel, e::AbstractChunkKeyEncoding, p, i) = read_items!(s, c, store_read_strategy(s), e, p, i) -function read_items!(s::AbstractStore, c::AbstractChannel, ::SequentialRead, e::AbstractChunkKeyEncoding, p, i) - for ii in i - res = store_readchunk(s, p, ii, e) - put!(c,(ii=>res)) - end -end -function read_items!(s::AbstractStore, c::AbstractChannel, r::ConcurrentRead, e::AbstractChunkKeyEncoding, p, i) - ntasks = r.ntasks - #@show ntasks - asyncmap(i,ntasks = ntasks) do ii - #@show ii,objectid(current_task),p - res = store_readchunk(s, p, ii, e) - #@show ii,length(res) - put!(c,(ii=>res)) - nothing - end -end - -write_items!(s::AbstractStore, c::AbstractChannel, e::AbstractChunkKeyEncoding, p, i) = write_items!(s, c, store_read_strategy(s), e, p, i) -function write_items!(s::AbstractStore, c::AbstractChannel, ::SequentialRead, e::AbstractChunkKeyEncoding, p, i) - for _ in 1:length(i) - ii,data = take!(c) - if data === nothing - if store_isinitialized(s, p, ii, e) - store_deletechunk(s, p, ii, e) - end - else - store_writechunk(s, data, p, ii, e) - end - end - close(c) -end - -function write_items!(s::AbstractStore, c::AbstractChannel, r::ConcurrentRead, e::AbstractChunkKeyEncoding, p, i) - ntasks = r.ntasks - asyncmap(i,ntasks = ntasks) do _ - ii,data = take!(c) - if data === nothing - if store_isinitialized(s, p, ii, e) - store_deletechunk(s, p, ii, e) - end - else - store_writechunk(s, data, p, ii, e) = data - end - nothing - end - close(c) -end - -isemptysub(s::AbstractStore, p) = isempty(subkeys(s,p)) && isempty(subdirs(s,p)) - -#Here different storage backends can register regexes that are checked against -#during auto-check of storage format when doing zopen -storageregexlist = Pair[] -push!(storageregexlist, r"^s3://" => S3Store) - -#include("formattedstore.jl") -include("directorystore.jl") -include("dictstore.jl") -include("gcstore.jl") -include("consolidated.jl") -include("http.jl") -include("zipstore.jl") diff --git a/src/Storage/consolidated.jl b/src/Storage/consolidated.jl deleted file mode 100644 index de1cc291..00000000 --- a/src/Storage/consolidated.jl +++ /dev/null @@ -1,101 +0,0 @@ -""" -A store that wraps any other AbstractStore but has access to the consolidated metadata -stored in the .zmetadata key. Whenever data attributes or metadata are accessed, the -data will be read from the dictionary instead. -""" -struct ConsolidatedStore{P} <: AbstractStore - parent::P - path::String - cons::Dict{String,Any} -end -function ConsolidatedStore(s::AbstractStore, p) - d = s[p, ".zmetadata"] - if d === nothing - throw(ArgumentError("Could not find consolidated metadata for store $s")) - end - meta = JSON.parse(String(copy(d)); dicttype = Dict{String,Any}) - ConsolidatedStore(s, p, meta["metadata"]) -end - -function Base.show(io::IO,d::ConsolidatedStore) - b = IOBuffer() - show(b,d.parent) - print(io, "Consolidated ", String(take!(b))) -end - -storagesize(d::ConsolidatedStore,p) = storagesize(d.parent,p) -function Base.getindex(d::ConsolidatedStore,i::String) - d.parent[i] -end -function getmetadata(::ZarrFormat{2}, d::ConsolidatedStore, p, fill_as_missing) - return Metadata(d.cons[_unconcpath(d, p, ".zarray")], fill_as_missing) -end -function getmetadata(::ZarrFormat{3}, d::ConsolidatedStore, p, fill_as_missing) - return Metadata(d.cons[_unconcpath(d, p, "zarr.json")], fill_as_missing) -end -function getattrs(::ZarrFormat{2}, d::ConsolidatedStore, p) - return get(d.cons, _unconcpath(d, p, ".zattrs"), Dict{String,Any}()) -end - -function getattrs(::ZarrFormat{3}, d::ConsolidatedStore, p) - return get(d.cons, _unconcpath(d, p, ".zattrs"), Dict{String,Any}()) -end -function _unconcpath(d,p) - startswith(p,d.path) || error("Requested key is not in consolidated path") - lstrip(replace(p,d.path=>"", count=1),'/') -end -_unconcpath(d,p,s) = _concatpath(_unconcpath(d,p),s) -is_zarray(::ZarrFormat{2}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, ".zarray")) -is_zgroup(::ZarrFormat{2}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, ".zgroup")) -is_zarray(::ZarrFormat{3}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, "zarr.json")) && get(d.cons[_unconcpath(d, p, "zarr.json")], "node_type", "") == "array" -is_zgroup(::ZarrFormat{3}, d::ConsolidatedStore, p) = haskey(d.cons, _unconcpath(d, p, "zarr.json")) && get(d.cons[_unconcpath(d, p, "zarr.json")], "node_type", "") == "group" -ZarrFormat(d::ConsolidatedStore, path) = ZarrFormat(d.parent, path) # detect format from parent, not cons -check_consolidated_write(i::String) = split(i,'/')[end] in (".zattrs",".zarray",".zgroup") && - throw(ArgumentError("Can not modify consolidated metadata, please re-open the dataset with `consolidated=false`")) - -_pdict(d::ConsolidatedStore,p) = filter(((k,v),)->startswith(k,p),d.cons) -function subdirs(d::ConsolidatedStore,p) - p2 = _unconcpath(d,p) - d2 = _pdict(d,p2) - _searchsubdict(d2, p2, (sp, lp) -> length(sp) > lp + 1) -end - -function subkeys(d::ConsolidatedStore,p) - subkeys(d.parent,p) -end - - - -function Base.setindex!(d::ConsolidatedStore,v,i::String) - #Here we check that we don't overwrite consolidated information - check_consolidated_write(i) - d.parent[i] = v -end -function Base.delete!(d::ConsolidatedStore,i::String) - check_consolidated_write(i) - delete!(d.parent,i) -end - -store_read_strategy(s::ConsolidatedStore) = store_read_strategy(s.parent) - -function consolidate_metadata(s::AbstractStore,d,prefix) - for k in (".zattrs",".zarray",".zgroup") - v = s[prefix,k] - if v !== nothing - d[_concatpath(prefix,k)] = JSON.parse(String(copy(v)); dicttype = Dict{String,Any}) - end - end - foreach(subdirs(s,prefix)) do subname - consolidate_metadata(s,d,string(prefix,subname,"/")) - end - d -end -function consolidate_metadata(s::AbstractStore,p) - d = consolidate_metadata(s,Dict{String,Any}(),p) - buf = IOBuffer() - JSON.print(buf,Dict("metadata"=>d,"zarr_consolidated_format"=>1),4) - s[p,".zmetadata"] = take!(buf) - ConsolidatedStore(s,p,d) -end - -consolidate_metadata(s) = consolidate_metadata(zopen(s,"w")) diff --git a/src/Storage/dictstore.jl b/src/Storage/dictstore.jl deleted file mode 100644 index d21bcb61..00000000 --- a/src/Storage/dictstore.jl +++ /dev/null @@ -1,54 +0,0 @@ -# Stores data in a simple dict in memory -struct DictStore <: AbstractStore - a::Dict{String,Vector{UInt8}} -end -DictStore() = DictStore(Dict{String,Vector{UInt8}}()) - -Base.show(io::IO,d::DictStore) = print(io,"Dictionary Storage") -function _pdict(d::DictStore,p) - p = (isempty(p) || endswith(p,'/')) ? p : p*'/' - filter(((k,v),)->startswith(k,p),d.a) -end -function storagesize(d::DictStore,p) - sum(i -> (last(split(i[1],'/')) ∉ (".zattrs",".zarray") ? sizeof(i[2]) : zero(sizeof(i[2]))), _pdict(d,p); init=0) -end - -function Base.getindex(d::DictStore,i::AbstractString) - get(d.a,i,nothing) -end -function Base.setindex!(d::DictStore,v,i::AbstractString) - d.a[i] = v -end -Base.delete!(d::DictStore, i::AbstractString) = delete!(d.a,i) - -function subdirs(d::DictStore,p) - d2 = _pdict(d,p) - _searchsubdict(d2,p,(sp,lp)->length(sp) > lp+1) -end - -function subkeys(d::DictStore,p) - d2 = _pdict(d,p) - _searchsubdict(d2,p,(sp,lp)->length(sp) == lp+1) -end - -function _searchsubdict(d2,p,condition) - o = Set{String}() - pspl = split(rstrip(p,'/'),'/') - lp = if length(pspl) == 1 && isempty(pspl[1]) - 0 - else - length(pspl) - end - for k in keys(d2) - sp = split(k,'/') - if condition(sp,lp) - push!(o,sp[lp+1]) - end - end - collect(o) -end - - -#getsub(d::DictStore, p, n) = _substore(d,p).subdirs[n] - -#path(d::DictStore) = "" diff --git a/src/Storage/directorystore.jl b/src/Storage/directorystore.jl deleted file mode 100644 index 2913176e..00000000 --- a/src/Storage/directorystore.jl +++ /dev/null @@ -1,59 +0,0 @@ -"Normalize logical storage path" -function normalize_path(p::AbstractString) - # \ to / since normpath on linux won't handle it - p = replace(p, '\\'=>'/') - p = normpath(p) - # \ to / again since normpath on windows creates \ - p = replace(p, '\\'=>'/') - p == "/" ? p : rstrip(p, '/') -end - -# Stores files in a regular file system -struct DirectoryStore <: AbstractStore - folder::String - function DirectoryStore(p) - mkpath(normalize_path(p)) - new(normalize_path(p)) - end -end - -function Base.getindex(d::DirectoryStore, i::String) - fname=joinpath(d.folder,i) - if isfile(fname) - read(fname) - else - nothing - end -end - -function Base.setindex!(d::DirectoryStore,v,i::String) - fname=d.folder * "/" * i - folder = dirname(fname) - isdir(folder) || mkpath(folder) - write(fname,v) - v -end - - -function storagesize(d::DirectoryStore,p) - sum(f -> filesize(d.folder * "/" * p * "/" * f), filter(i->i ∉ (".zattrs",".zarray"),readdir(d.folder * "/" * p)); init=0) -end - -function subdirs(s::DirectoryStore,p) - pbase = joinpath(s.folder,p) - if !isdir(pbase) - return String[] - else - return filter(i -> isdir(joinpath(s.folder,p, i)), readdir(pbase)) - end -end -function subkeys(s::DirectoryStore,p) - pbase = joinpath(s.folder,p) - if !isdir(pbase) - return String[] - else - return filter(i -> isfile(joinpath(s.folder,p, i)), readdir(pbase)) - end -end -Base.delete!(s::DirectoryStore, k::String) = isfile(joinpath(s.folder, k)) && rm(joinpath(s.folder, k)) - diff --git a/src/ZArray.jl b/src/ZArray.jl deleted file mode 100644 index af27d252..00000000 --- a/src/ZArray.jl +++ /dev/null @@ -1,507 +0,0 @@ -import JSON -import OffsetArrays: OffsetArray -import DiskArrays: AbstractDiskArray -import DiskArrays -using DateTimes64: DateTime64 -using Dates: Day, Millisecond - -""" -Number of tasks to use for async reading of chunks. Warning: setting this to very high values can lead to a large memory footprint. -""" -const concurrent_io_tasks = Ref(50) - -getfillval(::Type{T}, t::String) where {T <: Number} = parse(T, t) -getfillval(::Type{T}, t::Union{T,Nothing}) where {T} = t - -struct SenMissArray{T,N} <: AbstractArray{Union{T,Missing},N} - x::Array{T,N} - senval::T -end -SenMissArray(x::Array{T,N},v) where {T,N} = SenMissArray{T,N}(x,convert(T,v)) -Base.size(x::SenMissArray) = size(x.x) -senval(x::SenMissArray) = x.senval -function Base.getindex(x::SenMissArray,i::Int) - v = x.x[i] - isequal(v,senval(x)) ? missing : v -end -Base.setindex!(x::SenMissArray,v,i::Int) = x.x[i] = v -Base.setindex!(x::SenMissArray,::Missing,i::Int) = x.x[i] = senval(x) -Base.IndexStyle(::Type{<:SenMissArray})=Base.IndexLinear() - -# Struct representing a Zarr Array in Julia, note that -# chunks(chunk size) and size are always in Julia column-major order -struct ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata{T,N}} <: AbstractDiskArray{T,N} - metadata::M - storage::S - path::String - attrs::Dict - writeable::Bool -end - -Base.eltype(::ZArray{T}) where {T} = T -Base.ndims(::ZArray{<:Any,N}) where {N} = N -Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N, Int} -function Base.size(z::ZArray{<:Any,N}, i::Integer) where {N} - len = length(z.metadata.shape[]) - if 0 < i <= len - z.metadata.shape[][i]::Int - elseif i > len - 1 - else - error("arraysize: dimension out of range") - end -end -Base.length(z::ZArray) = prod(z.metadata.shape[])::Int -Base.lastindex(z::ZArray{<:Any,N}, n::Integer) where {N} = size(z, n)::Int -Base.lastindex(z::ZArray{<:Any,1}) = size(z, 1)::Int - -function Base.show(io::IO,z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) -end -function Base.show(io::IO,::MIME"text/plain",z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) -end - -zname(z::ZArray) = zname(z.path) - -function zname(s::String) - spl = split(rstrip(s,'/'),'/') - isempty(last(spl)) ? "root" : last(spl) -end - - -""" - storagesize(z::ZArray) - -Returns the size of the compressed data stored in the ZArray `z` in bytes -""" -storagesize(z::ZArray) = storagesize(z.storage,z.path) - -""" - storageratio(z::ZArray) - -Returns the ratio of the size of the uncompressed data in `z` and the size of the compressed data. -""" -storageratio(z::ZArray) = length(z)*sizeof(eltype(z))/storagesize(z) - -storageratio(z::ZArray{<:Vector}) = "unknown" - -nobytes(z::ZArray) = length(z)*sizeof(eltype(z)) -nobytes(z::ZArray{<:Vector}) = "unknown" -nobytes(z::ZArray{<:String}) = "unknown" - -zinfo(z::ZArray) = zinfo(stdout,z) -function zinfo(io::IO,z::ZArray) - ninit = sum(chunkindices(z)) do i - store_isinitialized(z.storage, z.path, i, z.metadata.chunk_key_encoding) - end - allinfos = [ - "Type" => "ZArray", - "Data type" => eltype(z), - "Shape" => size(z), - "Chunk Shape" => z.metadata.chunks, - "Order" => try get_order(z.metadata) catch e "unknown ($(e.msg))" end, - "Read-Only" => !z.writeable, - "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), - "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, - "Store type" => z.storage, - "No. bytes" => nobytes(z), - "No. bytes stored" => storagesize(z), - "Storage ratio" => storageratio(z), - "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" - ] - foreach(allinfos) do ii - println(io,rpad(ii[1],20),": ",ii[2]) - end -end - -function ZArray(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=false) where T<:AbstractStore - zv = if zarr_format == :auto - ZarrFormat(s, path) - else - ZarrFormat(zarr_format) - end - metadata = getmetadata(zv, s, path, fill_as_missing) - attrs = getattrs(zv, s, path) - writeable = mode == "w" - startswith(path,"/") && error("Paths should never start with a leading '/'") - ZArray(metadata, s, string(path), attrs, writeable) -end - -zarr_format(z::ZArray) = zarr_format(z.metadata) -dimension_separator(z::ZArray) = dimension_separator(z.metadata) - - -""" - trans_ind(r, bs) - -For a given index and blocksize determines which chunks of the Zarray will have to -be accessed. -""" -trans_ind(r::AbstractUnitRange, bs) = fld1(first(r),bs):fld1(last(r),bs) -trans_ind(r::Integer, bs) = fld1(r,bs) - -function boundint(r1, s2, o2) - r2 = range(o2+1,length=s2) - f1, f2 = first(r1), first(r2) - l1, l2 = last(r1),last(r2) - UnitRange(f1 > f2 ? f1 : f2, l1 < l2 ? l1 : l2) -end - -function getchunkarray(z::ZArray{>:Missing}) - # temporary workaround to use strings as data values - inner = fill(z.metadata.fill_value, z.metadata.chunks) - a = SenMissArray(inner,z.metadata.fill_value) -end -_zero(T) = zero(T) -_zero(T::Type{<:MaxLengthString}) = T("") -_zero(T::Type{ASCIIChar}) = ASCIIChar(0) -_zero(::Type{<:Vector{T}}) where T = T[] -_zero(::Type{Char}) = Char(0) -getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) - -maybeinner(a::Array) = a -maybeinner(a::SenMissArray) = a.x -resetbuffer!(fv,a::Array) = fv === nothing || fill!(a,fv) -resetbuffer!(_,a::SenMissArray) = fill!(a,missing) - -# Function to read or write from a zarr array. Could be refactored -# using type system to get rid of the `if readmode` statements. -function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} - - output_base_offsets = map(i->first(i)-1,r.indices) - # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) - # Allocate array of the size of a chunks where uncompressed data can be held - #bufferdict = IdDict((current_task()=>getchunkarray(z),)) - a = getchunkarray(z) - # Now loop through the chunks - c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - task = @async begin - read_items!($(z.storage), c, $(z.metadata.chunk_key_encoding), $(z.path), $(blockr)) - end - bind(c,task) - - try - for i in 1:length(blockr) - - bI,chunk_compressed = take!(c) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) - - uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) - nothing - end - finally - close(c) - end - - aout -end - -function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} - - z.writeable || error("Can not write to read-only ZArray") - input_base_offsets = map(i->first(i)-1,r.indices) - # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) - # Allocate array of the size of a chunks where uncompressed data can be held - #bufferdict = IdDict((current_task()=>getchunkarray(z),)) - a = getchunkarray(z) - # Now loop through the chunks - readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - readtask = @async begin - read_items!(z.storage, readchannel, z.metadata.chunk_key_encoding, z.path, blockr) - end - bind(readchannel,readtask) - - writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - writetask = @async begin - write_items!(z.storage, writechannel, z.metadata.chunk_key_encoding, z.path, blockr) - end - bind(writechannel,writetask) - - try - for i in 1:length(blockr) - - bI,chunk_compressed = take!(readchannel) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) - - if isnothing(chunk_compressed) || (length.(indranges) != size(a)) - resetbuffer!(z.metadata.fill_value,a) - end - - curchunk = if length.(indranges) != size(a) - view(a,dotminus.(indranges,current_chunk_offsets)...) - else - a - end - - if chunk_compressed !== nothing - uncompress_raw!(a,z,chunk_compressed) - end - - curchunk .= view(ain,dotminus.(indranges,input_base_offsets)...) - - put!(writechannel,bI=>compress_raw(maybeinner(a),z)) - nothing - end - finally - close(readchannel) - close(writechannel) - end - ain -end - -DiskArrays.readblock!(a::ZArray,aout,i::AbstractUnitRange...) = readblock!(aout,a,CartesianIndices(i)) -DiskArrays.writeblock!(a::ZArray,v,i::AbstractUnitRange...) = writeblock!(v,a,CartesianIndices(i)) -DiskArrays.haschunks(::ZArray) = DiskArrays.Chunked() -DiskArrays.eachchunk(a::ZArray) = DiskArrays.GridChunks(a,a.metadata.chunks) - -""" - uncompress_raw!(a::DenseArray{T},z::ZArray{T,N},i::CartesianIndex{N}) - -Read the chunk specified by `i` from the Zarray `z` and write its content to `a` -""" -function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N - if curchunk === nothing - if isnothing(z.metadata.fill_value) - throw(ArgumentError("The array $z got missing chunks and no fill_value")) - end - fill!(a, z.metadata.fill_value) - else - pipeline_decode!(get_pipeline(z.metadata), a, curchunk) - end - a -end - -dotminus(x,y) = x.-y - -function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) - - uncompress_raw!(a,z,chunk_compressed) - - if length.(indranges) == size(a) - aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a - else - curchunk = a[dotminus.(indranges,current_chunk_offsets)...] - aout[dotminus.(indranges, output_base_offsets)...] = curchunk - end -end - -function compress_raw(a,z) - length(a) == prod(z.metadata.chunks) || throw(DimensionMismatch("Array size does not equal chunk size")) - pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) -end - - - -""" - zcreate(T, dims...;kwargs) - -Creates a new empty zarr array with element type `T` and array dimensions `dims`. The following keyword arguments are accepted: - -* `path=""` directory name to store a persistent array. If left empty, an in-memory array will be created -* `name=""` name of the zarr array, defaults to the directory name -* `zarr_format`=$(DV) Zarr format version (2 or 3) -* `storagetype` determines the storage to use, current options are `DirectoryStore` or `DictStore` -* `chunks=dims` size of the individual array chunks, must be a tuple of length `length(dims)` -* `fill_value=nothing` value to represent missing values -* `fill_as_missing=false` set to `true` shall fillvalue s be converted to `missing`s -* `filters`=filters to be applied -* `compressor=BloscCompressor()` compressor type and properties -* `attrs=Dict()` a dict containing key-value pairs with metadata attributes associated to the array -* `writeable=true` determines if the array is opened in read-only or write mode -* `indent_json=false` determines if indents are added to format the json files `.zarray` and `.zattrs`. This makes them more readable, but increases file size. -* `dimension_separator='.'` sets how chunks are encoded. The Zarr v2 default is '.' such that the first 3D chunk would be `0.0.0`. The Zarr v3 default is `/`. -""" -function zcreate(::Type{T}, dims::Integer...; - name="", - path=nothing, - zarr_format=DV, - dimension_separator=default_sep(zarr_format), - kwargs... - ) where T - - if path===nothing - store = DictStore() - else - store = DirectoryStore(joinpath(path, name)) - end - zcreate(T, store, dims...; zarr_format, dimension_separator, kwargs...) -end - -function zcreate(::Type{T},storage::AbstractStore, - dims...; - path = "", - zarr_format = DV, - chunks=dims, - fill_value=nothing, - fill_as_missing=false, - compressor=BloscCompressor(), - filters = filterfromtype(T), - attrs=Dict(), - writeable=true, - indent_json=false, - dimension_separator=nothing - ) where {T} - - v = ZarrFormat(zarr_format) - if isnothing(dimension_separator) - dimension_separator = default_sep(v) - end - if dimension_separator isa AbstractString - # Convert AbstractString to Char - dimension_separator = only(dimension_separator) - end - chunk_key_encoding = ChunkKeyEncoding(dimension_separator, default_prefix(v)) - - length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) - N = length(dims) - C = typeof(compressor) - - # Create a dummy array to use with Metadata constructor - # This allows us to leverage the multiple dispatch in Metadata constructors - dummy_array = Array{T,N}(undef, dims...) - metadata = Metadata(dummy_array, chunks, v; - compressor=compressor, - fill_value=fill_value, - filters=filters, - fill_as_missing=fill_as_missing, - chunk_key_encoding=chunk_key_encoding - ) - - # Extract the element type from the metadata (handles T2 calculation) - T2 = eltype(metadata) - - isemptysub(storage,path) || error("$storage $path is not empty") - - writemetadata(v, storage, path, metadata, indent_json=indent_json) - - writeattrs(v, storage, path, attrs, indent_json=indent_json) - - ZArray(metadata, storage, path, attrs, writeable) -end - -filterfromtype(::Type{<:Any}) = nothing - -function filterfromtype(::Type{<:AbstractArray{T}}) where T - #Here we have to apply the vlenarray filter - (VLenArrayFilter{T}(),) -end - -filterfromtype(::Type{<:Union{<:AbstractString, Union{<:AbstractString, Missing}}}) = (VLenUTF8Filter(),) -filterfromtype(::Type{<:Union{MaxLengthString, Union{MaxLengthString, Missing}}}) = nothing - -#Not all Array types can be mapped directly to a valid ZArray encoding. -#Here we try to determine the correct element type -to_zarrtype(::AbstractArray{T}) where T = T -to_zarrtype(a::AbstractArray{<:Date}) = DateTime64{Day} -to_zarrtype(a::AbstractArray{<:DateTime}) = DateTime64{Millisecond} - -function ZArray(a::AbstractArray, args...; kwargs...) - z = zcreate(to_zarrtype(a), args..., size(a)...; kwargs...) - z[:]=a - z -end - -""" - chunkindices(z::ZArray) - -Returns the Cartesian Indices of the chunks of a given ZArray -""" -chunkindices(z::ZArray) = CartesianIndices(map((s, c) -> 1:ceil(Int, s/c), z.metadata.shape[], z.metadata.chunks)) - -""" - zzeros(T, dims...; kwargs... ) - -Creates a zarr array and initializes all values with zero. Accepts the same keyword arguments as `zcreate` -""" -function zzeros(T,dims...;kwargs...) - z = zcreate(T,dims...;kwargs...) - as = zeros(T, z.metadata.chunks...) - data_encoded = compress_raw(as,z) - p = z.path - if data_encoded !== nothing - for i in chunkindices(z) - store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) - end - end - z -end - -#Resizing Zarr arrays -""" - resize!(z::ZArray{T,N}, newsize::NTuple{N}) - -Resizes a `ZArray` to the new specified size. If the size along any of the -axes is decreased, unused chunks will be deleted from the store. -""" -function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N} - any(<(0), newsize) && throw(ArgumentError("Size must be positive")) - oldsize = z.metadata.shape[] - z.metadata.shape[] = newsize - #Check if array was shrunk - if any(map(<,newsize, oldsize)) - prune_oob_chunks(z.storage, z.path, oldsize, newsize, z.metadata.chunks, z.metadata.chunk_key_encoding) - end - writemetadata(zarr_format(z), z.storage, z.path, z.metadata) - nothing -end -Base.resize!(z::ZArray, newsize::Integer...) = resize!(z,newsize) - -""" - append!(z::ZArray{<:Any, N},a;dims = N) - -Appends an AbstractArray to an existinng `ZArray` along the dimension dims. The -size of the `ZArray` is increased accordingly and data appended. - -Example: - -````julia -z=zzeros(Int,5,3) -append!(z,[1,2,3],dims=1) #Add a new row -append!(z,ones(Int,6,2)) #Add two new columns -z[:,:] -```` -""" -function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N - #Determine how many entries to add to axis - otherdims = sort!(setdiff(1:N,dims)) - othersize = size(z)[otherdims] - if ndims(a)==N - nadd = size(a,dims) - size(a)[otherdims]==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) - elseif ndims(a)==N-1 - size(a)==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) - nadd = 1 - else - throw(DimensionMismatch("Number of dimensions of array must be either $N or $(N-1)")) - end - oldsize = size(z) - newsize = ntuple(i->i==dims ? oldsize[i]+nadd : oldsize[i], N) - resize!(z,newsize) - appendinds = ntuple(i->i==dims ? (oldsize[i]+1:newsize[i]) : Colon(),N) - z[appendinds...] = a - nothing -end - -function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks, chunk_key_encoding) - dimstoshorten = findall(map(<,newsize, oldsize)) - for idim in dimstoshorten - delrange = (fld1(newsize[idim],chunks[idim])+1):(fld1(oldsize[idim],chunks[idim])) - allchunkranges = map(i->1:fld1(oldsize[i],chunks[i]),1:length(oldsize)) - r = (allchunkranges[1:idim-1]..., delrange, allchunkranges[idim+1:end]...) - for cI in CartesianIndices(r) - store_deletechunk(s, path, cI, chunk_key_encoding) - end - end -end diff --git a/src/ZGroup.jl b/src/ZGroup.jl deleted file mode 100644 index 4b9324e8..00000000 --- a/src/ZGroup.jl +++ /dev/null @@ -1,206 +0,0 @@ -struct ZGroup{S<:AbstractStore} - storage::S - path::String - arrays::Dict{String, ZArray} - groups::Dict{String, ZGroup} - attrs::Dict - writeable::Bool -end - -# path can also be a SubString{String} -ZGroup(storage, path::AbstractString, arrays, groups, attrs, writeable) = - ZGroup(storage, String(path), arrays, groups, attrs, writeable) - -zname(g::ZGroup) = zname(g.path) - - - -#Open an existing ZGroup -function ZGroup(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=false) where T<:AbstractStore - arrays = Dict{String, ZArray}() - groups = Dict{String, ZGroup}() - zv = if zarr_format == :auto - ZarrFormat(s, path) - else - ZarrFormat(zarr_format) - end - for d in subdirs(s,path) - dshort = split(d,'/')[end] - subpath = _concatpath(path,dshort) - if is_zarray(zv, s, subpath) - m = zopen_noerr(s, mode, zv, path=_concatpath(path, dshort), fill_as_missing=fill_as_missing) - arrays[dshort] = m - elseif is_zgroup(zv, s, subpath) - m = zopen_noerr(s, mode, zv, path=_concatpath(path, dshort), fill_as_missing=fill_as_missing) - groups[dshort] = m - end - end - attrs = getattrs(zv, s, path) - startswith(path,"/") && error("Paths should never start with a leading '/'") - ZGroup(s, path, arrays, groups, attrs,mode=="w") -end - -#Function to guess a Zarr format from a store and a path, useful for guessing format when trying to open a group/array -ZarrFormat(s::AbstractStore, path) = is_zarr2(s, path) ? ZarrFormat(2) : - is_zarr3(s, path) ? ZarrFormat(3) : - throw(ArgumentError("Specified store $s in path $(path) is neither a ZArray nor a ZGroup in a recognized zarr format.")) - - -""" - zopen_noerr(AbstractStore, mode = "r"; consolidated = false) - -Works like `zopen` with the single difference that no error is thrown when -the path or store does not point to a valid zarr array or group, but nothing -is returned instead. -""" -function zopen_noerr(s::AbstractStore, mode, zv::ZarrFormat; - consolidated = false, - path="", - lru = 0, - fill_as_missing=false) - - consolidated && isinitialized(s, ".zmetadata") && return zopen(ConsolidatedStore(s, path), mode, path=path, lru=lru, fill_as_missing=fill_as_missing) - if lru !== 0 - error("LRU caches are not supported anymore by the current Zarr version. Please use an earlier version of Zarr for now and open an issue at Zarr.jl if you need this functionality") - end - if is_zarray(zv, s, path) - return ZArray(s, mode, path, zv; fill_as_missing=fill_as_missing) - elseif is_zgroup(zv, s, path) - return ZGroup(s, mode, path, zv; fill_as_missing=fill_as_missing) - else - return nothing - end -end - -function Base.show(io::IO, g::ZGroup) - print(io, "ZarrGroup at ", g.storage, " and path ", g.path) - !isempty(g.arrays) && print(io, "\nVariables: ", map(i -> string(zname(i), " "), values(g.arrays))...) - !isempty(g.groups) && print(io, "\nGroups: ", map(i -> string(zname(i), " "), values(g.groups))...) - nothing -end -Base.haskey(g::ZGroup,k)= haskey(g.groups,k) || haskey(g.arrays,k) - - -function Base.getindex(g::ZGroup, k) - if haskey(g.groups, k) - return g.groups[k] - elseif haskey(g.arrays, k) - return g.arrays[k] - else - throw(KeyError("Zarr Dataset does not contain $k")) - end -end - - -""" - zopen(s::AbstractStore, mode="r"; consolidated = false, path = "", lru = 0) - -Opens a zarr Array or Group at Store `s`. If `consolidated` is set to "true", -Zarr will search for a consolidated metadata field as created by the python zarr -`consolidate_metadata` function. This can substantially speed up metadata parsing -of large zarr groups. Setting `lru` to a value > 0 means that chunks that have been -accessed before will be cached and consecutive reads will happen from the cache. -Here, `lru` denotes the number of chunks that remain in memory. The expected zarr version -can be supplied through `zarr_format` and defaults to `:auto` which tries to detect -if the zarr version is v2 or v3. -""" -function zopen(s::AbstractStore, mode="r"; - zarr_format=:auto, - consolidated = false, - path = "", - lru = 0, - fill_as_missing = false) - - zarr_format = if zarr_format == :auto - ZarrFormat(s, path) - else - ZarrFormat(zarr_format) - end - # add interfaces to Stores later - r = zopen_noerr(s, mode, zarr_format; consolidated=consolidated, path=path, lru=lru, fill_as_missing=fill_as_missing) - if r === nothing - throw(ArgumentError("Specified store $s in path $(path) is neither a ZArray nor a ZGroup")) - else - return r - end -end - -""" - zopen(p::String, mode="r") - -Open a zarr Array or group at disc path p. -""" -function zopen(s::String, mode="r"; kwargs...) - store, path = storefromstring(s,false) - zopen(store, mode; path=path, kwargs...) -end - -function storefromstring(s, create=true) - for (r,t) in storageregexlist - if match(r,s) !== nothing - return storefromstring(t,s,create) - end - end - if create || isdir(s) - return DirectoryStore(s), "" - else - throw(ArgumentError("Path $s is not a directory.")) - end -end - -""" - zgroup(s::AbstractStore; attrs=Dict()) - -Create a new zgroup in the store `s` -""" -function zgroup(s::AbstractStore, path::String="", zarr_format=ZarrFormat(2); attrs=Dict(), indent_json::Bool=false) - zv = zarr_format isa ZarrFormat ? zarr_format : ZarrFormat(zarr_format) - isemptysub(s, path) || error("Store is not empty") - write_group_metadata(zv, s, path, attrs; indent_json=indent_json) - ZGroup(s, path, Dict{String,ZArray}(), Dict{String,ZGroup}(), attrs, true) -end - -function write_group_metadata(::ZarrFormat{2}, s::AbstractStore, path, attrs; indent_json::Bool=false) - d = Dict("zarr_format" => 2) - b = IOBuffer() - indent_json ? JSON.print(b, d, 4) : JSON.print(b, d) - s[path, ".zgroup"] = take!(b) - writeattrs(ZarrFormat(Val(2)), s, path, attrs, indent_json=indent_json) -end - -function write_group_metadata(::ZarrFormat{3}, s::AbstractStore, path, attrs; indent_json::Bool=false) - d = Dict{String,Any}("zarr_format" => 3, "node_type" => "group") - if !isempty(attrs) - d["attributes"] = attrs - end - b = IOBuffer() - indent_json ? JSON.print(b, d, 4) : JSON.print(b, d) - s[path, "zarr.json"] = take!(b) -end - -zgroup(s::String;kwargs...)=zgroup(storefromstring(s, true)...;kwargs...) - -"Create a subgroup of the group g" -function zgroup(g::ZGroup, name; attrs=Dict()) - g.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) - subpath = _concatpath(g.path, name) - # Detect format from parent - zv = is_zarr3(g.storage, g.path) ? ZarrFormat(3) : ZarrFormat(2) - g.groups[name] = zgroup(g.storage, subpath, zv; attrs=attrs) -end - -"Create a new subarray of the group g" -function zcreate(::Type{T},g::ZGroup, name::AbstractString, addargs...; kwargs...) where T - g.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) - name = string(name) - z = zcreate(T, g.storage, addargs...; path = _concatpath(g.path,name), kwargs...) - g.arrays[name] = z - return z -end - -HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) -writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) -function consolidate_metadata(z::Union{ZArray,ZGroup}) - z.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) - consolidate_metadata(z.storage,z.path) -end diff --git a/src/chunkkeyencoding.jl b/src/chunkkeyencoding.jl deleted file mode 100644 index 9a9be200..00000000 --- a/src/chunkkeyencoding.jl +++ /dev/null @@ -1,137 +0,0 @@ -abstract type AbstractChunkKeyEncoding end - -struct ChunkKeyEncoding <: AbstractChunkKeyEncoding - sep::Char - prefix::Bool -end - -# Default Zarr v2 separator -const DS2 = '.' -# Default Zarr v3 separator -const DS3 = '/' - -default_sep(::ZarrFormat{2}) = DS2 -default_sep(::ZarrFormat{3}) = DS3 -default_sep(v::Int) = default_sep(ZarrFormat(v)) -default_prefix(::ZarrFormat{2}) = false -default_prefix(::ZarrFormat{3}) = true -const DS = default_sep(DV) - -@inline function citostring(e::ChunkKeyEncoding, i::CartesianIndex) - if e.prefix - "c$(e.sep)" * join(reverse((i - oneunit(i)).I), e.sep) - else - join(reverse((i - oneunit(i)).I), e.sep) - end -end -@inline citostring(e::ChunkKeyEncoding, ::CartesianIndex{0}) = e.prefix ? "c$(e.sep)0" : "0" - -""" - SuffixChunkKeyEncoding{E<:AbstractChunkKeyEncoding} - -Chunk key encoding that appends a user-defined suffix string to the key -produced by a base chunk key encoding. The primary use case is adding file -extensions (e.g. `.tiff`, `.shard.zip`) so that individual chunk files are -directly usable by other software without Zarr-specific tooling. - -Per the zarr-extensions `suffix` chunk-key-encoding proposal. -""" -struct SuffixChunkKeyEncoding{E<:AbstractChunkKeyEncoding} <: AbstractChunkKeyEncoding - suffix::String - base_encoding::E -end - -SuffixChunkKeyEncoding(suffix::String; sep::Char='/', prefix::Bool=true) = - SuffixChunkKeyEncoding(suffix, ChunkKeyEncoding(sep, prefix)) - -@inline citostring(e::SuffixChunkKeyEncoding, i::CartesianIndex) = - citostring(e.base_encoding, i) * e.suffix - -"""Serialize an `AbstractChunkKeyEncoding` to a JSON-compatible dict.""" -function lower_chunk_key_encoding(e::ChunkKeyEncoding) - Dict{String,Any}( - "name" => e.prefix ? "default" : "v2", - "configuration" => Dict{String,Any}("separator" => string(e.sep)) - ) -end - -function lower_chunk_key_encoding(e::SuffixChunkKeyEncoding) - Dict{String,Any}( - "name" => "suffix", - "configuration" => Dict{String,Any}( - "suffix" => e.suffix, - "base_encoding" => lower_chunk_key_encoding(e.base_encoding) - ) - ) -end - -"""Stores a registered chunk key encoding parser together with its expected return type.""" -struct ChunkKeyEncodingEntry - return_type::Type{<:AbstractChunkKeyEncoding} - parser::Function -end - -""" -Registry mapping chunk key encoding names to `ChunkKeyEncodingEntry` values -(return type + parser function). Use `register_chunk_key_encoding` to add new entries. -""" -const chunk_key_encoding_parsers = Dict{String, ChunkKeyEncodingEntry}() - -""" - register_chunk_key_encoding(parser::Function, name::String[, ::Type{T}]) - -Register a chunk key encoding parser under `name`. The parser must accept a -`Dict{String,Any}` configuration and return an `AbstractChunkKeyEncoding`. - -The optional trailing `Type{T}` argument narrows the declared return type stored -in the registry (defaults to `AbstractChunkKeyEncoding`). Specifying it enables -a runtime assertion in `parse_chunk_key_encoding` and makes the registry -self-documenting. - -Supports do-block syntax: - - register_chunk_key_encoding("myenc") do config - MyEncoding(config["param"]) - end - - register_chunk_key_encoding("myenc", MyEncoding) do config - MyEncoding(config["param"]) - end -""" -function register_chunk_key_encoding(parser::Function, name::String, ::Type{T}) where {T<:AbstractChunkKeyEncoding} - chunk_key_encoding_parsers[name] = ChunkKeyEncodingEntry(T, parser) -end -register_chunk_key_encoding(parser::Function, name::String) = - register_chunk_key_encoding(parser, name, AbstractChunkKeyEncoding) - -""" - parse_chunk_key_encoding(d::AbstractDict) -> AbstractChunkKeyEncoding - -Parse a chunk key encoding dict (as found in `zarr.json`) into an -`AbstractChunkKeyEncoding` by looking up the registered parser for the encoding name. -""" -function parse_chunk_key_encoding(d::AbstractDict)::AbstractChunkKeyEncoding - name = d["name"] - config = get(d, "configuration", Dict{String,Any}())::Dict{String,Any} - haskey(chunk_key_encoding_parsers, name) || - throw(ArgumentError("Unknown chunk_key_encoding of name, $name")) - entry = chunk_key_encoding_parsers[name] - return entry.parser(config)::entry.return_type -end - -# Register built-in encodings -register_chunk_key_encoding("default", ChunkKeyEncoding) do config - ChunkKeyEncoding(only(get(config, "separator", '/')), true) -end - -register_chunk_key_encoding("v2", ChunkKeyEncoding) do config - ChunkKeyEncoding(only(get(config, "separator", '.')), false) -end - -register_chunk_key_encoding("suffix", SuffixChunkKeyEncoding) do config - suffix_str = config["suffix"] - base = parse_chunk_key_encoding(config["base_encoding"]) - SuffixChunkKeyEncoding(suffix_str, base) -end - -_concatpath(p,s) = isempty(p) ? s : rstrip(p,'/') * '/' * s diff --git a/src/metadata.jl b/src/metadata.jl deleted file mode 100644 index 7b36bbad..00000000 --- a/src/metadata.jl +++ /dev/null @@ -1,293 +0,0 @@ -import Dates: Date, DateTime -using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring - -"""NumPy array protocol type string (typestr) format - -A string providing the basic type of the homogeneous array. The basic string format -consists of 3 parts: a character describing the byteorder of the data -(<: little-endian, >: big-endian, |: not-relevant), a character code giving the basic -type of the array, and an integer providing the number of bytes the type uses. - -https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding -""" - -include("MaxLengthStrings.jl") -using .MaxLengthStrings: MaxLengthString - -primitive type ASCIIChar <: AbstractChar 8 end -ASCIIChar(x::UInt8) = reinterpret(ASCIIChar, x) -ASCIIChar(x::Integer) = ASCIIChar(UInt8(x)) -Base.UInt8(x::ASCIIChar) = reinterpret(UInt8, x) -Base.codepoint(x::ASCIIChar) = UInt8(x) -Base.show(io::IO, x::ASCIIChar) = print(io, Char(x)) -Base.zero(::Union{ASCIIChar,Type{ASCIIChar}}) = ASCIIChar(Base.zero(UInt8)) - -Base.zero(t::Union{String, Type{String}}) = "" - -typestr(t::Type) = string('<', 'V', sizeof(t)) -typestr(t::Type{>:Missing}) = typestr(Base.nonmissingtype(t)) -typestr(t::Type{Bool}) = string('|', 'b', sizeof(t)) -typestr(t::Type{<:Int8}) = string("|i1") -typestr(t::Type{<:Signed}) = string('<', 'i', sizeof(t)) -typestr(t::Type{<:UInt8}) = string("|u1") -typestr(t::Type{<:Unsigned}) = string('<', 'u', sizeof(t)) -typestr(t::Type{Complex{T}} where T<:AbstractFloat) = string('<', 'c', sizeof(t)) -typestr(t::Type{<:AbstractFloat}) = string('<', 'f', sizeof(t)) -typestr(::Type{MaxLengthString{N,UInt32}}) where N = string('<', 'U', N) -typestr(::Type{MaxLengthString{N,UInt8}}) where N = string('<', 'S', N) -typestr(::Type{<:Array}) = "|O" -typestr(t::Type{<:DateTime64}) = pydatetime_string(t) -typestr(::Type{<:AbstractString}) = "|O" - -const typestr_regex = r"^([<|>])([tbiufcmMOSUV])(\d*)(\[\w+\])?$" -const typemap = Dict{Tuple{Char, Int}, DataType}( - ('b', 1) => Bool, - ('S', 1) => ASCIIChar, - ('U', 1) => Char, -) -sizemapf(x::Type{<:Number}) = sizeof(x) -typecharf(::Type{<:Signed}) = 'i' -typecharf(::Type{<:Unsigned}) = 'u' -typecharf(::Type{<:AbstractFloat}) = 'f' -typecharf(::Type{<:Complex}) = 'c' -foreach([Float16,Float32,Float64,Int8,Int16,Int32,Int64,Int128, - UInt8,UInt16,UInt32,UInt64,UInt128, - Complex{Float16},Complex{Float32},Complex{Float64}]) do t - typemap[(typecharf(t),sizemapf(t))] = t -end - -function typestr(s::AbstractString, filterlist=nothing) - m = match(typestr_regex, s) - if m === nothing - throw(ArgumentError("$s is not a valid numpy typestr")) - else - - byteorder, typecode, typesize, typespec = m.captures - if typecode == "O" - if filterlist === nothing - throw(ArgumentError("Object array can only be parsed when an appropriate filter is defined")) - end - return sourcetype(first(filterlist)) - end - isempty(typesize) && throw((ArgumentError("$s is not a valid numpy typestr"))) - tc, ts = first(typecode), parse(Int, typesize) - if (tc in ('U','S')) && ts > 1 - return MaxLengthString{ts,tc=='U' ? UInt32 : UInt8} - end - if tc == 'M' && ts == 8 - #We have a datetime64 value - return datetime_from_pystring(s) - end - # convert typecode to Char and typesize to Int - typemap[(tc,ts)] - end -end - - -"""Metadata configuration of the stored array - -Each array requires essential configuration metadata to be stored, enabling correct -interpretation of the stored data. This metadata is encoded using JSON and stored as the -value of the ".zarray" key within an array store. - -# Type Parameters -* T - element type of the array -* N - dimensionality of the array -* C - compressor -* F - filters - -# See Also - -https://zarr.readthedocs.io/en/stable/spec/v2.html#metadata -""" -abstract type AbstractMetadata{T,N,E <: AbstractChunkKeyEncoding} end -Base.ndims(::AbstractMetadata{<:Any,N}) where N = N - - -"""Metadata for Zarr version 2 arrays""" -struct MetadataV2{T,N,C,F} <: AbstractMetadata{T,N,ChunkKeyEncoding} - zarr_format::Int - node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} - dtype::String # structured data types not yet supported - compressor::C - fill_value::Union{T, Nothing} - order::Char - filters::F # not yet supported - chunk_key_encoding::ChunkKeyEncoding - function MetadataV2{T2,N,C,F}(zarr_format, node_type, shape, chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F} - zarr_format == 2 || throw(ArgumentError("MetadataV2 only functions if zarr_format == 2")) - #Do some sanity checks to make sure we have a sane array - any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) - order === 'C' || throw(ArgumentError("Currently only 'C' storage order is supported")) - new{T2,N,C,F}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) - end -end -zarr_format(::MetadataV2) = ZarrFormat(Val(2)) - -# Type alias for backward compatibility -const Metadata = AbstractMetadata - -#To make unit tests pass with ref shape -function Base.:(==)(m1::MetadataV2, m2::MetadataV2) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.compressor == m2.compressor && - m1.fill_value == m2.fill_value && - m1.order == m2.order && - m1.filters == m2.filters && - m1.chunk_key_encoding == m2.chunk_key_encoding -end - - -"Construct Metadata based on your data" -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '.' - ) where {T, N, C} - return Metadata(A, chunks, ZarrFormat(zarr_format); - node_type=node_type, - compressor=compressor, - fill_value=fill_value, - order=order, - filters=filters, - fill_as_missing=fill_as_missing, - chunk_key_encoding=ChunkKeyEncoding(dimension_separator, default_prefix(ZarrFormat(zarr_format))) - ) -end - -# V2 constructor -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters::F=nothing, - fill_as_missing = false, - chunk_key_encoding=ChunkKeyEncoding('.', false) - ) where {T, N, C, F} - T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} - MetadataV2{T2,N,C,typeof(filters)}( - 2, - node_type, - size(A), - chunks, - typestr(eltype(A)), - compressor, - fill_value, - order, - filters, - chunk_key_encoding, - ) -end - -Metadata(s::Union{AbstractString, IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) - -"Construct Metadata from Dict" -function Metadata(d::AbstractDict, fill_as_missing) - zarr_format = d["zarr_format"]::Int - zarr_format ∉ (2, 3) && throw(ArgumentError("Zarr.jl currently only supports v2 or v3 of the specification")) - return Metadata(d, fill_as_missing, ZarrFormat(zarr_format)) -end - -# V2 constructor from Dict -function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{2}) - # Zarr v2 metadata is only for arrays - node_type = "array" - - compdict = d["compressor"] - if isnothing(compdict) - # try the last filter, for Kerchunk compat - if !isnothing(d["filters"]) && haskey(compressortypes, d["filters"][end]["id"]) - compdict = pop!(d["filters"]) # TODO: this will not work with JSON3! - end - end - compressor = getCompressor(compdict) - - filters = getfilters(d) - - T = typestr(d["dtype"], filters) - N = length(d["shape"]) - C = typeof(compressor) - F = typeof(filters) - - fv = fill_value_decoding(d["fill_value"], T) - - TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} - - dim_sep = only(get(d, "dimension_separator", '.')) - - MetadataV2{TU,N,C,F}( - d["zarr_format"], - node_type, - NTuple{N, Int}(d["shape"]) |> reverse, - NTuple{N, Int}(d["chunks"]) |> reverse, - d["dtype"], - compressor, - fv, - first(d["order"]), - filters, - ChunkKeyEncoding(dim_sep, false), - ) -end - - -"Describes how to lower Metadata to JSON, used in json(::Metadata)" -function JSON.lower(md::MetadataV2) - Dict{String, Any}( - "zarr_format" => Int(md.zarr_format), - "node_type" => md.node_type, - "shape" => md.shape[] |> reverse, - "chunks" => md.chunks |> reverse, - "dtype" => md.dtype, - "compressor" => md.compressor, - "fill_value" => fill_value_encoding(md.fill_value), - "order" => md.order, - "filters" => md.filters, - "dimension_separator" => md.chunk_key_encoding.sep - ) -end - - -# Fill value encoding and decoding as described in -# https://zarr.readthedocs.io/en/stable/spec/v2.html#fill-value-encoding - -fill_value_encoding(v) = v -fill_value_encoding(::Nothing)=nothing -function fill_value_encoding(v::AbstractFloat) - if isnan(v) - "NaN" - elseif isinf(v) - v>0 ? "Infinity" : "-Infinity" - else - v - end -end - -Base.eltype(::AbstractMetadata{T}) where T = T - -# this correctly parses "NaN" and "Infinity" -fill_value_decoding(v::AbstractString, T::Type{<:Number}) = parse(T, v) -fill_value_decoding(v::Nothing, ::Any) = v -fill_value_decoding(v, T) = T(v) -fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) -fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v -fill_value_decoding(v::Nothing, ::Type{Zarr.ASCIIChar}) = v -# Sometimes when translating between CF (climate and forecast) convention data -# and Zarr groups, fill values are left as "negative integers" to encode unsigned -# integers. So, we have to convert to the signed type with the same number of bytes -# as the unsigned integer, then reinterpret as unsigned. That's how a fill value -# of -1 can have a realistic meaning with an unsigned dtype. -# However, we have to apply this correction only if the integer is negative. -# If it's positive, then the value might be out of range of the signed integer type. -fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = sign(v) < 0 ? reinterpret(T, signed(T)(v)) : T(v) diff --git a/src/metadata3.jl b/src/metadata3.jl deleted file mode 100644 index 19c47c97..00000000 --- a/src/metadata3.jl +++ /dev/null @@ -1,456 +0,0 @@ -""" -Prototype Zarr version 3 support -""" - -const typemap3 = Dict{String, DataType}() -foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t - typemap3[lowercase(string(t))] = t -end -typemap3["complex64"] = ComplexF32 -typemap3["complex128"] = ComplexF64 - -function typestr3(t::Type) - return lowercase(string(t)) -end -# TODO: Check raw types -function typestr3(::Type{NTuple{N,UInt8}}) where {N} - return "r$(N*8)" -end - -function typestr3(s::AbstractString, codecs=nothing) - if !haskey(typemap3, s) - if startswith(s, "r") - num_bits = tryparse(Int, s[2:end]) - if isnothing(num_bits) - throw(ArgumentError("$s is not a known type")) - end - if mod(num_bits, 8) == 0 - return NTuple{num_bits÷8,UInt8} - else - throw(ArgumentError("$s must describe a raw type with bit size that is a multiple of 8 bits")) - end - end - end - return typemap3[s] -end - -function check_keys(d::AbstractDict, keys) - for key in keys - if !haskey(d, key) - throw(ArgumentError("Zarr v3 metadata must have a key called $key")) - end - end -end - -"""Metadata for Zarr version 3 arrays""" -struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} - zarr_format::Int - node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} - dtype::String # data_type in v3 - pipeline::P - fill_value::Union{T, Nothing} - chunk_key_encoding::E - function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} - zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) - #Do some sanity checks to make sure we have a sane array - any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) - new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) - end -end -MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) -zarr_format(::MetadataV3) = ZarrFormat(Val(3)) - -""" -Convenience constructor for MetadataV3 that builds the codec pipeline from -`order` (translated to a TransposeCodec), `endian` (translated to a BytesCodec), -and `compressor` (translated to bytes->bytes codecs). -""" -function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks::NTuple{N,Int}, - dtype::String, fill_value; - order::Char='C', - endian::Symbol=:little, - compressor=BloscCompressor(), - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T2, N, E} - T_base = Base.nonmissingtype(T2) - array_array_codecs = if order == 'F' - (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) - else - () - end - array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) - bytes_bytes_codecs = if compressor isa NoCompressor - () - elseif compressor isa BloscCompressor - (Codecs.V3Codecs.BloscV3Codec(compressor.cname, compressor.clevel, compressor.shuffle, compressor.blocksize, sizeof(T_base)),) - elseif compressor isa ZlibCompressor - # ZlibCompressor uses -1 to mean "default"; zarr v3 gzip spec requires 0-9 - level = compressor.config.level == -1 ? 6 : compressor.config.level - (Codecs.V3Codecs.GzipV3Codec(level),) - elseif compressor isa ZstdCompressor - (Codecs.V3Codecs.ZstdV3Codec(compressor.config.compressionLevel),) - else - throw(ArgumentError("Unsupported compressor type for v3: $(typeof(compressor))")) - end - pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) - return MetadataV3{T2,N,typeof(pipeline),E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) -end - -function Base.:(==)(m1::MetadataV3, m2::MetadataV3) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.fill_value == m2.fill_value && - m1.pipeline == m2.pipeline && - m1.chunk_key_encoding == m2.chunk_key_encoding -end - -""" -Derive the storage order ('C' or 'F') from the codec pipeline of a MetadataV3. - -Throws `ArgumentError` if the order cannot be unambiguously determined, which -occurs when: -- the pipeline contains more than one array->array codec, -- an array->array codec is not a `TransposeCodec` (unknown effect on order), or -- the `TransposeCodec` permutation is neither the identity (C order) nor the - full reversal (F order). -""" -function get_order(md::MetadataV3) - array_array = md.pipeline.array_array - if length(array_array) == 0 - return 'C' - end - if length(array_array) > 1 - throw(ArgumentError( - "Cannot determine storage order: pipeline has $(length(array_array)) " * - "array->array codecs; composed permutations yield an indeterminate order" - )) - end - codec = only(array_array) - if !(codec isa Codecs.V3Codecs.TransposeCodec) - throw(ArgumentError( - "Cannot determine storage order: unrecognized array->array codec $(typeof(codec))" - )) - end - N = ndims(md) - c_perm = ntuple(identity, N) - f_perm = ntuple(i -> N - i + 1, N) - if codec.order == c_perm - return 'C' - elseif codec.order == f_perm - return 'F' - else - throw(ArgumentError( - "Cannot determine storage order: TransposeCodec permutation $(codec.order) " * - "is neither C order $c_perm nor F order $f_perm" - )) - end -end -get_order(md::MetadataV2) = md.order - - - -function Metadata3(d::AbstractDict, fill_as_missing) - check_keys(d, ("zarr_format", "node_type")) - - zarr_format = d["zarr_format"]::Int - - node_type = d["node_type"]::String - if node_type ∉ ("group", "array") - throw(ArgumentError("Unknown node_type of $node_type")) - end - - zarr_format == 3 || throw(ArgumentError("Metadata3 only functions if zarr_format == 3")) - - # Groups - if node_type == "group" - # Groups only need zarr_format and node_type - # Optionally they can have attributes - for key in keys(d) - if key ∉ ("zarr_format", "node_type", "attributes") - throw(ArgumentError("Zarr v3 group metadata cannot have a key called $key")) - end - end - - group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) - return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) - end - - # Array keys - mandatory_keys = [ - "zarr_format", - "node_type", - "shape", - "data_type", - "chunk_grid", - "chunk_key_encoding", - "fill_value", - "codecs", - ] - optional_keys = [ - "attributes", - "storage_transformers", - "dimension_names", - ] - - check_keys(d, mandatory_keys) - for key in keys(d) - if key ∉ mandatory_keys && key ∉ optional_keys - throw(ArgumentError("Zarr v3 metadata cannot have a key called $key")) - end - end - - # Shape - shape = Int.(d["shape"]) - - # Datatype - data_type = d["data_type"]::String - - # Chunk Grid - chunk_grid = d["chunk_grid"] - if chunk_grid["name"] == "regular" - chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) - if length(shape) != length(chunks) - throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) - end - else - throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) - end - - # Chunk Key Encoding - chunk_key_encoding = d["chunk_key_encoding"] - - # Build V3Pipeline from codec chain - array_array_codecs = [] - array_bytes_codec = nothing - bytes_bytes_codecs = [] - order = 'C' # default - - for codec in d["codecs"] - codec_name = codec["name"] - config = get(codec, "configuration", Dict{String,Any}()) - if codec_name == "transpose" - _order = config["order"] - if _order isa AbstractString - n = length(shape) - if _order == "C" - @warn "Transpose codec dimension order of C is deprecated" - perm = ntuple(identity, n) - elseif _order == "F" - @warn "Transpose codec dimension order of F is deprecated" - perm = ntuple(i -> n - i + 1, n) - order = 'F' - else - throw(ArgumentError("Unknown transpose order string: $_order")) - end - else - perm = Tuple(Int.(_order) .+ 1) - default_perm = ntuple(identity, length(shape)) - rev_perm = ntuple(i -> length(shape) - i + 1, length(shape)) - if perm == default_perm - order = 'C' - elseif perm == rev_perm - order = 'F' - end - end - push!(array_array_codecs, Codecs.V3Codecs.TransposeCodec(perm)) - elseif codec_name == "bytes" - endian_str = get(config, "endian", "little") - endian = endian_str == "little" ? :little : - endian_str == "big" ? :big : - throw(ArgumentError("Unknown endian value: \"$endian_str\"")) - array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) - elseif codec_name == "sharding_indexed" - throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) - elseif codec_name == "gzip" - level = get(config, "level", 6) - push!(bytes_bytes_codecs, Codecs.V3Codecs.GzipV3Codec(level)) - elseif codec_name == "blosc" - cname = get(config, "cname", "lz4") - clevel = get(config, "clevel", 5) - shuffle_val = get(config, "shuffle", "noshuffle") - shuffle_int = shuffle_val isa Integer ? shuffle_val : - shuffle_val == "noshuffle" ? 0 : - shuffle_val == "shuffle" ? 1 : - shuffle_val == "bitshuffle" ? 2 : - throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) - blocksize = get(config, "blocksize", 0) - typesize = get(config, "typesize", 4) - push!(bytes_bytes_codecs, Codecs.V3Codecs.BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize)) - elseif codec_name == "zstd" - level = get(config, "level", 3) - push!(bytes_bytes_codecs, Codecs.V3Codecs.ZstdV3Codec(level)) - elseif codec_name == "crc32c" - push!(bytes_bytes_codecs, Codecs.V3Codecs.CRC32cV3Codec()) - else - throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) - end - end - - isnothing(array_bytes_codec) && throw(ArgumentError("V3 codec chain must contain a 'bytes' codec")) - pipeline = V3Pipeline(Tuple(array_array_codecs), array_bytes_codec, Tuple(bytes_bytes_codecs)) - - # Type Parameters - T = typestr3(data_type) - N = length(shape) - - fv = fill_value_decoding(d["fill_value"], T)::T - - TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} - - chunk_key_encoding = parse_chunk_key_encoding(chunk_key_encoding) - E = typeof(chunk_key_encoding) - - MetadataV3{TU, N, typeof(pipeline), E}( - zarr_format, - node_type, - NTuple{N, Int}(shape) |> reverse, - NTuple{N, Int}(chunks) |> reverse, - data_type, - pipeline, - fv, - chunk_key_encoding, - ) -end - -"Construct MetadataV3 based on your data" -function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; - node_type::String="array", - compressor=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '/' - ) where {T, N} - @warn("Zarr v3 support is experimental") - T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} - if fill_value === nothing - fill_value = zero(T) - end - return MetadataV3{T2, N}( - 3, - node_type, - size(A), - chunks, - typestr3(eltype(A)), - fill_value; - order=order, - endian=endian, - compressor=compressor, - chunk_key_encoding=ChunkKeyEncoding(dimension_separator, true) - ) -end - -function lower3(md::MetadataV3{T}) where T - chunk_grid = Dict{String,Any}( - "name" => "regular", - "configuration" => Dict{String,Any}( - "chunk_shape" => md.chunks |> reverse - ) - ) - - # chunk_key_encoding - chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) - - # Build codecs from pipeline - codecs = Dict{String,Any}[] - p = md.pipeline - - # array->array codecs - for codec in p.array_array - if codec isa Codecs.V3Codecs.TransposeCodec - push!(codecs, Dict{String,Any}( - "name" => "transpose", - "configuration" => Dict("order" => collect(codec.order .- 1)) - )) - end - end - - # array->bytes codec - if p.array_bytes isa Codecs.V3Codecs.BytesCodec - push!(codecs, Dict{String,Any}( - "name" => "bytes", - "configuration" => Dict{String,Any}("endian" => string(p.array_bytes.endian)) - )) - end - - # bytes->bytes codecs - for codec in p.bytes_bytes - if codec isa Codecs.V3Codecs.GzipV3Codec - push!(codecs, Dict{String,Any}( - "name" => "gzip", - "configuration" => Dict{String,Any}("level" => codec.level) - )) - elseif codec isa Codecs.V3Codecs.BloscV3Codec - push!(codecs, Dict{String,Any}( - "name" => "blosc", - "configuration" => Dict{String,Any}( - "cname" => codec.cname, - "clevel" => codec.clevel, - "shuffle" => codec.shuffle == 0 ? "noshuffle" : - codec.shuffle == 1 ? "shuffle" : - codec.shuffle == 2 ? "bitshuffle" : - throw(ArgumentError("Unknown shuffle integer: $(codec.shuffle)")), - "blocksize" => codec.blocksize, - "typesize" => codec.typesize - ) - )) - elseif codec isa Codecs.V3Codecs.ZstdV3Codec - push!(codecs, Dict{String,Any}( - "name" => "zstd", - "configuration" => Dict{String,Any}("level" => codec.level) - )) - elseif codec isa Codecs.V3Codecs.CRC32cV3Codec - push!(codecs, Dict{String,Any}("name" => "crc32c")) - end - end - - Dict{String, Any}( - "zarr_format" => Int(md.zarr_format), - "node_type" => md.node_type, - "shape" => md.shape[] |> reverse, - "data_type" => typestr3(T), - "chunk_grid" => chunk_grid, - "chunk_key_encoding" => chunk_key_encoding, - "fill_value" => fill_value_encoding(md.fill_value), - "codecs" => codecs - ) -end - -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters::F=nothing, - fill_as_missing = false, - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T, N, C, F, E} - return Metadata3(A, chunks; - node_type=node_type, - compressor=compressor, - fill_value=fill_value, - order=order, - endian=endian, - filters=filters, - fill_as_missing=fill_as_missing, - dimension_separator=chunk_key_encoding.sep - ) -end - -# V3 constructor from Dict - delegate to Metadata3 -function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{3}) - return Metadata3(d, fill_as_missing) -end - -function JSON.lower(md::MetadataV3) - return lower3(md) -end diff --git a/src/pipeline.jl b/src/pipeline.jl deleted file mode 100644 index c7eb9229..00000000 --- a/src/pipeline.jl +++ /dev/null @@ -1,77 +0,0 @@ -""" -V2Pipeline wraps the existing v2 compressor + filter pair. -Delegates to zcompress!/zuncompress! with zero behavior change. -""" -struct V2Pipeline{C<:Compressor, F} <: AbstractCodecPipeline - compressor::C - filters::F -end - -function pipeline_encode(p::V2Pipeline, data::AbstractArray, fill_value) - if fill_value !== nothing && all(isequal(fill_value), data) - return nothing - end - dtemp = UInt8[] - zcompress!(dtemp, data, p.compressor, p.filters) - return dtemp -end - -function pipeline_decode!(p::V2Pipeline, output::AbstractArray, compressed::Vector{UInt8}) - zuncompress!(output, compressed, p.compressor, p.filters) - return output -end - -""" -V3Pipeline holds a three-phase v3 codec chain: -- array_array: tuple of array->array codecs (e.g. transpose) -- array_bytes: single array->bytes codec (e.g. bytes, sharding) -- bytes_bytes: tuple of bytes->bytes codecs (e.g. gzip, blosc, crc32c) -""" -struct V3Pipeline{AA, AB, BB} <: AbstractCodecPipeline - array_array::AA - array_bytes::AB - bytes_bytes::BB -end - -function pipeline_encode(p::V3Pipeline, data::AbstractArray, fill_value) - if fill_value !== nothing && all(isequal(fill_value), data) - return nothing - end - # Phase 1: array->array codecs (forward order) - result = data - for codec in p.array_array - result = Codecs.V3Codecs.codec_encode(codec, result) - end - # Phase 2: array->bytes codec - bytes = Codecs.V3Codecs.codec_encode(p.array_bytes, result) - # Phase 3: bytes->bytes codecs (forward order) - for codec in p.bytes_bytes - bytes = Codecs.V3Codecs.codec_encode(codec, bytes) - end - return bytes -end - -function pipeline_decode!(p::V3Pipeline, output::AbstractArray, compressed::Vector{UInt8}) - # Phase 3 reverse: bytes->bytes codecs (reverse order) - bytes = compressed - for codec in reverse(collect(p.bytes_bytes)) - bytes = Codecs.V3Codecs.codec_decode(codec, bytes) - end - # Phase 2 reverse: bytes->array codec - # Compute the intermediate shape — the shape data has after array_array encoding - intermediate_shape = foldl( - (sz, codec) -> Codecs.V3Codecs.encoded_shape(codec, sz), - p.array_array; init=size(output) - ) - arr = Codecs.V3Codecs.codec_decode(p.array_bytes, bytes, eltype(output), intermediate_shape) - # Phase 1 reverse: array->array codecs (reverse order) - for codec in reverse(collect(p.array_array)) - arr = Codecs.V3Codecs.codec_decode(codec, arr) - end - copyto!(output, arr) - return output -end - -# Convenience: extract pipeline from metadata -get_pipeline(m::MetadataV2) = V2Pipeline(m.compressor, m.filters) -get_pipeline(m::MetadataV3) = m.pipeline From f7f0f41f9e2468fd418bcdcaf07108e09256968f Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:12:39 +0100 Subject: [PATCH 09/12] docs: update CLAUDE.md with ZarrCore package structure Add ZarrCore test command and package structure documentation describing the core/wrapper split, default_compressor override pattern, and v3_codec_parsers registry. Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 100 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index dfbdb435..b2c2a398 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Zarr.jl is a Julia implementation of the Zarr specification for chunked, compressed, N-dimensional arrays. It supports Zarr v2 (stable) and v3 (experimental, in development on `as/continue_v3` branch). The package provides multiple storage backends (filesystem, in-memory, S3, GCS, HTTP, ZIP) and compressors (Blosc, zlib, Zstandard). +Zarr.jl is a Julia implementation of the Zarr specification for chunked, compressed, N-dimensional arrays. It supports Zarr v2 (stable) and v3 (experimental). The package provides multiple storage backends (filesystem, in-memory, S3, GCS, HTTP, ZIP) and compressors (Blosc, zlib, Zstandard). ## Build & Test Commands @@ -12,32 +12,55 @@ Zarr.jl is a Julia implementation of the Zarr specification for chunked, compres # Run all tests julia --project -e 'using Pkg; Pkg.test()' -# Run a single test file interactively (use the test/ project environment) +# Run a single test file (use the test/ project environment) julia --project=test -e 'using Test, Zarr, JSON; include("test/v3_codecs.jl")' -# Instantiate test dependencies (after Julia version change or first time setup) +# Instantiate test dependencies (first time or after Julia version change) julia --project=test -e 'using Pkg; Pkg.instantiate()' + +# Run ZarrCore tests only +julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()' ``` Julia version requirement: 1.10+. CI tests against Julia LTS, stable (`1`), nightly, and pre-release on Ubuntu, macOS, and Windows. +### Test Structure + +- `test/runtests.jl` — Main test suite: ZArray, groups, metadata, indexing, resize, concat, strings, ragged arrays, fill values. Includes the other test files. +- `test/storage.jl` — Storage backend tests +- `test/Filters.jl` — Filter tests +- `test/python.jl` — Interoperability tests with Python-generated Zarr files +- `test/v3_codecs.jl` — V3 codec tests (BytesCodec, TransposeCodec, etc.) +- `test/v3_julia.jl` — Julia-generated V3 test fixtures +- `test/v3_python.jl` — Python-generated V3 fixtures (requires PythonCall + CondaPkg; CI generates these before running tests) + ## Architecture +### Package Structure +- `lib/ZarrCore/` — Minimal core package with types, registries, NoCompressor, BytesCodec, TransposeCodec, DirectoryStore, DictStore, ConsolidatedStore, all pure-Julia filters +- `src/` (Zarr.jl) — Wrapper that depends on ZarrCore, adds Blosc/Zlib/Zstd compressors, V3 compression codecs, HTTP/GCS/S3/ZIP stores +- ZarrCore's `default_compressor()` returns `NoCompressor()`; Zarr.jl overrides it to `BloscCompressor()` +- V3 codecs use `v3_codec_parsers` registry for extensible parsing; Zarr.jl registers compression codec parsers + ### Core Type Hierarchy ``` -AbstractMetadata{T,N} -├── MetadataV2{T,N,C,F} # .zarray + .zattrs (v2) -└── MetadataV3{T,N,P} # zarr.json (v3), P<:AbstractCodecPipeline +AbstractMetadata{T,N,E<:AbstractChunkKeyEncoding} +├── MetadataV2{T,N,C,F} # .zarray + .zattrs (v2) +└── MetadataV3{T,N,P,E} # zarr.json (v3), P<:AbstractCodecPipeline, E<:AbstractChunkKeyEncoding AbstractStore -├── DirectoryStore # Filesystem -├── DictStore # In-memory Dict{String,Vector{UInt8}} -├── S3Store # AWS S3 (via AWSS3 extension) -├── GCStore # Google Cloud Storage -├── ConsolidatedStore # Consolidated metadata wrapper -├── HTTPStore # HTTP-based read-only -└── ZipStore # ZIP archive read-only +├── DirectoryStore # Filesystem +├── DictStore # In-memory Dict{String,Vector{UInt8}} +├── S3Store # AWS S3 (via AWSS3 extension) +├── GCStore # Google Cloud Storage +├── ConsolidatedStore # Consolidated metadata wrapper +├── HTTPStore # HTTP-based read-only +└── ZipStore # ZIP archive read-only + +AbstractCodecPipeline +├── V2Pipeline{C,F} # compressor + filters +└── V3Pipeline{AA,AB,BB} # three-phase codec chain ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata} <: AbstractDiskArray{T,N} ZGroup{S<:AbstractStore} @@ -45,23 +68,24 @@ ZGroup{S<:AbstractStore} ### Module/File Layout -- `src/Zarr.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`) -- `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors for V2; dispatches V3 to `metadata3.jl` -- `src/metadata3.jl` — All V3-specific code: `MetadataV3` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, codec pipeline parsing, `get_order`, `JSON.lower(::MetadataV3)` -- `src/chunkencoding.jl` — `ChunkEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/Zarr.jl` — Module entry point. Defines `ZarrFormat{V}` (Val-parameterized version tag) and default `DV = ZarrFormat(Val(2))`. Default chunk separator constants: `DS = '.'` (v2), `DS2 = '.'`, `DS3 = '/'`. +- `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors; dispatches V3 to `metadata3.jl` +- `src/metadata3.jl` — `MetadataV3{T,N,P,E}` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, `get_order`, `JSON.lower(::MetadataV3)`. V3 type mappings use lowercase strings (e.g. `"float32"`, `"complex128"`) +- `src/chunkkeyencoding.jl` — `ChunkKeyEncoding` struct (separator char + prefix bool) and `SuffixChunkKeyEncoding{E}`, `citostring()` for chunk path generation, registry via `chunk_key_encoding_parsers` dict. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/pipeline.jl` — `V2Pipeline` (wraps compressor + filters) and `V3Pipeline` (three-phase: array→array, array→bytes, bytes→bytes). `pipeline_encode`/`pipeline_decode!` interface - `src/ZArray.jl` — Core array type, `readblock!`/`writeblock!` (DiskArrays interface), `zcreate`, `zzeros`, `zopen`, resize/append - `src/ZGroup.jl` — Hierarchical group support, `zopen`, `zgroup`, auto-detection of zarr version via `ZarrFormat(store, path)` - `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl`, `v3.jl` (v3 wrapper `Compressor_v3{C}`) -- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec` +- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec`. Registry: `codectypes` dict - `src/Filters/` — `Filter{T,TENC}` abstract type, implementations for variable-length arrays, strings, Fletcher32, shuffle, delta, quantize - `src/Storage/Storage.jl` — `AbstractStore` interface, I/O strategy (`SequentialRead`/`ConcurrentRead`), chunk read/write/delete helpers, metadata read/write dispatched on `ZarrFormat{2}` vs `ZarrFormat{3}` ### Key Design Patterns - **Format version dispatch**: `ZarrFormat{V}` (where V is 2 or 3) used throughout for multiple dispatch on version-specific behavior (metadata file names, chunk encoding, serialization format) +- **Registry pattern**: Used pervasively — `compressortypes` (compressors), `codectypes` (V3 codecs), `chunk_key_encoding_parsers` (chunk key encodings), filter dict. New implementations register via these dicts. - **Shape is mutable**: `metadata.shape` is `Base.RefValue{NTuple{N,Int}}` to allow `resize!` without replacing the metadata struct - **Column-major ↔ row-major**: Zarr stores shapes/chunks in row-major (C order); Julia uses column-major. All conversions happen via `reverse()` at metadata boundaries -- **Compressor registry**: `compressortypes` dict maps spec names (e.g., `"blosc"`, `"zlib"`) to compressor types. V3 uses `Compressor_v3{C}` wrapper to change JSON serialization format - **DiskArrays integration**: `ZArray <: AbstractDiskArray`, chunk-aware I/O via `readblock!`/`writeblock!`, `eachchunk`, `haschunks` - **Async chunk I/O**: Channel-based parallel chunk reads/writes for stores supporting `ConcurrentRead` @@ -69,32 +93,24 @@ ZGroup{S<:AbstractStore} New store backends must implement: `getindex(store, key)::Union{Vector{UInt8}, Nothing}`, `setindex!(store, data, key)`, `storagesize(store, path)`, `subdirs(store, path)`, `subkeys(store, path)`, `isinitialized(store, key)`, `storefromstring(Type, string, create)`. +### V3 Pipeline Architecture + +V3 encoding/decoding follows a three-phase codec chain in `V3Pipeline`: + +1. **array→array** codecs (e.g., `TransposeCodec`) — applied in forward order during encode +2. **array→bytes** codec (e.g., `BytesCodec`, `ShardingCodec`) — single codec, converts array to byte stream +3. **bytes→bytes** codecs (e.g., `GzipV3Codec`, `BloscV3Codec`, `CRC32cV3Codec`) — applied in forward order during encode + +Decoding reverses all three phases. The `MetadataV3` convenience constructor builds the pipeline from `order` (→ `TransposeCodec`), `endian` (→ `BytesCodec`), and `compressor` (→ bytes→bytes codecs). + ### V3 Status (Experimental) -V3 support is under active development. Current state: - -**Codecs (`src/Codecs/V3/V3.jl`)** -- `BytesCodec` — stores `endian::Symbol` (`:little` or `:big`); encode/decode byte-swap elements when the target endian differs from the system byte order (`Base.ENDIAN_BOM`). Default is `:little`. -- `TransposeCodec` — array→array permutation codec (renamed from `TransposeCodecImpl`) -- `BloscV3Codec` — shuffle stored as integer (0=noshuffle, 1=shuffle, 2=bitshuffle); parsed from spec strings (`"noshuffle"`, `"shuffle"`, `"bitshuffle"`) and serialized back to strings -- Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but is not yet wired into the main read/write pipeline (throws `ArgumentError` when encountered) -- `crc32c` codec has encode/decode implementations and is parseable from metadata - -**Metadata (`src/metadata3.jl`)** -- `MetadataV3{T,N,P}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec` -- Two constructors: - - Primary inner constructor: `MetadataV3{T,N,P}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_encoding)` — takes a pre-built pipeline, no `order` argument - - Convenience outer constructor: `MetadataV3{T,N}(...; order, endian, compressor, chunk_encoding)` — builds the pipeline from `order` (→ `TransposeCodec`), `endian` (→ `BytesCodec`), and `compressor` (→ bytes→bytes codecs) -- `get_order(md::MetadataV3)` — derives `'C'`/`'F'` from the pipeline; throws `ArgumentError` when order is ambiguous: multiple array→array codecs, an unrecognized array→array codec, or a `TransposeCodec` with a permutation that is neither the identity (C) nor the full reversal (F) -- `get_order(md::MetadataV2)` — returns `md.order` -- `==` for `MetadataV3` compares `pipeline` (not `order`) - -**Other V3 status** +- `MetadataV3{T,N,P,E}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec`. `get_order(md::MetadataV3)` derives `'C'`/`'F'` from the pipeline. +- Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but throws `ArgumentError` when encountered (not wired into main pipeline) +- Filters are not implemented for V3 (`filters = nothing` hardcoded) +- `zgroup()` always creates v2 groups (hardcoded `DV` format) +- `crc32c` codec has full encode/decode implementations - V3 groups work via `zarr.json` with `node_type: "group"` -- `zgroup()` currently always creates v2 groups (hardcoded `DV` format) -- Filters are not implemented for v3 (`filters = nothing` hardcoded in `Metadata3`) -- Test fixtures: `test/v3_julia.jl` (Julia-generated), `test/v3_python.jl` (Python-generated via PythonCall) -- V3 codec tests are in `test/v3_codecs.jl` ### Extension System From 7bcc7b422e913572f228dc17480c6ee888668382 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 19:12:46 +0100 Subject: [PATCH 10/12] docs: add ZarrCore extraction design and implementation plan - 2026-03-20-zarrcore-extraction-design.md: architectural decisions - 2026-03-20-zarrcore-extraction.md: 22-task implementation plan Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-03-20-zarrcore-extraction-design.md | 133 ++ docs/plans/2026-03-20-zarrcore-extraction.md | 1826 +++++++++++++++++ 2 files changed, 1959 insertions(+) create mode 100644 docs/plans/2026-03-20-zarrcore-extraction-design.md create mode 100644 docs/plans/2026-03-20-zarrcore-extraction.md diff --git a/docs/plans/2026-03-20-zarrcore-extraction-design.md b/docs/plans/2026-03-20-zarrcore-extraction-design.md new file mode 100644 index 00000000..f4201d9c --- /dev/null +++ b/docs/plans/2026-03-20-zarrcore-extraction-design.md @@ -0,0 +1,133 @@ +# ZarrCore.jl Extraction Design + +## Goal + +Factor Zarr.jl into two packages: +- **ZarrCore.jl** (`lib/ZarrCore/`) — minimal core sufficient to read/write uncompressed Zarr v2 and v3 arrays +- **Zarr.jl** — depends on ZarrCore, adds all compressors, V3 codecs, and network/archive storage backends + +## Design Decisions + +1. **Abstract pipeline in Core, concrete codecs split across packages.** ZarrCore defines `AbstractCodecPipeline`, `V2Pipeline`, `V3Pipeline`, and only `BytesCodec` + `TransposeCodec`. Zarr.jl registers compression codecs into the existing `codectypes` registry. Unknown codecs at parse time produce a clear error. + +2. **All pure-Julia filters in Core.** FixedScaleOffset, Fletcher32, Shuffle, Delta, Quantize — all tiny, no external deps. Avoids complex split of the Filters directory. + +3. **Core storage: DirectoryStore + DictStore + ConsolidatedStore.** No external deps. ConsolidatedStore is a thin JSON wrapper commonly used for read-only access. + +4. **Monorepo subdirectory package** at `lib/ZarrCore/`. Own `Project.toml`, publishable to General registry. Zarr.jl uses `path = "lib/ZarrCore"` during dev. + +5. **Zarr.jl re-exports everything from ZarrCore.** `using Zarr` is fully backward-compatible. `using ZarrCore` gives the minimal API. + +6. **Convenience functions in Core.** `zcreate`, `zopen`, `zzeros`, `zgroup` live in ZarrCore — without them Core has no user-facing functionality. + +7. **Extensible store resolution.** ZarrCore owns `zopen` dispatch through `storageregexlist`. Core handles local paths + DictStore. Zarr.jl registers HTTP/GCS/S3/Zip resolvers. + +## Package Structure + +### lib/ZarrCore/ + +``` +lib/ZarrCore/ + Project.toml # JSON, DiskArrays, OffsetArrays, DateTimes64, Dates + src/ + ZarrCore.jl # module entry, ZarrFormat, constants, includes, exports + chunkkeyencoding.jl + metadata.jl + metadata3.jl + MaxLengthStrings.jl + pipeline.jl + Compressors/ + Compressors.jl # Compressor, NoCompressor, compressortypes registry + Codecs/ + Codecs.jl # abstract Codec, zencode/zdecode + V3/ + V3.jl # BytesCodec, TransposeCodec only + Filters/ + Filters.jl # Filter + all pure-Julia filter implementations + fixedscaleoffset.jl, fletcher32.jl, shuffle.jl, delta.jl, quantize.jl, vlenfilters.jl + Storage/ + Storage.jl # AbstractStore, storageregexlist, metadata I/O + directorystore.jl + dictstore.jl + consolidated.jl + ZArray.jl + ZGroup.jl +``` + +### Zarr.jl (top-level, rewritten as thin wrapper) + +``` +src/ + Zarr.jl # using ZarrCore, re-exports, registrations + Compressors/ + blosc.jl + zlib.jl + zstd.jl + Codecs/ + V3/ + blosc.jl + gzip.jl + zstd.jl + crc32c.jl + sharding.jl + Storage/ + http.jl + gcstore.jl + zipstore.jl +``` + +## Registration Pattern + +Zarr.jl's `__init__` populates ZarrCore's registries: + +```julia +# Compressors +ZarrCore.compressortypes["blosc"] = BloscCompressor +ZarrCore.compressortypes["zlib"] = ZlibCompressor +ZarrCore.compressortypes["zstd"] = ZstdCompressor + +# V3 codecs +ZarrCore.codectypes["blosc"] = BloscV3Codec +ZarrCore.codectypes["gzip"] = GzipV3Codec +ZarrCore.codectypes["zstd"] = ZstdV3Codec +ZarrCore.codectypes["crc32c"] = CRC32cV3Codec + +# Store URL resolvers +push!(ZarrCore.storageregexlist, r"^https?://" => HTTPStore) +push!(ZarrCore.storageregexlist, r"^gs://" => GCStore) +``` + +## Migration Strategy + +- **Move** files from `src/` to `lib/ZarrCore/src/` (preserves git history) +- **Split** `Compressors/Compressors.jl`: abstract type + NoCompressor + registry to Core; blosc/zlib/zstd stay +- **Split** `Codecs/V3/V3.jl`: BytesCodec + TransposeCodec to Core; compression codecs stay +- **Split** `Storage/`: directorystore + dictstore + consolidated to Core; http/gcstore/zipstore stay +- **Rewrite** top-level `src/Zarr.jl` as thin wrapper + +## ZarrCore Dependencies + +```toml +[deps] +JSON = "..." +DiskArrays = "..." +OffsetArrays = "..." +DateTimes64 = "..." +Dates = "..." # stdlib +``` + +## Zarr.jl Dependencies (in addition to ZarrCore) + +```toml +[deps] +ZarrCore = "..." +Blosc = "..." +CRC32c = "..." +ChunkCodecCore = "..." +ChunkCodecLibZlib = "..." +ChunkCodecLibZstd = "..." +HTTP = "..." +OpenSSL = "..." +URIs = "..." +ZipArchives = "..." +``` diff --git a/docs/plans/2026-03-20-zarrcore-extraction.md b/docs/plans/2026-03-20-zarrcore-extraction.md new file mode 100644 index 00000000..3c56097a --- /dev/null +++ b/docs/plans/2026-03-20-zarrcore-extraction.md @@ -0,0 +1,1826 @@ +# ZarrCore.jl Extraction Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extract a minimal `ZarrCore.jl` package from `Zarr.jl` into `lib/ZarrCore/`, sufficient to read/write uncompressed Zarr v2 and v3 arrays, with Zarr.jl as a thin wrapper that registers compressors, codecs, and network stores. + +**Architecture:** ZarrCore owns all abstract types, registries, core stores (DirectoryStore, DictStore, ConsolidatedStore), all pure-Julia filters, and the minimal V3 codecs (BytesCodec, TransposeCodec). Zarr.jl depends on ZarrCore and populates its registries with Blosc/Zlib/Zstd compressors, compression V3 codecs, and HTTP/GCS/ZIP stores. A `default_compressor()` function allows Zarr.jl to override the default from `NoCompressor()` to `BloscCompressor()`. + +**Tech Stack:** Julia 1.10+, JSON, DiskArrays, OffsetArrays, DateTimes64 + +--- + +## Phase 1: ZarrCore Package Scaffold + +### Task 1: Create ZarrCore directory and Project.toml + +**Files:** +- Create: `lib/ZarrCore/Project.toml` + +**Step 1: Create the file** + +```toml +name = "ZarrCore" +uuid = "INSERT-GENERATED-UUID" +version = "0.1.0" + +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[compat] +JSON = "0.21, 1" +DiskArrays = "0.4.2" +OffsetArrays = "0.11, 1.0" +DateTimes64 = "1" +julia = "1.10" +``` + +Generate a real UUID with: `julia -e 'using UUIDs; println(uuid4())'` + +**Step 2: Verify directory structure** + +Run: `ls lib/ZarrCore/` +Expected: `Project.toml` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/Project.toml +git commit -m "scaffold: create ZarrCore package with Project.toml" +``` + +--- + +### Task 2: Create ZarrCore module entry point (stub) + +**Files:** +- Create: `lib/ZarrCore/src/ZarrCore.jl` + +**Step 1: Create a minimal module that loads** + +```julia +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +const DV = ZarrFormat(Val(2)) + +abstract type AbstractCodecPipeline end + +# Placeholder — will be populated in subsequent tasks +end # module +``` + +**Step 2: Verify it loads** + +Run: `julia --project=lib/ZarrCore -e 'using Pkg; Pkg.instantiate(); using ZarrCore; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/ZarrCore.jl +git commit -m "scaffold: add ZarrCore module entry point" +``` + +--- + +## Phase 2: Core Types & Registries + +### Task 3: Move chunkkeyencoding.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/chunkkeyencoding.jl` (copy from `src/chunkkeyencoding.jl`) + +This file has no external dependencies — it only references `ZarrFormat` which is defined in the module entry point. Copy it verbatim. + +**Step 1: Copy the file** + +```bash +cp src/chunkkeyencoding.jl lib/ZarrCore/src/chunkkeyencoding.jl +``` + +**Step 2: Add include to ZarrCore.jl** + +In `lib/ZarrCore/src/ZarrCore.jl`, add after the `AbstractCodecPipeline` line: + +```julia +include("chunkkeyencoding.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.ChunkKeyEncoding(".", false))'` +Expected: no error + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/chunkkeyencoding.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move chunkkeyencoding.jl to ZarrCore" +``` + +--- + +### Task 4: Move MaxLengthStrings.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/MaxLengthStrings.jl` (copy from `src/MaxLengthStrings.jl`) + +No external deps. Copy verbatim. + +**Step 1: Copy** + +```bash +cp src/MaxLengthStrings.jl lib/ZarrCore/src/MaxLengthStrings.jl +``` + +**Step 2: Verify** + +Run: `julia --project=lib/ZarrCore -e 'include("lib/ZarrCore/src/MaxLengthStrings.jl"); using .MaxLengthStrings; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/MaxLengthStrings.jl +git commit -m "core: move MaxLengthStrings.jl to ZarrCore" +``` + +--- + +### Task 5: Create ZarrCore Compressors (abstract type + NoCompressor + registry) + +**Files:** +- Create: `lib/ZarrCore/src/Compressors/Compressors.jl` + +Extract the abstract type, registry, `NoCompressor`, and all generic fallback methods from `src/Compressors/Compressors.jl`. Do NOT include `blosc.jl`, `zlib.jl`, `zstd.jl`. Also add `default_compressor()` function. + +**Step 1: Create the file** + +```julia +import JSON # for JSON.lower + +_reinterpret(::Type{T}, x::AbstractArray{S, 0}) where {T, S} = reinterpret(T, reshape(x, 1)) +_reinterpret(::Type{T}, x::AbstractArray) where T = reinterpret(T, x) + +""" + abstract type Compressor + +The abstract supertype for all Zarr compressors. + +## Interface + +All subtypes of `Compressor` SHALL implement the following methods: + +- `zcompress(a, c::Compressor)`: compress the array `a` using the compressor `c`. +- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` + and return an array of type `T`. +- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which + follows the Zarr specification for that compressor. +- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given + dictionary `d` which contains the compressor's parameters according to the Zarr spec. + +Subtypes of `Compressor` MAY also implement the following methods: + +- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the + compressor `c` and store the result in the array `compressed`. +- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` + using the compressor `c` and store the result in the array `data`. + +Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. +""" +abstract type Compressor end + +const compressortypes = Dict{Union{String,Nothing}, Type{<: Compressor}}() + +# Fallback definitions for the compressor interface +getCompressor(compdict::Dict) = haskey(compdict, "id") ? + getCompressor(compressortypes[compdict["id"]], compdict) : + getCompressor(compressortypes[compdict["name"]], compdict["configuration"]) +getCompressor(::Nothing) = NoCompressor() + +# Compression when no filter is given +zcompress!(compressed,data,c,::Nothing) = zcompress!(compressed,data,c) +zuncompress!(data,compressed,c,::Nothing) = zuncompress!(data,compressed,c) + +# Fallback definition of mutating form of compress and uncompress +function zcompress!(compressed, data, c) + empty!(compressed) + append!(compressed,zcompress(data, c)) +end +zuncompress!(data, compressed, c) = copyto!(data, zuncompress(compressed, c, eltype(data))) + + +# Function given a filter stack +function zcompress!(compressed, data, c, f) + a2 = foldl(f, init=data) do anow, fnow + zencode(anow,fnow) + end + zcompress!(compressed, a2, c) +end + +function zuncompress!(data, compressed, c, f) + data2 = zuncompress(compressed, c, desttype(last(f))) + a2 = foldr(f, init = data2) do fnow, anow + zdecode(anow, fnow) + end + copyto!(data, a2) +end + +# NoCompressor +""" + NoCompressor() + +Creates an object that can be passed to ZArray constructors without compression. +""" +struct NoCompressor <: Compressor end + +function zuncompress(a, ::NoCompressor, T) + _reinterpret(T,a) +end + +function zcompress(a, ::NoCompressor) + _reinterpret(UInt8,a) +end + +JSON.lower(::NoCompressor) = nothing + +compressortypes[nothing] = NoCompressor + +""" + default_compressor() + +Returns the default compressor used by `zcreate` and related functions. +ZarrCore returns `NoCompressor()`. Zarr.jl overrides this to `BloscCompressor()`. +""" +default_compressor() = NoCompressor() +``` + +**Step 2: Add include to ZarrCore.jl** + +After the `include("chunkkeyencoding.jl")` line add: + +```julia +include("Compressors/Compressors.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.NoCompressor())'` +Expected: `NoCompressor()` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/Compressors/Compressors.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add Compressors base with NoCompressor and registry" +``` + +--- + +### Task 6: Create ZarrCore Codecs with registry-based V3 infrastructure + +**Files:** +- Create: `lib/ZarrCore/src/Codecs/Codecs.jl` +- Create: `lib/ZarrCore/src/Codecs/V3/V3.jl` + +The Codecs.jl is copied from `src/Codecs/Codecs.jl` with its `include` pointing to the new V3. + +The V3.jl contains: +- `V3Codec{In,Out}` abstract type +- `v3_codec_parsers` registry (`Dict{String, Function}`) +- `codec_to_dict` generic function (for serialization) +- `codec_category` helper +- `BytesCodec` + `TransposeCodec` (structs, encode/decode, parse, serialize) +- `encoded_shape` generic function + +**Step 1: Create `lib/ZarrCore/src/Codecs/Codecs.jl`** + +```julia +module Codecs + +using JSON: JSON + +""" + abstract type Codec + +The abstract supertype for all Zarr codecs. +""" +abstract type Codec end + +zencode(a, c::Codec) = error("Unimplemented") +zencode!(encoded, data, c::Codec) = error("Unimplemented") +zdecode(a, c::Codec, T::Type) = error("Unimplemented") +zdecode!(data, encoded, c::Codec) = error("Unimplemented") +JSON.lower(c::Codec) = error("Unimplemented") +getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") + +include("V3/V3.jl") + +end +``` + +**Step 2: Create `lib/ZarrCore/src/Codecs/V3/V3.jl`** + +```julia +module V3Codecs + +import ..Codecs: zencode, zdecode, zencode!, zdecode! +using JSON: JSON + +abstract type V3Codec{In,Out} end + +"""Registry of V3 codec parsers: name -> (config_dict -> V3Codec instance)""" +const v3_codec_parsers = Dict{String, Function}() + +"""Each V3Codec must implement this for JSON serialization.""" +function codec_to_dict end + +"""Classify a V3Codec by its phase in the pipeline.""" +codec_category(::V3Codec{:array,:array}) = :array_array +codec_category(::V3Codec{:array,:bytes}) = :array_bytes +codec_category(::V3Codec{:bytes,:bytes}) = :bytes_bytes + +"""Return the shape of the output of `codec_encode(codec, data)` given the input shape.""" +encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz + +# --- BytesCodec (array -> bytes) --- + +struct BytesCodec <: V3Codec{:array, :bytes} + endian::Symbol # :little or :big + function BytesCodec(endian::Symbol) + endian ∈ (:little, :big) || + throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) + new(endian) + end +end +BytesCodec() = BytesCodec(:little) + +const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 +_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN + +function codec_encode(c::BytesCodec, data::AbstractArray) + if _needs_bswap(c.endian) + return reinterpret(UInt8, bswap.(vec(data))) |> collect + else + return reinterpret(UInt8, vec(data)) |> collect + end +end + +function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} + arr = collect(reinterpret(T, encoded)) + if _needs_bswap(c.endian) + arr = bswap.(arr) + end + return reshape(arr, shape) +end + +codec_to_dict(c::BytesCodec) = Dict{String,Any}( + "name" => "bytes", + "configuration" => Dict{String,Any}("endian" => string(c.endian)) +) + +v3_codec_parsers["bytes"] = function(config) + endian_str = get(config, "endian", "little") + endian = endian_str == "little" ? :little : + endian_str == "big" ? :big : + throw(ArgumentError("Unknown endian value: \"$endian_str\"")) + BytesCodec(endian) +end + +# --- TransposeCodec (array -> array) --- + +struct TransposeCodec{N} <: V3Codec{:array, :array} + order::NTuple{N, Int} # permutation (1-based Julia indexing) +end + +encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) + +function codec_encode(c::TransposeCodec, data::AbstractArray) + return permutedims(data, c.order) +end + +function codec_decode(c::TransposeCodec, encoded::AbstractArray) + inv_order = Tuple(invperm(collect(c.order))) + return permutedims(encoded, inv_order) +end + +codec_to_dict(c::TransposeCodec) = Dict{String,Any}( + "name" => "transpose", + "configuration" => Dict{String,Any}("order" => collect(c.order .- 1)) +) + +# TransposeCodec parser is registered in metadata3.jl because it needs +# the shape context to parse string orders like "C"/"F". +# The registry entry for numeric orders is here: +# NOTE: the parser receives (config, shape_length) — see parse_v3_codec. + +end +``` + +**Step 3: Add include to ZarrCore.jl** + +After `include("Compressors/Compressors.jl")`: + +```julia +include("Codecs/Codecs.jl") +``` + +**Step 4: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; c = ZarrCore.Codecs.V3Codecs.BytesCodec(); println(c)'` +Expected: `BytesCodec(:little)` + +**Step 5: Commit** + +```bash +git add lib/ZarrCore/src/Codecs/Codecs.jl lib/ZarrCore/src/Codecs/V3/V3.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add Codecs module with BytesCodec and TransposeCodec" +``` + +--- + +### Task 7: Move Filters to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/Filters/` (entire directory) + +Copy the entire Filters directory verbatim. These files only depend on JSON (imported by parent module) and have no external package deps. + +**Step 1: Copy all filter files** + +```bash +cp -r src/Filters lib/ZarrCore/src/Filters +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("Codecs/Codecs.jl")`: + +```julia +include("Filters/Filters.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.filterdict)'` +Expected: prints a Dict with filter entries + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/Filters/ lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move all Filters to ZarrCore" +``` + +--- + +## Phase 3: Metadata Layer + +### Task 8: Move metadata.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/metadata.jl` + +Copy `src/metadata.jl` with these changes: + +1. The `Metadata(A, chunks, zarr_format; compressor=BloscCompressor(), ...)` constructor — change default to `default_compressor()`. +2. The `Metadata(A, chunks, ::ZarrFormat{2}; compressor=BloscCompressor(), ...)` constructor — change default to `default_compressor()`. + +These are the ONLY changes. Everything else stays the same. + +**Step 1: Copy and modify** + +```bash +cp src/metadata.jl lib/ZarrCore/src/metadata.jl +``` + +Then edit `lib/ZarrCore/src/metadata.jl`: + +Replace both occurrences of `compressor::C=BloscCompressor()` with `compressor::C=default_compressor()`: + +- Line ~153 (in `Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; ...)`): + Change `compressor::C=BloscCompressor()` → `compressor::C=default_compressor()` + +- Line ~175 (in `Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; ...)`): + Change `compressor::C=BloscCompressor()` → `compressor::C=default_compressor()` + +**Step 2: Add include to ZarrCore.jl** + +After `include("chunkkeyencoding.jl")` and before `include("Compressors/Compressors.jl")`, add: + +```julia +include("metadata.jl") +``` + +Wait — `metadata.jl` uses `BloscCompressor` and `getCompressor` which are defined in Compressors. And it uses `getfilters` from Filters. So the include order must be: + +```julia +include("chunkkeyencoding.jl") +abstract type AbstractCodecPipeline end +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +``` + +Actually, `metadata.jl` only needs `Compressor` types for the constructor default and `getCompressor`/`getfilters` for parsing. Both are already included before metadata.jl. This matches the original `src/Zarr.jl` include order. + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.MetadataV2)'` +Expected: prints the type + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/metadata.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move metadata.jl to ZarrCore with default_compressor()" +``` + +--- + +### Task 9: Refactor and move metadata3.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/metadata3.jl` + +This is the biggest refactoring. The current `metadata3.jl` has: + +1. **`Metadata3(d::AbstractDict, ...)`** — hardcodes codec parsing with if/elseif chain +2. **`lower3(md::MetadataV3)`** — hardcodes codec serialization with isa checks +3. **`MetadataV3{T2,N}` convenience constructor** — hardcodes compressor-to-codec mapping + +Changes needed: + +**A. Replace hardcoded codec parsing with registry lookup:** + +The current code: +```julia +if codec_name == "transpose" ... +elseif codec_name == "bytes" ... +elseif codec_name == "gzip" ... +``` + +Becomes: +```julia +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + # Special case: transpose needs shape_length for "C"/"F" string orders + if codec_name == "transpose" + return _parse_transpose(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end +``` + +**B. Replace hardcoded serialization with `codec_to_dict` dispatch:** + +The current `lower3` builds codec dicts inline with isa checks. Replace with: +```julia +for codec in p.array_array + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) +end +``` + +**C. Replace hardcoded compressor-to-codec in convenience constructor:** + +The `MetadataV3{T2,N}(...)` constructor maps compressors to V3 codecs. This constructor moves to Zarr.jl (not ZarrCore) since it references specific compressor types. ZarrCore keeps only the inner constructor and `Metadata3(d::AbstractDict, ...)`. + +**Step 1: Create the refactored file** + +Create `lib/ZarrCore/src/metadata3.jl` with this content: + +```julia +""" +Prototype Zarr version 3 support +""" + +const typemap3 = Dict{String, DataType}() +foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t + typemap3[lowercase(string(t))] = t +end +typemap3["complex64"] = ComplexF32 +typemap3["complex128"] = ComplexF64 + +function typestr3(t::Type) + return lowercase(string(t)) +end +function typestr3(::Type{NTuple{N,UInt8}}) where {N} + return "r$(N*8)" +end + +function typestr3(s::AbstractString, codecs=nothing) + if !haskey(typemap3, s) + if startswith(s, "r") + num_bits = tryparse(Int, s[2:end]) + if isnothing(num_bits) + throw(ArgumentError("$s is not a known type")) + end + if mod(num_bits, 8) == 0 + return NTuple{num_bits÷8,UInt8} + else + throw(ArgumentError("$s must describe a raw type with bit size that is a multiple of 8 bits")) + end + end + end + return typemap3[s] +end + +function check_keys(d::AbstractDict, keys) + for key in keys + if !haskey(d, key) + throw(ArgumentError("Zarr v3 metadata must have a key called $key")) + end + end +end + +"""Metadata for Zarr version 3 arrays""" +struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} + zarr_format::Int + node_type::String + shape::Base.RefValue{NTuple{N, Int}} + chunks::NTuple{N, Int} + dtype::String # data_type in v3 + pipeline::P + fill_value::Union{T, Nothing} + chunk_key_encoding::E + function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} + zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) + any(<(0), shape) && throw(ArgumentError("Size must be positive")) + any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + end +end +MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) +zarr_format(::MetadataV3) = ZarrFormat(Val(3)) + +function Base.:(==)(m1::MetadataV3, m2::MetadataV3) + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.fill_value == m2.fill_value && + m1.pipeline == m2.pipeline && + m1.chunk_key_encoding == m2.chunk_key_encoding +end + +""" +Derive the storage order ('C' or 'F') from the codec pipeline of a MetadataV3. +""" +function get_order(md::MetadataV3) + array_array = md.pipeline.array_array + if length(array_array) == 0 + return 'C' + end + if length(array_array) > 1 + throw(ArgumentError( + "Cannot determine storage order: pipeline has $(length(array_array)) " * + "array->array codecs; composed permutations yield an indeterminate order" + )) + end + codec = only(array_array) + if !(codec isa Codecs.V3Codecs.TransposeCodec) + throw(ArgumentError( + "Cannot determine storage order: unrecognized array->array codec $(typeof(codec))" + )) + end + N = ndims(md) + c_perm = ntuple(identity, N) + f_perm = ntuple(i -> N - i + 1, N) + if codec.order == c_perm + return 'C' + elseif codec.order == f_perm + return 'F' + else + throw(ArgumentError( + "Cannot determine storage order: TransposeCodec permutation $(codec.order) " * + "is neither C order $c_perm nor F order $f_perm" + )) + end +end +get_order(md::MetadataV2) = md.order + +# --- Registry-based codec parsing --- + +""" +Parse a single V3 codec entry from a zarr.json codecs array. +`shape_length` is needed for transpose codec string order parsing ("C"/"F"). +""" +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + # Transpose needs shape_length for "C"/"F" string parsing + if codec_name == "transpose" + return _parse_transpose_codec(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end + +function _parse_transpose_codec(config::Dict, shape_length::Int) + _order = config["order"] + if _order isa AbstractString + n = shape_length + if _order == "C" + @warn "Transpose codec dimension order of C is deprecated" + perm = ntuple(identity, n) + elseif _order == "F" + @warn "Transpose codec dimension order of F is deprecated" + perm = ntuple(i -> n - i + 1, n) + else + throw(ArgumentError("Unknown transpose order string: $_order")) + end + else + perm = Tuple(Int.(_order) .+ 1) + end + return Codecs.V3Codecs.TransposeCodec(perm) +end + + +function Metadata3(d::AbstractDict, fill_as_missing) + check_keys(d, ("zarr_format", "node_type")) + + zarr_format = d["zarr_format"]::Int + node_type = d["node_type"]::String + if node_type ∉ ("group", "array") + throw(ArgumentError("Unknown node_type of $node_type")) + end + zarr_format == 3 || throw(ArgumentError("Metadata3 only functions if zarr_format == 3")) + + # Groups + if node_type == "group" + for key in keys(d) + if key ∉ ("zarr_format", "node_type", "attributes") + throw(ArgumentError("Zarr v3 group metadata cannot have a key called $key")) + end + end + group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) + return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) + end + + # Array keys + mandatory_keys = [ + "zarr_format", "node_type", "shape", "data_type", + "chunk_grid", "chunk_key_encoding", "fill_value", "codecs", + ] + optional_keys = ["attributes", "storage_transformers", "dimension_names"] + check_keys(d, mandatory_keys) + for key in keys(d) + if key ∉ mandatory_keys && key ∉ optional_keys + throw(ArgumentError("Zarr v3 metadata cannot have a key called $key")) + end + end + + shape = Int.(d["shape"]) + data_type = d["data_type"]::String + + chunk_grid = d["chunk_grid"] + if chunk_grid["name"] == "regular" + chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) + if length(shape) != length(chunks) + throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) + end + else + throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) + end + + # Build V3Pipeline from codec chain using registry + array_array_codecs = [] + array_bytes_codec = nothing + bytes_bytes_codecs = [] + + for codec in d["codecs"] + codec_name = codec["name"] + config = get(codec, "configuration", Dict{String,Any}()) + parsed = parse_v3_codec(codec_name, config, length(shape)) + cat = Codecs.V3Codecs.codec_category(parsed) + if cat == :array_array + push!(array_array_codecs, parsed) + elseif cat == :array_bytes + array_bytes_codec = parsed + elseif cat == :bytes_bytes + push!(bytes_bytes_codecs, parsed) + end + end + + isnothing(array_bytes_codec) && throw(ArgumentError("V3 codec chain must contain a 'bytes' codec")) + pipeline = V3Pipeline(Tuple(array_array_codecs), array_bytes_codec, Tuple(bytes_bytes_codecs)) + + T = typestr3(data_type) + N = length(shape) + fv = fill_value_decoding(d["fill_value"], T)::T + TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} + + chunk_key_encoding = parse_chunk_key_encoding(d["chunk_key_encoding"]) + E = typeof(chunk_key_encoding) + + MetadataV3{TU, N, typeof(pipeline), E}( + zarr_format, node_type, + NTuple{N, Int}(shape) |> reverse, + NTuple{N, Int}(chunks) |> reverse, + data_type, pipeline, fv, chunk_key_encoding, + ) +end + +"Construct MetadataV3 based on your data (minimal — no compressor mapping)" +function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; + node_type::String="array", + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + fill_as_missing = false, + dimension_separator::Char = '/', + bytes_bytes_codecs::Tuple=() + ) where {T, N} + @warn("Zarr v3 support is experimental") + T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} + if fill_value === nothing + fill_value = zero(T) + end + array_array_codecs = if order == 'F' + (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) + else + () + end + array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) + pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) + chunk_key_encoding = ChunkKeyEncoding(dimension_separator, true) + E = typeof(chunk_key_encoding) + return MetadataV3{T2,N,typeof(pipeline),E}( + 3, node_type, size(A), chunks, typestr3(eltype(A)), + pipeline, fill_value, chunk_key_encoding + ) +end + +# --- Registry-based serialization --- + +function lower3(md::MetadataV3{T}) where T + chunk_grid = Dict{String,Any}( + "name" => "regular", + "configuration" => Dict{String,Any}( + "chunk_shape" => md.chunks |> reverse + ) + ) + chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) + + codecs = Dict{String,Any}[] + p = md.pipeline + + for codec in p.array_array + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + push!(codecs, Codecs.V3Codecs.codec_to_dict(p.array_bytes)) + for codec in p.bytes_bytes + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + + Dict{String, Any}( + "zarr_format" => Int(md.zarr_format), + "node_type" => md.node_type, + "shape" => md.shape[] |> reverse, + "data_type" => typestr3(T), + "chunk_grid" => chunk_grid, + "chunk_key_encoding" => chunk_key_encoding, + "fill_value" => fill_value_encoding(md.fill_value), + "codecs" => codecs + ) +end + +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; + node_type::String="array", + compressor::C=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + filters::F=nothing, + fill_as_missing = false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) + ) where {T, N, C, F, E} + # Map compressor to bytes_bytes_codecs using compressor_to_v3_codecs registry + bytes_bytes = compressor_to_v3_bytes_codecs(compressor) + return Metadata3(A, chunks; + node_type=node_type, + fill_value=fill_value, + order=order, + endian=endian, + fill_as_missing=fill_as_missing, + dimension_separator=chunk_key_encoding.sep, + bytes_bytes_codecs=bytes_bytes + ) +end + +# V3 constructor from Dict +function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{3}) + return Metadata3(d, fill_as_missing) +end + +function JSON.lower(md::MetadataV3) + return lower3(md) +end + +""" + compressor_to_v3_bytes_codecs(c::Compressor) -> Tuple + +Convert a v2 Compressor to a tuple of V3 bytes->bytes codecs. +ZarrCore handles NoCompressor. Zarr.jl adds methods for BloscCompressor, etc. +""" +compressor_to_v3_bytes_codecs(::NoCompressor) = () +compressor_to_v3_bytes_codecs(c::Compressor) = throw(ArgumentError( + "Unsupported compressor type for v3: $(typeof(c)). Load Zarr.jl for compression support." +)) +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("metadata.jl")`: + +```julia +include("metadata3.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.MetadataV3)'` +Expected: prints the type + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/metadata3.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add refactored metadata3.jl with registry-based codec parsing" +``` + +--- + +### Task 10: Move pipeline.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/pipeline.jl` + +Copy `src/pipeline.jl` verbatim. It references `Codecs.V3Codecs.codec_encode` and `codec_decode` which exist in ZarrCore's V3Codecs module. + +**Step 1: Copy** + +```bash +cp src/pipeline.jl lib/ZarrCore/src/pipeline.jl +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("metadata3.jl")`: + +```julia +include("pipeline.jl") +``` + +Note: in the original `src/Zarr.jl`, `pipeline.jl` is included after `ZArray.jl`. However, `pipeline.jl` only depends on types already defined (V3Pipeline, Codecs.V3Codecs, Compressors). `ZArray.jl` calls `pipeline_encode`/`pipeline_decode!` which are defined in `pipeline.jl`. So `pipeline.jl` must come before `ZArray.jl`. Check that this doesn't break any circular dependencies — it shouldn't because `pipeline.jl` doesn't reference ZArray. + +Actually wait, looking at the original include order in `src/Zarr.jl`: +```julia +include("ZArray.jl") +include("pipeline.jl") +``` + +But `ZArray.jl` calls `pipeline_encode`/`pipeline_decode!`/`get_pipeline`. These are defined in `pipeline.jl`. How does this work? Because Julia resolves function calls at runtime, not at include time. The struct `V2Pipeline`/`V3Pipeline` are used as types in `V2Pipeline{C,F}` etc. But `V3Pipeline` is defined... where? Let me check. + +`V3Pipeline` is defined in `pipeline.jl` (line 30). But `MetadataV3` references `pipeline::P` where `P<:AbstractCodecPipeline`, and `metadata3.jl` constructs `V3Pipeline(...)`. So `pipeline.jl` must be included BEFORE `metadata3.jl`. + +Looking at the original order: `metadata3.jl` is included at line 18, `pipeline.jl` at line 24. But `metadata3.jl` uses `V3Pipeline(...)`. How? Because in the original code, `V3Pipeline` is defined in `pipeline.jl` which is included AFTER `metadata3.jl`. This works because the `Metadata3` function that constructs `V3Pipeline` isn't called until runtime. + +So the include order doesn't matter for function bodies, only for type definitions and struct references. Since `AbstractCodecPipeline` is defined before `metadata3.jl`, and `V3Pipeline` is only used in function bodies (not in struct fields — `MetadataV3` uses `P<:AbstractCodecPipeline`), this works. + +For ZarrCore, include `pipeline.jl` BEFORE `ZArray.jl` to match the logical dependency order. But technically it works either way. + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; p = ZarrCore.V3Pipeline((), ZarrCore.Codecs.V3Codecs.BytesCodec(), ()); println(p)'` +Expected: prints the pipeline + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/pipeline.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move pipeline.jl to ZarrCore" +``` + +--- + +## Phase 4: Storage & Array + +### Task 11: Move core Storage to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/Storage/Storage.jl` +- Create: `lib/ZarrCore/src/Storage/directorystore.jl` +- Create: `lib/ZarrCore/src/Storage/dictstore.jl` +- Create: `lib/ZarrCore/src/Storage/consolidated.jl` + +Copy `Storage.jl` but remove the includes for `gcstore.jl`, `http.jl`, `zipstore.jl`. Also remove the `S3Store` stub (move it to Zarr.jl) and the S3 regex registration. + +**Step 1: Copy store implementation files verbatim** + +```bash +cp src/Storage/directorystore.jl lib/ZarrCore/src/Storage/directorystore.jl +cp src/Storage/dictstore.jl lib/ZarrCore/src/Storage/dictstore.jl +cp src/Storage/consolidated.jl lib/ZarrCore/src/Storage/consolidated.jl +``` + +**Step 2: Create modified `lib/ZarrCore/src/Storage/Storage.jl`** + +Copy `src/Storage/Storage.jl` with these changes: + +1. Remove the `S3Store` struct definition and constructor (lines 36-43) +2. Remove `push!(storageregexlist, r"^s3://" => S3Store)` (line 283) +3. Remove `include("gcstore.jl")`, `include("http.jl")`, `include("zipstore.jl")` (lines 288-291) + +The remaining includes are: `directorystore.jl`, `dictstore.jl`, `consolidated.jl`. + +**Step 3: Add include to ZarrCore.jl** + +After `include("pipeline.jl")`: + +```julia +include("Storage/Storage.jl") +``` + +**Step 4: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; d = ZarrCore.DictStore(); println(d)'` +Expected: `Dictionary Storage` + +**Step 5: Commit** + +```bash +git add lib/ZarrCore/src/Storage/ lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move Storage with DirectoryStore, DictStore, ConsolidatedStore" +``` + +--- + +### Task 12: Move ZArray.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/ZArray.jl` + +Copy `src/ZArray.jl` with these changes: + +1. `zcreate(::Type{T}, storage::AbstractStore, ...)` — change `compressor=BloscCompressor()` → `compressor=default_compressor()` + +That's the only change needed. The rest (imports, types, functions) stays the same. + +**Step 1: Copy and modify** + +```bash +cp src/ZArray.jl lib/ZarrCore/src/ZArray.jl +``` + +Edit `lib/ZarrCore/src/ZArray.jl`: +- Change `compressor=BloscCompressor()` to `compressor=default_compressor()` in the `zcreate(::Type{T},storage::AbstractStore, ...)` function (around line 349). + +**Step 2: Add include to ZarrCore.jl** + +After `include("Storage/Storage.jl")`: + +```julia +include("ZArray.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; z = ZarrCore.zzeros(Int, 4, 4); println(z)'` +Expected: `ZArray{Int64} of size 4 x 4` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/ZArray.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move ZArray.jl to ZarrCore with default_compressor()" +``` + +--- + +### Task 13: Move ZGroup.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/ZGroup.jl` + +Copy `src/ZGroup.jl` with these changes: + +1. Remove `HTTP.serve(...)` line (line 201) — move to Zarr.jl +2. Remove `writezip(...)` line (line 202) — move to Zarr.jl + +These are the only changes. `consolidate_metadata` stays (it only depends on Store and JSON). + +**Step 1: Copy and modify** + +```bash +cp src/ZGroup.jl lib/ZarrCore/src/ZGroup.jl +``` + +Edit `lib/ZarrCore/src/ZGroup.jl`: +- Remove the line: `HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...)` +- Remove the line: `writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...)` + +**Step 2: Add include to ZarrCore.jl** + +After `include("ZArray.jl")`: + +```julia +include("ZGroup.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; g = ZarrCore.zgroup(ZarrCore.DictStore(), "", ZarrCore.ZarrFormat(2)); println(g)'` +Expected: `ZarrGroup at Dictionary Storage and path ` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/ZGroup.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move ZGroup.jl to ZarrCore (without HTTP.serve, writezip)" +``` + +--- + +## Phase 5: Module Entry Points & Wiring + +### Task 14: Finalize ZarrCore.jl module entry with exports + +**Files:** +- Modify: `lib/ZarrCore/src/ZarrCore.jl` + +Write the complete module entry point with all includes and exports. + +**Step 1: Write the final ZarrCore.jl** + +```julia +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +#Default Zarr Version +const DV = ZarrFormat(Val(2)) + +include("chunkkeyencoding.jl") +abstract type AbstractCodecPipeline end +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +include("metadata3.jl") +include("pipeline.jl") +include("Storage/Storage.jl") +include("ZArray.jl") +include("ZGroup.jl") + +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +end # module +``` + +**Step 2: Verify full load** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; z = zzeros(Int, 4, 4); println(z); println(z[1:2, 1:2])'` +Expected: creates a ZArray and reads zeros back + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: finalize ZarrCore module with all exports" +``` + +--- + +### Task 15: Rewrite Zarr.jl as wrapper + +**Files:** +- Modify: `src/Zarr.jl` +- Keep: `src/Compressors/blosc.jl`, `src/Compressors/zlib.jl`, `src/Compressors/zstd.jl` +- Keep: `src/Storage/gcstore.jl`, `src/Storage/http.jl`, `src/Storage/zipstore.jl` + +The new `src/Zarr.jl` imports ZarrCore, re-exports its names, includes the compression and network store files, and registers everything into ZarrCore's registries. + +**Step 1: Rewrite `src/Zarr.jl`** + +```julia +module Zarr + +using ZarrCore +import JSON +import Blosc + +# Re-export all ZarrCore public names +using ZarrCore: ZarrCore, + # Types + ZArray, ZGroup, AbstractStore, DirectoryStore, DictStore, ConsolidatedStore, + AbstractMetadata, MetadataV2, MetadataV3, + AbstractCodecPipeline, V2Pipeline, V3Pipeline, + Compressor, NoCompressor, + Filter, + ZarrFormat, + AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + ASCIIChar, + # Functions + zcreate, zopen, zzeros, zgroup, + storagesize, storageratio, zinfo, zname, + pipeline_encode, pipeline_decode!, get_pipeline, get_order, + Metadata, Metadata3, + typestr, typestr3, + fill_value_encoding, fill_value_decoding, + getCompressor, compressortypes, + getfilters, filterdict, + zencode, zdecode, + citostring, + default_sep, default_prefix, + storageregexlist, storefromstring, + chunkindices, + consolidate_metadata, + # Pipeline/Codec internals used by tests + _reinterpret, + # Constants + DV, DS, DS2, DS3 + +# Re-export the same symbols as before +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +# Bring in the MaxLengthStrings submodule +using ZarrCore: MaxLengthStrings +using .MaxLengthStrings: MaxLengthString + +# Include compressor implementations (they register into ZarrCore.compressortypes) +include("Compressors/blosc.jl") +include("Compressors/zlib.jl") +include("Compressors/zstd.jl") + +# Override default compressor +ZarrCore.default_compressor() = BloscCompressor() + +# Override compressor_to_v3_bytes_codecs for specific compressor types +function ZarrCore.compressor_to_v3_bytes_codecs(c::BloscCompressor) + T_base = UInt8 # will be overridden by actual element type at call site + (ZarrCore.Codecs.V3Codecs.BloscV3Codec(c.cname, c.clevel, c.shuffle, c.blocksize, sizeof(T_base)),) +end +function ZarrCore.compressor_to_v3_bytes_codecs(c::ZlibCompressor) + level = c.config.level == -1 ? 6 : c.config.level + (ZarrCore.Codecs.V3Codecs.GzipV3Codec(level),) +end +function ZarrCore.compressor_to_v3_bytes_codecs(c::ZstdCompressor) + (ZarrCore.Codecs.V3Codecs.ZstdV3Codec(c.config.compressionLevel),) +end + +# Include V3 compression codec implementations +include("Codecs/V3/compression_codecs.jl") + +# Include network/archive storage backends +include("Storage/gcstore.jl") +include("Storage/http.jl") +include("Storage/zipstore.jl") + +# Register S3Store stub +struct S3Store <: AbstractStore + bucket::String + aws::Any +end +function S3Store(args...) + error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") +end + +# Register store URL resolvers +push!(storageregexlist, r"^s3://" => S3Store) + +# HTTP.serve and writezip for ZArray/ZGroup +HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) +writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) + +export S3Store, GCStore + +end # module +``` + +**Important:** The compressor files (`blosc.jl`, `zlib.jl`, `zstd.jl`) need to be updated to reference `ZarrCore.compressortypes` instead of `Zarr.compressortypes` since the registry lives in ZarrCore. However, since `Zarr` uses `ZarrCore` and imports `compressortypes`, the line `Zarr.compressortypes["blosc"] = BloscCompressor` will still work (it references the same dict object). No change needed in the compressor files. + +Wait — actually, the compressor files do `Zarr.compressortypes["blosc"] = BloscCompressor`. Since `compressortypes` is imported into `Zarr` from `ZarrCore`, `Zarr.compressortypes` IS `ZarrCore.compressortypes`. So this works without changes. + +**Step 2: Verify** + +Run: `julia --project -e 'using Zarr; z = zzeros(Int, 4, 4); println(z)'` +Expected: `ZArray{Int64} of size 4 x 4` (with Blosc compression as before) + +**Step 3: Commit** + +```bash +git add src/Zarr.jl +git commit -m "wrapper: rewrite Zarr.jl to use ZarrCore with re-exports" +``` + +--- + +### Task 16: Create V3 compression codec file for Zarr.jl + +**Files:** +- Create: `src/Codecs/V3/compression_codecs.jl` + +Extract the compression-related V3 codecs from the original `src/Codecs/V3/V3.jl` into a new file. This includes: `GzipV3Codec`, `BloscV3Codec`, `ZstdV3Codec`, `CRC32cV3Codec`, and their `codec_encode`/`codec_decode`/`codec_to_dict` methods, plus the `CRC32cCodec` helper, `ShardingCodec`, and all sharding infrastructure. + +Each codec must also register its parser in `ZarrCore.Codecs.V3Codecs.v3_codec_parsers`. + +**Step 1: Create `src/Codecs/V3/compression_codecs.jl`** + +```julia +# V3 compression codecs that depend on Blosc, ChunkCodecLibZlib, ChunkCodecLibZstd, CRC32c +# These register into ZarrCore's V3 codec registry. + +using CRC32c: CRC32c +using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions +using ChunkCodecCore: encode as cc_encode, decode as cc_decode + +import ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode, zencode, zdecode, zencode!, zdecode! + +# --- CRC32c internal codec (used by CRC32cV3Codec) --- + +struct CRC32cCodec +end + +function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) + hash::UInt32 = 0x00000000 + while(bytesavailable(input) > 0) + sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) + read!(input, sized_buffer) + write(output, sized_buffer) + hash = CRC32c.crc32c(sized_buffer, hash) + end + return hash +end + +function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=false) + zencode!(output, input, c) + return take!(output) +end +function zencode!(output::IO, input::IO, c::CRC32cCodec) + hash = crc32c_stream!(output, input) + write(output, hash) + return output +end +function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=true) + zdecode!(output, input, c) + return take!(output) +end +function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) + input_vec = take!(input) + truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) + hash = crc32c_stream!(output, truncated_input) + if input_vec[end-3:end] != reinterpret(UInt8, [hash]) + throw(IOError("CRC32c hash does not match")) + end + return output +end + +# --- GzipV3Codec --- + +struct GzipV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +GzipV3Codec() = GzipV3Codec(6) + +function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) + opts = GzipEncodeOptions(; level=c.level) + return cc_encode(opts, data) +end +function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) + return cc_decode(LibZGzipCodec(), encoded) +end +codec_to_dict(c::GzipV3Codec) = Dict{String,Any}( + "name" => "gzip", + "configuration" => Dict{String,Any}("level" => c.level) +) + +v3_codec_parsers["gzip"] = function(config) + level = get(config, "level", 6) + GzipV3Codec(level) +end + +# --- BloscV3Codec --- + +struct BloscV3Codec <: V3Codec{:bytes, :bytes} + cname::String + clevel::Int + shuffle::Int + blocksize::Int + typesize::Int +end +BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) + +function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) + comp = Zarr.BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return ZarrCore.zcompress(data, comp) +end +function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) + comp = Zarr.BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return collect(ZarrCore.zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::BloscV3Codec) = Dict{String,Any}( + "name" => "blosc", + "configuration" => Dict{String,Any}( + "cname" => c.cname, + "clevel" => c.clevel, + "shuffle" => c.shuffle == 0 ? "noshuffle" : + c.shuffle == 1 ? "shuffle" : + c.shuffle == 2 ? "bitshuffle" : + throw(ArgumentError("Unknown shuffle integer: $(c.shuffle)")), + "blocksize" => c.blocksize, + "typesize" => c.typesize + ) +) + +v3_codec_parsers["blosc"] = function(config) + cname = get(config, "cname", "lz4") + clevel = get(config, "clevel", 5) + shuffle_val = get(config, "shuffle", "noshuffle") + shuffle_int = shuffle_val isa Integer ? shuffle_val : + shuffle_val == "noshuffle" ? 0 : + shuffle_val == "shuffle" ? 1 : + shuffle_val == "bitshuffle" ? 2 : + throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) + blocksize = get(config, "blocksize", 0) + typesize = get(config, "typesize", 4) + BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize) +end + +# --- ZstdV3Codec --- + +struct ZstdV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +ZstdV3Codec() = ZstdV3Codec(3) + +function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) + comp = Zarr.ZstdCompressor(level=c.level) + return ZarrCore.zcompress(data, comp) +end +function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) + comp = Zarr.ZstdCompressor(level=c.level) + return collect(ZarrCore.zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::ZstdV3Codec) = Dict{String,Any}( + "name" => "zstd", + "configuration" => Dict{String,Any}("level" => c.level) +) + +v3_codec_parsers["zstd"] = function(config) + level = get(config, "level", 3) + ZstdV3Codec(level) +end + +# --- CRC32cV3Codec --- + +struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} +end + +function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) + out = UInt8[] + return zencode!(out, data, CRC32cCodec()) +end +function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) + out = UInt8[] + return zdecode!(out, encoded, CRC32cCodec()) +end +codec_to_dict(::CRC32cV3Codec) = Dict{String,Any}("name" => "crc32c") + +v3_codec_parsers["crc32c"] = function(config) + CRC32cV3Codec() +end + +# --- ShardingCodec (stub — throws at parse time) --- + +v3_codec_parsers["sharding_indexed"] = function(config) + throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) +end +``` + +**Note:** The full `ShardingCodec` struct and all its encode/decode/index infrastructure from the original V3.jl should also be moved here. For brevity, the plan shows the registry stub. Copy the full ShardingCodec code (lines 106-539 of the original `src/Codecs/V3/V3.jl`) into this file. + +**Step 2: Verify** + +Run: `julia --project -e 'using Zarr; println(haskey(ZarrCore.Codecs.V3Codecs.v3_codec_parsers, "blosc"))'` +Expected: `true` + +**Step 3: Commit** + +```bash +git add src/Codecs/V3/compression_codecs.jl +git commit -m "wrapper: add V3 compression codecs with registry entries" +``` + +--- + +### Task 17: Update Project.toml files + +**Files:** +- Modify: `Project.toml` (top-level Zarr.jl) +- Modify: `lib/ZarrCore/Project.toml` (if UUID not yet set) + +**Step 1: Add ZarrCore as a dependency to Zarr.jl** + +Add to `Project.toml` `[deps]` section: +```toml +ZarrCore = "" +``` + +Add to `[sources]` (Julia 1.11+ for path deps, or use `[extras]` + dev approach): +```toml +[sources.ZarrCore] +path = "lib/ZarrCore" +``` + +For Julia < 1.11 compatibility, you may need to use `Pkg.develop(path="lib/ZarrCore")` instead. Check the Julia version requirements. + +**Step 2: Remove dependencies that moved to ZarrCore** + +From Zarr.jl's `Project.toml`, the deps that are ONLY used by ZarrCore files should be removed. However, some deps are used by both. Keep all deps for now to avoid breakage; optimize later. + +Actually — since Zarr.jl `using ZarrCore` and ZarrCore has its own deps, Zarr.jl only needs to declare deps that IT directly uses (not transitive deps from ZarrCore). But for safety during initial migration, keep all deps. Can be cleaned up in a follow-up. + +**Step 3: Verify** + +Run: `julia --project -e 'using Pkg; Pkg.instantiate(); using Zarr; println("OK")'` +Expected: `OK` + +**Step 4: Commit** + +```bash +git add Project.toml lib/ZarrCore/Project.toml +git commit -m "deps: wire ZarrCore as dependency of Zarr.jl" +``` + +--- + +### Task 18: Clean up original src/ files + +**Files:** +- Remove: Files that were moved to ZarrCore (they're now dead code in `src/`) + +After Zarr.jl is rewritten as a wrapper, the old files in `src/` that were moved to ZarrCore are no longer included. Remove them to avoid confusion: + +``` +src/chunkkeyencoding.jl → moved to lib/ZarrCore/src/ +src/MaxLengthStrings.jl → moved to lib/ZarrCore/src/ +src/metadata.jl → moved to lib/ZarrCore/src/ +src/metadata3.jl → moved to lib/ZarrCore/src/ +src/pipeline.jl → moved to lib/ZarrCore/src/ +src/ZArray.jl → moved to lib/ZarrCore/src/ +src/ZGroup.jl → moved to lib/ZarrCore/src/ +src/Filters/ → moved to lib/ZarrCore/src/ +src/Storage/Storage.jl → moved to lib/ZarrCore/src/ +src/Storage/directorystore.jl → moved to lib/ZarrCore/src/ +src/Storage/dictstore.jl → moved to lib/ZarrCore/src/ +src/Storage/consolidated.jl → moved to lib/ZarrCore/src/ +src/Compressors/Compressors.jl → split; core moved to lib/ZarrCore/src/ +src/Codecs/Codecs.jl → split; core moved to lib/ZarrCore/src/ +src/Codecs/V3/V3.jl → split; core moved to lib/ZarrCore/src/ +``` + +**Step 1: Use git rm to remove moved files** + +```bash +git rm src/chunkkeyencoding.jl src/MaxLengthStrings.jl src/metadata.jl src/metadata3.jl src/pipeline.jl src/ZArray.jl src/ZGroup.jl +git rm -r src/Filters/ +git rm src/Storage/Storage.jl src/Storage/directorystore.jl src/Storage/dictstore.jl src/Storage/consolidated.jl +git rm src/Compressors/Compressors.jl src/Codecs/Codecs.jl src/Codecs/V3/V3.jl +``` + +**Step 2: Verify Zarr.jl still loads** + +Run: `julia --project -e 'using Zarr; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add -A +git commit -m "cleanup: remove files that moved to ZarrCore" +``` + +--- + +## Phase 6: Tests & Verification + +### Task 19: Update test Project.toml + +**Files:** +- Modify: `test/Project.toml` + +**Step 1: Add ZarrCore as a test dependency** + +Add to `test/Project.toml` `[deps]`: +```toml +ZarrCore = "" +``` + +Add source path: +```toml +[sources.ZarrCore] +path = "../lib/ZarrCore" +``` + +**Step 2: Commit** + +```bash +git add test/Project.toml +git commit -m "test: add ZarrCore to test dependencies" +``` + +--- + +### Task 20: Run the existing test suite + +**Step 1: Instantiate and run** + +Run: `julia --project -e 'using Pkg; Pkg.test()'` + +Expected: All existing tests pass. If any fail, investigate: + +- **`Zarr.BloscCompressor` not found**: Ensure the re-export in `src/Zarr.jl` includes `BloscCompressor` +- **`Zarr.compressortypes` not found**: Ensure `compressortypes` is imported from ZarrCore +- **`Codecs.V3Codecs.XxxCodec` not found**: Ensure all V3 codec types are accessible through the module path +- **Codec parsing fails**: Ensure all V3 codec parsers are registered in `v3_codec_parsers` + +Fix any failures before proceeding. + +**Step 2: Commit any fixes** + +```bash +git add -A +git commit -m "fix: resolve test failures from ZarrCore extraction" +``` + +--- + +### Task 21: Add ZarrCore-only smoke test + +**Files:** +- Create: `lib/ZarrCore/test/runtests.jl` +- Create: `lib/ZarrCore/test/Project.toml` + +**Step 1: Create `lib/ZarrCore/test/Project.toml`** + +```toml +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +ZarrCore = "" + +[sources.ZarrCore] +path = ".." +``` + +**Step 2: Create `lib/ZarrCore/test/runtests.jl`** + +```julia +using Test +using ZarrCore +using JSON + +@testset "ZarrCore" begin + @testset "NoCompressor roundtrip" begin + z = zzeros(Int64, 4, 4) + @test z isa ZArray + @test size(z) == (4, 4) + @test z[1,1] == 0 + z[2,3] = 42 + @test z[2,3] == 42 + end + + @testset "DirectoryStore" begin + p = mktempdir() + z = zcreate(Float64, 10, 10; path=p, chunks=(5,5)) + z[:] = reshape(1.0:100.0, 10, 10) + z2 = zopen(p) + @test z2[1,1] == 1.0 + @test z2[10,10] == 100.0 + end + + @testset "DictStore" begin + z = zcreate(Int32, 6, 6; chunks=(3,3)) + z[:] = ones(Int32, 6, 6) + @test z[1,1] == 1 + end + + @testset "V3 uncompressed roundtrip" begin + z = zcreate(Float32, 8, 8; + zarr_format=3, + chunks=(4,4), + compressor=ZarrCore.NoCompressor()) + z[:] = reshape(Float32.(1:64), 8, 8) + z2 = zopen(z.storage, path=z.path) + @test z2[1,1] == 1.0f0 + @test z2[8,8] == 64.0f0 + end + + @testset "default_compressor is NoCompressor" begin + @test ZarrCore.default_compressor() isa ZarrCore.NoCompressor + end +end +``` + +**Step 3: Run ZarrCore tests** + +Run: `julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()'` +Expected: All pass + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/test/ +git commit -m "test: add ZarrCore smoke tests" +``` + +--- + +### Task 22: Update CLAUDE.md + +**Files:** +- Modify: `CLAUDE.md` + +Add information about the ZarrCore package structure to the Architecture section. Add test commands for ZarrCore. + +**Step 1: Add ZarrCore section** + +Add to Build & Test Commands: +```bash +# Run ZarrCore tests only +julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()' +``` + +Add to Architecture section: +``` +### Package Structure +- `lib/ZarrCore/` — Minimal core package with types, registries, NoCompressor, BytesCodec, TransposeCodec, DirectoryStore, DictStore, ConsolidatedStore, all pure-Julia filters +- `src/` (Zarr.jl) — Wrapper that depends on ZarrCore, adds Blosc/Zlib/Zstd compressors, V3 compression codecs, HTTP/GCS/S3/ZIP stores +- ZarrCore's `default_compressor()` returns `NoCompressor()`; Zarr.jl overrides it to `BloscCompressor()` +- V3 codecs use `v3_codec_parsers` registry for extensible parsing; Zarr.jl registers compression codec parsers +``` + +**Step 2: Commit** + +```bash +git add CLAUDE.md +git commit -m "docs: update CLAUDE.md with ZarrCore package structure" +``` + +--- + +## Important Notes for Implementation + +### Module Path References + +In tests and existing code, `Zarr.SomeType` must still work. Since `Zarr.jl` does `using ZarrCore: SomeType`, the name is available as `Zarr.SomeType`. But for internal module paths like `Zarr.Codecs.V3Codecs.BloscV3Codec`, these need careful handling: + +- Core V3 types (`BytesCodec`, `TransposeCodec`) are at `ZarrCore.Codecs.V3Codecs.BytesCodec` +- Compression V3 types (`BloscV3Codec`, etc.) are defined in Zarr.jl but registered into `ZarrCore.Codecs.V3Codecs.v3_codec_parsers` +- Tests that reference `Zarr.Codecs.V3Codecs.XxxCodec` will need updates to use either the Zarr-local name or the ZarrCore path + +### The Compressor File Imports + +The existing `src/Compressors/blosc.jl` has `import Blosc` and `Zarr.compressortypes["blosc"] = BloscCompressor`. Since `Zarr` module now imports `compressortypes` from ZarrCore, `Zarr.compressortypes` refers to the same dict. This should work without changes to the compressor files. + +### S3 Extension + +The `ext/ZarrAWSS3Ext.jl` extension references `Zarr.S3Store`, `Zarr.AbstractStore`, etc. Since `S3Store` is now defined in `Zarr.jl` (not ZarrCore) and `AbstractStore` is re-exported from ZarrCore, the extension should work. Verify by checking the extension file. + +### Iterative Debugging + +This refactoring touches many files. Expect some iteration in Task 20. Common issues: +- Missing re-exports in `Zarr.jl` +- Module path mismatches (`Zarr.X` vs `ZarrCore.X`) +- Function signatures referencing types not yet imported +- `using` vs `import` confusion (Julia is strict about these) + +The debugging approach: run `using Zarr`, fix the first error, repeat until it loads. Then run the test suite. From ba8197387060c2afc9587b3a0da90c1b4c5ef263 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 23 Mar 2026 20:14:46 +0100 Subject: [PATCH 11/12] ci: dev ZarrCore before build for Julia 1.10 compatibility Julia LTS (1.10) doesn't support [sources] in Project.toml for local path dependencies. Add explicit Pkg.develop step to resolve ZarrCore from lib/ZarrCore/ before the build step. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index be047993..c4e23806 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,6 +31,8 @@ jobs: arch: ${{ matrix.arch }} show-versioninfo: true - uses: julia-actions/cache@v2 + - name: Dev ZarrCore local package + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: PYTHON: From fd4d5e75cdcb2d3d84c540023561581be85e5a28 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Fri, 27 Mar 2026 12:49:35 +0000 Subject: [PATCH 12/12] ci: only run ZarrCore dev step on Julia LTS The Pkg.develop workaround is only needed for Julia < 1.11 which lacks [sources] support. Restrict to lts matrix version. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c4e23806..3837e089 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,7 +31,8 @@ jobs: arch: ${{ matrix.arch }} show-versioninfo: true - uses: julia-actions/cache@v2 - - name: Dev ZarrCore local package + - name: Dev ZarrCore local package (Julia < 1.11 lacks [sources] support) + if: matrix.version == 'lts' run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: