diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index be047993..3837e089 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,6 +31,9 @@ jobs: arch: ${{ matrix.arch }} show-versioninfo: true - uses: julia-actions/cache@v2 + - name: Dev ZarrCore local package (Julia < 1.11 lacks [sources] support) + if: matrix.version == 'lts' + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: PYTHON: diff --git a/CLAUDE.md b/CLAUDE.md index dfbdb435..b2c2a398 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Zarr.jl is a Julia implementation of the Zarr specification for chunked, compressed, N-dimensional arrays. It supports Zarr v2 (stable) and v3 (experimental, in development on `as/continue_v3` branch). The package provides multiple storage backends (filesystem, in-memory, S3, GCS, HTTP, ZIP) and compressors (Blosc, zlib, Zstandard). +Zarr.jl is a Julia implementation of the Zarr specification for chunked, compressed, N-dimensional arrays. It supports Zarr v2 (stable) and v3 (experimental). The package provides multiple storage backends (filesystem, in-memory, S3, GCS, HTTP, ZIP) and compressors (Blosc, zlib, Zstandard). ## Build & Test Commands @@ -12,32 +12,55 @@ Zarr.jl is a Julia implementation of the Zarr specification for chunked, compres # Run all tests julia --project -e 'using Pkg; Pkg.test()' -# Run a single test file interactively (use the test/ project environment) +# Run a single test file (use the test/ project environment) julia --project=test -e 'using Test, Zarr, JSON; include("test/v3_codecs.jl")' -# Instantiate test dependencies (after Julia version change or first time setup) +# Instantiate test dependencies (first time or after Julia version change) julia --project=test -e 'using Pkg; Pkg.instantiate()' + +# Run ZarrCore tests only +julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()' ``` Julia version requirement: 1.10+. CI tests against Julia LTS, stable (`1`), nightly, and pre-release on Ubuntu, macOS, and Windows. +### Test Structure + +- `test/runtests.jl` — Main test suite: ZArray, groups, metadata, indexing, resize, concat, strings, ragged arrays, fill values. Includes the other test files. +- `test/storage.jl` — Storage backend tests +- `test/Filters.jl` — Filter tests +- `test/python.jl` — Interoperability tests with Python-generated Zarr files +- `test/v3_codecs.jl` — V3 codec tests (BytesCodec, TransposeCodec, etc.) +- `test/v3_julia.jl` — Julia-generated V3 test fixtures +- `test/v3_python.jl` — Python-generated V3 fixtures (requires PythonCall + CondaPkg; CI generates these before running tests) + ## Architecture +### Package Structure +- `lib/ZarrCore/` — Minimal core package with types, registries, NoCompressor, BytesCodec, TransposeCodec, DirectoryStore, DictStore, ConsolidatedStore, all pure-Julia filters +- `src/` (Zarr.jl) — Wrapper that depends on ZarrCore, adds Blosc/Zlib/Zstd compressors, V3 compression codecs, HTTP/GCS/S3/ZIP stores +- ZarrCore's `default_compressor()` returns `NoCompressor()`; Zarr.jl overrides it to `BloscCompressor()` +- V3 codecs use `v3_codec_parsers` registry for extensible parsing; Zarr.jl registers compression codec parsers + ### Core Type Hierarchy ``` -AbstractMetadata{T,N} -├── MetadataV2{T,N,C,F} # .zarray + .zattrs (v2) -└── MetadataV3{T,N,P} # zarr.json (v3), P<:AbstractCodecPipeline +AbstractMetadata{T,N,E<:AbstractChunkKeyEncoding} +├── MetadataV2{T,N,C,F} # .zarray + .zattrs (v2) +└── MetadataV3{T,N,P,E} # zarr.json (v3), P<:AbstractCodecPipeline, E<:AbstractChunkKeyEncoding AbstractStore -├── DirectoryStore # Filesystem -├── DictStore # In-memory Dict{String,Vector{UInt8}} -├── S3Store # AWS S3 (via AWSS3 extension) -├── GCStore # Google Cloud Storage -├── ConsolidatedStore # Consolidated metadata wrapper -├── HTTPStore # HTTP-based read-only -└── ZipStore # ZIP archive read-only +├── DirectoryStore # Filesystem +├── DictStore # In-memory Dict{String,Vector{UInt8}} +├── S3Store # AWS S3 (via AWSS3 extension) +├── GCStore # Google Cloud Storage +├── ConsolidatedStore # Consolidated metadata wrapper +├── HTTPStore # HTTP-based read-only +└── ZipStore # ZIP archive read-only + +AbstractCodecPipeline +├── V2Pipeline{C,F} # compressor + filters +└── V3Pipeline{AA,AB,BB} # three-phase codec chain ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata} <: AbstractDiskArray{T,N} ZGroup{S<:AbstractStore} @@ -45,23 +68,24 @@ ZGroup{S<:AbstractStore} ### Module/File Layout -- `src/Zarr.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`) -- `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors for V2; dispatches V3 to `metadata3.jl` -- `src/metadata3.jl` — All V3-specific code: `MetadataV3` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, codec pipeline parsing, `get_order`, `JSON.lower(::MetadataV3)` -- `src/chunkencoding.jl` — `ChunkEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/Zarr.jl` — Module entry point. Defines `ZarrFormat{V}` (Val-parameterized version tag) and default `DV = ZarrFormat(Val(2))`. Default chunk separator constants: `DS = '.'` (v2), `DS2 = '.'`, `DS3 = '/'`. +- `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors; dispatches V3 to `metadata3.jl` +- `src/metadata3.jl` — `MetadataV3{T,N,P,E}` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, `get_order`, `JSON.lower(::MetadataV3)`. V3 type mappings use lowercase strings (e.g. `"float32"`, `"complex128"`) +- `src/chunkkeyencoding.jl` — `ChunkKeyEncoding` struct (separator char + prefix bool) and `SuffixChunkKeyEncoding{E}`, `citostring()` for chunk path generation, registry via `chunk_key_encoding_parsers` dict. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/pipeline.jl` — `V2Pipeline` (wraps compressor + filters) and `V3Pipeline` (three-phase: array→array, array→bytes, bytes→bytes). `pipeline_encode`/`pipeline_decode!` interface - `src/ZArray.jl` — Core array type, `readblock!`/`writeblock!` (DiskArrays interface), `zcreate`, `zzeros`, `zopen`, resize/append - `src/ZGroup.jl` — Hierarchical group support, `zopen`, `zgroup`, auto-detection of zarr version via `ZarrFormat(store, path)` - `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl`, `v3.jl` (v3 wrapper `Compressor_v3{C}`) -- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec` +- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec`. Registry: `codectypes` dict - `src/Filters/` — `Filter{T,TENC}` abstract type, implementations for variable-length arrays, strings, Fletcher32, shuffle, delta, quantize - `src/Storage/Storage.jl` — `AbstractStore` interface, I/O strategy (`SequentialRead`/`ConcurrentRead`), chunk read/write/delete helpers, metadata read/write dispatched on `ZarrFormat{2}` vs `ZarrFormat{3}` ### Key Design Patterns - **Format version dispatch**: `ZarrFormat{V}` (where V is 2 or 3) used throughout for multiple dispatch on version-specific behavior (metadata file names, chunk encoding, serialization format) +- **Registry pattern**: Used pervasively — `compressortypes` (compressors), `codectypes` (V3 codecs), `chunk_key_encoding_parsers` (chunk key encodings), filter dict. New implementations register via these dicts. - **Shape is mutable**: `metadata.shape` is `Base.RefValue{NTuple{N,Int}}` to allow `resize!` without replacing the metadata struct - **Column-major ↔ row-major**: Zarr stores shapes/chunks in row-major (C order); Julia uses column-major. All conversions happen via `reverse()` at metadata boundaries -- **Compressor registry**: `compressortypes` dict maps spec names (e.g., `"blosc"`, `"zlib"`) to compressor types. V3 uses `Compressor_v3{C}` wrapper to change JSON serialization format - **DiskArrays integration**: `ZArray <: AbstractDiskArray`, chunk-aware I/O via `readblock!`/`writeblock!`, `eachchunk`, `haschunks` - **Async chunk I/O**: Channel-based parallel chunk reads/writes for stores supporting `ConcurrentRead` @@ -69,32 +93,24 @@ ZGroup{S<:AbstractStore} New store backends must implement: `getindex(store, key)::Union{Vector{UInt8}, Nothing}`, `setindex!(store, data, key)`, `storagesize(store, path)`, `subdirs(store, path)`, `subkeys(store, path)`, `isinitialized(store, key)`, `storefromstring(Type, string, create)`. +### V3 Pipeline Architecture + +V3 encoding/decoding follows a three-phase codec chain in `V3Pipeline`: + +1. **array→array** codecs (e.g., `TransposeCodec`) — applied in forward order during encode +2. **array→bytes** codec (e.g., `BytesCodec`, `ShardingCodec`) — single codec, converts array to byte stream +3. **bytes→bytes** codecs (e.g., `GzipV3Codec`, `BloscV3Codec`, `CRC32cV3Codec`) — applied in forward order during encode + +Decoding reverses all three phases. The `MetadataV3` convenience constructor builds the pipeline from `order` (→ `TransposeCodec`), `endian` (→ `BytesCodec`), and `compressor` (→ bytes→bytes codecs). + ### V3 Status (Experimental) -V3 support is under active development. Current state: - -**Codecs (`src/Codecs/V3/V3.jl`)** -- `BytesCodec` — stores `endian::Symbol` (`:little` or `:big`); encode/decode byte-swap elements when the target endian differs from the system byte order (`Base.ENDIAN_BOM`). Default is `:little`. -- `TransposeCodec` — array→array permutation codec (renamed from `TransposeCodecImpl`) -- `BloscV3Codec` — shuffle stored as integer (0=noshuffle, 1=shuffle, 2=bitshuffle); parsed from spec strings (`"noshuffle"`, `"shuffle"`, `"bitshuffle"`) and serialized back to strings -- Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but is not yet wired into the main read/write pipeline (throws `ArgumentError` when encountered) -- `crc32c` codec has encode/decode implementations and is parseable from metadata - -**Metadata (`src/metadata3.jl`)** -- `MetadataV3{T,N,P}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec` -- Two constructors: - - Primary inner constructor: `MetadataV3{T,N,P}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_encoding)` — takes a pre-built pipeline, no `order` argument - - Convenience outer constructor: `MetadataV3{T,N}(...; order, endian, compressor, chunk_encoding)` — builds the pipeline from `order` (→ `TransposeCodec`), `endian` (→ `BytesCodec`), and `compressor` (→ bytes→bytes codecs) -- `get_order(md::MetadataV3)` — derives `'C'`/`'F'` from the pipeline; throws `ArgumentError` when order is ambiguous: multiple array→array codecs, an unrecognized array→array codec, or a `TransposeCodec` with a permutation that is neither the identity (C) nor the full reversal (F) -- `get_order(md::MetadataV2)` — returns `md.order` -- `==` for `MetadataV3` compares `pipeline` (not `order`) - -**Other V3 status** +- `MetadataV3{T,N,P,E}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec`. `get_order(md::MetadataV3)` derives `'C'`/`'F'` from the pipeline. +- Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but throws `ArgumentError` when encountered (not wired into main pipeline) +- Filters are not implemented for V3 (`filters = nothing` hardcoded) +- `zgroup()` always creates v2 groups (hardcoded `DV` format) +- `crc32c` codec has full encode/decode implementations - V3 groups work via `zarr.json` with `node_type: "group"` -- `zgroup()` currently always creates v2 groups (hardcoded `DV` format) -- Filters are not implemented for v3 (`filters = nothing` hardcoded in `Metadata3`) -- Test fixtures: `test/v3_julia.jl` (Julia-generated), `test/v3_python.jl` (Python-generated via PythonCall) -- V3 codec tests are in `test/v3_codecs.jl` ### Extension System diff --git a/Project.toml b/Project.toml index cfe36f77..4a1554a9 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" @@ -19,8 +18,12 @@ OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +[sources.ZarrCore] +path = "lib/ZarrCore" + [weakdeps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -34,7 +37,6 @@ ChunkCodecCore = "1" ChunkCodecLibZlib = "1" ChunkCodecLibZstd = "1" CRC32c = "1.10, 1.11" -DataStructures = "0.17, 0.18, 0.19" DateTimes64 = "1" DiskArrays = "0.4.2" HTTP = "^1.3" diff --git a/docs/plans/2026-03-20-zarrcore-extraction-design.md b/docs/plans/2026-03-20-zarrcore-extraction-design.md new file mode 100644 index 00000000..f4201d9c --- /dev/null +++ b/docs/plans/2026-03-20-zarrcore-extraction-design.md @@ -0,0 +1,133 @@ +# ZarrCore.jl Extraction Design + +## Goal + +Factor Zarr.jl into two packages: +- **ZarrCore.jl** (`lib/ZarrCore/`) — minimal core sufficient to read/write uncompressed Zarr v2 and v3 arrays +- **Zarr.jl** — depends on ZarrCore, adds all compressors, V3 codecs, and network/archive storage backends + +## Design Decisions + +1. **Abstract pipeline in Core, concrete codecs split across packages.** ZarrCore defines `AbstractCodecPipeline`, `V2Pipeline`, `V3Pipeline`, and only `BytesCodec` + `TransposeCodec`. Zarr.jl registers compression codecs into the existing `codectypes` registry. Unknown codecs at parse time produce a clear error. + +2. **All pure-Julia filters in Core.** FixedScaleOffset, Fletcher32, Shuffle, Delta, Quantize — all tiny, no external deps. Avoids complex split of the Filters directory. + +3. **Core storage: DirectoryStore + DictStore + ConsolidatedStore.** No external deps. ConsolidatedStore is a thin JSON wrapper commonly used for read-only access. + +4. **Monorepo subdirectory package** at `lib/ZarrCore/`. Own `Project.toml`, publishable to General registry. Zarr.jl uses `path = "lib/ZarrCore"` during dev. + +5. **Zarr.jl re-exports everything from ZarrCore.** `using Zarr` is fully backward-compatible. `using ZarrCore` gives the minimal API. + +6. **Convenience functions in Core.** `zcreate`, `zopen`, `zzeros`, `zgroup` live in ZarrCore — without them Core has no user-facing functionality. + +7. **Extensible store resolution.** ZarrCore owns `zopen` dispatch through `storageregexlist`. Core handles local paths + DictStore. Zarr.jl registers HTTP/GCS/S3/Zip resolvers. + +## Package Structure + +### lib/ZarrCore/ + +``` +lib/ZarrCore/ + Project.toml # JSON, DiskArrays, OffsetArrays, DateTimes64, Dates + src/ + ZarrCore.jl # module entry, ZarrFormat, constants, includes, exports + chunkkeyencoding.jl + metadata.jl + metadata3.jl + MaxLengthStrings.jl + pipeline.jl + Compressors/ + Compressors.jl # Compressor, NoCompressor, compressortypes registry + Codecs/ + Codecs.jl # abstract Codec, zencode/zdecode + V3/ + V3.jl # BytesCodec, TransposeCodec only + Filters/ + Filters.jl # Filter + all pure-Julia filter implementations + fixedscaleoffset.jl, fletcher32.jl, shuffle.jl, delta.jl, quantize.jl, vlenfilters.jl + Storage/ + Storage.jl # AbstractStore, storageregexlist, metadata I/O + directorystore.jl + dictstore.jl + consolidated.jl + ZArray.jl + ZGroup.jl +``` + +### Zarr.jl (top-level, rewritten as thin wrapper) + +``` +src/ + Zarr.jl # using ZarrCore, re-exports, registrations + Compressors/ + blosc.jl + zlib.jl + zstd.jl + Codecs/ + V3/ + blosc.jl + gzip.jl + zstd.jl + crc32c.jl + sharding.jl + Storage/ + http.jl + gcstore.jl + zipstore.jl +``` + +## Registration Pattern + +Zarr.jl's `__init__` populates ZarrCore's registries: + +```julia +# Compressors +ZarrCore.compressortypes["blosc"] = BloscCompressor +ZarrCore.compressortypes["zlib"] = ZlibCompressor +ZarrCore.compressortypes["zstd"] = ZstdCompressor + +# V3 codecs +ZarrCore.codectypes["blosc"] = BloscV3Codec +ZarrCore.codectypes["gzip"] = GzipV3Codec +ZarrCore.codectypes["zstd"] = ZstdV3Codec +ZarrCore.codectypes["crc32c"] = CRC32cV3Codec + +# Store URL resolvers +push!(ZarrCore.storageregexlist, r"^https?://" => HTTPStore) +push!(ZarrCore.storageregexlist, r"^gs://" => GCStore) +``` + +## Migration Strategy + +- **Move** files from `src/` to `lib/ZarrCore/src/` (preserves git history) +- **Split** `Compressors/Compressors.jl`: abstract type + NoCompressor + registry to Core; blosc/zlib/zstd stay +- **Split** `Codecs/V3/V3.jl`: BytesCodec + TransposeCodec to Core; compression codecs stay +- **Split** `Storage/`: directorystore + dictstore + consolidated to Core; http/gcstore/zipstore stay +- **Rewrite** top-level `src/Zarr.jl` as thin wrapper + +## ZarrCore Dependencies + +```toml +[deps] +JSON = "..." +DiskArrays = "..." +OffsetArrays = "..." +DateTimes64 = "..." +Dates = "..." # stdlib +``` + +## Zarr.jl Dependencies (in addition to ZarrCore) + +```toml +[deps] +ZarrCore = "..." +Blosc = "..." +CRC32c = "..." +ChunkCodecCore = "..." +ChunkCodecLibZlib = "..." +ChunkCodecLibZstd = "..." +HTTP = "..." +OpenSSL = "..." +URIs = "..." +ZipArchives = "..." +``` diff --git a/docs/plans/2026-03-20-zarrcore-extraction.md b/docs/plans/2026-03-20-zarrcore-extraction.md new file mode 100644 index 00000000..3c56097a --- /dev/null +++ b/docs/plans/2026-03-20-zarrcore-extraction.md @@ -0,0 +1,1826 @@ +# ZarrCore.jl Extraction Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extract a minimal `ZarrCore.jl` package from `Zarr.jl` into `lib/ZarrCore/`, sufficient to read/write uncompressed Zarr v2 and v3 arrays, with Zarr.jl as a thin wrapper that registers compressors, codecs, and network stores. + +**Architecture:** ZarrCore owns all abstract types, registries, core stores (DirectoryStore, DictStore, ConsolidatedStore), all pure-Julia filters, and the minimal V3 codecs (BytesCodec, TransposeCodec). Zarr.jl depends on ZarrCore and populates its registries with Blosc/Zlib/Zstd compressors, compression V3 codecs, and HTTP/GCS/ZIP stores. A `default_compressor()` function allows Zarr.jl to override the default from `NoCompressor()` to `BloscCompressor()`. + +**Tech Stack:** Julia 1.10+, JSON, DiskArrays, OffsetArrays, DateTimes64 + +--- + +## Phase 1: ZarrCore Package Scaffold + +### Task 1: Create ZarrCore directory and Project.toml + +**Files:** +- Create: `lib/ZarrCore/Project.toml` + +**Step 1: Create the file** + +```toml +name = "ZarrCore" +uuid = "INSERT-GENERATED-UUID" +version = "0.1.0" + +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[compat] +JSON = "0.21, 1" +DiskArrays = "0.4.2" +OffsetArrays = "0.11, 1.0" +DateTimes64 = "1" +julia = "1.10" +``` + +Generate a real UUID with: `julia -e 'using UUIDs; println(uuid4())'` + +**Step 2: Verify directory structure** + +Run: `ls lib/ZarrCore/` +Expected: `Project.toml` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/Project.toml +git commit -m "scaffold: create ZarrCore package with Project.toml" +``` + +--- + +### Task 2: Create ZarrCore module entry point (stub) + +**Files:** +- Create: `lib/ZarrCore/src/ZarrCore.jl` + +**Step 1: Create a minimal module that loads** + +```julia +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +const DV = ZarrFormat(Val(2)) + +abstract type AbstractCodecPipeline end + +# Placeholder — will be populated in subsequent tasks +end # module +``` + +**Step 2: Verify it loads** + +Run: `julia --project=lib/ZarrCore -e 'using Pkg; Pkg.instantiate(); using ZarrCore; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/ZarrCore.jl +git commit -m "scaffold: add ZarrCore module entry point" +``` + +--- + +## Phase 2: Core Types & Registries + +### Task 3: Move chunkkeyencoding.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/chunkkeyencoding.jl` (copy from `src/chunkkeyencoding.jl`) + +This file has no external dependencies — it only references `ZarrFormat` which is defined in the module entry point. Copy it verbatim. + +**Step 1: Copy the file** + +```bash +cp src/chunkkeyencoding.jl lib/ZarrCore/src/chunkkeyencoding.jl +``` + +**Step 2: Add include to ZarrCore.jl** + +In `lib/ZarrCore/src/ZarrCore.jl`, add after the `AbstractCodecPipeline` line: + +```julia +include("chunkkeyencoding.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.ChunkKeyEncoding(".", false))'` +Expected: no error + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/chunkkeyencoding.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move chunkkeyencoding.jl to ZarrCore" +``` + +--- + +### Task 4: Move MaxLengthStrings.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/MaxLengthStrings.jl` (copy from `src/MaxLengthStrings.jl`) + +No external deps. Copy verbatim. + +**Step 1: Copy** + +```bash +cp src/MaxLengthStrings.jl lib/ZarrCore/src/MaxLengthStrings.jl +``` + +**Step 2: Verify** + +Run: `julia --project=lib/ZarrCore -e 'include("lib/ZarrCore/src/MaxLengthStrings.jl"); using .MaxLengthStrings; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/MaxLengthStrings.jl +git commit -m "core: move MaxLengthStrings.jl to ZarrCore" +``` + +--- + +### Task 5: Create ZarrCore Compressors (abstract type + NoCompressor + registry) + +**Files:** +- Create: `lib/ZarrCore/src/Compressors/Compressors.jl` + +Extract the abstract type, registry, `NoCompressor`, and all generic fallback methods from `src/Compressors/Compressors.jl`. Do NOT include `blosc.jl`, `zlib.jl`, `zstd.jl`. Also add `default_compressor()` function. + +**Step 1: Create the file** + +```julia +import JSON # for JSON.lower + +_reinterpret(::Type{T}, x::AbstractArray{S, 0}) where {T, S} = reinterpret(T, reshape(x, 1)) +_reinterpret(::Type{T}, x::AbstractArray) where T = reinterpret(T, x) + +""" + abstract type Compressor + +The abstract supertype for all Zarr compressors. + +## Interface + +All subtypes of `Compressor` SHALL implement the following methods: + +- `zcompress(a, c::Compressor)`: compress the array `a` using the compressor `c`. +- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` + and return an array of type `T`. +- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which + follows the Zarr specification for that compressor. +- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given + dictionary `d` which contains the compressor's parameters according to the Zarr spec. + +Subtypes of `Compressor` MAY also implement the following methods: + +- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the + compressor `c` and store the result in the array `compressed`. +- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` + using the compressor `c` and store the result in the array `data`. + +Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. +""" +abstract type Compressor end + +const compressortypes = Dict{Union{String,Nothing}, Type{<: Compressor}}() + +# Fallback definitions for the compressor interface +getCompressor(compdict::Dict) = haskey(compdict, "id") ? + getCompressor(compressortypes[compdict["id"]], compdict) : + getCompressor(compressortypes[compdict["name"]], compdict["configuration"]) +getCompressor(::Nothing) = NoCompressor() + +# Compression when no filter is given +zcompress!(compressed,data,c,::Nothing) = zcompress!(compressed,data,c) +zuncompress!(data,compressed,c,::Nothing) = zuncompress!(data,compressed,c) + +# Fallback definition of mutating form of compress and uncompress +function zcompress!(compressed, data, c) + empty!(compressed) + append!(compressed,zcompress(data, c)) +end +zuncompress!(data, compressed, c) = copyto!(data, zuncompress(compressed, c, eltype(data))) + + +# Function given a filter stack +function zcompress!(compressed, data, c, f) + a2 = foldl(f, init=data) do anow, fnow + zencode(anow,fnow) + end + zcompress!(compressed, a2, c) +end + +function zuncompress!(data, compressed, c, f) + data2 = zuncompress(compressed, c, desttype(last(f))) + a2 = foldr(f, init = data2) do fnow, anow + zdecode(anow, fnow) + end + copyto!(data, a2) +end + +# NoCompressor +""" + NoCompressor() + +Creates an object that can be passed to ZArray constructors without compression. +""" +struct NoCompressor <: Compressor end + +function zuncompress(a, ::NoCompressor, T) + _reinterpret(T,a) +end + +function zcompress(a, ::NoCompressor) + _reinterpret(UInt8,a) +end + +JSON.lower(::NoCompressor) = nothing + +compressortypes[nothing] = NoCompressor + +""" + default_compressor() + +Returns the default compressor used by `zcreate` and related functions. +ZarrCore returns `NoCompressor()`. Zarr.jl overrides this to `BloscCompressor()`. +""" +default_compressor() = NoCompressor() +``` + +**Step 2: Add include to ZarrCore.jl** + +After the `include("chunkkeyencoding.jl")` line add: + +```julia +include("Compressors/Compressors.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.NoCompressor())'` +Expected: `NoCompressor()` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/Compressors/Compressors.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add Compressors base with NoCompressor and registry" +``` + +--- + +### Task 6: Create ZarrCore Codecs with registry-based V3 infrastructure + +**Files:** +- Create: `lib/ZarrCore/src/Codecs/Codecs.jl` +- Create: `lib/ZarrCore/src/Codecs/V3/V3.jl` + +The Codecs.jl is copied from `src/Codecs/Codecs.jl` with its `include` pointing to the new V3. + +The V3.jl contains: +- `V3Codec{In,Out}` abstract type +- `v3_codec_parsers` registry (`Dict{String, Function}`) +- `codec_to_dict` generic function (for serialization) +- `codec_category` helper +- `BytesCodec` + `TransposeCodec` (structs, encode/decode, parse, serialize) +- `encoded_shape` generic function + +**Step 1: Create `lib/ZarrCore/src/Codecs/Codecs.jl`** + +```julia +module Codecs + +using JSON: JSON + +""" + abstract type Codec + +The abstract supertype for all Zarr codecs. +""" +abstract type Codec end + +zencode(a, c::Codec) = error("Unimplemented") +zencode!(encoded, data, c::Codec) = error("Unimplemented") +zdecode(a, c::Codec, T::Type) = error("Unimplemented") +zdecode!(data, encoded, c::Codec) = error("Unimplemented") +JSON.lower(c::Codec) = error("Unimplemented") +getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") + +include("V3/V3.jl") + +end +``` + +**Step 2: Create `lib/ZarrCore/src/Codecs/V3/V3.jl`** + +```julia +module V3Codecs + +import ..Codecs: zencode, zdecode, zencode!, zdecode! +using JSON: JSON + +abstract type V3Codec{In,Out} end + +"""Registry of V3 codec parsers: name -> (config_dict -> V3Codec instance)""" +const v3_codec_parsers = Dict{String, Function}() + +"""Each V3Codec must implement this for JSON serialization.""" +function codec_to_dict end + +"""Classify a V3Codec by its phase in the pipeline.""" +codec_category(::V3Codec{:array,:array}) = :array_array +codec_category(::V3Codec{:array,:bytes}) = :array_bytes +codec_category(::V3Codec{:bytes,:bytes}) = :bytes_bytes + +"""Return the shape of the output of `codec_encode(codec, data)` given the input shape.""" +encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz + +# --- BytesCodec (array -> bytes) --- + +struct BytesCodec <: V3Codec{:array, :bytes} + endian::Symbol # :little or :big + function BytesCodec(endian::Symbol) + endian ∈ (:little, :big) || + throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) + new(endian) + end +end +BytesCodec() = BytesCodec(:little) + +const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 +_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN + +function codec_encode(c::BytesCodec, data::AbstractArray) + if _needs_bswap(c.endian) + return reinterpret(UInt8, bswap.(vec(data))) |> collect + else + return reinterpret(UInt8, vec(data)) |> collect + end +end + +function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} + arr = collect(reinterpret(T, encoded)) + if _needs_bswap(c.endian) + arr = bswap.(arr) + end + return reshape(arr, shape) +end + +codec_to_dict(c::BytesCodec) = Dict{String,Any}( + "name" => "bytes", + "configuration" => Dict{String,Any}("endian" => string(c.endian)) +) + +v3_codec_parsers["bytes"] = function(config) + endian_str = get(config, "endian", "little") + endian = endian_str == "little" ? :little : + endian_str == "big" ? :big : + throw(ArgumentError("Unknown endian value: \"$endian_str\"")) + BytesCodec(endian) +end + +# --- TransposeCodec (array -> array) --- + +struct TransposeCodec{N} <: V3Codec{:array, :array} + order::NTuple{N, Int} # permutation (1-based Julia indexing) +end + +encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) + +function codec_encode(c::TransposeCodec, data::AbstractArray) + return permutedims(data, c.order) +end + +function codec_decode(c::TransposeCodec, encoded::AbstractArray) + inv_order = Tuple(invperm(collect(c.order))) + return permutedims(encoded, inv_order) +end + +codec_to_dict(c::TransposeCodec) = Dict{String,Any}( + "name" => "transpose", + "configuration" => Dict{String,Any}("order" => collect(c.order .- 1)) +) + +# TransposeCodec parser is registered in metadata3.jl because it needs +# the shape context to parse string orders like "C"/"F". +# The registry entry for numeric orders is here: +# NOTE: the parser receives (config, shape_length) — see parse_v3_codec. + +end +``` + +**Step 3: Add include to ZarrCore.jl** + +After `include("Compressors/Compressors.jl")`: + +```julia +include("Codecs/Codecs.jl") +``` + +**Step 4: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; c = ZarrCore.Codecs.V3Codecs.BytesCodec(); println(c)'` +Expected: `BytesCodec(:little)` + +**Step 5: Commit** + +```bash +git add lib/ZarrCore/src/Codecs/Codecs.jl lib/ZarrCore/src/Codecs/V3/V3.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add Codecs module with BytesCodec and TransposeCodec" +``` + +--- + +### Task 7: Move Filters to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/Filters/` (entire directory) + +Copy the entire Filters directory verbatim. These files only depend on JSON (imported by parent module) and have no external package deps. + +**Step 1: Copy all filter files** + +```bash +cp -r src/Filters lib/ZarrCore/src/Filters +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("Codecs/Codecs.jl")`: + +```julia +include("Filters/Filters.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.filterdict)'` +Expected: prints a Dict with filter entries + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/Filters/ lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move all Filters to ZarrCore" +``` + +--- + +## Phase 3: Metadata Layer + +### Task 8: Move metadata.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/metadata.jl` + +Copy `src/metadata.jl` with these changes: + +1. The `Metadata(A, chunks, zarr_format; compressor=BloscCompressor(), ...)` constructor — change default to `default_compressor()`. +2. The `Metadata(A, chunks, ::ZarrFormat{2}; compressor=BloscCompressor(), ...)` constructor — change default to `default_compressor()`. + +These are the ONLY changes. Everything else stays the same. + +**Step 1: Copy and modify** + +```bash +cp src/metadata.jl lib/ZarrCore/src/metadata.jl +``` + +Then edit `lib/ZarrCore/src/metadata.jl`: + +Replace both occurrences of `compressor::C=BloscCompressor()` with `compressor::C=default_compressor()`: + +- Line ~153 (in `Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; ...)`): + Change `compressor::C=BloscCompressor()` → `compressor::C=default_compressor()` + +- Line ~175 (in `Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; ...)`): + Change `compressor::C=BloscCompressor()` → `compressor::C=default_compressor()` + +**Step 2: Add include to ZarrCore.jl** + +After `include("chunkkeyencoding.jl")` and before `include("Compressors/Compressors.jl")`, add: + +```julia +include("metadata.jl") +``` + +Wait — `metadata.jl` uses `BloscCompressor` and `getCompressor` which are defined in Compressors. And it uses `getfilters` from Filters. So the include order must be: + +```julia +include("chunkkeyencoding.jl") +abstract type AbstractCodecPipeline end +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +``` + +Actually, `metadata.jl` only needs `Compressor` types for the constructor default and `getCompressor`/`getfilters` for parsing. Both are already included before metadata.jl. This matches the original `src/Zarr.jl` include order. + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.MetadataV2)'` +Expected: prints the type + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/metadata.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move metadata.jl to ZarrCore with default_compressor()" +``` + +--- + +### Task 9: Refactor and move metadata3.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/metadata3.jl` + +This is the biggest refactoring. The current `metadata3.jl` has: + +1. **`Metadata3(d::AbstractDict, ...)`** — hardcodes codec parsing with if/elseif chain +2. **`lower3(md::MetadataV3)`** — hardcodes codec serialization with isa checks +3. **`MetadataV3{T2,N}` convenience constructor** — hardcodes compressor-to-codec mapping + +Changes needed: + +**A. Replace hardcoded codec parsing with registry lookup:** + +The current code: +```julia +if codec_name == "transpose" ... +elseif codec_name == "bytes" ... +elseif codec_name == "gzip" ... +``` + +Becomes: +```julia +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + # Special case: transpose needs shape_length for "C"/"F" string orders + if codec_name == "transpose" + return _parse_transpose(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end +``` + +**B. Replace hardcoded serialization with `codec_to_dict` dispatch:** + +The current `lower3` builds codec dicts inline with isa checks. Replace with: +```julia +for codec in p.array_array + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) +end +``` + +**C. Replace hardcoded compressor-to-codec in convenience constructor:** + +The `MetadataV3{T2,N}(...)` constructor maps compressors to V3 codecs. This constructor moves to Zarr.jl (not ZarrCore) since it references specific compressor types. ZarrCore keeps only the inner constructor and `Metadata3(d::AbstractDict, ...)`. + +**Step 1: Create the refactored file** + +Create `lib/ZarrCore/src/metadata3.jl` with this content: + +```julia +""" +Prototype Zarr version 3 support +""" + +const typemap3 = Dict{String, DataType}() +foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t + typemap3[lowercase(string(t))] = t +end +typemap3["complex64"] = ComplexF32 +typemap3["complex128"] = ComplexF64 + +function typestr3(t::Type) + return lowercase(string(t)) +end +function typestr3(::Type{NTuple{N,UInt8}}) where {N} + return "r$(N*8)" +end + +function typestr3(s::AbstractString, codecs=nothing) + if !haskey(typemap3, s) + if startswith(s, "r") + num_bits = tryparse(Int, s[2:end]) + if isnothing(num_bits) + throw(ArgumentError("$s is not a known type")) + end + if mod(num_bits, 8) == 0 + return NTuple{num_bits÷8,UInt8} + else + throw(ArgumentError("$s must describe a raw type with bit size that is a multiple of 8 bits")) + end + end + end + return typemap3[s] +end + +function check_keys(d::AbstractDict, keys) + for key in keys + if !haskey(d, key) + throw(ArgumentError("Zarr v3 metadata must have a key called $key")) + end + end +end + +"""Metadata for Zarr version 3 arrays""" +struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} + zarr_format::Int + node_type::String + shape::Base.RefValue{NTuple{N, Int}} + chunks::NTuple{N, Int} + dtype::String # data_type in v3 + pipeline::P + fill_value::Union{T, Nothing} + chunk_key_encoding::E + function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} + zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) + any(<(0), shape) && throw(ArgumentError("Size must be positive")) + any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + end +end +MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) +zarr_format(::MetadataV3) = ZarrFormat(Val(3)) + +function Base.:(==)(m1::MetadataV3, m2::MetadataV3) + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.fill_value == m2.fill_value && + m1.pipeline == m2.pipeline && + m1.chunk_key_encoding == m2.chunk_key_encoding +end + +""" +Derive the storage order ('C' or 'F') from the codec pipeline of a MetadataV3. +""" +function get_order(md::MetadataV3) + array_array = md.pipeline.array_array + if length(array_array) == 0 + return 'C' + end + if length(array_array) > 1 + throw(ArgumentError( + "Cannot determine storage order: pipeline has $(length(array_array)) " * + "array->array codecs; composed permutations yield an indeterminate order" + )) + end + codec = only(array_array) + if !(codec isa Codecs.V3Codecs.TransposeCodec) + throw(ArgumentError( + "Cannot determine storage order: unrecognized array->array codec $(typeof(codec))" + )) + end + N = ndims(md) + c_perm = ntuple(identity, N) + f_perm = ntuple(i -> N - i + 1, N) + if codec.order == c_perm + return 'C' + elseif codec.order == f_perm + return 'F' + else + throw(ArgumentError( + "Cannot determine storage order: TransposeCodec permutation $(codec.order) " * + "is neither C order $c_perm nor F order $f_perm" + )) + end +end +get_order(md::MetadataV2) = md.order + +# --- Registry-based codec parsing --- + +""" +Parse a single V3 codec entry from a zarr.json codecs array. +`shape_length` is needed for transpose codec string order parsing ("C"/"F"). +""" +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + # Transpose needs shape_length for "C"/"F" string parsing + if codec_name == "transpose" + return _parse_transpose_codec(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end + +function _parse_transpose_codec(config::Dict, shape_length::Int) + _order = config["order"] + if _order isa AbstractString + n = shape_length + if _order == "C" + @warn "Transpose codec dimension order of C is deprecated" + perm = ntuple(identity, n) + elseif _order == "F" + @warn "Transpose codec dimension order of F is deprecated" + perm = ntuple(i -> n - i + 1, n) + else + throw(ArgumentError("Unknown transpose order string: $_order")) + end + else + perm = Tuple(Int.(_order) .+ 1) + end + return Codecs.V3Codecs.TransposeCodec(perm) +end + + +function Metadata3(d::AbstractDict, fill_as_missing) + check_keys(d, ("zarr_format", "node_type")) + + zarr_format = d["zarr_format"]::Int + node_type = d["node_type"]::String + if node_type ∉ ("group", "array") + throw(ArgumentError("Unknown node_type of $node_type")) + end + zarr_format == 3 || throw(ArgumentError("Metadata3 only functions if zarr_format == 3")) + + # Groups + if node_type == "group" + for key in keys(d) + if key ∉ ("zarr_format", "node_type", "attributes") + throw(ArgumentError("Zarr v3 group metadata cannot have a key called $key")) + end + end + group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) + return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) + end + + # Array keys + mandatory_keys = [ + "zarr_format", "node_type", "shape", "data_type", + "chunk_grid", "chunk_key_encoding", "fill_value", "codecs", + ] + optional_keys = ["attributes", "storage_transformers", "dimension_names"] + check_keys(d, mandatory_keys) + for key in keys(d) + if key ∉ mandatory_keys && key ∉ optional_keys + throw(ArgumentError("Zarr v3 metadata cannot have a key called $key")) + end + end + + shape = Int.(d["shape"]) + data_type = d["data_type"]::String + + chunk_grid = d["chunk_grid"] + if chunk_grid["name"] == "regular" + chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) + if length(shape) != length(chunks) + throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) + end + else + throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) + end + + # Build V3Pipeline from codec chain using registry + array_array_codecs = [] + array_bytes_codec = nothing + bytes_bytes_codecs = [] + + for codec in d["codecs"] + codec_name = codec["name"] + config = get(codec, "configuration", Dict{String,Any}()) + parsed = parse_v3_codec(codec_name, config, length(shape)) + cat = Codecs.V3Codecs.codec_category(parsed) + if cat == :array_array + push!(array_array_codecs, parsed) + elseif cat == :array_bytes + array_bytes_codec = parsed + elseif cat == :bytes_bytes + push!(bytes_bytes_codecs, parsed) + end + end + + isnothing(array_bytes_codec) && throw(ArgumentError("V3 codec chain must contain a 'bytes' codec")) + pipeline = V3Pipeline(Tuple(array_array_codecs), array_bytes_codec, Tuple(bytes_bytes_codecs)) + + T = typestr3(data_type) + N = length(shape) + fv = fill_value_decoding(d["fill_value"], T)::T + TU = (fv === nothing || !fill_as_missing) ? T : Union{T,Missing} + + chunk_key_encoding = parse_chunk_key_encoding(d["chunk_key_encoding"]) + E = typeof(chunk_key_encoding) + + MetadataV3{TU, N, typeof(pipeline), E}( + zarr_format, node_type, + NTuple{N, Int}(shape) |> reverse, + NTuple{N, Int}(chunks) |> reverse, + data_type, pipeline, fv, chunk_key_encoding, + ) +end + +"Construct MetadataV3 based on your data (minimal — no compressor mapping)" +function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; + node_type::String="array", + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + fill_as_missing = false, + dimension_separator::Char = '/', + bytes_bytes_codecs::Tuple=() + ) where {T, N} + @warn("Zarr v3 support is experimental") + T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} + if fill_value === nothing + fill_value = zero(T) + end + array_array_codecs = if order == 'F' + (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) + else + () + end + array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) + pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) + chunk_key_encoding = ChunkKeyEncoding(dimension_separator, true) + E = typeof(chunk_key_encoding) + return MetadataV3{T2,N,typeof(pipeline),E}( + 3, node_type, size(A), chunks, typestr3(eltype(A)), + pipeline, fill_value, chunk_key_encoding + ) +end + +# --- Registry-based serialization --- + +function lower3(md::MetadataV3{T}) where T + chunk_grid = Dict{String,Any}( + "name" => "regular", + "configuration" => Dict{String,Any}( + "chunk_shape" => md.chunks |> reverse + ) + ) + chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) + + codecs = Dict{String,Any}[] + p = md.pipeline + + for codec in p.array_array + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + push!(codecs, Codecs.V3Codecs.codec_to_dict(p.array_bytes)) + for codec in p.bytes_bytes + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) + end + + Dict{String, Any}( + "zarr_format" => Int(md.zarr_format), + "node_type" => md.node_type, + "shape" => md.shape[] |> reverse, + "data_type" => typestr3(T), + "chunk_grid" => chunk_grid, + "chunk_key_encoding" => chunk_key_encoding, + "fill_value" => fill_value_encoding(md.fill_value), + "codecs" => codecs + ) +end + +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; + node_type::String="array", + compressor::C=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + filters::F=nothing, + fill_as_missing = false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) + ) where {T, N, C, F, E} + # Map compressor to bytes_bytes_codecs using compressor_to_v3_codecs registry + bytes_bytes = compressor_to_v3_bytes_codecs(compressor) + return Metadata3(A, chunks; + node_type=node_type, + fill_value=fill_value, + order=order, + endian=endian, + fill_as_missing=fill_as_missing, + dimension_separator=chunk_key_encoding.sep, + bytes_bytes_codecs=bytes_bytes + ) +end + +# V3 constructor from Dict +function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{3}) + return Metadata3(d, fill_as_missing) +end + +function JSON.lower(md::MetadataV3) + return lower3(md) +end + +""" + compressor_to_v3_bytes_codecs(c::Compressor) -> Tuple + +Convert a v2 Compressor to a tuple of V3 bytes->bytes codecs. +ZarrCore handles NoCompressor. Zarr.jl adds methods for BloscCompressor, etc. +""" +compressor_to_v3_bytes_codecs(::NoCompressor) = () +compressor_to_v3_bytes_codecs(c::Compressor) = throw(ArgumentError( + "Unsupported compressor type for v3: $(typeof(c)). Load Zarr.jl for compression support." +)) +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("metadata.jl")`: + +```julia +include("metadata3.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; println(ZarrCore.MetadataV3)'` +Expected: prints the type + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/metadata3.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: add refactored metadata3.jl with registry-based codec parsing" +``` + +--- + +### Task 10: Move pipeline.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/pipeline.jl` + +Copy `src/pipeline.jl` verbatim. It references `Codecs.V3Codecs.codec_encode` and `codec_decode` which exist in ZarrCore's V3Codecs module. + +**Step 1: Copy** + +```bash +cp src/pipeline.jl lib/ZarrCore/src/pipeline.jl +``` + +**Step 2: Add include to ZarrCore.jl** + +After `include("metadata3.jl")`: + +```julia +include("pipeline.jl") +``` + +Note: in the original `src/Zarr.jl`, `pipeline.jl` is included after `ZArray.jl`. However, `pipeline.jl` only depends on types already defined (V3Pipeline, Codecs.V3Codecs, Compressors). `ZArray.jl` calls `pipeline_encode`/`pipeline_decode!` which are defined in `pipeline.jl`. So `pipeline.jl` must come before `ZArray.jl`. Check that this doesn't break any circular dependencies — it shouldn't because `pipeline.jl` doesn't reference ZArray. + +Actually wait, looking at the original include order in `src/Zarr.jl`: +```julia +include("ZArray.jl") +include("pipeline.jl") +``` + +But `ZArray.jl` calls `pipeline_encode`/`pipeline_decode!`/`get_pipeline`. These are defined in `pipeline.jl`. How does this work? Because Julia resolves function calls at runtime, not at include time. The struct `V2Pipeline`/`V3Pipeline` are used as types in `V2Pipeline{C,F}` etc. But `V3Pipeline` is defined... where? Let me check. + +`V3Pipeline` is defined in `pipeline.jl` (line 30). But `MetadataV3` references `pipeline::P` where `P<:AbstractCodecPipeline`, and `metadata3.jl` constructs `V3Pipeline(...)`. So `pipeline.jl` must be included BEFORE `metadata3.jl`. + +Looking at the original order: `metadata3.jl` is included at line 18, `pipeline.jl` at line 24. But `metadata3.jl` uses `V3Pipeline(...)`. How? Because in the original code, `V3Pipeline` is defined in `pipeline.jl` which is included AFTER `metadata3.jl`. This works because the `Metadata3` function that constructs `V3Pipeline` isn't called until runtime. + +So the include order doesn't matter for function bodies, only for type definitions and struct references. Since `AbstractCodecPipeline` is defined before `metadata3.jl`, and `V3Pipeline` is only used in function bodies (not in struct fields — `MetadataV3` uses `P<:AbstractCodecPipeline`), this works. + +For ZarrCore, include `pipeline.jl` BEFORE `ZArray.jl` to match the logical dependency order. But technically it works either way. + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; p = ZarrCore.V3Pipeline((), ZarrCore.Codecs.V3Codecs.BytesCodec(), ()); println(p)'` +Expected: prints the pipeline + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/pipeline.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move pipeline.jl to ZarrCore" +``` + +--- + +## Phase 4: Storage & Array + +### Task 11: Move core Storage to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/Storage/Storage.jl` +- Create: `lib/ZarrCore/src/Storage/directorystore.jl` +- Create: `lib/ZarrCore/src/Storage/dictstore.jl` +- Create: `lib/ZarrCore/src/Storage/consolidated.jl` + +Copy `Storage.jl` but remove the includes for `gcstore.jl`, `http.jl`, `zipstore.jl`. Also remove the `S3Store` stub (move it to Zarr.jl) and the S3 regex registration. + +**Step 1: Copy store implementation files verbatim** + +```bash +cp src/Storage/directorystore.jl lib/ZarrCore/src/Storage/directorystore.jl +cp src/Storage/dictstore.jl lib/ZarrCore/src/Storage/dictstore.jl +cp src/Storage/consolidated.jl lib/ZarrCore/src/Storage/consolidated.jl +``` + +**Step 2: Create modified `lib/ZarrCore/src/Storage/Storage.jl`** + +Copy `src/Storage/Storage.jl` with these changes: + +1. Remove the `S3Store` struct definition and constructor (lines 36-43) +2. Remove `push!(storageregexlist, r"^s3://" => S3Store)` (line 283) +3. Remove `include("gcstore.jl")`, `include("http.jl")`, `include("zipstore.jl")` (lines 288-291) + +The remaining includes are: `directorystore.jl`, `dictstore.jl`, `consolidated.jl`. + +**Step 3: Add include to ZarrCore.jl** + +After `include("pipeline.jl")`: + +```julia +include("Storage/Storage.jl") +``` + +**Step 4: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; d = ZarrCore.DictStore(); println(d)'` +Expected: `Dictionary Storage` + +**Step 5: Commit** + +```bash +git add lib/ZarrCore/src/Storage/ lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move Storage with DirectoryStore, DictStore, ConsolidatedStore" +``` + +--- + +### Task 12: Move ZArray.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/ZArray.jl` + +Copy `src/ZArray.jl` with these changes: + +1. `zcreate(::Type{T}, storage::AbstractStore, ...)` — change `compressor=BloscCompressor()` → `compressor=default_compressor()` + +That's the only change needed. The rest (imports, types, functions) stays the same. + +**Step 1: Copy and modify** + +```bash +cp src/ZArray.jl lib/ZarrCore/src/ZArray.jl +``` + +Edit `lib/ZarrCore/src/ZArray.jl`: +- Change `compressor=BloscCompressor()` to `compressor=default_compressor()` in the `zcreate(::Type{T},storage::AbstractStore, ...)` function (around line 349). + +**Step 2: Add include to ZarrCore.jl** + +After `include("Storage/Storage.jl")`: + +```julia +include("ZArray.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; z = ZarrCore.zzeros(Int, 4, 4); println(z)'` +Expected: `ZArray{Int64} of size 4 x 4` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/ZArray.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move ZArray.jl to ZarrCore with default_compressor()" +``` + +--- + +### Task 13: Move ZGroup.jl to ZarrCore + +**Files:** +- Create: `lib/ZarrCore/src/ZGroup.jl` + +Copy `src/ZGroup.jl` with these changes: + +1. Remove `HTTP.serve(...)` line (line 201) — move to Zarr.jl +2. Remove `writezip(...)` line (line 202) — move to Zarr.jl + +These are the only changes. `consolidate_metadata` stays (it only depends on Store and JSON). + +**Step 1: Copy and modify** + +```bash +cp src/ZGroup.jl lib/ZarrCore/src/ZGroup.jl +``` + +Edit `lib/ZarrCore/src/ZGroup.jl`: +- Remove the line: `HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...)` +- Remove the line: `writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...)` + +**Step 2: Add include to ZarrCore.jl** + +After `include("ZArray.jl")`: + +```julia +include("ZGroup.jl") +``` + +**Step 3: Verify** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; g = ZarrCore.zgroup(ZarrCore.DictStore(), "", ZarrCore.ZarrFormat(2)); println(g)'` +Expected: `ZarrGroup at Dictionary Storage and path ` + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/src/ZGroup.jl lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: move ZGroup.jl to ZarrCore (without HTTP.serve, writezip)" +``` + +--- + +## Phase 5: Module Entry Points & Wiring + +### Task 14: Finalize ZarrCore.jl module entry with exports + +**Files:** +- Modify: `lib/ZarrCore/src/ZarrCore.jl` + +Write the complete module entry point with all includes and exports. + +**Step 1: Write the final ZarrCore.jl** + +```julia +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +#Default Zarr Version +const DV = ZarrFormat(Val(2)) + +include("chunkkeyencoding.jl") +abstract type AbstractCodecPipeline end +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +include("metadata3.jl") +include("pipeline.jl") +include("Storage/Storage.jl") +include("ZArray.jl") +include("ZGroup.jl") + +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +end # module +``` + +**Step 2: Verify full load** + +Run: `julia --project=lib/ZarrCore -e 'using ZarrCore; z = zzeros(Int, 4, 4); println(z); println(z[1:2, 1:2])'` +Expected: creates a ZArray and reads zeros back + +**Step 3: Commit** + +```bash +git add lib/ZarrCore/src/ZarrCore.jl +git commit -m "core: finalize ZarrCore module with all exports" +``` + +--- + +### Task 15: Rewrite Zarr.jl as wrapper + +**Files:** +- Modify: `src/Zarr.jl` +- Keep: `src/Compressors/blosc.jl`, `src/Compressors/zlib.jl`, `src/Compressors/zstd.jl` +- Keep: `src/Storage/gcstore.jl`, `src/Storage/http.jl`, `src/Storage/zipstore.jl` + +The new `src/Zarr.jl` imports ZarrCore, re-exports its names, includes the compression and network store files, and registers everything into ZarrCore's registries. + +**Step 1: Rewrite `src/Zarr.jl`** + +```julia +module Zarr + +using ZarrCore +import JSON +import Blosc + +# Re-export all ZarrCore public names +using ZarrCore: ZarrCore, + # Types + ZArray, ZGroup, AbstractStore, DirectoryStore, DictStore, ConsolidatedStore, + AbstractMetadata, MetadataV2, MetadataV3, + AbstractCodecPipeline, V2Pipeline, V3Pipeline, + Compressor, NoCompressor, + Filter, + ZarrFormat, + AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + ASCIIChar, + # Functions + zcreate, zopen, zzeros, zgroup, + storagesize, storageratio, zinfo, zname, + pipeline_encode, pipeline_decode!, get_pipeline, get_order, + Metadata, Metadata3, + typestr, typestr3, + fill_value_encoding, fill_value_decoding, + getCompressor, compressortypes, + getfilters, filterdict, + zencode, zdecode, + citostring, + default_sep, default_prefix, + storageregexlist, storefromstring, + chunkindices, + consolidate_metadata, + # Pipeline/Codec internals used by tests + _reinterpret, + # Constants + DV, DS, DS2, DS3 + +# Re-export the same symbols as before +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +# Bring in the MaxLengthStrings submodule +using ZarrCore: MaxLengthStrings +using .MaxLengthStrings: MaxLengthString + +# Include compressor implementations (they register into ZarrCore.compressortypes) +include("Compressors/blosc.jl") +include("Compressors/zlib.jl") +include("Compressors/zstd.jl") + +# Override default compressor +ZarrCore.default_compressor() = BloscCompressor() + +# Override compressor_to_v3_bytes_codecs for specific compressor types +function ZarrCore.compressor_to_v3_bytes_codecs(c::BloscCompressor) + T_base = UInt8 # will be overridden by actual element type at call site + (ZarrCore.Codecs.V3Codecs.BloscV3Codec(c.cname, c.clevel, c.shuffle, c.blocksize, sizeof(T_base)),) +end +function ZarrCore.compressor_to_v3_bytes_codecs(c::ZlibCompressor) + level = c.config.level == -1 ? 6 : c.config.level + (ZarrCore.Codecs.V3Codecs.GzipV3Codec(level),) +end +function ZarrCore.compressor_to_v3_bytes_codecs(c::ZstdCompressor) + (ZarrCore.Codecs.V3Codecs.ZstdV3Codec(c.config.compressionLevel),) +end + +# Include V3 compression codec implementations +include("Codecs/V3/compression_codecs.jl") + +# Include network/archive storage backends +include("Storage/gcstore.jl") +include("Storage/http.jl") +include("Storage/zipstore.jl") + +# Register S3Store stub +struct S3Store <: AbstractStore + bucket::String + aws::Any +end +function S3Store(args...) + error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") +end + +# Register store URL resolvers +push!(storageregexlist, r"^s3://" => S3Store) + +# HTTP.serve and writezip for ZArray/ZGroup +HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) +writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) + +export S3Store, GCStore + +end # module +``` + +**Important:** The compressor files (`blosc.jl`, `zlib.jl`, `zstd.jl`) need to be updated to reference `ZarrCore.compressortypes` instead of `Zarr.compressortypes` since the registry lives in ZarrCore. However, since `Zarr` uses `ZarrCore` and imports `compressortypes`, the line `Zarr.compressortypes["blosc"] = BloscCompressor` will still work (it references the same dict object). No change needed in the compressor files. + +Wait — actually, the compressor files do `Zarr.compressortypes["blosc"] = BloscCompressor`. Since `compressortypes` is imported into `Zarr` from `ZarrCore`, `Zarr.compressortypes` IS `ZarrCore.compressortypes`. So this works without changes. + +**Step 2: Verify** + +Run: `julia --project -e 'using Zarr; z = zzeros(Int, 4, 4); println(z)'` +Expected: `ZArray{Int64} of size 4 x 4` (with Blosc compression as before) + +**Step 3: Commit** + +```bash +git add src/Zarr.jl +git commit -m "wrapper: rewrite Zarr.jl to use ZarrCore with re-exports" +``` + +--- + +### Task 16: Create V3 compression codec file for Zarr.jl + +**Files:** +- Create: `src/Codecs/V3/compression_codecs.jl` + +Extract the compression-related V3 codecs from the original `src/Codecs/V3/V3.jl` into a new file. This includes: `GzipV3Codec`, `BloscV3Codec`, `ZstdV3Codec`, `CRC32cV3Codec`, and their `codec_encode`/`codec_decode`/`codec_to_dict` methods, plus the `CRC32cCodec` helper, `ShardingCodec`, and all sharding infrastructure. + +Each codec must also register its parser in `ZarrCore.Codecs.V3Codecs.v3_codec_parsers`. + +**Step 1: Create `src/Codecs/V3/compression_codecs.jl`** + +```julia +# V3 compression codecs that depend on Blosc, ChunkCodecLibZlib, ChunkCodecLibZstd, CRC32c +# These register into ZarrCore's V3 codec registry. + +using CRC32c: CRC32c +using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions +using ChunkCodecCore: encode as cc_encode, decode as cc_decode + +import ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode, zencode, zdecode, zencode!, zdecode! + +# --- CRC32c internal codec (used by CRC32cV3Codec) --- + +struct CRC32cCodec +end + +function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) + hash::UInt32 = 0x00000000 + while(bytesavailable(input) > 0) + sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) + read!(input, sized_buffer) + write(output, sized_buffer) + hash = CRC32c.crc32c(sized_buffer, hash) + end + return hash +end + +function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=false) + zencode!(output, input, c) + return take!(output) +end +function zencode!(output::IO, input::IO, c::CRC32cCodec) + hash = crc32c_stream!(output, input) + write(output, hash) + return output +end +function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=true) + zdecode!(output, input, c) + return take!(output) +end +function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) + input_vec = take!(input) + truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) + hash = crc32c_stream!(output, truncated_input) + if input_vec[end-3:end] != reinterpret(UInt8, [hash]) + throw(IOError("CRC32c hash does not match")) + end + return output +end + +# --- GzipV3Codec --- + +struct GzipV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +GzipV3Codec() = GzipV3Codec(6) + +function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) + opts = GzipEncodeOptions(; level=c.level) + return cc_encode(opts, data) +end +function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) + return cc_decode(LibZGzipCodec(), encoded) +end +codec_to_dict(c::GzipV3Codec) = Dict{String,Any}( + "name" => "gzip", + "configuration" => Dict{String,Any}("level" => c.level) +) + +v3_codec_parsers["gzip"] = function(config) + level = get(config, "level", 6) + GzipV3Codec(level) +end + +# --- BloscV3Codec --- + +struct BloscV3Codec <: V3Codec{:bytes, :bytes} + cname::String + clevel::Int + shuffle::Int + blocksize::Int + typesize::Int +end +BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) + +function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) + comp = Zarr.BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return ZarrCore.zcompress(data, comp) +end +function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) + comp = Zarr.BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return collect(ZarrCore.zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::BloscV3Codec) = Dict{String,Any}( + "name" => "blosc", + "configuration" => Dict{String,Any}( + "cname" => c.cname, + "clevel" => c.clevel, + "shuffle" => c.shuffle == 0 ? "noshuffle" : + c.shuffle == 1 ? "shuffle" : + c.shuffle == 2 ? "bitshuffle" : + throw(ArgumentError("Unknown shuffle integer: $(c.shuffle)")), + "blocksize" => c.blocksize, + "typesize" => c.typesize + ) +) + +v3_codec_parsers["blosc"] = function(config) + cname = get(config, "cname", "lz4") + clevel = get(config, "clevel", 5) + shuffle_val = get(config, "shuffle", "noshuffle") + shuffle_int = shuffle_val isa Integer ? shuffle_val : + shuffle_val == "noshuffle" ? 0 : + shuffle_val == "shuffle" ? 1 : + shuffle_val == "bitshuffle" ? 2 : + throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) + blocksize = get(config, "blocksize", 0) + typesize = get(config, "typesize", 4) + BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize) +end + +# --- ZstdV3Codec --- + +struct ZstdV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +ZstdV3Codec() = ZstdV3Codec(3) + +function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) + comp = Zarr.ZstdCompressor(level=c.level) + return ZarrCore.zcompress(data, comp) +end +function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) + comp = Zarr.ZstdCompressor(level=c.level) + return collect(ZarrCore.zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::ZstdV3Codec) = Dict{String,Any}( + "name" => "zstd", + "configuration" => Dict{String,Any}("level" => c.level) +) + +v3_codec_parsers["zstd"] = function(config) + level = get(config, "level", 3) + ZstdV3Codec(level) +end + +# --- CRC32cV3Codec --- + +struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} +end + +function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) + out = UInt8[] + return zencode!(out, data, CRC32cCodec()) +end +function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) + out = UInt8[] + return zdecode!(out, encoded, CRC32cCodec()) +end +codec_to_dict(::CRC32cV3Codec) = Dict{String,Any}("name" => "crc32c") + +v3_codec_parsers["crc32c"] = function(config) + CRC32cV3Codec() +end + +# --- ShardingCodec (stub — throws at parse time) --- + +v3_codec_parsers["sharding_indexed"] = function(config) + throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) +end +``` + +**Note:** The full `ShardingCodec` struct and all its encode/decode/index infrastructure from the original V3.jl should also be moved here. For brevity, the plan shows the registry stub. Copy the full ShardingCodec code (lines 106-539 of the original `src/Codecs/V3/V3.jl`) into this file. + +**Step 2: Verify** + +Run: `julia --project -e 'using Zarr; println(haskey(ZarrCore.Codecs.V3Codecs.v3_codec_parsers, "blosc"))'` +Expected: `true` + +**Step 3: Commit** + +```bash +git add src/Codecs/V3/compression_codecs.jl +git commit -m "wrapper: add V3 compression codecs with registry entries" +``` + +--- + +### Task 17: Update Project.toml files + +**Files:** +- Modify: `Project.toml` (top-level Zarr.jl) +- Modify: `lib/ZarrCore/Project.toml` (if UUID not yet set) + +**Step 1: Add ZarrCore as a dependency to Zarr.jl** + +Add to `Project.toml` `[deps]` section: +```toml +ZarrCore = "" +``` + +Add to `[sources]` (Julia 1.11+ for path deps, or use `[extras]` + dev approach): +```toml +[sources.ZarrCore] +path = "lib/ZarrCore" +``` + +For Julia < 1.11 compatibility, you may need to use `Pkg.develop(path="lib/ZarrCore")` instead. Check the Julia version requirements. + +**Step 2: Remove dependencies that moved to ZarrCore** + +From Zarr.jl's `Project.toml`, the deps that are ONLY used by ZarrCore files should be removed. However, some deps are used by both. Keep all deps for now to avoid breakage; optimize later. + +Actually — since Zarr.jl `using ZarrCore` and ZarrCore has its own deps, Zarr.jl only needs to declare deps that IT directly uses (not transitive deps from ZarrCore). But for safety during initial migration, keep all deps. Can be cleaned up in a follow-up. + +**Step 3: Verify** + +Run: `julia --project -e 'using Pkg; Pkg.instantiate(); using Zarr; println("OK")'` +Expected: `OK` + +**Step 4: Commit** + +```bash +git add Project.toml lib/ZarrCore/Project.toml +git commit -m "deps: wire ZarrCore as dependency of Zarr.jl" +``` + +--- + +### Task 18: Clean up original src/ files + +**Files:** +- Remove: Files that were moved to ZarrCore (they're now dead code in `src/`) + +After Zarr.jl is rewritten as a wrapper, the old files in `src/` that were moved to ZarrCore are no longer included. Remove them to avoid confusion: + +``` +src/chunkkeyencoding.jl → moved to lib/ZarrCore/src/ +src/MaxLengthStrings.jl → moved to lib/ZarrCore/src/ +src/metadata.jl → moved to lib/ZarrCore/src/ +src/metadata3.jl → moved to lib/ZarrCore/src/ +src/pipeline.jl → moved to lib/ZarrCore/src/ +src/ZArray.jl → moved to lib/ZarrCore/src/ +src/ZGroup.jl → moved to lib/ZarrCore/src/ +src/Filters/ → moved to lib/ZarrCore/src/ +src/Storage/Storage.jl → moved to lib/ZarrCore/src/ +src/Storage/directorystore.jl → moved to lib/ZarrCore/src/ +src/Storage/dictstore.jl → moved to lib/ZarrCore/src/ +src/Storage/consolidated.jl → moved to lib/ZarrCore/src/ +src/Compressors/Compressors.jl → split; core moved to lib/ZarrCore/src/ +src/Codecs/Codecs.jl → split; core moved to lib/ZarrCore/src/ +src/Codecs/V3/V3.jl → split; core moved to lib/ZarrCore/src/ +``` + +**Step 1: Use git rm to remove moved files** + +```bash +git rm src/chunkkeyencoding.jl src/MaxLengthStrings.jl src/metadata.jl src/metadata3.jl src/pipeline.jl src/ZArray.jl src/ZGroup.jl +git rm -r src/Filters/ +git rm src/Storage/Storage.jl src/Storage/directorystore.jl src/Storage/dictstore.jl src/Storage/consolidated.jl +git rm src/Compressors/Compressors.jl src/Codecs/Codecs.jl src/Codecs/V3/V3.jl +``` + +**Step 2: Verify Zarr.jl still loads** + +Run: `julia --project -e 'using Zarr; println("OK")'` +Expected: `OK` + +**Step 3: Commit** + +```bash +git add -A +git commit -m "cleanup: remove files that moved to ZarrCore" +``` + +--- + +## Phase 6: Tests & Verification + +### Task 19: Update test Project.toml + +**Files:** +- Modify: `test/Project.toml` + +**Step 1: Add ZarrCore as a test dependency** + +Add to `test/Project.toml` `[deps]`: +```toml +ZarrCore = "" +``` + +Add source path: +```toml +[sources.ZarrCore] +path = "../lib/ZarrCore" +``` + +**Step 2: Commit** + +```bash +git add test/Project.toml +git commit -m "test: add ZarrCore to test dependencies" +``` + +--- + +### Task 20: Run the existing test suite + +**Step 1: Instantiate and run** + +Run: `julia --project -e 'using Pkg; Pkg.test()'` + +Expected: All existing tests pass. If any fail, investigate: + +- **`Zarr.BloscCompressor` not found**: Ensure the re-export in `src/Zarr.jl` includes `BloscCompressor` +- **`Zarr.compressortypes` not found**: Ensure `compressortypes` is imported from ZarrCore +- **`Codecs.V3Codecs.XxxCodec` not found**: Ensure all V3 codec types are accessible through the module path +- **Codec parsing fails**: Ensure all V3 codec parsers are registered in `v3_codec_parsers` + +Fix any failures before proceeding. + +**Step 2: Commit any fixes** + +```bash +git add -A +git commit -m "fix: resolve test failures from ZarrCore extraction" +``` + +--- + +### Task 21: Add ZarrCore-only smoke test + +**Files:** +- Create: `lib/ZarrCore/test/runtests.jl` +- Create: `lib/ZarrCore/test/Project.toml` + +**Step 1: Create `lib/ZarrCore/test/Project.toml`** + +```toml +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +ZarrCore = "" + +[sources.ZarrCore] +path = ".." +``` + +**Step 2: Create `lib/ZarrCore/test/runtests.jl`** + +```julia +using Test +using ZarrCore +using JSON + +@testset "ZarrCore" begin + @testset "NoCompressor roundtrip" begin + z = zzeros(Int64, 4, 4) + @test z isa ZArray + @test size(z) == (4, 4) + @test z[1,1] == 0 + z[2,3] = 42 + @test z[2,3] == 42 + end + + @testset "DirectoryStore" begin + p = mktempdir() + z = zcreate(Float64, 10, 10; path=p, chunks=(5,5)) + z[:] = reshape(1.0:100.0, 10, 10) + z2 = zopen(p) + @test z2[1,1] == 1.0 + @test z2[10,10] == 100.0 + end + + @testset "DictStore" begin + z = zcreate(Int32, 6, 6; chunks=(3,3)) + z[:] = ones(Int32, 6, 6) + @test z[1,1] == 1 + end + + @testset "V3 uncompressed roundtrip" begin + z = zcreate(Float32, 8, 8; + zarr_format=3, + chunks=(4,4), + compressor=ZarrCore.NoCompressor()) + z[:] = reshape(Float32.(1:64), 8, 8) + z2 = zopen(z.storage, path=z.path) + @test z2[1,1] == 1.0f0 + @test z2[8,8] == 64.0f0 + end + + @testset "default_compressor is NoCompressor" begin + @test ZarrCore.default_compressor() isa ZarrCore.NoCompressor + end +end +``` + +**Step 3: Run ZarrCore tests** + +Run: `julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()'` +Expected: All pass + +**Step 4: Commit** + +```bash +git add lib/ZarrCore/test/ +git commit -m "test: add ZarrCore smoke tests" +``` + +--- + +### Task 22: Update CLAUDE.md + +**Files:** +- Modify: `CLAUDE.md` + +Add information about the ZarrCore package structure to the Architecture section. Add test commands for ZarrCore. + +**Step 1: Add ZarrCore section** + +Add to Build & Test Commands: +```bash +# Run ZarrCore tests only +julia --project=lib/ZarrCore -e 'using Pkg; Pkg.test()' +``` + +Add to Architecture section: +``` +### Package Structure +- `lib/ZarrCore/` — Minimal core package with types, registries, NoCompressor, BytesCodec, TransposeCodec, DirectoryStore, DictStore, ConsolidatedStore, all pure-Julia filters +- `src/` (Zarr.jl) — Wrapper that depends on ZarrCore, adds Blosc/Zlib/Zstd compressors, V3 compression codecs, HTTP/GCS/S3/ZIP stores +- ZarrCore's `default_compressor()` returns `NoCompressor()`; Zarr.jl overrides it to `BloscCompressor()` +- V3 codecs use `v3_codec_parsers` registry for extensible parsing; Zarr.jl registers compression codec parsers +``` + +**Step 2: Commit** + +```bash +git add CLAUDE.md +git commit -m "docs: update CLAUDE.md with ZarrCore package structure" +``` + +--- + +## Important Notes for Implementation + +### Module Path References + +In tests and existing code, `Zarr.SomeType` must still work. Since `Zarr.jl` does `using ZarrCore: SomeType`, the name is available as `Zarr.SomeType`. But for internal module paths like `Zarr.Codecs.V3Codecs.BloscV3Codec`, these need careful handling: + +- Core V3 types (`BytesCodec`, `TransposeCodec`) are at `ZarrCore.Codecs.V3Codecs.BytesCodec` +- Compression V3 types (`BloscV3Codec`, etc.) are defined in Zarr.jl but registered into `ZarrCore.Codecs.V3Codecs.v3_codec_parsers` +- Tests that reference `Zarr.Codecs.V3Codecs.XxxCodec` will need updates to use either the Zarr-local name or the ZarrCore path + +### The Compressor File Imports + +The existing `src/Compressors/blosc.jl` has `import Blosc` and `Zarr.compressortypes["blosc"] = BloscCompressor`. Since `Zarr` module now imports `compressortypes` from ZarrCore, `Zarr.compressortypes` refers to the same dict. This should work without changes to the compressor files. + +### S3 Extension + +The `ext/ZarrAWSS3Ext.jl` extension references `Zarr.S3Store`, `Zarr.AbstractStore`, etc. Since `S3Store` is now defined in `Zarr.jl` (not ZarrCore) and `AbstractStore` is re-exported from ZarrCore, the extension should work. Verify by checking the extension file. + +### Iterative Debugging + +This refactoring touches many files. Expect some iteration in Task 20. Common issues: +- Missing re-exports in `Zarr.jl` +- Module path mismatches (`Zarr.X` vs `ZarrCore.X`) +- Function signatures referencing types not yet imported +- `using` vs `import` confusion (Julia is strict about these) + +The debugging approach: run `using Zarr`, fix the first error, repeat until it loads. Then run the test suite. diff --git a/lib/ZarrCore/Project.toml b/lib/ZarrCore/Project.toml new file mode 100644 index 00000000..a155517c --- /dev/null +++ b/lib/ZarrCore/Project.toml @@ -0,0 +1,17 @@ +name = "ZarrCore" +uuid = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" +version = "0.1.0" + +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[compat] +JSON = "0.21, 1" +DiskArrays = "0.4.2" +OffsetArrays = "0.11, 1.0" +DateTimes64 = "1" +julia = "1.10" diff --git a/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl similarity index 95% rename from src/Codecs/Codecs.jl rename to lib/ZarrCore/src/Codecs/Codecs.jl index ec6e6205..9f5e775f 100644 --- a/src/Codecs/Codecs.jl +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -12,18 +12,18 @@ The abstract supertype for all Zarr codecs All subtypes of `Codec` SHALL implement the following methods: - `zencode(a, c::Codec)`: compress the array `a` using the codec `c`. -- `zdecode(a, c::Codec, T)`: decode the array `a` using the codec `c` +- `zdecode(a, c::Codec, T)`: decode the array `a` using the codec `c` and return an array of type `T`. -- `JSON.lower(c::Codec)`: return a JSON representation of the codec `c`, which +- `JSON.lower(c::Codec)`: return a JSON representation of the codec `c`, which follows the Zarr specification for that codec. -- `getCodec(::Type{<:Codec}, d::Dict)`: return a codec object from a given +- `getCodec(::Type{<:Codec}, d::Dict)`: return a codec object from a given dictionary `d` which contains the codec's parameters according to the Zarr spec. Subtypes of `Codec` MAY also implement the following methods: -- `zencode!(encoded, data, c::Codec)`: encode the array `data` using the +- `zencode!(encoded, data, c::Codec)`: encode the array `data` using the codec `c` and store the result in the array `encoded`. -- `zdecode!(data, encoded, c::Codec)`: decode the array `encoded` +- `zdecode!(data, encoded, c::Codec)`: decode the array `encoded` using the codec `c` and store the result in the array `data`. Finally, an entry MUST be added to the `VN.codectypes` dictionary for each codec type where N is the @@ -31,7 +31,7 @@ Zarr format version. This must also follow the Zarr specification's name for that compressor. The name of the compressor is the key, and the value is the compressor type (e.g. `BloscCodec` or `NoCodec`). -For example, the Blosc codec is named "blosc" in the Zarr spec, so the entry for [`BloscCodec`](@ref) +For example, the Blosc codec is named "blosc" in the Zarr spec, so the entry for [`BloscCodec`](@ref) must be added to `codectypes` as `codectypes["blosc"] = BloscCodec`. """ diff --git a/lib/ZarrCore/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl new file mode 100644 index 00000000..e49e1cc8 --- /dev/null +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -0,0 +1,109 @@ +module V3Codecs + +import ..Codecs: zencode, zdecode, zencode!, zdecode! +using JSON: JSON + +abstract type V3Codec{In,Out} end + +""" +Registry mapping V3 codec names (strings) to parser functions. +Each parser function takes a config dict and returns a V3Codec instance. +""" +const v3_codec_parsers = Dict{String, Function}() + +""" + codec_to_dict(c::V3Codec) -> Dict{String,Any} + +Serialize a V3Codec to a JSON-compatible dictionary. +All V3Codec subtypes should implement this method. +""" +function codec_to_dict end + +""" + codec_category(::V3Codec{In,Out}) -> Tuple{Symbol,Symbol} + +Return the (input, output) category of a codec, e.g. (:array, :bytes). +""" +codec_category(::V3Codec{In,Out}) where {In,Out} = (In, Out) + +""" + encoded_shape(codec::V3Codec, sz::NTuple{N,Int}) -> NTuple{N,Int} + +Return the shape of the output of `codec_encode(codec, data)` given the input shape. +Default implementation returns the input shape unchanged. +""" +encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz + +# --- BytesCodec (array -> bytes) --- + +struct BytesCodec <: V3Codec{:array, :bytes} + endian::Symbol # :little or :big + function BytesCodec(endian::Symbol) + endian in (:little, :big) || + throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) + new(endian) + end +end +BytesCodec() = BytesCodec(:little) + +const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 +_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN + +function codec_encode(c::BytesCodec, data::AbstractArray) + if _needs_bswap(c.endian) + return reinterpret(UInt8, bswap.(vec(data))) |> collect + else + return reinterpret(UInt8, vec(data)) |> collect + end +end + +function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} + arr = collect(reinterpret(T, encoded)) + if _needs_bswap(c.endian) + arr = bswap.(arr) + end + return reshape(arr, shape) +end + +function codec_to_dict(c::BytesCodec) + Dict{String,Any}( + "name" => "bytes", + "configuration" => Dict{String,Any}("endian" => string(c.endian)) + ) +end + +# Register BytesCodec parser +v3_codec_parsers["bytes"] = function(config) + endian = Symbol(get(config, "endian", "little")) + BytesCodec(endian) +end + +# --- TransposeCodec (array -> array) --- + +struct TransposeCodec{N} <: V3Codec{:array, :array} + order::NTuple{N, Int} # permutation (1-based Julia indexing) +end + +encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) + +function codec_encode(c::TransposeCodec, data::AbstractArray) + return permutedims(data, c.order) +end + +function codec_decode(c::TransposeCodec, encoded::AbstractArray) + inv_order = Tuple(invperm(collect(c.order))) + return permutedims(encoded, inv_order) +end + +function codec_to_dict(c::TransposeCodec) + # Zarr v3 spec uses 0-based C-order for the transpose order + Dict{String,Any}( + "name" => "transpose", + "configuration" => Dict{String,Any}("order" => collect(c.order .- 1)) + ) +end + +# Note: TransposeCodec is NOT registered in v3_codec_parsers here because +# parsing requires shape context from metadata3.jl + +end diff --git a/src/Compressors/Compressors.jl b/lib/ZarrCore/src/Compressors/Compressors.jl similarity index 80% rename from src/Compressors/Compressors.jl rename to lib/ZarrCore/src/Compressors/Compressors.jl index 58a80109..59accd55 100644 --- a/src/Compressors/Compressors.jl +++ b/lib/ZarrCore/src/Compressors/Compressors.jl @@ -13,43 +13,31 @@ The abstract supertype for all Zarr compressors. All subtypes of `Compressor` SHALL implement the following methods: - `zcompress(a, c::Compressor)`: compress the array `a` using the compressor `c`. -- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` +- `zuncompress(a, c::Compressor, T)`: uncompress the array `a` using the compressor `c` and return an array of type `T`. -- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which +- `JSON.lower(c::Compressor)`: return a JSON representation of the compressor `c`, which follows the Zarr specification for that compressor. -- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given +- `getCompressor(::Type{<:Compressor}, d::Dict)`: return a compressor object from a given dictionary `d` which contains the compressor's parameters according to the Zarr spec. Subtypes of `Compressor` MAY also implement the following methods: -- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the +- `zcompress!(compressed, data, c::Compressor)`: compress the array `data` using the compressor `c` and store the result in the array `compressed`. -- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` +- `zuncompress!(data, compressed, c::Compressor)`: uncompress the array `compressed` using the compressor `c` and store the result in the array `data`. -Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. +Finally, an entry MUST be added to the `compressortypes` dictionary for each compressor type. This must also follow the Zarr specification's name for that compressor. The name of the compressor is the key, and the value is the compressor type (e.g. `BloscCompressor` or `NoCompressor`). -For example, the Blosc compressor is named "blosc" in the Zarr spec, so the entry for [`BloscCompressor`](@ref) +For example, the Blosc compressor is named "blosc" in the Zarr spec, so the entry for [`BloscCompressor`](@ref) must be added to `compressortypes` as `compressortypes["blosc"] = BloscCompressor`. """ abstract type Compressor end const compressortypes = Dict{Union{String,Nothing}, Type{<: Compressor}}() -# function getCompressor end -# function zcompress end -# function zuncompress end -# function zcompress! end -# function zuncompress! end -# JSON.lower is neither defined nor documented here, since that would be documentation piracy :yarr: - -# Include the compressor implementations -include("blosc.jl") -include("zlib.jl") -include("zstd.jl") - # ## Fallback definitions for the compressor interface # Define fallbacks and generic methods for the compressor interface getCompressor(compdict::Dict) = haskey(compdict, "id") ? @@ -62,7 +50,7 @@ zcompress!(compressed,data,c,::Nothing) = zcompress!(compressed,data,c) zuncompress!(data,compressed,c,::Nothing) = zuncompress!(data,compressed,c) # Fallback definition of mutating form of compress and uncompress -function zcompress!(compressed, data, c) +function zcompress!(compressed, data, c) empty!(compressed) append!(compressed,zcompress(data, c)) end @@ -78,7 +66,7 @@ function zcompress!(compressed, data, c, f) end function zuncompress!(data, compressed, c, f) - data2 = zuncompress(compressed, c, desttype(last(f))) + data2 = zuncompress(compressed, c, desttype(last(f))) a2 = foldr(f, init = data2) do fnow, anow zdecode(anow, fnow) end @@ -107,3 +95,19 @@ end JSON.lower(::NoCompressor) = nothing compressortypes[nothing] = NoCompressor + +""" + DEFAULT_COMPRESSOR_FACTORY + +A Ref holding a zero-argument function that returns the default compressor. +Override by setting `ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> MyCompressor()`. +""" +const DEFAULT_COMPRESSOR_FACTORY = Ref{Function}(() -> NoCompressor()) + +""" + default_compressor() + +Return the default compressor. By default returns `NoCompressor()`. +Override by setting `ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> MyCompressor()`. +""" +default_compressor() = DEFAULT_COMPRESSOR_FACTORY[]() diff --git a/src/Filters/Filters.jl b/lib/ZarrCore/src/Filters/Filters.jl similarity index 100% rename from src/Filters/Filters.jl rename to lib/ZarrCore/src/Filters/Filters.jl diff --git a/src/Filters/delta.jl b/lib/ZarrCore/src/Filters/delta.jl similarity index 100% rename from src/Filters/delta.jl rename to lib/ZarrCore/src/Filters/delta.jl diff --git a/src/Filters/fixedscaleoffset.jl b/lib/ZarrCore/src/Filters/fixedscaleoffset.jl similarity index 100% rename from src/Filters/fixedscaleoffset.jl rename to lib/ZarrCore/src/Filters/fixedscaleoffset.jl diff --git a/src/Filters/fletcher32.jl b/lib/ZarrCore/src/Filters/fletcher32.jl similarity index 100% rename from src/Filters/fletcher32.jl rename to lib/ZarrCore/src/Filters/fletcher32.jl diff --git a/src/Filters/quantize.jl b/lib/ZarrCore/src/Filters/quantize.jl similarity index 100% rename from src/Filters/quantize.jl rename to lib/ZarrCore/src/Filters/quantize.jl diff --git a/src/Filters/shuffle.jl b/lib/ZarrCore/src/Filters/shuffle.jl similarity index 100% rename from src/Filters/shuffle.jl rename to lib/ZarrCore/src/Filters/shuffle.jl diff --git a/src/Filters/vlenfilters.jl b/lib/ZarrCore/src/Filters/vlenfilters.jl similarity index 100% rename from src/Filters/vlenfilters.jl rename to lib/ZarrCore/src/Filters/vlenfilters.jl diff --git a/src/MaxLengthStrings.jl b/lib/ZarrCore/src/MaxLengthStrings.jl similarity index 100% rename from src/MaxLengthStrings.jl rename to lib/ZarrCore/src/MaxLengthStrings.jl diff --git a/src/Storage/Storage.jl b/lib/ZarrCore/src/Storage/Storage.jl similarity index 94% rename from src/Storage/Storage.jl rename to lib/ZarrCore/src/Storage/Storage.jl index 0845c604..918c6db6 100644 --- a/src/Storage/Storage.jl +++ b/lib/ZarrCore/src/Storage/Storage.jl @@ -3,7 +3,7 @@ # and Dictionaries are supported """ - abstract type AbstractStore + abstract type AbstractStore This the abstract supertype for all Zarr store implementations. Currently only regular files ([`DirectoryStore`](@ref)) and Dictionaries are supported. @@ -28,20 +28,6 @@ abstract type AbstractStore end # Define the interface -""" - S3Store(bucket::String; aws=nothing) - -An S3-backed Zarr store. Available after loading the `ZarrAWSS3Ext` extension. -""" -struct S3Store <: AbstractStore - bucket::String - aws::Any -end - -function S3Store(args...) - error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") -end - """ storagesize(d::AbstractStore, p::AbstractString) @@ -76,7 +62,7 @@ function subdirs end Returns the keys of files in the given store. """ -function subkeys end +function subkeys end # Function to construct the full path to a chunk given the base path, Cartesian Index i, and the chunk ecoding store_readchunk(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = s[p, citostring(e, i)] @@ -85,7 +71,7 @@ store_writechunk(s::AbstractStore, v, p, i::CartesianIndex, e::AbstractChunkKeyE store_isinitialized(s::AbstractStore, p, i::CartesianIndex, e::AbstractChunkKeyEncoding) = isinitialized(s, p, citostring(e, i)) -#Functions to concat path and key +#Functions to concat path and key Base.getindex(s::AbstractStore, p, i::AbstractString) = s[_concatpath(p, i)] Base.delete!(s::AbstractStore, p, i::AbstractString) = delete!(s, _concatpath(p, i)) Base.haskey(s::AbstractStore, k::AbstractString) = isinitialized(s, k) @@ -195,7 +181,7 @@ function writemetadata(::ZarrFormat{2}, s::AbstractStore, p, m::AbstractMetadata else JSON.print(met,m) end - + s[p,".zarray"] = take!(met) m end @@ -280,12 +266,8 @@ isemptysub(s::AbstractStore, p) = isempty(subkeys(s,p)) && isempty(subdirs(s,p)) #Here different storage backends can register regexes that are checked against #during auto-check of storage format when doing zopen storageregexlist = Pair[] -push!(storageregexlist, r"^s3://" => S3Store) #include("formattedstore.jl") include("directorystore.jl") include("dictstore.jl") -include("gcstore.jl") include("consolidated.jl") -include("http.jl") -include("zipstore.jl") diff --git a/src/Storage/consolidated.jl b/lib/ZarrCore/src/Storage/consolidated.jl similarity index 100% rename from src/Storage/consolidated.jl rename to lib/ZarrCore/src/Storage/consolidated.jl diff --git a/src/Storage/dictstore.jl b/lib/ZarrCore/src/Storage/dictstore.jl similarity index 100% rename from src/Storage/dictstore.jl rename to lib/ZarrCore/src/Storage/dictstore.jl diff --git a/src/Storage/directorystore.jl b/lib/ZarrCore/src/Storage/directorystore.jl similarity index 100% rename from src/Storage/directorystore.jl rename to lib/ZarrCore/src/Storage/directorystore.jl diff --git a/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl similarity index 98% rename from src/ZArray.jl rename to lib/ZarrCore/src/ZArray.jl index af27d252..72219492 100644 --- a/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -6,7 +6,7 @@ using DateTimes64: DateTime64 using Dates: Day, Millisecond """ -Number of tasks to use for async reading of chunks. Warning: setting this to very high values can lead to a large memory footprint. +Number of tasks to use for async reading of chunks. Warning: setting this to very high values can lead to a large memory footprint. """ const concurrent_io_tasks = Ref(50) @@ -168,7 +168,7 @@ resetbuffer!(_,a::SenMissArray) = fill!(a,missing) # Function to read or write from a zarr array. Could be refactored # using type system to get rid of the `if readmode` statements. function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} - + output_base_offsets = map(i->first(i)-1,r.indices) # Determines which chunks are affected blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) @@ -177,33 +177,33 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = getchunkarray(z) # Now loop through the chunks c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - + task = @async begin read_items!($(z.storage), c, $(z.metadata.chunk_key_encoding), $(z.path), $(blockr)) end bind(c,task) - try + try for i in 1:length(blockr) - + bI,chunk_compressed = take!(c) - + current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) indranges = map(boundint,r.indices,size(a),current_chunk_offsets) - + uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) nothing end finally close(c) end - + aout end function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} - + z.writeable || error("Can not write to read-only ZArray") input_base_offsets = map(i->first(i)-1,r.indices) # Determines which chunks are affected @@ -213,8 +213,8 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = getchunkarray(z) # Now loop through the chunks readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - readtask = @async begin + + readtask = @async begin read_items!(z.storage, readchannel, z.metadata.chunk_key_encoding, z.path, blockr) end bind(readchannel,readtask) @@ -225,12 +225,12 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes write_items!(z.storage, writechannel, z.metadata.chunk_key_encoding, z.path, blockr) end bind(writechannel,writetask) - - try + + try for i in 1:length(blockr) - + bI,chunk_compressed = take!(readchannel) - + current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) indranges = map(boundint,r.indices,size(a),current_chunk_offsets) @@ -244,7 +244,7 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes else a end - + if chunk_compressed !== nothing uncompress_raw!(a,z,chunk_compressed) end @@ -273,7 +273,7 @@ Read the chunk specified by `i` from the Zarray `z` and write its content to `a` """ function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N if curchunk === nothing - if isnothing(z.metadata.fill_value) + if isnothing(z.metadata.fill_value) throw(ArgumentError("The array $z got missing chunks and no fill_value")) end fill!(a, z.metadata.fill_value) @@ -286,9 +286,9 @@ end dotminus(x,y) = x.-y function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) - + uncompress_raw!(a,z,chunk_compressed) - + if length.(indranges) == size(a) aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a else @@ -317,7 +317,7 @@ Creates a new empty zarr array with element type `T` and array dimensions `dims` * `fill_value=nothing` value to represent missing values * `fill_as_missing=false` set to `true` shall fillvalue s be converted to `missing`s * `filters`=filters to be applied -* `compressor=BloscCompressor()` compressor type and properties +* `compressor=default_compressor()` compressor type and properties * `attrs=Dict()` a dict containing key-value pairs with metadata attributes associated to the array * `writeable=true` determines if the array is opened in read-only or write mode * `indent_json=false` determines if indents are added to format the json files `.zarray` and `.zattrs`. This makes them more readable, but increases file size. @@ -346,8 +346,8 @@ function zcreate(::Type{T},storage::AbstractStore, chunks=dims, fill_value=nothing, fill_as_missing=false, - compressor=BloscCompressor(), - filters = filterfromtype(T), + compressor=default_compressor(), + filters = filterfromtype(T), attrs=Dict(), writeable=true, indent_json=false, @@ -363,11 +363,11 @@ function zcreate(::Type{T},storage::AbstractStore, dimension_separator = only(dimension_separator) end chunk_key_encoding = ChunkKeyEncoding(dimension_separator, default_prefix(v)) - + length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) N = length(dims) C = typeof(compressor) - + # Create a dummy array to use with Metadata constructor # This allows us to leverage the multiple dispatch in Metadata constructors dummy_array = Array{T,N}(undef, dims...) @@ -378,16 +378,16 @@ function zcreate(::Type{T},storage::AbstractStore, fill_as_missing=fill_as_missing, chunk_key_encoding=chunk_key_encoding ) - + # Extract the element type from the metadata (handles T2 calculation) T2 = eltype(metadata) - + isemptysub(storage,path) || error("$storage $path is not empty") - + writemetadata(v, storage, path, metadata, indent_json=indent_json) - + writeattrs(v, storage, path, attrs, indent_json=indent_json) - + ZArray(metadata, storage, path, attrs, writeable) end diff --git a/src/ZGroup.jl b/lib/ZarrCore/src/ZGroup.jl similarity index 93% rename from src/ZGroup.jl rename to lib/ZarrCore/src/ZGroup.jl index 4b9324e8..90c05247 100644 --- a/src/ZGroup.jl +++ b/lib/ZarrCore/src/ZGroup.jl @@ -49,13 +49,13 @@ ZarrFormat(s::AbstractStore, path) = is_zarr2(s, path) ? ZarrFormat(2) : """ zopen_noerr(AbstractStore, mode = "r"; consolidated = false) -Works like `zopen` with the single difference that no error is thrown when -the path or store does not point to a valid zarr array or group, but nothing +Works like `zopen` with the single difference that no error is thrown when +the path or store does not point to a valid zarr array or group, but nothing is returned instead. """ function zopen_noerr(s::AbstractStore, mode, zv::ZarrFormat; - consolidated = false, - path="", + consolidated = false, + path="", lru = 0, fill_as_missing=false) @@ -99,15 +99,15 @@ Opens a zarr Array or Group at Store `s`. If `consolidated` is set to "true", Zarr will search for a consolidated metadata field as created by the python zarr `consolidate_metadata` function. This can substantially speed up metadata parsing of large zarr groups. Setting `lru` to a value > 0 means that chunks that have been -accessed before will be cached and consecutive reads will happen from the cache. +accessed before will be cached and consecutive reads will happen from the cache. Here, `lru` denotes the number of chunks that remain in memory. The expected zarr version -can be supplied through `zarr_format` and defaults to `:auto` which tries to detect +can be supplied through `zarr_format` and defaults to `:auto` which tries to detect if the zarr version is v2 or v3. """ -function zopen(s::AbstractStore, mode="r"; +function zopen(s::AbstractStore, mode="r"; zarr_format=:auto, - consolidated = false, - path = "", + consolidated = false, + path = "", lru = 0, fill_as_missing = false) @@ -116,7 +116,7 @@ function zopen(s::AbstractStore, mode="r"; else ZarrFormat(zarr_format) end - # add interfaces to Stores later + # add interfaces to Stores later r = zopen_noerr(s, mode, zarr_format; consolidated=consolidated, path=path, lru=lru, fill_as_missing=fill_as_missing) if r === nothing throw(ArgumentError("Specified store $s in path $(path) is neither a ZArray nor a ZGroup")) @@ -198,9 +198,7 @@ function zcreate(::Type{T},g::ZGroup, name::AbstractString, addargs...; kwargs.. return z end -HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) -writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) -function consolidate_metadata(z::Union{ZArray,ZGroup}) +function consolidate_metadata(z::Union{ZArray,ZGroup}) z.writeable || throw(ArgumentError("This Zarr group is not writeable. Please re-open in write mode to create an array")) consolidate_metadata(z.storage,z.path) end diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl new file mode 100644 index 00000000..c63b9aae --- /dev/null +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -0,0 +1,29 @@ +module ZarrCore + +import JSON + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +const DV = ZarrFormat(Val(2)) + +abstract type AbstractCodecPipeline end + +include("chunkkeyencoding.jl") +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Filters/Filters.jl") +include("metadata.jl") +include("metadata3.jl") +include("pipeline.jl") +include("Storage/Storage.jl") +include("ZArray.jl") +include("ZGroup.jl") + +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +end # module diff --git a/src/chunkkeyencoding.jl b/lib/ZarrCore/src/chunkkeyencoding.jl similarity index 100% rename from src/chunkkeyencoding.jl rename to lib/ZarrCore/src/chunkkeyencoding.jl diff --git a/src/metadata.jl b/lib/ZarrCore/src/metadata.jl similarity index 98% rename from src/metadata.jl rename to lib/ZarrCore/src/metadata.jl index 7b36bbad..09859610 100644 --- a/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -1,5 +1,5 @@ import Dates: Date, DateTime -using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring +using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring """NumPy array protocol type string (typestr) format @@ -148,7 +148,7 @@ end "Construct Metadata based on your data" function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; node_type::String="array", - compressor::C=BloscCompressor(), + compressor::C=default_compressor(), fill_value::Union{T, Nothing}=nothing, order::Char='C', filters=nothing, @@ -169,7 +169,7 @@ end # V2 constructor function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; node_type::String="array", - compressor::C=BloscCompressor(), + compressor::C=default_compressor(), fill_value::Union{T, Nothing}=nothing, order::Char='C', filters::F=nothing, @@ -282,12 +282,12 @@ fill_value_decoding(v::Nothing, ::Any) = v fill_value_decoding(v, T) = T(v) fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v -fill_value_decoding(v::Nothing, ::Type{Zarr.ASCIIChar}) = v +fill_value_decoding(v::Nothing, ::Type{ZarrCore.ASCIIChar}) = v # Sometimes when translating between CF (climate and forecast) convention data # and Zarr groups, fill values are left as "negative integers" to encode unsigned # integers. So, we have to convert to the signed type with the same number of bytes -# as the unsigned integer, then reinterpret as unsigned. That's how a fill value +# as the unsigned integer, then reinterpret as unsigned. That's how a fill value # of -1 can have a realistic meaning with an unsigned dtype. -# However, we have to apply this correction only if the integer is negative. +# However, we have to apply this correction only if the integer is negative. # If it's positive, then the value might be out of range of the signed integer type. fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = sign(v) < 0 ? reinterpret(T, signed(T)(v)) : T(v) diff --git a/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl similarity index 60% rename from src/metadata3.jl rename to lib/ZarrCore/src/metadata3.jl index 19c47c97..555ee772 100644 --- a/src/metadata3.jl +++ b/lib/ZarrCore/src/metadata3.jl @@ -63,42 +63,6 @@ end MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) zarr_format(::MetadataV3) = ZarrFormat(Val(3)) -""" -Convenience constructor for MetadataV3 that builds the codec pipeline from -`order` (translated to a TransposeCodec), `endian` (translated to a BytesCodec), -and `compressor` (translated to bytes->bytes codecs). -""" -function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks::NTuple{N,Int}, - dtype::String, fill_value; - order::Char='C', - endian::Symbol=:little, - compressor=BloscCompressor(), - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T2, N, E} - T_base = Base.nonmissingtype(T2) - array_array_codecs = if order == 'F' - (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) - else - () - end - array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) - bytes_bytes_codecs = if compressor isa NoCompressor - () - elseif compressor isa BloscCompressor - (Codecs.V3Codecs.BloscV3Codec(compressor.cname, compressor.clevel, compressor.shuffle, compressor.blocksize, sizeof(T_base)),) - elseif compressor isa ZlibCompressor - # ZlibCompressor uses -1 to mean "default"; zarr v3 gzip spec requires 0-9 - level = compressor.config.level == -1 ? 6 : compressor.config.level - (Codecs.V3Codecs.GzipV3Codec(level),) - elseif compressor isa ZstdCompressor - (Codecs.V3Codecs.ZstdV3Codec(compressor.config.compressionLevel),) - else - throw(ArgumentError("Unsupported compressor type for v3: $(typeof(compressor))")) - end - pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) - return MetadataV3{T2,N,typeof(pipeline),E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) -end - function Base.:(==)(m1::MetadataV3, m2::MetadataV3) m1.zarr_format == m2.zarr_format && m1.node_type == m2.node_type && @@ -153,7 +117,37 @@ function get_order(md::MetadataV3) end get_order(md::MetadataV2) = md.order +# --- Registry-based codec parsing --- +function parse_v3_codec(codec_name::String, config::Dict, shape_length::Int) + if codec_name == "transpose" + return _parse_transpose_codec(config, shape_length) + end + haskey(Codecs.V3Codecs.v3_codec_parsers, codec_name) || + throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + return Codecs.V3Codecs.v3_codec_parsers[codec_name](config) +end + +function _parse_transpose_codec(config::Dict, shape_length::Int) + _order = config["order"] + if _order isa AbstractString + n = shape_length + if _order == "C" + @warn "Transpose codec dimension order of C is deprecated" + perm = ntuple(identity, n) + elseif _order == "F" + @warn "Transpose codec dimension order of F is deprecated" + perm = ntuple(i -> n - i + 1, n) + else + throw(ArgumentError("Unknown transpose order string: $_order")) + end + else + perm = Tuple(Int.(_order) .+ 1) + end + return Codecs.V3Codecs.TransposeCodec(perm) +end + +# --- Metadata3 from Dict --- function Metadata3(d::AbstractDict, fill_as_missing) check_keys(d, ("zarr_format", "node_type")) @@ -225,70 +219,22 @@ function Metadata3(d::AbstractDict, fill_as_missing) # Chunk Key Encoding chunk_key_encoding = d["chunk_key_encoding"] - # Build V3Pipeline from codec chain + # Build V3Pipeline from codec chain using registry-based parsing array_array_codecs = [] array_bytes_codec = nothing bytes_bytes_codecs = [] - order = 'C' # default for codec in d["codecs"] codec_name = codec["name"] config = get(codec, "configuration", Dict{String,Any}()) - if codec_name == "transpose" - _order = config["order"] - if _order isa AbstractString - n = length(shape) - if _order == "C" - @warn "Transpose codec dimension order of C is deprecated" - perm = ntuple(identity, n) - elseif _order == "F" - @warn "Transpose codec dimension order of F is deprecated" - perm = ntuple(i -> n - i + 1, n) - order = 'F' - else - throw(ArgumentError("Unknown transpose order string: $_order")) - end - else - perm = Tuple(Int.(_order) .+ 1) - default_perm = ntuple(identity, length(shape)) - rev_perm = ntuple(i -> length(shape) - i + 1, length(shape)) - if perm == default_perm - order = 'C' - elseif perm == rev_perm - order = 'F' - end - end - push!(array_array_codecs, Codecs.V3Codecs.TransposeCodec(perm)) - elseif codec_name == "bytes" - endian_str = get(config, "endian", "little") - endian = endian_str == "little" ? :little : - endian_str == "big" ? :big : - throw(ArgumentError("Unknown endian value: \"$endian_str\"")) - array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) - elseif codec_name == "sharding_indexed" - throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) - elseif codec_name == "gzip" - level = get(config, "level", 6) - push!(bytes_bytes_codecs, Codecs.V3Codecs.GzipV3Codec(level)) - elseif codec_name == "blosc" - cname = get(config, "cname", "lz4") - clevel = get(config, "clevel", 5) - shuffle_val = get(config, "shuffle", "noshuffle") - shuffle_int = shuffle_val isa Integer ? shuffle_val : - shuffle_val == "noshuffle" ? 0 : - shuffle_val == "shuffle" ? 1 : - shuffle_val == "bitshuffle" ? 2 : - throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) - blocksize = get(config, "blocksize", 0) - typesize = get(config, "typesize", 4) - push!(bytes_bytes_codecs, Codecs.V3Codecs.BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize)) - elseif codec_name == "zstd" - level = get(config, "level", 3) - push!(bytes_bytes_codecs, Codecs.V3Codecs.ZstdV3Codec(level)) - elseif codec_name == "crc32c" - push!(bytes_bytes_codecs, Codecs.V3Codecs.CRC32cV3Codec()) - else - throw(ArgumentError("Zarr.jl currently does not support the $codec_name codec")) + parsed = parse_v3_codec(codec_name, config, length(shape)) + cat = Codecs.V3Codecs.codec_category(parsed) + if cat == (:array, :array) + push!(array_array_codecs, parsed) + elseif cat == (:array, :bytes) + array_bytes_codec = parsed + elseif cat == (:bytes, :bytes) + push!(bytes_bytes_codecs, parsed) end end @@ -318,14 +264,16 @@ function Metadata3(d::AbstractDict, fill_as_missing) ) end +# --- Metadata3 from Array (takes bytes_bytes_codecs directly) --- + "Construct MetadataV3 based on your data" function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; node_type::String="array", - compressor=BloscCompressor(), fill_value::Union{T, Nothing}=nothing, order::Char='C', endian::Symbol=:little, - filters=nothing, + compressor=nothing, + bytes_bytes_codecs::Tuple=(compressor === nothing ? () : compressor_to_v3_bytes_codecs(compressor)), fill_as_missing = false, dimension_separator::Char = '/' ) where {T, N} @@ -334,20 +282,64 @@ function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; if fill_value === nothing fill_value = zero(T) end - return MetadataV3{T2, N}( + T_base = Base.nonmissingtype(T2) + array_array_codecs = if order == 'F' + (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) + else + () + end + array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) + pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) + chunk_key_encoding = ChunkKeyEncoding(dimension_separator, true) + return MetadataV3{T2, N, typeof(pipeline), typeof(chunk_key_encoding)}( 3, node_type, size(A), chunks, typestr3(eltype(A)), - fill_value; + pipeline, + fill_value, + chunk_key_encoding, + ) +end + +# --- Compressor to V3 bytes codecs dispatch --- + +""" + compressor_to_v3_bytes_codecs(compressor) -> Tuple + +Convert a compressor to a tuple of V3 bytes->bytes codecs. +Subtypes should add methods for their specific compressor types. +""" +compressor_to_v3_bytes_codecs(::NoCompressor) = () +compressor_to_v3_bytes_codecs(c) = throw(ArgumentError("Unsupported compressor type for v3: $(typeof(c)). Register a method for compressor_to_v3_bytes_codecs.")) + +# --- Metadata(A, chunks, ::ZarrFormat{3}) using compressor_to_v3_bytes_codecs --- + +function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; + node_type::String="array", + compressor=default_compressor(), + fill_value::Union{T, Nothing}=nothing, + order::Char='C', + endian::Symbol=:little, + filters=nothing, + fill_as_missing = false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) + ) where {T, N, E} + bb_codecs = compressor_to_v3_bytes_codecs(compressor) + return Metadata3(A, chunks; + node_type=node_type, + fill_value=fill_value, order=order, endian=endian, - compressor=compressor, - chunk_key_encoding=ChunkKeyEncoding(dimension_separator, true) + bytes_bytes_codecs=bb_codecs, + fill_as_missing=fill_as_missing, + dimension_separator=chunk_key_encoding.sep ) end +# --- lower3: serialize MetadataV3 using codec_to_dict dispatch --- + function lower3(md::MetadataV3{T}) where T chunk_grid = Dict{String,Any}( "name" => "regular", @@ -359,57 +351,21 @@ function lower3(md::MetadataV3{T}) where T # chunk_key_encoding chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) - # Build codecs from pipeline + # Build codecs from pipeline using codec_to_dict dispatch codecs = Dict{String,Any}[] p = md.pipeline # array->array codecs for codec in p.array_array - if codec isa Codecs.V3Codecs.TransposeCodec - push!(codecs, Dict{String,Any}( - "name" => "transpose", - "configuration" => Dict("order" => collect(codec.order .- 1)) - )) - end + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) end # array->bytes codec - if p.array_bytes isa Codecs.V3Codecs.BytesCodec - push!(codecs, Dict{String,Any}( - "name" => "bytes", - "configuration" => Dict{String,Any}("endian" => string(p.array_bytes.endian)) - )) - end + push!(codecs, Codecs.V3Codecs.codec_to_dict(p.array_bytes)) # bytes->bytes codecs for codec in p.bytes_bytes - if codec isa Codecs.V3Codecs.GzipV3Codec - push!(codecs, Dict{String,Any}( - "name" => "gzip", - "configuration" => Dict{String,Any}("level" => codec.level) - )) - elseif codec isa Codecs.V3Codecs.BloscV3Codec - push!(codecs, Dict{String,Any}( - "name" => "blosc", - "configuration" => Dict{String,Any}( - "cname" => codec.cname, - "clevel" => codec.clevel, - "shuffle" => codec.shuffle == 0 ? "noshuffle" : - codec.shuffle == 1 ? "shuffle" : - codec.shuffle == 2 ? "bitshuffle" : - throw(ArgumentError("Unknown shuffle integer: $(codec.shuffle)")), - "blocksize" => codec.blocksize, - "typesize" => codec.typesize - ) - )) - elseif codec isa Codecs.V3Codecs.ZstdV3Codec - push!(codecs, Dict{String,Any}( - "name" => "zstd", - "configuration" => Dict{String,Any}("level" => codec.level) - )) - elseif codec isa Codecs.V3Codecs.CRC32cV3Codec - push!(codecs, Dict{String,Any}("name" => "crc32c")) - end + push!(codecs, Codecs.V3Codecs.codec_to_dict(codec)) end Dict{String, Any}( @@ -424,28 +380,6 @@ function lower3(md::MetadataV3{T}) where T ) end -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters::F=nothing, - fill_as_missing = false, - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T, N, C, F, E} - return Metadata3(A, chunks; - node_type=node_type, - compressor=compressor, - fill_value=fill_value, - order=order, - endian=endian, - filters=filters, - fill_as_missing=fill_as_missing, - dimension_separator=chunk_key_encoding.sep - ) -end - # V3 constructor from Dict - delegate to Metadata3 function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{3}) return Metadata3(d, fill_as_missing) diff --git a/src/pipeline.jl b/lib/ZarrCore/src/pipeline.jl similarity index 100% rename from src/pipeline.jl rename to lib/ZarrCore/src/pipeline.jl diff --git a/lib/ZarrCore/test/Project.toml b/lib/ZarrCore/test/Project.toml new file mode 100644 index 00000000..86e1c9d3 --- /dev/null +++ b/lib/ZarrCore/test/Project.toml @@ -0,0 +1,7 @@ +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" + +[sources.ZarrCore] +path = ".." diff --git a/lib/ZarrCore/test/runtests.jl b/lib/ZarrCore/test/runtests.jl new file mode 100644 index 00000000..db6051c7 --- /dev/null +++ b/lib/ZarrCore/test/runtests.jl @@ -0,0 +1,43 @@ +using Test +using ZarrCore +using JSON + +@testset "ZarrCore" begin + @testset "NoCompressor roundtrip" begin + z = zzeros(Int64, 4, 4) + @test z isa ZArray + @test size(z) == (4, 4) + @test z[1,1] == 0 + z[2,3] = 42 + @test z[2,3] == 42 + end + + @testset "DirectoryStore" begin + p = mktempdir() + z = zcreate(Float64, 10, 10; path=p, chunks=(5,5)) + z[:] = reshape(1.0:100.0, 10, 10) + z2 = zopen(p) + @test z2[1,1] == 1.0 + @test z2[10,10] == 100.0 + end + + @testset "DictStore" begin + z = zcreate(Int32, 6, 6; chunks=(3,3)) + z[:] = ones(Int32, 6, 6) + @test z[1,1] == 1 + end + + @testset "V3 uncompressed roundtrip" begin + z = zcreate(Float32, 8, 8; + zarr_format=3, + chunks=(4,4), + compressor=ZarrCore.NoCompressor()) + z[:] = reshape(Float32.(1:64), 8, 8) + @test z[1,1] == 1.0f0 + @test z[8,8] == 64.0f0 + end + + @testset "default_compressor is NoCompressor" begin + @test ZarrCore.default_compressor() isa ZarrCore.NoCompressor + end +end diff --git a/src/Codecs/V3/V3.jl b/src/Codecs/V3/V3.jl deleted file mode 100644 index 5c27690d..00000000 --- a/src/Codecs/V3/V3.jl +++ /dev/null @@ -1,650 +0,0 @@ -module V3Codecs - -import ..Codecs: zencode, zdecode, zencode!, zdecode! -# Import compressor types and functions from Zarr (grandparent module) -import ...Zarr: ZlibCompressor, ZstdCompressor, zcompress, zuncompress -import ...Zarr: BloscCompressor as ZarrBloscCompressor -using CRC32c: CRC32c -using JSON: JSON -using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions -using ChunkCodecCore: encode as cc_encode, decode as cc_decode - -abstract type V3Codec{In,Out} end -const codectypes = Dict{String, V3Codec}() - -@enum BloscCompressor begin - lz4 - lz4hc - blosclz - zstd - snappy - zlib -end - -@enum BloscShuffle begin - noshuffle - shuffle - bitshuffle -end - -struct BloscCodec <: V3Codec{:bytes, :bytes} - cname::BloscCompressor - clevel::Int64 - shuffle::BloscShuffle - typesize::UInt8 - blocksize::UInt -end -name(::BloscCodec) = "blosc" - -struct BytesCodec <: V3Codec{:array, :bytes} - endian::Symbol # :little or :big - function BytesCodec(endian::Symbol) - endian ∈ (:little, :big) || - throw(ArgumentError("BytesCodec endian must be :little or :big, got :$endian")) - new(endian) - end -end -BytesCodec() = BytesCodec(:little) -name(::BytesCodec) = "bytes" - -const _SYSTEM_LITTLE_ENDIAN = Base.ENDIAN_BOM == 0x04030201 -_needs_bswap(endian::Symbol) = (endian == :little) != _SYSTEM_LITTLE_ENDIAN - -struct CRC32cCodec <: V3Codec{:bytes, :bytes} -end -name(::CRC32cCodec) = "crc32c" - -struct GzipCodec <: V3Codec{:bytes, :bytes} -end -name(::GzipCodec) = "gzip" - - -#= -zencode(a, c::Codec) = error("Unimplemented") -zencode!(encoded, data, c::Codec) = error("Unimplemented") -zdecode(a, c::Codec, T::Type) = error("Unimplemented") -zdecode!(data, encoded, c::Codec) = error("Unimplemented") -=# - -function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) - hash::UInt32 = 0x00000000 - while(bytesavailable(input) > 0) - sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) - read!(input, sized_buffer) - write(output, sized_buffer) - hash = CRC32c.crc32c(sized_buffer, hash) - end - return hash -end -function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) - output = IOBuffer(encoded, read=false, write=true) - input = IOBuffer(data, read=true, write=false) - zencode!(output, input, c) - return take!(output) -end -function zencode!(output::IO, input::IO, c::CRC32cCodec) - hash = crc32c_stream!(output, input) - write(output, hash) - return output -end -function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) - output = IOBuffer(encoded, read=false, write=true) - input = IOBuffer(data, read=true, write=true) - zdecode!(output, input, c) - return take!(output) -end -function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) - input_vec = take!(input) - truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) - hash = crc32c_stream!(output, truncated_input) - if input_vec[end-3:end] != reinterpret(UInt8, [hash]) - throw(IOError("CRC32c hash does not match")) - end - return output -end - -""" - ShardingCodec{N} - -Sharding codec for Zarr v3. Sharding splits chunks into smaller "shards" and stores them -in a single file with an index mapping chunk coordinates to shard locations. - -# Fields -- `chunk_shape`: Shape of each shard (NTuple{N,Int}) -- `codecs`: Vector of codecs to apply to shard data (e.g., [BytesCodec(), GzipCodec()]) -- `index_codecs`: Vector of codecs to apply to the index (e.g., [BytesCodec()]) -- `index_location`: Location of index in shard file, either `:start` or `:end` - -# Implementation Notes -Sharding works by: -1. Taking a chunk of data and splitting it into shards based on `chunk_shape` -2. Encoding each shard using the `codecs` pipeline -3. Creating an index that maps (chunk_coords, shard_coords) -> (offset, size) in the shard file -4. Encoding the index using `index_codecs` -5. Writing the shard file with index at `index_location` (start or end) - -""" -struct ShardingCodec{N} <: V3Codec{:array, :bytes} - chunk_shape::NTuple{N,Int} # Shape of each shard - codecs::Vector{V3Codec} # Codecs to apply to shard data - index_codecs::Vector{V3Codec} # Codecs to apply to the index - index_location::Symbol # :start or :end -end -name(::ShardingCodec) = "sharding_indexed" - -""" - JSON.lower(c::ShardingCodec) - -Serialize ShardingCodec to JSON format for Zarr v3 metadata. -""" -function JSON.lower(c::ShardingCodec) - return Dict( - "name" => "sharding_indexed", - "configuration" => Dict( - "chunk_shape" => collect(c.chunk_shape), - "codecs" => [JSON.lower(codec) for codec in c.codecs], - "index_codecs" => [JSON.lower(codec) for codec in c.index_codecs], - "index_location" => string(c.index_location) - ) - ) -end - -""" - getCodec(::Type{ShardingCodec}, d::Dict) - -Deserialize ShardingCodec from JSON configuration dict. -""" -function getCodec(::Type{ShardingCodec}, d::Dict) - config = d["configuration"] - N = length(config["chunk_shape"]) - chunk_shape = NTuple{N,Int}(config["chunk_shape"]) - codecs = [getCodec(codec_dict) for codec_dict in config["codecs"]] - index_codecs = [getCodec(codec_dict) for codec_dict in config["index_codecs"]] - index_location = Symbol(get(config, "index_location", "end")) - return ShardingCodec{N}(chunk_shape, codecs, index_codecs, index_location) -end - -const MAX_UINT64 = typemax(UInt64) - -""" - ChunkShardInfo - -Information about a chunk's location within a shard. -""" -struct ChunkShardInfo - offset::UInt64 # Byte offset within shard where chunk begins - nbytes::UInt64 # Number of bytes the chunk occupies -end - -ChunkShardInfo() = ChunkShardInfo(MAX_UINT64, MAX_UINT64) # Empty chunk marker - -""" - ShardIndex{N} - -Internal structure representing the shard index. -Stores chunk location info for an N-dimensional grid of chunks. -Empty chunks are marked with ChunkShardInfo(MAX_UINT64, MAX_UINT64) -""" -struct ShardIndex{N} - chunks::Array{ChunkShardInfo, N} # N-dimensional array of chunk info -end - -""" - ShardIndex(chunks_per_shard::NTuple{N,Int}) - -Create an empty shard index with all chunks marked as empty. -""" -function ShardIndex(chunks_per_shard::NTuple{N,Int}) where N - chunks = fill(ChunkShardInfo(), chunks_per_shard) - return ShardIndex{N}(chunks) -end - -""" - get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) - -Get the byte range (offset, offset+nbytes) for a chunk, or nothing if empty. -""" -function get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N - info = idx.chunks[chunk_coords...] - - if info.offset == MAX_UINT64 && info.nbytes == MAX_UINT64 - return nothing - end - - return (Int(info.offset), Int(info.offset + info.nbytes)) -end - -""" - set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) - -Set the byte range for a chunk in the index. -""" -function set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) where N - idx.chunks[chunk_coords...] = ChunkShardInfo(UInt64(offset), UInt64(nbytes)) -end - -""" - set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) - -Mark a chunk as empty in the index. -""" -function set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N - idx.chunks[chunk_coords...] = ChunkShardInfo() -end - -""" - calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) - -Calculate how many chunks fit in each shard dimension. -""" -function calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) where N - return ntuple(i -> div(shard_shape[i], chunk_shape[i]), N) -end - -""" - get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) - -Get the array slice ranges for a chunk within a shard. -chunk_coords are 1-based indices. -""" -function get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) where N - return ntuple(N) do i - start_idx = (chunk_coords[i] - 1) * chunk_shape[i] + 1 - end_idx = min(chunk_coords[i] * chunk_shape[i], shard_shape[i]) - start_idx:end_idx - end -end - -""" - apply_codec_chain(data, codecs::Vector{V3Codec}) - -Apply codec pipeline in forward order (encoding). -""" -function apply_codec_chain(data, codecs::Vector{V3Codec}) - result = data - for codec in codecs - result = zencode(result, codec) - end - return result -end - -""" - reverse_codec_chain(data, codecs::Vector{V3Codec}) - -Apply codec pipeline in reverse order (decoding). -""" -function reverse_codec_chain(data, codecs::Vector{V3Codec}) - result = data - for codec in reverse(codecs) - result = zdecode(result, codec) - end - return result -end - -""" - encode_shard_index(index::ShardIndex, index_codecs::Vector{V3Codec}) - -Encode the shard index using the index codec pipeline. - -Per Zarr v3 spec, the index is linearized in C-order (row-major) with alternating -offset/nbytes values: [chunk_0_offset, chunk_0_nbytes, chunk_1_offset, chunk_1_nbytes, ...] -``` -""" -function encode_shard_index(index::ShardIndex{N}, index_codecs::Vector{V3Codec}) where N - # Pre-allocate buffer for index data - n_chunks = length(index.chunks) - index_data = Vector{UInt64}(undef, 2 * n_chunks) - - # Iterate in C-order (row-major) and interleave offset/nbytes - idx = 1 - for cart_idx in CartesianIndices(index.chunks) - info = index.chunks[cart_idx] - index_data[idx] = info.offset - index_data[idx + 1] = info.nbytes - idx += 2 - end - - # Convert to bytes - index_bytes = reinterpret(UInt8, index_data) - - # Apply index codecs - encoded = apply_codec_chain(index_bytes, index_codecs) - - return encoded -end - -""" - decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) - -Decode the shard index from bytes. - -The bytes are in C-order with alternating offset/nbytes: -[offset0, nbytes0, offset1, nbytes1, ...] -""" -function decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N - # Decode using index codecs (in reverse order) - decoded_bytes = reverse_codec_chain(index_bytes, index_codecs) - - # Expected size: 16 bytes (2 * UInt64) per chunk - n_chunks = prod(chunks_per_shard) - expected_length = n_chunks * 2 * sizeof(UInt64) - - if length(decoded_bytes) != expected_length - throw(DimensionMismatch("Index size mismatch: expected $expected_length, got $(length(decoded_bytes))")) - end - - # Reinterpret as UInt64 array: [offset1, nbytes1, offset1, nbytes1, ...] - index_data = reinterpret(UInt64, decoded_bytes) - - # Reconstruct the N-dimensional array of ChunkShardInfo - chunks = Array{ChunkShardInfo, N}(undef, chunks_per_shard) - - idx = 1 - for cart_idx in CartesianIndices(chunks) - offset = index_data[idx] - nbytes = index_data[idx + 1] - chunks[cart_idx] = ChunkShardInfo(offset, nbytes) - idx += 2 - end - - return ShardIndex{N}(chunks) -end - -""" - compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) - -Compute the byte size of the encoded shard index. -Per spec: "The size of the index can be determined by applying c.compute_encoded_size -for each index codec recursively. The initial size is the byte size of the index array, -i.e. 16 * chunks per shard." -""" -function compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N - # Initial size: 16 bytes per chunk (2 * UInt64) - n_chunks = prod(chunks_per_shard) - size = n_chunks * 16 - - # Apply each codec's size transformation - # For most codecs, we need to actually encode to know the size - # For simplicity, we encode an empty index - index = ShardIndex(chunks_per_shard) - encoded = encode_shard_index(index, index_codecs) - - return length(encoded) -end - -""" - zencode!(encoded::Vector{UInt8}, data::AbstractArray, c::ShardingCodec) - -Encode array data using sharding codec following Zarr v3 spec. - -Per spec: "In the sharding_indexed binary format, inner chunks are written successively -in a shard, where unused space between them is allowed, followed by an index referencing them." -""" -function zencode!(encoded::Vector{UInt8}, data::AbstractArray, c::ShardingCodec{N}) where N - shard_shape = size(data) - chunks_per_shard = calculate_chunks_per_shard(shard_shape, c.chunk_shape) - - # Create empty index - index = ShardIndex(chunks_per_shard) - - # Buffers for encoded chunks - chunk_buffers = Vector{UInt8}[] - current_offset = 0 - - # Process chunks in C order (row-major) - # Per spec: "The actual order of the chunk content is not fixed" - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - - # Extract chunk data from shard - slice_ranges = get_chunk_slice_in_shard(chunk_coords, c.chunk_shape, shard_shape) - chunk_data = data[slice_ranges...] - - # Encode chunk using codec pipeline - encoded_chunk = apply_codec_chain(chunk_data, c.codecs) - - # Skip if chunk is empty (no bytes) - if isempty(encoded_chunk) - set_chunk_empty!(index, chunk_coords) - continue - end - - nbytes = length(encoded_chunk) - - # Record offset and length in index - set_chunk_slice!(index, chunk_coords, current_offset, nbytes) - - push!(chunk_buffers, encoded_chunk) - current_offset += nbytes - end - - # Encode the index - encoded_index = encode_shard_index(index, c.index_codecs) - index_size = length(encoded_index) - - # If index is at start, adjust all offsets to account for index size - if c.index_location == :start - # Add index_size to all non-empty chunk offsets - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - info = index.chunks[cart_idx] - if info.offset != MAX_UINT64 - index.chunks[cart_idx] = ChunkShardInfo(info.offset + index_size, info.nbytes) - end - end - # Re-encode index with corrected offsets - encoded_index = encode_shard_index(index, c.index_codecs) - end - - # If all chunks are empty, return empty buffer (no shard) - if isempty(chunk_buffers) - resize!(encoded, 0) - return encoded - end - - # Assemble final shard: [index] + chunks or chunks + [index] - total_size = (c.index_location == :start ? index_size : 0) + - current_offset + - (c.index_location == :end ? index_size : 0) - - resize!(encoded, total_size) - output = IOBuffer(encoded, write=true) - - if c.index_location == :start - write(output, encoded_index) - for buf in chunk_buffers - write(output, buf) - end - else # :end - for buf in chunk_buffers - write(output, buf) - end - write(output, encoded_index) - end - - return encoded -end - -""" - zdecode!(data::AbstractArray, encoded::Vector{UInt8}, c::ShardingCodec) - -Decode sharded data back to array following Zarr v3 spec. - -Per spec: "A simple implementation to decode inner chunks in a shard would -(a) read the entire value from the store into a byte buffer, -(b) parse the shard index from the beginning or end of the buffer and -(c) cut out the relevant bytes that belong to the requested chunk." -""" -function zdecode!(data::AbstractArray, encoded::Vector{UInt8}, c::ShardingCodec{N}) where N - # Handle empty shard (no data) - if isempty(encoded) - fill!(data, zero(eltype(data))) # Fill with zeros (or should use fill_value from spec) - return data - end - - shard_shape = size(data) - chunks_per_shard = calculate_chunks_per_shard(shard_shape, c.chunk_shape) - - # Compute encoded index size - index_size = compute_encoded_index_size(chunks_per_shard, c.index_codecs) - - # Extract index bytes based on location - if c.index_location == :start - index_bytes = encoded[1:index_size] - chunk_data_offset = index_size - else # :end - index_bytes = encoded[end-index_size+1:end] - chunk_data_offset = 0 - end - - # Decode the index - index = decode_shard_index(index_bytes, chunks_per_shard, c.index_codecs) - - # Decode each chunk and place into output array - for cart_idx in CartesianIndices(chunks_per_shard) - chunk_coords = Tuple(cart_idx) - - # Get chunk byte range from index - chunk_slice = get_chunk_slice(index, chunk_coords) - - # Get array slice for this chunk - array_slice = get_chunk_slice_in_shard(chunk_coords, c.chunk_shape, shard_shape) - - if chunk_slice === nothing - # Empty chunk - fill with zeros (or fill_value) - # Per spec: "Empty inner chunks are interpreted as being filled with the fill value" - data[array_slice...] .= zero(eltype(data)) - continue - end - - # Extract chunk bytes - # Offsets in index are relative to start of chunk data - offset_start, offset_end = chunk_slice - - # Adjust for where chunk data begins in the shard - byte_start = chunk_data_offset + offset_start + 1 # Julia 1-based indexing - byte_end = chunk_data_offset + offset_end - - encoded_chunk = encoded[byte_start:byte_end] - - # Decode chunk using codec pipeline (in reverse) - decoded_chunk = reverse_codec_chain(encoded_chunk, c.codecs) - - # Place decoded chunk into output array - expected_shape = length.(array_slice) - data[array_slice...] = reshape(decoded_chunk, expected_shape) - end - - return data -end - -struct TransposeCodec{N} <: V3Codec{:array, :array} - order::NTuple{N, Int} # permutation (1-based Julia indexing) -end -name(::TransposeCodec) = "transpose" - -# codec_encode / codec_decode methods for V3 codecs - -# --- BytesCodec (array -> bytes) --- - -function codec_encode(c::BytesCodec, data::AbstractArray) - if _needs_bswap(c.endian) - return reinterpret(UInt8, bswap.(vec(data))) |> collect - else - return reinterpret(UInt8, vec(data)) |> collect - end -end - -function codec_decode(c::BytesCodec, encoded::Vector{UInt8}, ::Type{T}, shape::NTuple{N,Int}) where {T, N} - arr = collect(reinterpret(T, encoded)) - if _needs_bswap(c.endian) - arr = bswap.(arr) - end - return reshape(arr, shape) -end - -# --- TransposeCodec (array -> array) --- - -"""Return the shape of the output of `codec_encode(codec, data)` given the input shape.""" -encoded_shape(::V3Codec, sz::NTuple{N,Int}) where {N} = sz -encoded_shape(c::TransposeCodec, sz::NTuple{N,Int}) where {N} = ntuple(i -> sz[c.order[i]], Val{N}()) - -function codec_encode(c::TransposeCodec, data::AbstractArray) - return permutedims(data, c.order) -end - -function codec_decode(c::TransposeCodec, encoded::AbstractArray) - inv_order = Tuple(invperm(collect(c.order))) - return permutedims(encoded, inv_order) -end - -# --- Wrapper codecs (bytes -> bytes) that delegate to Zarr compressors --- -# These use fully-qualified names to avoid conflicts with the BloscCompressor -# enum and other names already defined in this module. - -struct GzipV3Codec <: V3Codec{:bytes, :bytes} - level::Int -end -GzipV3Codec() = GzipV3Codec(6) -name(::GzipV3Codec) = "gzip" - -function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) - opts = GzipEncodeOptions(; level=c.level) - return cc_encode(opts, data) -end - -function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) - return cc_decode(LibZGzipCodec(), encoded) -end - -struct BloscV3Codec <: V3Codec{:bytes, :bytes} - cname::String - clevel::Int - shuffle::Int - blocksize::Int - typesize::Int -end -BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) -name(::BloscV3Codec) = "blosc" - -function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) - comp = ZarrBloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) - return zcompress(data, comp) -end - -function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) - comp = ZarrBloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) - return collect(zuncompress(encoded, comp, UInt8)) -end - -struct ZstdV3Codec <: V3Codec{:bytes, :bytes} - level::Int -end -ZstdV3Codec() = ZstdV3Codec(3) -name(::ZstdV3Codec) = "zstd" - -function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) - comp = ZstdCompressor(level=c.level) - return zcompress(data, comp) -end - -function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) - comp = ZstdCompressor(level=c.level) - return collect(zuncompress(encoded, comp, UInt8)) -end - -struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} -end -name(::CRC32cV3Codec) = "crc32c" - -function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) - out = UInt8[] - return zencode!(out, data, CRC32cCodec()) -end - -function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) - out = UInt8[] - return zdecode!(out, encoded, CRC32cCodec()) -end - -end diff --git a/src/Codecs/V3/compression_codecs.jl b/src/Codecs/V3/compression_codecs.jl new file mode 100644 index 00000000..40938f42 --- /dev/null +++ b/src/Codecs/V3/compression_codecs.jl @@ -0,0 +1,323 @@ +# V3 compression codecs that depend on Blosc, ChunkCodecLibZlib, ChunkCodecLibZstd, CRC32c +# These register into ZarrCore's V3 codec registry. + +using CRC32c: CRC32c +using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions +using ChunkCodecCore: encode as cc_encode, decode as cc_decode + +import ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode + +# --- CRC32cCodec (internal helper) --- + +struct CRC32cCodec +end + +function crc32c_stream!(output::IO, input::IO; buffer = Vector{UInt8}(undef, 1024*32)) + hash::UInt32 = 0x00000000 + while(bytesavailable(input) > 0) + sized_buffer = @view(buffer[1:min(length(buffer), bytesavailable(input))]) + read!(input, sized_buffer) + write(output, sized_buffer) + hash = CRC32c.crc32c(sized_buffer, hash) + end + return hash +end + +function zencode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=false) + zencode!(output, input, c) + return take!(output) +end +function zencode!(output::IO, input::IO, c::CRC32cCodec) + hash = crc32c_stream!(output, input) + write(output, hash) + return output +end +function zdecode!(encoded::Vector{UInt8}, data::Vector{UInt8}, c::CRC32cCodec) + output = IOBuffer(encoded, read=false, write=true) + input = IOBuffer(data, read=true, write=true) + zdecode!(output, input, c) + return take!(output) +end +function zdecode!(output::IOBuffer, input::IOBuffer, c::CRC32cCodec) + input_vec = take!(input) + truncated_input = IOBuffer(@view(input_vec[1:end-4]); read=true, write=false) + hash = crc32c_stream!(output, truncated_input) + if input_vec[end-3:end] != reinterpret(UInt8, [hash]) + throw(IOError("CRC32c hash does not match")) + end + return output +end + +# --- GzipV3Codec --- + +struct GzipV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +GzipV3Codec() = GzipV3Codec(6) + +function codec_encode(c::GzipV3Codec, data::Vector{UInt8}) + opts = GzipEncodeOptions(; level=c.level) + return cc_encode(opts, data) +end +function codec_decode(c::GzipV3Codec, encoded::Vector{UInt8}) + return cc_decode(LibZGzipCodec(), encoded) +end +codec_to_dict(c::GzipV3Codec) = Dict{String,Any}( + "name" => "gzip", + "configuration" => Dict{String,Any}("level" => c.level) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- BloscV3Codec --- + +struct BloscV3Codec <: V3Codec{:bytes, :bytes} + cname::String + clevel::Int + shuffle::Int + blocksize::Int + typesize::Int +end +BloscV3Codec() = BloscV3Codec("lz4", 5, 1, 0, 4) + +function codec_encode(c::BloscV3Codec, data::Vector{UInt8}) + comp = BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return zcompress(data, comp) +end +function codec_decode(c::BloscV3Codec, encoded::Vector{UInt8}) + comp = BloscCompressor(blocksize=c.blocksize, clevel=c.clevel, cname=c.cname, shuffle=c.shuffle) + return collect(zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::BloscV3Codec) = Dict{String,Any}( + "name" => "blosc", + "configuration" => Dict{String,Any}( + "cname" => c.cname, + "clevel" => c.clevel, + "shuffle" => c.shuffle == 0 ? "noshuffle" : + c.shuffle == 1 ? "shuffle" : + c.shuffle == 2 ? "bitshuffle" : + throw(ArgumentError("Unknown shuffle integer: $(c.shuffle)")), + "blocksize" => c.blocksize, + "typesize" => c.typesize + ) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- ZstdV3Codec --- + +struct ZstdV3Codec <: V3Codec{:bytes, :bytes} + level::Int +end +ZstdV3Codec() = ZstdV3Codec(3) + +function codec_encode(c::ZstdV3Codec, data::Vector{UInt8}) + comp = ZstdCompressor(level=c.level) + return zcompress(data, comp) +end +function codec_decode(c::ZstdV3Codec, encoded::Vector{UInt8}) + comp = ZstdCompressor(level=c.level) + return collect(zuncompress(encoded, comp, UInt8)) +end +codec_to_dict(c::ZstdV3Codec) = Dict{String,Any}( + "name" => "zstd", + "configuration" => Dict{String,Any}("level" => c.level) +) + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- CRC32cV3Codec --- + +struct CRC32cV3Codec <: V3Codec{:bytes, :bytes} +end + +function codec_encode(c::CRC32cV3Codec, data::Vector{UInt8}) + out = UInt8[] + return zencode!(out, data, CRC32cCodec()) +end +function codec_decode(c::CRC32cV3Codec, encoded::Vector{UInt8}) + out = UInt8[] + return zdecode!(out, encoded, CRC32cCodec()) +end +codec_to_dict(::CRC32cV3Codec) = Dict{String,Any}("name" => "crc32c") + +# Parser registration moved to _register_v3_codec_parsers!() + +# --- ShardingCodec --- + +""" + ShardingCodec{N} + +Sharding codec for Zarr v3. Sharding splits chunks into smaller "shards" and stores them +in a single file with an index mapping chunk coordinates to shard locations. + +# Fields +- `chunk_shape`: Shape of each shard (NTuple{N,Int}) +- `codecs`: Vector of codecs to apply to shard data (e.g., [BytesCodec(), GzipCodec()]) +- `index_codecs`: Vector of codecs to apply to the index (e.g., [BytesCodec()]) +- `index_location`: Location of index in shard file, either `:start` or `:end` +""" +struct ShardingCodec{N} <: V3Codec{:array, :bytes} + chunk_shape::NTuple{N,Int} + codecs::Vector{V3Codec} + index_codecs::Vector{V3Codec} + index_location::Symbol +end + +const MAX_UINT64 = typemax(UInt64) + +""" + ChunkShardInfo + +Information about a chunk's location within a shard. +""" +struct ChunkShardInfo + offset::UInt64 + nbytes::UInt64 +end + +ChunkShardInfo() = ChunkShardInfo(MAX_UINT64, MAX_UINT64) + +""" + ShardIndex{N} + +Internal structure representing the shard index. +Stores chunk location info for an N-dimensional grid of chunks. +Empty chunks are marked with ChunkShardInfo(MAX_UINT64, MAX_UINT64) +""" +struct ShardIndex{N} + chunks::Array{ChunkShardInfo, N} +end + +function ShardIndex(chunks_per_shard::NTuple{N,Int}) where N + chunks = fill(ChunkShardInfo(), chunks_per_shard) + return ShardIndex{N}(chunks) +end + +function get_chunk_slice(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N + info = idx.chunks[chunk_coords...] + if info.offset == MAX_UINT64 && info.nbytes == MAX_UINT64 + return nothing + end + return (Int(info.offset), Int(info.offset + info.nbytes)) +end + +function set_chunk_slice!(idx::ShardIndex, chunk_coords::NTuple{N,Int}, offset::Int, nbytes::Int) where N + idx.chunks[chunk_coords...] = ChunkShardInfo(UInt64(offset), UInt64(nbytes)) +end + +function set_chunk_empty!(idx::ShardIndex, chunk_coords::NTuple{N,Int}) where N + idx.chunks[chunk_coords...] = ChunkShardInfo() +end + +function calculate_chunks_per_shard(shard_shape::NTuple{N,Int}, chunk_shape::NTuple{N,Int}) where N + return ntuple(i -> div(shard_shape[i], chunk_shape[i]), N) +end + +function get_chunk_slice_in_shard(chunk_coords::NTuple{N,Int}, chunk_shape::NTuple{N,Int}, shard_shape::NTuple{N,Int}) where N + return ntuple(N) do i + start_idx = (chunk_coords[i] - 1) * chunk_shape[i] + 1 + end_idx = min(chunk_coords[i] * chunk_shape[i], shard_shape[i]) + start_idx:end_idx + end +end + +function apply_codec_chain(data, codecs::Vector{V3Codec}) + result = data + for codec in codecs + result = zencode(result, codec) + end + return result +end + +function reverse_codec_chain(data, codecs::Vector{V3Codec}) + result = data + for codec in reverse(codecs) + result = zdecode(result, codec) + end + return result +end + +function encode_shard_index(index::ShardIndex{N}, index_codecs::Vector{V3Codec}) where N + n_chunks = length(index.chunks) + index_data = Vector{UInt64}(undef, 2 * n_chunks) + idx = 1 + for cart_idx in CartesianIndices(index.chunks) + info = index.chunks[cart_idx] + index_data[idx] = info.offset + index_data[idx + 1] = info.nbytes + idx += 2 + end + index_bytes = reinterpret(UInt8, index_data) + encoded = apply_codec_chain(index_bytes, index_codecs) + return encoded +end + +function decode_shard_index(index_bytes::Vector{UInt8}, chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N + decoded_bytes = reverse_codec_chain(index_bytes, index_codecs) + n_chunks = prod(chunks_per_shard) + expected_length = n_chunks * 2 * sizeof(UInt64) + if length(decoded_bytes) != expected_length + throw(DimensionMismatch("Index size mismatch: expected $expected_length, got $(length(decoded_bytes))")) + end + index_data = reinterpret(UInt64, decoded_bytes) + chunks = Array{ChunkShardInfo, N}(undef, chunks_per_shard) + idx = 1 + for cart_idx in CartesianIndices(chunks) + offset = index_data[idx] + nbytes = index_data[idx + 1] + chunks[cart_idx] = ChunkShardInfo(offset, nbytes) + idx += 2 + end + return ShardIndex{N}(chunks) +end + +function compute_encoded_index_size(chunks_per_shard::NTuple{N,Int}, index_codecs::Vector{V3Codec}) where N + index = ShardIndex(chunks_per_shard) + encoded = encode_shard_index(index, index_codecs) + return length(encoded) +end + +# Sharding codec parser registration moved to _register_v3_codec_parsers!() + +""" + _register_v3_codec_parsers!() + +Register all V3 codec parsers into ZarrCore's registry. Called from Zarr.__init__(). +""" +function _register_v3_codec_parsers!() + v3_codec_parsers["gzip"] = function(config) + level = get(config, "level", 6) + GzipV3Codec(level) + end + + v3_codec_parsers["blosc"] = function(config) + cname = get(config, "cname", "lz4") + clevel = get(config, "clevel", 5) + shuffle_val = get(config, "shuffle", "noshuffle") + shuffle_int = shuffle_val isa Integer ? shuffle_val : + shuffle_val == "noshuffle" ? 0 : + shuffle_val == "shuffle" ? 1 : + shuffle_val == "bitshuffle" ? 2 : + throw(ArgumentError("Unknown shuffle: \"$shuffle_val\".")) + blocksize = get(config, "blocksize", 0) + typesize = get(config, "typesize", 4) + BloscV3Codec(string(cname), clevel, shuffle_int, blocksize, typesize) + end + + v3_codec_parsers["zstd"] = function(config) + level = get(config, "level", 3) + ZstdV3Codec(level) + end + + v3_codec_parsers["crc32c"] = function(config) + CRC32cV3Codec() + end + + v3_codec_parsers["sharding_indexed"] = function(config) + throw(ArgumentError("Zarr.jl currently does not support the sharding_indexed codec")) + end +end diff --git a/src/Compressors/blosc.jl b/src/Compressors/blosc.jl index 789a298f..efe3c770 100644 --- a/src/Compressors/blosc.jl +++ b/src/Compressors/blosc.jl @@ -67,4 +67,4 @@ end JSON.lower(c::BloscCompressor) = Dict("id"=>"blosc", "cname"=>c.cname, "clevel"=>c.clevel, "shuffle"=>c.shuffle, "blocksize"=>c.blocksize) -Zarr.compressortypes["blosc"] = BloscCompressor \ No newline at end of file +# Registration moved to Zarr.__init__ \ No newline at end of file diff --git a/src/Compressors/zlib.jl b/src/Compressors/zlib.jl index 6b82feef..5977ff48 100644 --- a/src/Compressors/zlib.jl +++ b/src/Compressors/zlib.jl @@ -42,4 +42,4 @@ end JSON.lower(z::ZlibCompressor) = Dict("id"=>"zlib", "level" => z.config.level) -Zarr.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file +# Registration moved to Zarr.__init__ \ No newline at end of file diff --git a/src/Compressors/zstd.jl b/src/Compressors/zstd.jl index 6cd80a08..7b6e7c9d 100644 --- a/src/Compressors/zstd.jl +++ b/src/Compressors/zstd.jl @@ -52,4 +52,4 @@ function JSON.lower(z::ZstdCompressor) end end -Zarr.compressortypes["zstd"] = ZstdCompressor +# Registration moved to Zarr.__init__ diff --git a/src/Storage/gcstore.jl b/src/Storage/gcstore.jl index 02a2051c..cf2d320d 100644 --- a/src/Storage/gcstore.jl +++ b/src/Storage/gcstore.jl @@ -130,9 +130,7 @@ function subdirs(s::GCStore, p) return dirs end -pushfirst!(storageregexlist,r"^https://storage.googleapis.com"=>GCStore) -pushfirst!(storageregexlist,r"^http://storage.googleapis.com"=>GCStore) -push!(storageregexlist,r"^gs://"=>GCStore) +# Storage regex registration moved to Zarr.__init__ function storefromstring(::Type{<:GCStore}, url,_) uri = URI(url) diff --git a/src/Storage/http.jl b/src/Storage/http.jl index 980284f2..7286531b 100644 --- a/src/Storage/http.jl +++ b/src/Storage/http.jl @@ -37,18 +37,13 @@ end end -push!(storageregexlist,r"^https://"=>HTTPStore) -push!(storageregexlist,r"^http://"=>HTTPStore) +# Storage regex registration moved to Zarr.__init__ function storefromstring(::Type{<:HTTPStore}, s,_) http_store = HTTPStore(s) try if http_store["", ".zmetadata"] !== nothing http_store = ConsolidatedStore(http_store,"") end - if is_zarray(http_store, "") - meta = getmetadata(http_store, "", false) - http_store = FormattedStore{meta.zarr_format, meta.dimension_separator}(http_store) - end catch err @warn exception=err "Additional metadata was not available for HTTPStore." end diff --git a/src/Zarr.jl b/src/Zarr.jl index 76fc25bf..8639d114 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -1,30 +1,151 @@ module Zarr +using ZarrCore import JSON import Blosc -struct ZarrFormat{V} - version::Val{V} -end -Base.Int(v::ZarrFormat{V}) where V = V -@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) -ZarrFormat(v::ZarrFormat) = v -#Default Zarr Version -const DV = ZarrFormat(Val(2)) - -include("chunkkeyencoding.jl") -abstract type AbstractCodecPipeline end -include("metadata.jl") -include("metadata3.jl") -include("Compressors/Compressors.jl") -include("Codecs/Codecs.jl") -include("Storage/Storage.jl") -include("Filters/Filters.jl") -include("ZArray.jl") -include("pipeline.jl") -include("ZGroup.jl") +# Import names that will be extended by included files (import allows method extension) +import ZarrCore: getCompressor, zuncompress, zuncompress!, zcompress, zcompress!, + storagesize, storefromstring, store_read_strategy, + subdirs, subkeys, isinitialized, + consolidate_metadata, compressor_to_v3_bytes_codecs +# Import types, non-extended functions, and constants via using +using ZarrCore: ZarrCore, + # Types + ZArray, ZGroup, AbstractStore, DirectoryStore, DictStore, ConsolidatedStore, + AbstractMetadata, MetadataV2, MetadataV3, + AbstractCodecPipeline, V2Pipeline, V3Pipeline, + Compressor, NoCompressor, + Filter, + ZarrFormat, + AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + ASCIIChar, + # Functions (not extended) + zcreate, zopen, zzeros, zgroup, + storageratio, zinfo, zname, + pipeline_encode, pipeline_decode!, get_pipeline, get_order, + Metadata, Metadata3, + typestr, typestr3, + fill_value_encoding, fill_value_decoding, + compressortypes, default_compressor, + getfilters, filterdict, + zencode, zdecode, + citostring, + default_sep, default_prefix, + storageregexlist, + chunkindices, + _reinterpret, + # Storage functions (not extended) + SequentialRead, ConcurrentRead, + concurrent_io_tasks, + is_zarray, is_zgroup, is_zarr2, is_zarr3, + getmetadata, writemetadata, getattrs, writeattrs, + isemptysub, _concatpath, + store_readchunk, store_writechunk, store_deletechunk, store_isinitialized, + read_items!, write_items!, channelsize, maybecopy, + # Storage utility functions + normalize_path, + # Filter types + VLenArrayFilter, VLenUTF8Filter, + Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, + QuantizeFilter, DeltaFilter, + # DateTime support + DateTime64, + # Constants + DV, DS, DS2, DS3 + +# Re-export the same symbols as before export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, - zinfo, DirectoryStore, S3Store, GCStore, zgroup + zinfo, DirectoryStore, DictStore, ConsolidatedStore, zgroup + +# Bring in the MaxLengthStrings submodule +using ZarrCore: MaxLengthStrings +using .MaxLengthStrings: MaxLengthString + +# Include compressor implementations (they register into ZarrCore.compressortypes) +include("Compressors/blosc.jl") +include("Compressors/zlib.jl") +include("Compressors/zstd.jl") + +# Override default compressor and register compressors/codecs/stores at module init time +# (Dict/Array mutations during precompile don't persist, so they must happen in __init__) +function __init__() + ZarrCore.DEFAULT_COMPRESSOR_FACTORY[] = () -> BloscCompressor() + + # Register compressor types + compressortypes["blosc"] = BloscCompressor + compressortypes["zlib"] = ZlibCompressor + compressortypes["zstd"] = ZstdCompressor + + # Register V3 codec parsers + _register_v3_codec_parsers!() + + # Register storage URL resolvers + _register_storage_regexes!() +end + +function _register_storage_regexes!() + # GCStore regexes (must be first to match before generic HTTP) + pushfirst!(storageregexlist, r"^https://storage.googleapis.com" => GCStore) + pushfirst!(storageregexlist, r"^http://storage.googleapis.com" => GCStore) + push!(storageregexlist, r"^gs://" => GCStore) + # HTTPStore regexes + push!(storageregexlist, r"^https://" => HTTPStore) + push!(storageregexlist, r"^http://" => HTTPStore) + # S3Store regex + push!(storageregexlist, r"^s3://" => S3Store) +end + +# Include V3 compression codec implementations +include("Codecs/V3/compression_codecs.jl") + +# Now define the compressor-to-v3 codec mappings +function compressor_to_v3_bytes_codecs(c::BloscCompressor) + (BloscV3Codec(c.cname, c.clevel, c.shuffle, c.blocksize, sizeof(UInt8)),) +end +function compressor_to_v3_bytes_codecs(c::ZlibCompressor) + level = c.config.level == -1 ? 6 : c.config.level + (GzipV3Codec(level),) +end +function compressor_to_v3_bytes_codecs(c::ZstdCompressor) + (ZstdV3Codec(c.config.compressionLevel),) +end + +# Create a Codecs module wrapper so that Zarr.Codecs.V3Codecs.XXX paths work for tests. +# This re-exports everything from ZarrCore.Codecs.V3Codecs plus the compression codecs +# defined in Zarr. +module Codecs + module V3Codecs + # Re-export everything from ZarrCore.Codecs.V3Codecs + using ZarrCore.Codecs.V3Codecs: V3Codec, v3_codec_parsers, codec_to_dict, + codec_encode, codec_decode, codec_category, encoded_shape, + BytesCodec, TransposeCodec + # Re-export the compression codecs defined in Zarr + using ...Zarr: BloscV3Codec, GzipV3Codec, ZstdV3Codec, CRC32cV3Codec, ShardingCodec + end +end + +# Include network/archive storage backends +include("Storage/gcstore.jl") +include("Storage/http.jl") +include("Storage/zipstore.jl") + +# Register S3Store stub +struct S3Store <: AbstractStore + bucket::String + aws::Any +end +function S3Store(args...) + error("AWSS3 must be loaded to use S3Store. Try `using AWSS3`.") +end + +# Store URL resolvers registered in __init__ + +# HTTP.serve and writezip for ZArray/ZGroup +HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...) +writezip(io::IO, s::Union{ZArray,ZGroup}; kwargs...) = writezip(io, s.storage, s.path; kwargs...) + +export S3Store, GCStore end # module diff --git a/test/Project.toml b/test/Project.toml index ff8f8c15..51cf9837 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -11,7 +11,11 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" +ZarrCore = "d6e7b3f4-6eca-4ed7-8aa6-35dc33f59f61" [sources] CondaPkg = {rev = "manifest-check", url = "https://github.com/JamesWrigley/CondaPkg.jl"} Zarr = {path = ".."} + +[sources.ZarrCore] +path = "../lib/ZarrCore"