From 0407f55dbddc183fdf60eaf5c95842b18bdd455e Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 17:55:42 -0400 Subject: [PATCH 01/11] Add async feature --- Cargo.toml | 2 ++ src/array/mod.rs | 2 ++ src/group/mod.rs | 2 ++ src/lib.rs | 2 ++ src/storage/mod.rs | 2 ++ 5 files changed, 10 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 6cfccd4..b7fce9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,11 +16,13 @@ name = "_zarrista" crate-type = ["cdylib"] [features] +default = ["async"] # Build a single abi3 wheel compatible with GIL-enabled Python 3.11+. abi3-py311 = ["pyo3/abi3-py311"] # Build a single abi3t wheel (free-threaded stable ABI, PEP 803) loadable by # both GIL-enabled and free-threaded Python 3.15+. abi3t-py315 = ["pyo3/abi3t-py315"] +async = [] # https://www.maturin.rs/distribution.html#cross-compile-to-windows generate-import-lib = ["pyo3/generate-import-lib"] diff --git a/src/array/mod.rs b/src/array/mod.rs index f3f49aa..987bea0 100644 --- a/src/array/mod.rs +++ b/src/array/mod.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "async")] mod r#async; mod builder; mod chunk_grid; @@ -8,6 +9,7 @@ mod shared; mod sync; mod type_wrappers; +#[cfg(feature = "async")] pub use r#async::PyAsyncArray; pub use builder::PyArrayBuilder; pub use chunk_grid::PyChunkGrid; diff --git a/src/group/mod.rs b/src/group/mod.rs index b501ad8..e066bae 100644 --- a/src/group/mod.rs +++ b/src/group/mod.rs @@ -1,7 +1,9 @@ +#[cfg(feature = "async")] mod r#async; mod shared; mod sync; +#[cfg(feature = "async")] pub use r#async::PyAsyncGroup; pub use sync::PyGroup; diff --git a/src/lib.rs b/src/lib.rs index 08ab8b9..bb28062 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,9 @@ fn _zarrista(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + #[cfg(feature = "async")] m.add_class::()?; + #[cfg(feature = "async")] m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 6f48a54..825d962 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,7 +1,9 @@ +#[cfg(feature = "async")] mod r#async; mod sync; mod type_wrappers; +#[cfg(feature = "async")] pub use r#async::{AsyncReadOnlyStorageAdapter, PyAsyncStorage}; pub(crate) use sync::PySyncStorage; pub use sync::{PyFilesystemStore, PyMemoryStore, ReadOnlyStorageAdapter}; From 00a82ac611780577255d7a03eba49273ed228d8b Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 18:02:17 -0400 Subject: [PATCH 02/11] more conditional --- Cargo.toml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b7fce9c..e6de802 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,14 @@ abi3-py311 = ["pyo3/abi3-py311"] # Build a single abi3t wheel (free-threaded stable ABI, PEP 803) loadable by # both GIL-enabled and free-threaded Python 3.15+. abi3t-py315 = ["pyo3/abi3t-py315"] -async = [] +async = [ + "dep:icechunk", + "dep:object_store", + "dep:pyo3-async-runtimes", + "zarrs/async", + "dep:zarrs_icechunk", + "dep:zarrs_object_store", +] # https://www.maturin.rs/distribution.html#cross-compile-to-windows generate-import-lib = ["pyo3/generate-import-lib"] @@ -35,24 +42,21 @@ bytes = "1.12.0" dlpark = { git = "https://github.com/kylebarron/dlpark", rev = "31c6f49c064e634326c97172d39a00acecd854b6", features = [ "pyo3", ] } -icechunk = { version = "2.0.0", default-features = false } -object_store = "0.13" +icechunk = { version = "2.0.0", default-features = false, optional = true } +object_store = { version = "0.13", optional = true } pyo3 = { version = "0.29", features = ["macros", "multiple-pymethods"] } pyo3-arrow = "0.19" -pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } +pyo3-async-runtimes = { version = "0.29", features = [ + "tokio-runtime", +], optional = true } pyo3-bytes = "0.7.1" -pyo3-object_store = { version = "0.11.0", default-features = false } +pyo3-object_store = { version = "0.11.0", default-features = false, optional = true } pythonize = "0.29" serde_json = "1" thiserror = "2.0.18" -zarrs = { version = "0.23", features = [ - "async", - "bitround", - "ndarray", - "sharding", -] } -zarrs_icechunk = "0.5" -zarrs_object_store = { version = "0.6.2" } +zarrs = { version = "0.23", features = ["bitround", "ndarray", "sharding"] } +zarrs_icechunk = { version = "0.5", optional = true } +zarrs_object_store = { version = "0.6.2", optional = true } [dev-dependencies] # `auto-initialize` lets `cargo test` start a Python interpreter for `Python::attach`. From 9a6bf3588201fa7d3c700ec63259511dd7ab0766 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 18:04:34 -0400 Subject: [PATCH 03/11] cfg feature --- src/array/builder.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/array/builder.rs b/src/array/builder.rs index a56e954..11ec511 100644 --- a/src/array/builder.rs +++ b/src/array/builder.rs @@ -1,18 +1,15 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3_async_runtimes::tokio::future_into_py; use zarrs::array::ArrayBuilder; use crate::array::type_wrappers::PyDimensionName; -use crate::array::{ - PyArray, PyArrayShape, PyAsyncArray, PyChunkGrid, PyChunkKeyEncoding, PyFillValue, -}; +use crate::array::{PyArray, PyArrayShape, PyChunkGrid, PyChunkKeyEncoding, PyFillValue}; use crate::codec::{PyArrayToArrayCodec, PyArrayToBytesCodec, PyBytesToBytesCodec}; use crate::dtype::PyDataType; use crate::error::ZarristaError; use crate::error::ZarristaResult; use crate::metadata::{PyArrayMetadataV3, PyAttributes}; -use crate::storage::{PyAsyncStorage, PySyncStorage}; +use crate::storage::PySyncStorage; #[pyclass(module = "zarrista.array", frozen, name = "ArrayBuilder")] pub struct PyArrayBuilder(ArrayBuilder); @@ -40,9 +37,12 @@ impl PyArrayBuilder { fn like<'py>(array: Bound<'py, PyAny>) -> ZarristaResult { if let Ok(array) = array.cast::() { Ok(Self(ArrayBuilder::from_array(array.get().inner()))) - } else if let Ok(array) = array.cast::() { - Ok(Self(ArrayBuilder::from_array(array.get().inner()))) } else { + #[cfg(feature = "async")] + if let Ok(array) = array.cast::() { + return Ok(Self(ArrayBuilder::from_array(array.get().inner()))); + } + Err(PyTypeError::new_err(format!( "expected an Array or AsyncArray, got {}", array.get_type().name()? @@ -85,10 +85,11 @@ impl PyArrayBuilder { Ok(array.into()) } + #[cfg(feature = "async")] fn create_async<'py>( &self, py: Python<'py>, - store: PyAsyncStorage, + store: crate::storage::PyAsyncStorage, path: &str, ) -> PyResult> { let array = self @@ -96,12 +97,12 @@ impl PyArrayBuilder { .build_arc(store.into_inner(), path) .map_err(ZarristaError::from)?; - future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { array .async_store_metadata() .await .map_err(ZarristaError::from)?; - Ok(PyAsyncArray::from(array)) + Ok(crate::array::PyAsyncArray::from(array)) }) } From fec96a0f1b7e9a8cd76e84638fd4b21bf4b3efd4 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 18:08:53 -0400 Subject: [PATCH 04/11] split sync and async nodes --- src/lib.rs | 10 +++--- src/node/{node.rs => async.rs} | 61 ++----------------------------- src/node/mod.rs | 9 +++-- src/node/sync.rs | 66 ++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 67 deletions(-) rename src/node/{node.rs => async.rs} (52%) create mode 100644 src/node/sync.rs diff --git a/src/lib.rs b/src/lib.rs index bb28062..0b01266 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,15 +17,13 @@ mod storage; use pyo3::prelude::*; -use crate::array::{ - PyArray, PyArrayBuilder, PyAsyncArray, PyChunkGrid, PyChunkKeyEncoding, PyFillValue, -}; +use crate::array::{PyArray, PyArrayBuilder, PyChunkGrid, PyChunkKeyEncoding, PyFillValue}; use crate::array_bytes::PyArrayBytes; use crate::codec::register_codec_module; use crate::data::{PyMaskedTensor, PyMaskedVariableArray, PyTensor, PyVariableArray}; use crate::dtype::PyDataType; use crate::exceptions::register_exceptions_module; -use crate::group::{PyAsyncGroup, PyGroup}; +use crate::group::PyGroup; use crate::storage::{PyFilesystemStore, PyMemoryStore}; /// The compiled core of zarrista, imported as `zarrista._zarrista`. @@ -37,9 +35,9 @@ fn _zarrista(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; #[cfg(feature = "async")] - m.add_class::()?; + m.add_class::()?; #[cfg(feature = "async")] - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/node/node.rs b/src/node/async.rs similarity index 52% rename from src/node/node.rs rename to src/node/async.rs index 5bb3fae..d32dd71 100644 --- a/src/node/node.rs +++ b/src/node/async.rs @@ -2,52 +2,15 @@ use std::sync::Arc; -use crate::array::{PyArray, PyAsyncArray}; +use crate::array::PyAsyncArray; use crate::error::{ZarristaError, ZarristaResult}; -use crate::group::{PyAsyncGroup, PyGroup}; +use crate::group::PyAsyncGroup; use pyo3::IntoPyObjectExt; use pyo3::prelude::*; use zarrs::array::Array; use zarrs::group::Group; use zarrs::node::{Node, NodeMetadata}; -use zarrs::storage::{ - AsyncReadableWritableListableStorageTraits, ReadableWritableListableStorageTraits, -}; - -/// An opened node: either an array or a group. -pub(crate) struct PyNode { - node: Node, - storage: Arc, -} - -impl PyNode { - pub fn new(node: Node, storage: Arc) -> Self { - Self { node, storage } - } -} - -// TODO: remove this if we make `Node` a full pyclass -impl<'py> IntoPyObject<'py> for PyNode { - type Target = PyAny; - type Error = ZarristaError; - type Output = Bound<'py, Self::Target>; - - fn into_pyobject(self, py: Python<'py>) -> Result { - let storage = self.storage; - let path = self.node.path().clone(); - let node_metadata = NodeMetadata::from(self.node); - match node_metadata { - NodeMetadata::Array(array_metadata) => { - let array = Array::new_with_metadata(storage, path.as_str(), array_metadata)?; - Ok(PyArray::new(Arc::new(array)).into_bound_py_any(py)?) - } - NodeMetadata::Group(group_metadata) => { - let group = Group::new_with_metadata(storage, path.as_str(), group_metadata)?; - Ok(PyGroup::new(Arc::new(group)).into_bound_py_any(py)?) - } - } - } -} +use zarrs::storage::AsyncReadableWritableListableStorageTraits; /// An opened node from an async store: either an array or a group. pub(crate) struct PyAsyncNode { @@ -87,24 +50,6 @@ impl<'py> IntoPyObject<'py> for PyAsyncNode { } } -#[derive(IntoPyObject)] -pub enum PyArrayOrGroup { - Array(PyArray), - Group(PyGroup), -} - -impl From for PyArrayOrGroup { - fn from(array: PyArray) -> Self { - Self::Array(array) - } -} - -impl From for PyArrayOrGroup { - fn from(group: PyGroup) -> Self { - Self::Group(group) - } -} - #[derive(IntoPyObject)] pub enum PyAsyncArrayOrGroup { Array(PyAsyncArray), diff --git a/src/node/mod.rs b/src/node/mod.rs index f511cc0..32a0264 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -1,6 +1,9 @@ -#[allow(clippy::module_inception)] -mod node; +#[cfg(feature = "async")] +mod r#async; mod node_path; +mod sync; -pub(crate) use node::{PyArrayOrGroup, PyAsyncArrayOrGroup, PyAsyncNode, PyNode}; +#[cfg(feature = "async")] +pub(crate) use r#async::{PyAsyncArrayOrGroup, PyAsyncNode}; pub(crate) use node_path::PyNodePath; +pub(crate) use sync::{PyArrayOrGroup, PyNode}; diff --git a/src/node/sync.rs b/src/node/sync.rs new file mode 100644 index 0000000..16cad39 --- /dev/null +++ b/src/node/sync.rs @@ -0,0 +1,66 @@ +//! Opening a node (array or group) at a path, with auto-detection. + +use std::sync::Arc; + +use crate::array::PyArray; +use crate::error::ZarristaError; +use crate::group::PyGroup; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; +use zarrs::array::Array; +use zarrs::group::Group; +use zarrs::node::{Node, NodeMetadata}; +use zarrs::storage::ReadableWritableListableStorageTraits; + +/// An opened node: either an array or a group. +pub(crate) struct PyNode { + node: Node, + storage: Arc, +} + +impl PyNode { + pub fn new(node: Node, storage: Arc) -> Self { + Self { node, storage } + } +} + +// TODO: remove this if we make `Node` a full pyclass +impl<'py> IntoPyObject<'py> for PyNode { + type Target = PyAny; + type Error = ZarristaError; + type Output = Bound<'py, Self::Target>; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let storage = self.storage; + let path = self.node.path().clone(); + let node_metadata = NodeMetadata::from(self.node); + match node_metadata { + NodeMetadata::Array(array_metadata) => { + let array = Array::new_with_metadata(storage, path.as_str(), array_metadata)?; + Ok(PyArray::new(Arc::new(array)).into_bound_py_any(py)?) + } + NodeMetadata::Group(group_metadata) => { + let group = Group::new_with_metadata(storage, path.as_str(), group_metadata)?; + Ok(PyGroup::new(Arc::new(group)).into_bound_py_any(py)?) + } + } + } +} + +#[derive(IntoPyObject)] +pub enum PyArrayOrGroup { + Array(PyArray), + Group(PyGroup), +} + +impl From for PyArrayOrGroup { + fn from(array: PyArray) -> Self { + Self::Array(array) + } +} + +impl From for PyArrayOrGroup { + fn from(group: PyGroup) -> Self { + Self::Group(group) + } +} From 08b5c0a28367371c5a77db81c2368b37f64c1fca Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 18:14:39 -0400 Subject: [PATCH 05/11] add develop --- Cargo.toml | 3 +- DEVELOP.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 DEVELOP.md diff --git a/Cargo.toml b/Cargo.toml index e6de802..72c51e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ abi3-py311 = ["pyo3/abi3-py311"] # both GIL-enabled and free-threaded Python 3.15+. abi3t-py315 = ["pyo3/abi3t-py315"] async = [ + "dep:async-trait", "dep:icechunk", "dep:object_store", "dep:pyo3-async-runtimes", @@ -37,7 +38,7 @@ generate-import-lib = ["pyo3/generate-import-lib"] arrow-array = "59" arrow-buffer = "59" arrow-schema = "59" -async-trait = "0.1" +async-trait = { version = "0.1", optional = true } bytes = "1.12.0" dlpark = { git = "https://github.com/kylebarron/dlpark", rev = "31c6f49c064e634326c97172d39a00acecd854b6", features = [ "pyo3", diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 0000000..45175d5 --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,100 @@ +# Development + +Requires a Rust toolchain and Python 3.11+. We use +[uv](https://docs.astral.sh/uv/) and [maturin](https://www.maturin.rs/). + +```bash +# Create a dev environment and install the dev dependencies +uv sync --no-install-package zarrista + +# Build the Rust extension and install it into the environment (debug build) +uv run --no-project maturin develop --uv + +# Or, in release mode: +uv run --no-project maturin develop --uv --release + +# Run the tests +uv run --no-project pytest +``` + +The `--no-project` is annoying but unavoidable in our current setup. Otherwise `uv` will try to build the rust library _in release mode, as a dependency of the project_ before reaching `uv sync` or `uv run`. + +## Docs Website + +```bash +uv run --group docs mkdocs serve +``` + +## Emscripten Python wheels + +Emscripten wheels (PEP 783) are built once per Python version. The entire +toolchain config (Rust toolchain, Emscripten version, ABI tag, rustflags) is +defined by `pyodide-build` *running under that same Python version* — e.g. +Python 3.13 maps to ABI `2025_0`/Emscripten 4.0.9 while Python 3.14 maps to +ABI `2026_0`/Emscripten 5.0.3. Use `uvx -p` to query the config for a given +Python version without touching the project venv: + +```bash +PYTHON_VERSION=3.14 # or 3.13 +# The `pyodide` executable lives in pyodide-cli; most subcommands (config, +# xbuildenv) are plugins provided by pyodide-build, so both packages are +# needed. +pyodide_cmd() { + uvx -p "$PYTHON_VERSION" --from pyodide-cli --with pyodide-build pyodide "$@" +} +RUST_TOOLCHAIN=$(pyodide_cmd config get rust_toolchain) +PYODIDE_ABI_VERSION=$(pyodide_cmd config get pyodide_abi_version) +PYODIDE_RUSTFLAGS=$(pyodide_cmd config get rustflags) +PYODIDE_CFLAGS=$(pyodide_cmd config get cflags) + +echo "RUST_TOOLCHAIN: $RUST_TOOLCHAIN" +echo "PYODIDE_ABI_VERSION: $PYODIDE_ABI_VERSION" +echo "PYODIDE_RUSTFLAGS: $PYODIDE_RUSTFLAGS" +echo "PYODIDE_CFLAGS: $PYODIDE_CFLAGS" +``` + +Install the matching Rust toolchain and wasm target: + +```bash +rustup toolchain install $RUST_TOOLCHAIN +rustup target add --toolchain $RUST_TOOLCHAIN wasm32-unknown-emscripten +``` + +Install Emscripten via the Pyodide cross-build environment rather than a +stock emsdk. This pins the Emscripten version matching the target Pyodide ABI +automatically, and applies [Pyodide's patches to +Emscripten](https://github.com/pyodide/pyodide/tree/main/emsdk/patches) — +several of which affect dynamic linking of Rust side modules: + +```bash +export PYODIDE_XBUILDENV_PATH="$HOME/.cache/pyodide-xbuildenv" +pyodide_cmd xbuildenv install +pyodide_cmd xbuildenv install-emscripten +source "$PYODIDE_XBUILDENV_PATH/$(pyodide_cmd xbuildenv version)/emsdk/emsdk_env.sh" +``` + +Build the wheel. Notes on the environment variables: + +- `MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION` is required for the wheel to get the + PyPI-accepted `pyemscripten_*` platform tag instead of the legacy + `emscripten_x_y_z` tag PyPI rejects (this also needs a recent maturin, hence + `uvx maturin` rather than the project venv's maturin). +- `CFLAGS_wasm32_unknown_emscripten` is needed for crates that compile C code + (e.g. zstd-sys in arro3-io): Pyodide's cflags include `-fPIC`, without which + the C objects can't be linked into a `SIDE_MODULE` (errors like "relocation + R_WASM_MEMORY_ADDR_SLEB cannot be used ...; recompile with -fPIC"). +- Always build with `--release`: debug builds are ~10x larger (full DWARF) and + slow. + +```bash +RUSTUP_TOOLCHAIN=$RUST_TOOLCHAIN \ +CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS="$PYODIDE_RUSTFLAGS" \ +CFLAGS_wasm32_unknown_emscripten="$PYODIDE_CFLAGS" \ +MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION=$PYODIDE_ABI_VERSION \ + uvx maturin build \ + --release \ + --no-default-features \ + -o dist \ + --target wasm32-unknown-emscripten \ + -i python$PYTHON_VERSION +``` From 9b8b82f640034a921847e723f76324eefe43d4c9 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 8 Jul 2026 18:34:12 -0400 Subject: [PATCH 06/11] unsafe impl send sync --- src/array/builder.rs | 5 ++++- src/array/chunk_grid.rs | 2 ++ src/array/chunk_key_encoding.rs | 2 ++ src/array/sync.rs | 2 ++ src/codec/array_to_array.rs | 2 ++ src/codec/array_to_bytes/mod.rs | 2 ++ src/codec/bytes_to_bytes/mod.rs | 2 ++ src/data/tensor.rs | 4 ++++ src/data/variable.rs | 4 ++++ src/dtype.rs | 2 ++ src/group/sync.rs | 2 ++ src/lib.rs | 1 + src/storage/sync.rs | 4 ++++ src/wasm.rs | 26 ++++++++++++++++++++++++++ 14 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/wasm.rs diff --git a/src/array/builder.rs b/src/array/builder.rs index 11ec511..660f255 100644 --- a/src/array/builder.rs +++ b/src/array/builder.rs @@ -6,7 +6,6 @@ use crate::array::type_wrappers::PyDimensionName; use crate::array::{PyArray, PyArrayShape, PyChunkGrid, PyChunkKeyEncoding, PyFillValue}; use crate::codec::{PyArrayToArrayCodec, PyArrayToBytesCodec, PyBytesToBytesCodec}; use crate::dtype::PyDataType; -use crate::error::ZarristaError; use crate::error::ZarristaResult; use crate::metadata::{PyArrayMetadataV3, PyAttributes}; use crate::storage::PySyncStorage; @@ -14,6 +13,8 @@ use crate::storage::PySyncStorage; #[pyclass(module = "zarrista.array", frozen, name = "ArrayBuilder")] pub struct PyArrayBuilder(ArrayBuilder); +crate::wasm_send_sync!(PyArrayBuilder); + impl PyArrayBuilder { fn with(&self, f: impl FnOnce(&mut ArrayBuilder)) -> Self { let mut b = self.0.clone(); @@ -92,6 +93,8 @@ impl PyArrayBuilder { store: crate::storage::PyAsyncStorage, path: &str, ) -> PyResult> { + use crate::error::ZarristaError; + let array = self .0 .build_arc(store.into_inner(), path) diff --git a/src/array/chunk_grid.rs b/src/array/chunk_grid.rs index db267b1..b6b8daa 100644 --- a/src/array/chunk_grid.rs +++ b/src/array/chunk_grid.rs @@ -16,6 +16,8 @@ use crate::metadata::PyMetadataV3; #[pyclass(module = "zarrista", frozen, name = "ChunkGrid", from_py_object)] pub struct PyChunkGrid(ChunkGrid); +crate::wasm_send_sync!(PyChunkGrid); + impl PyChunkGrid { pub fn new(chunk_grid: ChunkGrid) -> Self { Self(chunk_grid) diff --git a/src/array/chunk_key_encoding.rs b/src/array/chunk_key_encoding.rs index 327e804..aa1c235 100644 --- a/src/array/chunk_key_encoding.rs +++ b/src/array/chunk_key_encoding.rs @@ -14,6 +14,8 @@ use crate::metadata::PyMetadataV3; #[pyclass(module = "zarrista", frozen, name = "ChunkKeyEncoding", from_py_object)] pub struct PyChunkKeyEncoding(ChunkKeyEncoding); +crate::wasm_send_sync!(PyChunkKeyEncoding); + impl PyChunkKeyEncoding { pub fn into_inner(self) -> ChunkKeyEncoding { self.0 diff --git a/src/array/sync.rs b/src/array/sync.rs index 0c1fdb1..58e35a4 100644 --- a/src/array/sync.rs +++ b/src/array/sync.rs @@ -23,6 +23,8 @@ pub struct PyArray { pub(crate) inner: Arc>, } +crate::wasm_send_sync!(PyArray); + impl PyArray { pub(crate) fn new(inner: Arc>) -> Self { Self { inner } diff --git a/src/codec/array_to_array.rs b/src/codec/array_to_array.rs index 0aeef57..7adf706 100644 --- a/src/codec/array_to_array.rs +++ b/src/codec/array_to_array.rs @@ -35,6 +35,8 @@ pub fn bitround(keepbits: u32) -> PyArrayToArrayCodec { )] pub struct PyArrayToArrayCodec(Arc); +crate::wasm_send_sync!(PyArrayToArrayCodec); + impl PyArrayToArrayCodec { pub fn into_inner(self) -> Arc { self.0 diff --git a/src/codec/array_to_bytes/mod.rs b/src/codec/array_to_bytes/mod.rs index 4b6128b..e6d2daf 100644 --- a/src/codec/array_to_bytes/mod.rs +++ b/src/codec/array_to_bytes/mod.rs @@ -20,6 +20,8 @@ use crate::metadata::{PyConfiguration, PyMetadataV3}; )] pub struct PyArrayToBytesCodec(Arc); +crate::wasm_send_sync!(PyArrayToBytesCodec); + impl PyArrayToBytesCodec { pub fn new(codec: Arc) -> Self { Self(codec) diff --git a/src/codec/bytes_to_bytes/mod.rs b/src/codec/bytes_to_bytes/mod.rs index bc3db1d..b779e88 100644 --- a/src/codec/bytes_to_bytes/mod.rs +++ b/src/codec/bytes_to_bytes/mod.rs @@ -26,6 +26,8 @@ use crate::metadata::{PyConfiguration, PyMetadataV3}; )] pub struct PyBytesToBytesCodec(Arc); +crate::wasm_send_sync!(PyBytesToBytesCodec); + impl PyBytesToBytesCodec { pub fn new(codec: Arc) -> Self { Self(codec) diff --git a/src/data/tensor.rs b/src/data/tensor.rs index 31b5843..8713fac 100644 --- a/src/data/tensor.rs +++ b/src/data/tensor.rs @@ -28,6 +28,8 @@ pub struct PyTensor { shape: Arc<[u64]>, } +crate::wasm_send_sync!(PyTensor); + impl PyTensor { /// Construct a new PyTensor from the given bytes, data type, and shape. pub fn new(bytes: Bytes, data_type: DataType, shape: Arc<[u64]>) -> Self { @@ -214,6 +216,8 @@ pub struct PyMaskedTensor { mask: PyTensor, } +crate::wasm_send_sync!(PyMaskedTensor); + impl PyMaskedTensor { /// Construct a new PyMaskedTensor from the given bytes, mask, data type, and shape. pub fn new(data: PyTensor, mask: PyTensor) -> Self { diff --git a/src/data/variable.rs b/src/data/variable.rs index a11d9d4..72483e5 100644 --- a/src/data/variable.rs +++ b/src/data/variable.rs @@ -22,6 +22,8 @@ pub struct PyVariableArray { shape: Arc<[u64]>, } +crate::wasm_send_sync!(PyVariableArray); + impl PyVariableArray { pub fn new(bytes: Bytes, offsets: Vec, data_type: DataType, shape: Arc<[u64]>) -> Self { Self { @@ -126,6 +128,8 @@ pub struct PyMaskedVariableArray { shape: Arc<[u64]>, } +crate::wasm_send_sync!(PyMaskedVariableArray); + impl PyMaskedVariableArray { /// Construct a new PyMaskedVariableArray from the given bytes, offsets, mask, data type, and shape. pub fn new( diff --git a/src/dtype.rs b/src/dtype.rs index 41142af..be53588 100644 --- a/src/dtype.rs +++ b/src/dtype.rs @@ -15,6 +15,8 @@ pub struct PyDataType { inner: DataType, } +crate::wasm_send_sync!(PyDataType); + impl PyDataType { pub(crate) fn inner(&self) -> &DataType { &self.inner diff --git a/src/group/sync.rs b/src/group/sync.rs index 27cd322..8613bf6 100644 --- a/src/group/sync.rs +++ b/src/group/sync.rs @@ -22,6 +22,8 @@ pub struct PyGroup { pub(crate) inner: Arc>, } +crate::wasm_send_sync!(PyGroup); + impl PyGroup { pub(crate) fn new(inner: Arc>) -> Self { Self { inner } diff --git a/src/lib.rs b/src/lib.rs index 0b01266..2320f23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod group; mod metadata; mod node; mod storage; +mod wasm; use pyo3::prelude::*; diff --git a/src/storage/sync.rs b/src/storage/sync.rs index 1fec4d6..9356590 100644 --- a/src/storage/sync.rs +++ b/src/storage/sync.rs @@ -48,6 +48,8 @@ pub struct PyFilesystemStore { pub(crate) storage: Arc, } +crate::wasm_send_sync!(PyFilesystemStore); + #[pymethods] impl PyFilesystemStore { /// Open a filesystem store rooted at `path`. @@ -70,6 +72,8 @@ pub struct PyMemoryStore { pub(crate) storage: Arc, } +crate::wasm_send_sync!(PyMemoryStore); + #[pymethods] impl PyMemoryStore { #[new] diff --git a/src/wasm.rs b/src/wasm.rs new file mode 100644 index 0000000..de2a342 --- /dev/null +++ b/src/wasm.rs @@ -0,0 +1,26 @@ +//! wasm32-specific glue for the Python bindings. + +/// Assert `Send + Sync` for a `#[pyclass]` on `wasm32`. +/// +/// zarrs relaxes its `Send`/`Sync` bounds on `wasm32` (via its +/// `MaybeSend`/`MaybeSync` shim), so any type wrapping an `Arc` +/// is neither `Send` nor `Sync` there. pyo3 nonetheless requires every +/// `#[pyclass]` to be `Send + Sync`, and `Bound::get` on a `frozen` class +/// requires `Sync`. `wasm32` targets are single-threaded, so these bounds can +/// never be exercised, making the assertion sound. On native the bounds hold +/// for real and this macro expands to nothing. +/// +/// The `target_arch = "wasm32"` gate intentionally mirrors zarrs' own gate so +/// the assertion is active on exactly the targets where the bounds are relaxed. +#[macro_export] +macro_rules! wasm_send_sync { + ($ty:ty) => { + // SAFETY: `wasm32` is single-threaded, so `Send`/`Sync` are never + // exercised across threads. + #[cfg(target_arch = "wasm32")] + unsafe impl Send for $ty {} + // SAFETY: see above. + #[cfg(target_arch = "wasm32")] + unsafe impl Sync for $ty {} + }; +} From 7d16ef86d15d9f3ca1dc18812e93ce9cc08e26fe Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jul 2026 12:32:55 -0400 Subject: [PATCH 07/11] add wheel build ci --- .github/workflows/python-wheels.yml | 10 ++++++++++ pyproject.toml | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 08ffe3e..8facb35 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -11,11 +11,17 @@ name: Build wheels # scaffolded but commented out below: it needs a 3.15 interpreter, and 3.15 is # still in beta. Uncomment the `abi3t` steps (and the `3.15t` interpreter # installs) once 3.15 is released. +# +# On pull requests, only the `emscripten` job runs (every other job is gated +# with `if: github.event_name != 'pull_request'`). This catches regressions in +# the emscripten/Pyodide build — the most fragile target — without rebuilding +# every native wheel on every PR. on: push: tags: - "v*" + pull_request: workflow_dispatch: permissions: @@ -27,6 +33,7 @@ concurrency: jobs: linux: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -78,6 +85,7 @@ jobs: path: dist musllinux: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -129,6 +137,7 @@ jobs: path: dist windows: + if: github.event_name != 'pull_request' runs-on: windows-latest steps: - uses: actions/checkout@v6 @@ -169,6 +178,7 @@ jobs: path: dist macos: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: diff --git a/pyproject.toml b/pyproject.toml index efec9f7..62fee80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,14 @@ docs = [ # installed to format signatures in the docs "black>=26", ] +# Toolchain for building PEP 783 (pyemscripten) wheels for Pyodide; see the +# `emscripten` job in `.github/workflows/python-wheels.yml` and `DEVELOP.md`. +# `pyodide-cli` provides the `pyodide` entrypoint; `pyodide-build` provides the +# `config` and `xbuildenv` subcommands. Both require Python 3.12+. +pyodide = [ + "pyodide-build>=0.35.0 ; python_version >= '3.12'", + "pyodide-cli>=0.2.2 ; python_version >= '3.12'", +] [tool.ruff] # A REPL/notebook-style scratch file using top-level `await`; not a real test. From 6630a202da4d579a0041344ee6bbde2f6c20bb2d Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jul 2026 12:36:54 -0400 Subject: [PATCH 08/11] add wheel build --- .github/workflows/python-wheels.yml | 101 +++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 8facb35..2601974 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -226,7 +226,106 @@ jobs: name: wheels-macos-${{ matrix.platform.target }} path: dist + # PEP 783 (pyemscripten) wheel for Pyodide, built for Python 3.14 only. + # + # The Emscripten version, rustflags, and platform tag are all defined by + # pyodide-build for the target Python version. Two things differ from the + # native jobs: + # + # 1. The wheel is built `--no-default-features` (sync-only). The default + # `async` feature pulls in object_store / tokio / icechunk, which don't + # build for emscripten. + # 2. We override the Rust toolchain via `matrix.rust-toolchain-override` + # (falling back to the one Pyodide pins). zarrista's MSRV (1.94) is newer + # than the toolchain Pyodide pins for 3.14 (1.93), so we override to the + # MSRV. 3.14's ABI needs no nightly-only rustflags, so stable works. Drop + # the override once Pyodide's pinned toolchain reaches the MSRV. + # + # Unlike the native jobs, this also runs on every pull request to catch + # emscripten build regressions (e.g. a dependency update that no longer + # compiles or links for this target) before release time. + emscripten: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.14"] + include: + - python-version: "3.14" + rust-toolchain-override: "1.94" + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + + - name: Install pyodide-build + run: uv sync --only-group pyodide + + - name: Get pyodide config + id: pyodide-config + run: | + { + echo "rust-toolchain=$(uv run --no-sync pyodide config get rust_toolchain)" + echo "pyodide-abi-version=$(uv run --no-sync pyodide config get pyodide_abi_version)" + echo "rustflags=$(uv run --no-sync pyodide config get rustflags)" + echo "cflags=$(uv run --no-sync pyodide config get cflags)" + } >> "$GITHUB_OUTPUT" + + # Install Emscripten via the Pyodide cross-build environment rather than a + # stock emsdk: this pins the Emscripten version matching the target + # Pyodide ABI, and applies Pyodide's patches to Emscripten, several of + # which affect dynamic linking of Rust side modules. + # https://github.com/pyodide/pyodide/tree/main/emsdk/patches + - name: Install emsdk (Pyodide xbuildenv) + env: + PYODIDE_XBUILDENV_PATH: ${{ github.workspace }}/.pyodide-xbuildenv + run: | + uv run --no-sync pyodide xbuildenv install + uv run --no-sync pyodide xbuildenv install-emscripten + # shellcheck disable=SC1090 + source "$PYODIDE_XBUILDENV_PATH/$(uv run --no-sync pyodide xbuildenv version)/emsdk/emsdk_env.sh" + { + echo "EMSDK=$EMSDK" + echo "EMSDK_NODE=$EMSDK_NODE" + } >> "$GITHUB_ENV" + { + echo "$EMSDK" + echo "$EMSDK/upstream/emscripten" + dirname "$EMSDK_NODE" + } >> "$GITHUB_PATH" + + - name: Build emscripten wheel + uses: PyO3/maturin-action@v1 + env: + CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS: ${{ steps.pyodide-config.outputs.rustflags }} + CFLAGS_wasm32_unknown_emscripten: ${{ steps.pyodide-config.outputs.cflags }} + MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION: ${{ steps.pyodide-config.outputs.pyodide-abi-version }} + with: + target: wasm32-unknown-emscripten + rust-toolchain: ${{ matrix.rust-toolchain-override || steps.pyodide-config.outputs.rust-toolchain }} + args: --release --out dist -i ${{ matrix.python-version }} --no-default-features + sccache: "true" + + - name: Verify pyemscripten platform tag + # PyPI rejects the legacy `emscripten_x_y_z` tag; fail before upload if + # maturin didn't emit the PEP 783 `pyemscripten_*` tag (this needs + # `MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION` and a recent maturin). + run: | + ls -lh dist/ + ls dist/*-pyemscripten_*_wasm32.whl >/dev/null + + - name: Upload wheels + uses: actions/upload-artifact@v7 + with: + name: wheels-emscripten-py${{ matrix.python-version }} + path: dist + sdist: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -253,7 +352,7 @@ jobs: # IMPORTANT: this permission is mandatory for trusted publishing id-token: write if: startsWith(github.ref, 'refs/tags/') - needs: [linux, musllinux, windows, macos, sdist] + needs: [linux, musllinux, windows, macos, emscripten, sdist] steps: - uses: actions/download-artifact@v8 with: From 6a694114809f706a2398972f7a8a6f07778d0cd0 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jul 2026 12:38:48 -0400 Subject: [PATCH 09/11] update develop.md --- DEVELOP.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index 45175d5..964e79b 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -30,32 +30,35 @@ uv run --group docs mkdocs serve Emscripten wheels (PEP 783) are built once per Python version. The entire toolchain config (Rust toolchain, Emscripten version, ABI tag, rustflags) is defined by `pyodide-build` *running under that same Python version* — e.g. -Python 3.13 maps to ABI `2025_0`/Emscripten 4.0.9 while Python 3.14 maps to -ABI `2026_0`/Emscripten 5.0.3. Use `uvx -p` to query the config for a given -Python version without touching the project venv: +Python 3.14 maps to ABI `2026_0`/Emscripten 5.0.3. Use `uvx -p` to query the +config for a given Python version without touching the project venv: ```bash -PYTHON_VERSION=3.14 # or 3.13 +PYTHON_VERSION=3.14 # The `pyodide` executable lives in pyodide-cli; most subcommands (config, # xbuildenv) are plugins provided by pyodide-build, so both packages are # needed. pyodide_cmd() { uvx -p "$PYTHON_VERSION" --from pyodide-cli --with pyodide-build pyodide "$@" } -RUST_TOOLCHAIN=$(pyodide_cmd config get rust_toolchain) +PYODIDE_RUST_TOOLCHAIN=$(pyodide_cmd config get rust_toolchain) PYODIDE_ABI_VERSION=$(pyodide_cmd config get pyodide_abi_version) PYODIDE_RUSTFLAGS=$(pyodide_cmd config get rustflags) PYODIDE_CFLAGS=$(pyodide_cmd config get cflags) -echo "RUST_TOOLCHAIN: $RUST_TOOLCHAIN" -echo "PYODIDE_ABI_VERSION: $PYODIDE_ABI_VERSION" -echo "PYODIDE_RUSTFLAGS: $PYODIDE_RUSTFLAGS" -echo "PYODIDE_CFLAGS: $PYODIDE_CFLAGS" +echo "PYODIDE_RUST_TOOLCHAIN: $PYODIDE_RUST_TOOLCHAIN" +echo "PYODIDE_ABI_VERSION: $PYODIDE_ABI_VERSION" +echo "PYODIDE_RUSTFLAGS: $PYODIDE_RUSTFLAGS" +echo "PYODIDE_CFLAGS: $PYODIDE_CFLAGS" ``` -Install the matching Rust toolchain and wasm target: +Install a Rust toolchain and the wasm target. + +Note we do **not** use the toolchain Pyodide pins for 3.14 (rustc 1.93): it is +older than zarrista's MSRV (1.94). Force a newer version instead ```bash +RUST_TOOLCHAIN=1.94 rustup toolchain install $RUST_TOOLCHAIN rustup target add --toolchain $RUST_TOOLCHAIN wasm32-unknown-emscripten ``` @@ -80,7 +83,7 @@ Build the wheel. Notes on the environment variables: `emscripten_x_y_z` tag PyPI rejects (this also needs a recent maturin, hence `uvx maturin` rather than the project venv's maturin). - `CFLAGS_wasm32_unknown_emscripten` is needed for crates that compile C code - (e.g. zstd-sys in arro3-io): Pyodide's cflags include `-fPIC`, without which + (e.g. zstd-sys, pulled in by zarrs): Pyodide's cflags include `-fPIC`, without which the C objects can't be linked into a `SIDE_MODULE` (errors like "relocation R_WASM_MEMORY_ADDR_SLEB cannot be used ...; recompile with -fPIC"). - Always build with `--release`: debug builds are ~10x larger (full DWARF) and From 8cfc85977113cd556347e0fc2def0904a6f9d319 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jul 2026 12:39:28 -0400 Subject: [PATCH 10/11] fix feature flag --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 72c51e7..1bf7906 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ async = [ "dep:icechunk", "dep:object_store", "dep:pyo3-async-runtimes", + "dep:pyo3-object_store", "zarrs/async", "dep:zarrs_icechunk", "dep:zarrs_object_store", From e4812d20cc4991b66159259b8619ee9876293a6c Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jul 2026 12:42:40 -0400 Subject: [PATCH 11/11] build in debug mode on PRs --- .github/workflows/python-wheels.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 2601974..08c20db 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -246,6 +246,10 @@ jobs: # compiles or links for this target) before release time. emscripten: runs-on: ubuntu-latest + # Debug build on pull requests (faster, only checks it compiles and links); + # release build on tag pushes and manual dispatch. + env: + RELEASE_FLAG: ${{ github.event_name != 'pull_request' && '--release' || '' }} strategy: fail-fast: false matrix: @@ -307,7 +311,7 @@ jobs: with: target: wasm32-unknown-emscripten rust-toolchain: ${{ matrix.rust-toolchain-override || steps.pyodide-config.outputs.rust-toolchain }} - args: --release --out dist -i ${{ matrix.python-version }} --no-default-features + args: --out dist -i ${{ matrix.python-version }} --no-default-features ${{ env.RELEASE_FLAG }} sccache: "true" - name: Verify pyemscripten platform tag