diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3585299..f08bed2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,39 +46,6 @@ jobs: cargo test --all cargo test --all --all-features - check-emscripten: - name: Check compile for wasm32-unknown-emscripten - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # Keep toolchains in sync with the emscripten job in - # .github/workflows/wheels.yml: - # the 3.13 line's nightly override and the 3.14 line's - # pyodide-pinned stable (`pyodide config get rust_toolchain`). - include: - - rust-toolchain: nightly-2026-06-01 - python-version: "3.13" - - rust-toolchain: "1.93.0" - python-version: "3.14" - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.rust-toolchain }} - targets: wasm32-unknown-emscripten - - - uses: Swatinem/rust-cache@v2 - - - name: cargo check - run: cargo check --target wasm32-unknown-emscripten -p arro3-core -p arro3-compute --features pyo3/extension-module - env: - PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} - check-features_pyo3_arrow: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e1a4cd9..7fe1ce3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -9,6 +9,11 @@ on: push: tags: - "py-v*" + # On pull requests, only the emscripten job runs (the others are gated with + # `if: github.event_name != 'pull_request'`). This catches regressions in + # the emscripten/Pyodide build, which is the most fragile target, without + # building every native wheel on every PR. + pull_request: workflow_dispatch: permissions: @@ -20,6 +25,7 @@ concurrency: jobs: linux: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -83,6 +89,7 @@ jobs: path: dist musllinux: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -136,6 +143,7 @@ jobs: path: dist windows: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -181,6 +189,7 @@ jobs: path: dist macos: + if: github.event_name != 'pull_request' runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -230,7 +239,11 @@ jobs: # PEP 783 emscripten wheels for Pyodide. # # The Rust toolchain, Emscripten version, rustflags, and platform tag are all - # defined by pyodide-build + # defined by pyodide-build. + # + # Unlike the other jobs, this also runs on every pull request, to catch + # emscripten build regressions (e.g. a dependency update that doesn't + # compile or link for this target) before release time. emscripten: runs-on: ubuntu-latest strategy: @@ -240,11 +253,16 @@ jobs: module: - arro3-core - arro3-compute + - arro3-io include: # Pyodide's pinned toolchain for Python 3.13 (nightly-2025-02-01, # rustc 1.86) is very old. We override with a newer nightly - python-version: "3.13" rust-toolchain-override: nightly-2026-06-01 + # arro3-io's default "async" feature (object_store/tokio) doesn't + # compile for emscripten; build a sync-only wheel without it. + - module: arro3-io + extra-build-args: --no-default-features steps: - uses: actions/checkout@v4 @@ -265,6 +283,7 @@ jobs: echo "rust-toolchain=$(uv run --no-sync pyodide config get rust_toolchain)" echo "pyodide-abi-version=$(uv run --no-sync pyodide config get pyodide_abi_version)" echo "rustflags=$(uv run --no-sync pyodide config get rustflags)" + echo "cflags=$(uv run --no-sync pyodide config get cflags)" } >> "$GITHUB_OUTPUT" # Install Emscripten via the Pyodide cross-build environment rather @@ -295,11 +314,12 @@ jobs: uses: PyO3/maturin-action@v1 env: CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS: ${{ steps.pyodide-config.outputs.rustflags }} + CFLAGS_wasm32_unknown_emscripten: ${{ steps.pyodide-config.outputs.cflags }} MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION: ${{ steps.pyodide-config.outputs.pyodide-abi-version }} with: target: wasm32-unknown-emscripten rust-toolchain: ${{ matrix.rust-toolchain-override || steps.pyodide-config.outputs.rust-toolchain }} - args: --release --out dist -i ${{ matrix.python-version }} --manifest-path ${{ matrix.module }}/Cargo.toml + args: --release --out dist -i ${{ matrix.python-version }} --manifest-path ${{ matrix.module }}/Cargo.toml ${{ matrix.extra-build-args }} sccache: "true" # PyPI rejects legacy emscripten_x_y_z tags; fail before upload if @@ -315,6 +335,7 @@ jobs: path: dist sdist: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest strategy: matrix: diff --git a/DEVELOP.md b/DEVELOP.md index 282c1b5..66a5ce8 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -35,10 +35,12 @@ pyodide_cmd() { RUST_TOOLCHAIN=$(pyodide_cmd config get rust_toolchain) PYODIDE_ABI_VERSION=$(pyodide_cmd config get pyodide_abi_version) PYODIDE_RUSTFLAGS=$(pyodide_cmd config get rustflags) +PYODIDE_CFLAGS=$(pyodide_cmd config get cflags) echo "RUST_TOOLCHAIN: $RUST_TOOLCHAIN" echo "PYODIDE_ABI_VERSION: $PYODIDE_ABI_VERSION" echo "PYODIDE_RUSTFLAGS: $PYODIDE_RUSTFLAGS" +echo "PYODIDE_CFLAGS: $PYODIDE_CFLAGS" ``` Install the matching Rust toolchain and wasm target: @@ -61,14 +63,24 @@ pyodide_cmd xbuildenv install-emscripten source "$PYODIDE_XBUILDENV_PATH/$(pyodide_cmd xbuildenv version)/emsdk/emsdk_env.sh" ``` -Build `arro3-core`. `MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION` is required for the -wheel to get the PyPI-accepted `pyemscripten_*` platform tag instead of the -legacy `emscripten_x_y_z` tag PyPI rejects (this also needs a recent maturin, -hence `uvx maturin` rather than the project venv's maturin): +Build the wheel. Notes on the environment variables: + +- `MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION` is required for the wheel to get the + PyPI-accepted `pyemscripten_*` platform tag instead of the legacy + `emscripten_x_y_z` tag PyPI rejects (this also needs a recent maturin, hence + `uvx maturin` rather than the project venv's maturin). +- `CFLAGS_wasm32_unknown_emscripten` is needed for crates that compile C code + (e.g. zstd-sys in arro3-io): Pyodide's cflags include `-fPIC`, without which + the C objects can't be linked into a `SIDE_MODULE` (errors like "relocation + R_WASM_MEMORY_ADDR_SLEB cannot be used ...; recompile with -fPIC"). +- Always build with `--release`: debug builds are ~10x larger (full DWARF) and + slow. ```bash +# arro3-core RUSTUP_TOOLCHAIN=$RUST_TOOLCHAIN \ CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS="$PYODIDE_RUSTFLAGS" \ +CFLAGS_wasm32_unknown_emscripten="$PYODIDE_CFLAGS" \ MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION=$PYODIDE_ABI_VERSION \ uvx maturin build \ --release \ @@ -76,8 +88,36 @@ MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION=$PYODIDE_ABI_VERSION \ -m arro3-core/Cargo.toml \ --target wasm32-unknown-emscripten \ -i python$PYTHON_VERSION + +# arro3-compute +RUSTUP_TOOLCHAIN=$RUST_TOOLCHAIN \ +CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS="$PYODIDE_RUSTFLAGS" \ +CFLAGS_wasm32_unknown_emscripten="$PYODIDE_CFLAGS" \ +MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION=$PYODIDE_ABI_VERSION \ + uvx maturin build \ + --release \ + -o dist \ + -m arro3-compute/Cargo.toml \ + --target wasm32-unknown-emscripten \ + -i python$PYTHON_VERSION + +# arro3-io +RUSTUP_TOOLCHAIN=$RUST_TOOLCHAIN \ +CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUSTFLAGS="$PYODIDE_RUSTFLAGS" \ +CFLAGS_wasm32_unknown_emscripten="$PYODIDE_CFLAGS" \ +MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION=$PYODIDE_ABI_VERSION \ + uvx maturin build \ + --release \ + -o dist \ + -m arro3-io/Cargo.toml \ + --target wasm32-unknown-emscripten \ + -i python$PYTHON_VERSION \ + --no-default-features ``` +`--no-default-features` applies to `arro3-io` only: it disables the `async` +feature, which doesn't compile for emscripten. + Verify the wheel filename ends in `pyemscripten_${PYODIDE_ABI_VERSION}_wasm32.whl` before considering it publishable. ## Updating pyo3 version diff --git a/arro3-io/python/arro3/io/__init__.py b/arro3-io/python/arro3/io/__init__.py index f6a853d..2a9ca92 100644 --- a/arro3-io/python/arro3/io/__init__.py +++ b/arro3-io/python/arro3/io/__init__.py @@ -1,4 +1,11 @@ from ._io import * -from ._io import ___version, store +from ._io import ___version + +try: + # The store module only exists when arro3-io is compiled with the "async" + # feature; emscripten/pyodide wheels are built without it. + from ._io import store +except ImportError: + pass __version__: str = ___version() diff --git a/arro3-io/src/error.rs b/arro3-io/src/error.rs index fc7ac5f..a217df0 100644 --- a/arro3-io/src/error.rs +++ b/arro3-io/src/error.rs @@ -14,6 +14,7 @@ pub enum Arro3IoError { ArrowError(#[from] arrow_schema::ArrowError), /// A wrapped [object_store::Error] + #[cfg(feature = "async")] #[error(transparent)] ObjectStoreError(#[from] object_store::Error), @@ -31,6 +32,7 @@ impl From for PyErr { match error { Arro3IoError::PyErr(err) => err, Arro3IoError::ArrowError(err) => PyException::new_err(err.to_string()), + #[cfg(feature = "async")] Arro3IoError::ObjectStoreError(err) => PyException::new_err(err.to_string()), Arro3IoError::ParquetError(err) => PyException::new_err(err.to_string()), } @@ -46,4 +48,6 @@ impl<'a, 'py> From> for Arro3IoError { } /// A type wrapper around `Result`. +// Currently only referenced from async code paths. +#[cfg(feature = "async")] pub type Arro3IoResult = Result; diff --git a/arro3-io/src/lib.rs b/arro3-io/src/lib.rs index 2a33cc2..37408fa 100644 --- a/arro3-io/src/lib.rs +++ b/arro3-io/src/lib.rs @@ -40,7 +40,9 @@ fn _io(py: Python, m: &Bound) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(___version))?; + #[cfg(feature = "async")] pyo3_object_store::register_store_module(py, m, "arro3.io", "store")?; + #[cfg(feature = "async")] pyo3_object_store::register_exceptions_module(py, m, "arro3.io", "exceptions")?; m.add_wrapped(wrap_pyfunction!(csv::infer_csv_schema))?; @@ -58,6 +60,7 @@ fn _io(py: Python, m: &Bound) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(ipc::write_ipc_stream))?; m.add_wrapped(wrap_pyfunction!(parquet::read_parquet))?; + #[cfg(feature = "async")] m.add_wrapped(wrap_pyfunction!(parquet::read_parquet_async))?; m.add_wrapped(wrap_pyfunction!(parquet::write_parquet))?; diff --git a/arro3-io/src/parquet.rs b/arro3-io/src/parquet.rs index 303c99a..0ca068c 100644 --- a/arro3-io/src/parquet.rs +++ b/arro3-io/src/parquet.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use arrow_array::{RecordBatchIterator, RecordBatchReader}; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::arrow_writer::ArrowWriterOptions; -use parquet::arrow::async_reader::ParquetObjectReader; use parquet::arrow::ArrowWriter; use parquet::basic::{Compression, Encoding}; use parquet::file::metadata::KeyValue; @@ -17,10 +16,10 @@ use pyo3::pybacked::PyBackedStr; use pyo3_arrow::error::PyArrowResult; use pyo3_arrow::export::Arro3RecordBatchReader; use pyo3_arrow::input::AnyRecordBatch; -use pyo3_arrow::{PyRecordBatchReader, PyTable}; +use pyo3_arrow::PyRecordBatchReader; +#[cfg(feature = "async")] use pyo3_object_store::PyObjectStore; -use crate::error::Arro3IoResult; use crate::utils::{FileReader, FileWriter}; #[pyfunction] @@ -45,6 +44,7 @@ pub fn read_parquet(file: FileReader) -> PyArrowResult { Ok(PyRecordBatchReader::new(iter).into()) } +#[cfg(feature = "async")] #[pyfunction] #[pyo3(signature = (path, *, store))] pub fn read_parquet_async<'py>( @@ -59,11 +59,13 @@ pub fn read_parquet_async<'py>( Ok(fut) } +#[cfg(feature = "async")] async fn read_parquet_async_inner( store: Arc, path: String, -) -> Arro3IoResult { +) -> crate::error::Arro3IoResult { use futures::TryStreamExt; + use parquet::arrow::async_reader::ParquetObjectReader; use parquet::arrow::ParquetRecordBatchStreamBuilder; let object_reader = ParquetObjectReader::new(store, path.into()); @@ -75,7 +77,7 @@ async fn read_parquet_async_inner( let arrow_schema = Arc::new(reader.schema().as_ref().clone().with_metadata(metadata)); let batches = reader.try_collect::>().await?; - Ok(PyTable::try_new(batches, arrow_schema)?) + Ok(pyo3_arrow::PyTable::try_new(batches, arrow_schema)?) } pub(crate) struct PyWriterVersion(WriterVersion);