From b17f602d20d4ae9ddecb479ef8ce8a56d36ee39b Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 25 Jun 2026 16:28:40 -0400 Subject: [PATCH 1/5] run tests with pytest-run-parallel --- .github/workflows/test-python.yml | 9 ++++++++- pyproject.toml | 1 + uv.lock | 23 ++++++++++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 33ed0c3..2b62f73 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14t"] steps: - uses: actions/checkout@v6 @@ -74,5 +74,12 @@ jobs: - name: maturin develop run: uv run --no-project maturin develop + # On free-threaded (`t`) builds, run each test across multiple threads + # (via pytest-run-parallel) to surface thread-safety issues. - name: Run pytest + if: ${{ !endsWith(matrix.python-version, 't') }} run: uv run --no-project pytest --verbose + + - name: Run pytest (free-threaded) + if: ${{ endsWith(matrix.python-version, 't') }} + run: uv run --no-project pytest --verbose --parallel-threads=2 diff --git a/pyproject.toml b/pyproject.toml index 0365061..3b9c3c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dev = [ "obstore>=0.10.1", "pytest>=8.0", "pytest-asyncio>=1.4", + "pytest-run-parallel>=0.3.0;python_version>='3.13'", "ruff>=0.6", "zarr>=3", ] diff --git a/uv.lock b/uv.lock index 60f138a..82ebaa6 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] @@ -591,7 +592,8 @@ name = "icechunk" version = "2.0.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", ] dependencies = [ { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1499,6 +1501,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-run-parallel" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/c2/e94d5db442c975425ec7da0f66d29e191371a3aad4f4d9872bfeaf506cf7/pytest_run_parallel-0.9.1.tar.gz", hash = "sha256:9972ad8e66340a83819abae9cb3cefd7b3ca7b69b884d19521137767c6a69b58", size = 66001, upload-time = "2026-06-03T20:14:11.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/88/b09e7e1be1847e61f0602fe691cefb3363dc45bb08564568999a23e484c4/pytest_run_parallel-0.9.1-py3-none-any.whl", hash = "sha256:623d1fc7f7a2bec5487addf791f559e7814de5b135c43434e1740fd63eff3c7f", size = 19506, upload-time = "2026-06-03T20:14:10.328Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1868,7 +1882,8 @@ name = "zarr" version = "3.2.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", ] dependencies = [ { name = "donfig", marker = "python_full_version >= '3.12'" }, @@ -1913,6 +1928,7 @@ dev = [ { name = "obstore" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'" }, { name = "ruff" }, { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1940,6 +1956,7 @@ dev = [ { name = "obstore", specifier = ">=0.10.1" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=1.4" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'", specifier = ">=0.3.0" }, { name = "ruff", specifier = ">=0.6" }, { name = "zarr", specifier = ">=3" }, ] From 8ad742c5da1a46ebda9dcbb8a66e8e8f4a2a3ca6 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 25 Jun 2026 16:29:38 -0400 Subject: [PATCH 2/5] mark parallel tests as thread unsafe --- pyproject.toml | 9 ++++++++- tests/conftest.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py diff --git a/pyproject.toml b/pyproject.toml index 3b9c3c7..89f0659 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ ] [tool.maturin] -features = ["pyo3/extension-module", "abi3-py311"] +features = ["pyo3/extension-module"] module-name = "zarrista._zarrista" python-source = "python" @@ -96,3 +96,10 @@ known-first-party = ["zarrista"] [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" +markers = [ + # Registered by pytest-run-parallel; declared here so the markers resolve + # even when the plugin isn't installed (e.g. on Python < 3.13). `tests/ + # conftest.py` applies `thread_unsafe` to async tests. + "parallel_threads: mark test for concurrent thread execution", + "thread_unsafe: run the test single-threaded under --parallel-threads", +] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..535b4dd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +"""Shared pytest configuration.""" + +import inspect + + +def pytest_collection_modifyitems(items): + """ + Bridges pytest-asyncio and pytest-run-parallel: under `--parallel-threads`, + pytest-run-parallel calls each test directly inside worker threads, which leaves + `async def` tests' coroutines un-awaited (pytest-asyncio never gets to drive + them). Mark every coroutine test as `thread_unsafe` so it runs single-threaded + on the normal pytest-asyncio path, while synchronous tests still get the + free-threaded parallel stress. + + The marker is a no-op when pytest-run-parallel isn't installed (Python < 3.13), + so this is safe across the whole test matrix. + """ + for item in items: + if inspect.iscoroutinefunction(getattr(item, "obj", None)): + item.add_marker("thread_unsafe") From 802a9bcd99e583668dbd7458e8b6cfd6bfdf6a69 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 25 Jun 2026 16:31:02 -0400 Subject: [PATCH 3/5] build free-threaded wheels --- .github/workflows/python-wheels.yml | 111 ++++++++++++++++++++++++++-- Cargo.toml | 13 ++-- 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 45a9951..08ffe3e 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -1,5 +1,17 @@ name: Build wheels +# Each platform builds two wheels from the same crate: +# 1. an abi3 wheel (`--features abi3-py311`, built with a GIL-enabled 3.11) +# that works on every GIL-enabled CPython 3.11+, and +# 2. a version-specific free-threaded `cp314-cp314t` wheel (`-i 3.14t`, no +# abi3) — free-threaded CPython 3.14 has no stable ABI, so it needs its own +# wheel. +# +# A third, abi3t wheel (free-threaded stable ABI, PEP 803) for Python 3.15+ is +# scaffolded but commented out below: it needs a 3.15 interpreter, and 3.15 is +# still in beta. Uncomment the `abi3t` steps (and the `3.15t` interpreter +# installs) once 3.15 is released. + on: push: tags: @@ -26,14 +38,39 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install Python versions + run: uv python install 3.11 3.14t + # When 3.15 is released, add `3.15t` above for the abi3t build. + - name: Build abi3 wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist + args: --release --out dist --features abi3-py311 -i 3.11 + sccache: "true" + manylinux: "2_28" + + - name: Build free-threaded (cp314t) wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist -i 3.14t sccache: "true" manylinux: "2_28" + # - name: Build abi3t wheel (Python 3.15+) + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.platform.target }} + # args: --release --out dist --features abi3t-py315 -i 3.15t + # sccache: "true" + # manylinux: "2_28" + - name: Upload wheels uses: actions/upload-artifact@v7 with: @@ -52,14 +89,39 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install Python versions + run: uv python install 3.11 3.14t + # When 3.15 is released, add `3.15t` above for the abi3t build. + - name: Build abi3 wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist + args: --release --out dist --features abi3-py311 -i 3.11 + sccache: "true" + manylinux: musllinux_1_2 + + - name: Build free-threaded (cp314t) wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist -i 3.14t sccache: "true" manylinux: musllinux_1_2 + # - name: Build abi3t wheel (Python 3.15+) + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.platform.target }} + # args: --release --out dist --features abi3t-py315 -i 3.15t + # sccache: "true" + # manylinux: musllinux_1_2 + - name: Upload wheels uses: actions/upload-artifact@v7 with: @@ -72,7 +134,9 @@ jobs: - uses: actions/checkout@v6 # uv-provided Python has linking issues on Windows with maturin, so use - # the GitHub-actions-provided interpreter. + # the GitHub-actions-provided interpreter. `generate-import-lib` lets + # maturin build the version-specific wheels (e.g. 3.14t) without each of + # those interpreters installed. - uses: actions/setup-python@v6 with: python-version: "3.12" @@ -81,9 +145,23 @@ jobs: uses: PyO3/maturin-action@v1 with: target: x64 - args: --release --out dist + args: --release --out dist --features abi3-py311 --features generate-import-lib -i 3.11 sccache: "true" + - name: Build free-threaded (cp314t) wheel + uses: PyO3/maturin-action@v1 + with: + target: x64 + args: --release --out dist --features generate-import-lib -i 3.14t + sccache: "true" + + # - name: Build abi3t wheel (Python 3.15+) + # uses: PyO3/maturin-action@v1 + # with: + # target: x64 + # args: --release --out dist --features abi3t-py315 --features generate-import-lib -i 3.15t + # sccache: "true" + - name: Upload wheels uses: actions/upload-artifact@v7 with: @@ -102,17 +180,36 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - python-version: "3.12" + enable-cache: true + + - name: Install Python versions + run: uv python install 3.11 3.14t + # When 3.15 is released, add `3.15t` above for the abi3t build. - name: Build abi3 wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist + args: --release --out dist --features abi3-py311 -i 3.11 sccache: "true" + - name: Build free-threaded (cp314t) wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist -i 3.14t + sccache: "true" + + # - name: Build abi3t wheel (Python 3.15+) + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.platform.target }} + # args: --release --out dist --features abi3t-py315 -i 3.15t + # sccache: "true" + - name: Upload wheels uses: actions/upload-artifact@v7 with: diff --git a/Cargo.toml b/Cargo.toml index 14f3569..12e1c73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,13 @@ name = "_zarrista" crate-type = ["cdylib"] [features] -# Build a single abi3 wheel compatible with Python 3.11+. +# Build a single abi3 wheel compatible with GIL-enabled Python 3.11+. abi3-py311 = ["pyo3/abi3-py311"] +# Build a single abi3t wheel (free-threaded stable ABI, PEP 803) loadable by +# both GIL-enabled and free-threaded Python 3.15+. +abi3t-py315 = ["pyo3/abi3t-py315"] +# https://www.maturin.rs/distribution.html#cross-compile-to-windows +generate-import-lib = ["pyo3/generate-import-lib"] [dependencies] arrow-array = "59" @@ -33,11 +38,7 @@ icechunk = { version = "2.0.0", default-features = false } ndarray = "0.17.2" numpy = { version = "0.29", features = ["half"] } object_store = "0.13.2" -pyo3 = { version = "0.29", features = [ - "macros", - "abi3-py311", - "multiple-pymethods", -] } +pyo3 = { version = "0.29", features = ["macros", "multiple-pymethods"] } pyo3-arrow = "0.19" pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } pyo3-bytes = "0.7.1" From ee30759f2134d12e46ccfa0a07c444fe4689de85 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 25 Jun 2026 16:50:27 -0400 Subject: [PATCH 4/5] revert test --- .github/workflows/test-python.yml | 9 +-------- pyproject.toml | 8 -------- uv.lock | 23 +++-------------------- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 2b62f73..33ed0c3 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.12", "3.13", "3.14t"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 @@ -74,12 +74,5 @@ jobs: - name: maturin develop run: uv run --no-project maturin develop - # On free-threaded (`t`) builds, run each test across multiple threads - # (via pytest-run-parallel) to surface thread-safety issues. - name: Run pytest - if: ${{ !endsWith(matrix.python-version, 't') }} run: uv run --no-project pytest --verbose - - - name: Run pytest (free-threaded) - if: ${{ endsWith(matrix.python-version, 't') }} - run: uv run --no-project pytest --verbose --parallel-threads=2 diff --git a/pyproject.toml b/pyproject.toml index 89f0659..efec9f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ dev = [ "obstore>=0.10.1", "pytest>=8.0", "pytest-asyncio>=1.4", - "pytest-run-parallel>=0.3.0;python_version>='3.13'", "ruff>=0.6", "zarr>=3", ] @@ -96,10 +95,3 @@ known-first-party = ["zarrista"] [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" -markers = [ - # Registered by pytest-run-parallel; declared here so the markers resolve - # even when the plugin isn't installed (e.g. on Python < 3.13). `tests/ - # conftest.py` applies `thread_unsafe` to async tests. - "parallel_threads: mark test for concurrent thread execution", - "thread_unsafe: run the test single-threaded under --parallel-threads", -] diff --git a/uv.lock b/uv.lock index 82ebaa6..60f138a 100644 --- a/uv.lock +++ b/uv.lock @@ -2,8 +2,7 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", + "python_full_version >= '3.12'", "python_full_version < '3.12'", ] @@ -592,8 +591,7 @@ name = "icechunk" version = "2.0.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", + "python_full_version >= '3.12'", ] dependencies = [ { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1501,18 +1499,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] -[[package]] -name = "pytest-run-parallel" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version >= '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/c2/e94d5db442c975425ec7da0f66d29e191371a3aad4f4d9872bfeaf506cf7/pytest_run_parallel-0.9.1.tar.gz", hash = "sha256:9972ad8e66340a83819abae9cb3cefd7b3ca7b69b884d19521137767c6a69b58", size = 66001, upload-time = "2026-06-03T20:14:11.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/88/b09e7e1be1847e61f0602fe691cefb3363dc45bb08564568999a23e484c4/pytest_run_parallel-0.9.1-py3-none-any.whl", hash = "sha256:623d1fc7f7a2bec5487addf791f559e7814de5b135c43434e1740fd63eff3c7f", size = 19506, upload-time = "2026-06-03T20:14:10.328Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1882,8 +1868,7 @@ name = "zarr" version = "3.2.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", + "python_full_version >= '3.12'", ] dependencies = [ { name = "donfig", marker = "python_full_version >= '3.12'" }, @@ -1928,7 +1913,6 @@ dev = [ { name = "obstore" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'" }, { name = "ruff" }, { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1956,7 +1940,6 @@ dev = [ { name = "obstore", specifier = ">=0.10.1" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=1.4" }, - { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'", specifier = ">=0.3.0" }, { name = "ruff", specifier = ">=0.6" }, { name = "zarr", specifier = ">=3" }, ] From 696e5218484064b6c0681b3d36eedcd11c55828a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 25 Jun 2026 16:50:34 -0400 Subject: [PATCH 5/5] remove conftest --- tests/conftest.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 535b4dd..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Shared pytest configuration.""" - -import inspect - - -def pytest_collection_modifyitems(items): - """ - Bridges pytest-asyncio and pytest-run-parallel: under `--parallel-threads`, - pytest-run-parallel calls each test directly inside worker threads, which leaves - `async def` tests' coroutines un-awaited (pytest-asyncio never gets to drive - them). Mark every coroutine test as `thread_unsafe` so it runs single-threaded - on the normal pytest-asyncio path, while synchronous tests still get the - free-threaded parallel stress. - - The marker is a no-op when pytest-run-parallel isn't installed (Python < 3.13), - so this is safe across the whole test matrix. - """ - for item in items: - if inspect.iscoroutinefunction(getattr(item, "obj", None)): - item.add_marker("thread_unsafe")