diff --git a/.github/workflows/ci-emscripten.yaml b/.github/workflows/ci-emscripten.yaml new file mode 100644 index 00000000..e5b02f54 --- /dev/null +++ b/.github/workflows/ci-emscripten.yaml @@ -0,0 +1,31 @@ +name: Tests on Pyodide + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + build: + name: Build numcodecs WASM distribution + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: recursive + fetch-depth: 0 + # https://github.com/actions/checkout/issues/2041 isn't released yet + # fetch-tags: true # required for version resolution + + # TODO: build for Pyodide 314.0.0 stable when released and added to cibuildwheel + # hopefully this will happen for us in sync with cibuildwheel v4.0.0 stable as well + - name: Build and test numcodecs for Pyodide + uses: pypa/cibuildwheel@54327ab9d35de03b359ac25c97de9417d94639c0 # v4.0.0rc1 + env: + CIBW_ENABLE: pyodide-prerelease + with: + only: cp314-pyodide_wasm32 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c286ceba..8b189274 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,7 +25,9 @@ jobs: uses: actions/checkout@v5 with: submodules: recursive - fetch-depth: 0 # required for version resolution + fetch-depth: 0 + # https://github.com/actions/checkout/issues/2041 isn't released yet + # fetch-tags: true # required for version resolution - name: Set up Python uses: actions/setup-python@v6 diff --git a/.gitignore b/.gitignore index 14facfe7..59766e27 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ coverage.xml *,cover .hypothesis/ cover/ +.pytest_cache/ # Translations @@ -99,4 +100,4 @@ ENV/ .pixi/* *.egg-info pixi.lock -uv.lock \ No newline at end of file +uv.lock diff --git a/meson.build b/meson.build index 8d90d9c7..9b1a0f41 100644 --- a/meson.build +++ b/meson.build @@ -15,6 +15,18 @@ py = import('python').find_installation(pure: false) cc = meson.get_compiler('c') # NumPy include directory (needed for vlen extension) -numpy_dep = dependency('numpy') +# When cross-compiling for Emscripten/WASM, numpy-config is not available on +# the target because of https://github.com/pyodide/pyodide-build/pull/21, so +# we have tofall back to finding headers. +if host_machine.cpu_family() == 'wasm32' + _numpy_inc = run_command( + find_program('python', native: true), + ['-c', 'import numpy; print(numpy.get_include())'], + check: true, + ).stdout().strip() + numpy_dep = declare_dependency(include_directories: include_directories(_numpy_inc)) +else + numpy_dep = dependency('numpy') +endif subdir('src/numcodecs') diff --git a/pyproject.toml b/pyproject.toml index 878c912a..5d7f016d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,6 +163,23 @@ config-settings = "setup-args=-Davx2=disabled" select = "*-macosx_arm64" config-settings = {setup-args = ["-Davx2=disabled", "-Dsse2=disabled"]} +[[tool.cibuildwheel.overrides]] +select = "*-pyodide_wasm32" +before-build = "patch -p1 < tools/ci/patches/0001-disable-multiprocessing-and-pthreads.patch && patch -p1 < tools/ci/patches/0002-add-missing-unistd-headers.patch -d c-blosc/internal-complibs/zlib-1.3.1/" +config-settings = {setup-args = ["-Dsse2=disabled", "-Davx2=disabled"]} +test-requires = [ + "pytest", +# "pytest-cov", +# "pyzstd", # TODO: add back once Pyodide 314.0.0 stable includes compression.zstd builtin + "msgpack", + "crc32c", + "zfpy", + "zarr>=3", + "importlib_metadata", +] +# Override addopts over to speed up test suite and not run coverage +test-command = "python -m pytest -p no:cacheprovider --override-ini=addopts= -svra {project}/tests" + [tool.ruff] line-length = 100 extend-exclude = ["c-blosc"] diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f7358..047ab997 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -85,6 +85,8 @@ def get_mutex(): mutex = None except ImportError: mutex = None + except ModuleNotFoundError: + mutex = None _MUTEX = mutex _MUTEX_IS_INIT = True return _MUTEX diff --git a/tests/common.py b/tests/common.py index e3c9cfb3..de099c95 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,8 @@ import array import json as _json import os +import platform +import sys import warnings from glob import glob @@ -27,6 +29,8 @@ 'เฮลโลเวิลด์', ] +is_wasm = (sys.platform == 'emscripten') or (platform.machine() in ['wasm32', 'wasm64']) + def compare_arrays(arr, res, precision=None): # ensure numpy array with matching dtype diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 537dc350..7a258a80 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -4,14 +4,6 @@ import numpy as np import pytest -try: - from numcodecs.blosc import Blosc - - from numcodecs import blosc -except ImportError: # pragma: no cover - pytest.skip("numcodecs.blosc not available", allow_module_level=True) - - from tests.common import ( check_backwards_compatibility, check_config, @@ -19,8 +11,12 @@ check_err_decode_object_buffer, check_err_encode_object_buffer, check_max_buffer_size, + is_wasm, ) +blosc = pytest.importorskip("numcodecs.blosc") +Blosc = blosc.Blosc + codecs = [ Blosc(shuffle=Blosc.SHUFFLE), Blosc(clevel=0, shuffle=Blosc.SHUFFLE), @@ -210,6 +206,7 @@ def _decode_worker(enc): return compressor.decode(enc) +@pytest.mark.skipif(is_wasm, reason="WASM/Pyodide does not support multiprocessing") @pytest.mark.parametrize('pool', [Pool, ThreadPool]) def test_multiprocessing(use_threads, pool): data = np.arange(1000000) diff --git a/tests/test_entrypoints_backport.py b/tests/test_entrypoints_backport.py index 7e1c32bc..09a0918c 100644 --- a/tests/test_entrypoints_backport.py +++ b/tests/test_entrypoints_backport.py @@ -6,6 +6,7 @@ import pytest import numcodecs.registry +from tests.common import is_wasm importlib_spec = importlib.util.find_spec("importlib_metadata") if importlib_spec is None or importlib_spec.loader is None: # pragma: no cover @@ -29,6 +30,7 @@ def get_entrypoints_with_importlib_metadata_loaded(): assert cls.codec_id == "test" +@pytest.mark.skipif(is_wasm, reason="Spawning processes is not supported in Pyodide/WASM") def test_entrypoint_codec_with_importlib_metadata(): p = Process(target=get_entrypoints_with_importlib_metadata_loaded) p.start() diff --git a/tests/test_lz4.py b/tests/test_lz4.py index c6b8faec..65984118 100644 --- a/tests/test_lz4.py +++ b/tests/test_lz4.py @@ -3,12 +3,6 @@ import numpy as np import pytest -try: - from numcodecs.lz4 import LZ4 -except ImportError: # pragma: no cover - pytest.skip("numcodecs.lz4 not available", allow_module_level=True) - - from tests.common import ( check_backwards_compatibility, check_config, @@ -19,6 +13,8 @@ check_repr, ) +LZ4 = pytest.importorskip("numcodecs.lz4").LZ4 + codecs = [ LZ4(), LZ4(acceleration=-1), diff --git a/tests/test_lzma.py b/tests/test_lzma.py index ff36cc42..e36e067c 100644 --- a/tests/test_lzma.py +++ b/tests/test_lzma.py @@ -1,18 +1,10 @@ import itertools -import unittest from types import ModuleType from typing import cast import numpy as np import pytest -try: - # noinspection PyProtectedMember - from numcodecs.lzma import LZMA, _lzma -except ImportError as e: # pragma: no cover - raise unittest.SkipTest("LZMA not available") from e - - from tests.common import ( check_backwards_compatibility, check_config, @@ -22,6 +14,9 @@ check_repr, ) +pytest.importorskip("lzma") +from numcodecs.lzma import LZMA, _lzma + _lzma = cast(ModuleType, _lzma) codecs = [ diff --git a/tests/test_msgpacks.py b/tests/test_msgpacks.py index 0792cdfd..eb670781 100644 --- a/tests/test_msgpacks.py +++ b/tests/test_msgpacks.py @@ -1,14 +1,6 @@ -import unittest - import numpy as np import pytest -try: - from numcodecs.msgpacks import MsgPack -except ImportError as e: # pragma: no cover - raise unittest.SkipTest("msgpack not available") from e - - from tests.common import ( check_backwards_compatibility, check_config, @@ -17,6 +9,8 @@ greetings, ) +MsgPack = pytest.importorskip("numcodecs.msgpacks").MsgPack + # object array with strings # object array with mix strings / nans # object array with mix of string, int, float diff --git a/tests/test_pcodec.py b/tests/test_pcodec.py index e34d6658..f89f78e2 100644 --- a/tests/test_pcodec.py +++ b/tests/test_pcodec.py @@ -1,11 +1,6 @@ import numpy as np import pytest -try: - from numcodecs.pcodec import PCodec -except ImportError: # pragma: no cover - pytest.skip("pcodec not available", allow_module_level=True) - from tests.common import ( check_backwards_compatibility, check_config, @@ -15,6 +10,10 @@ check_repr, ) +# pcodec is not installed in the Pyodide CI environment because Pyodide ships +# pcodec>=1.0, which has an incompatible API with what we require, i.e., <0.4. +PCodec = pytest.importorskip("numcodecs.pcodec").PCodec + codecs = [ PCodec(), PCodec(level=1), diff --git a/tests/test_pyzstd.py b/tests/test_pyzstd.py index e6df84a7..2757dd44 100644 --- a/tests/test_pyzstd.py +++ b/tests/test_pyzstd.py @@ -2,9 +2,11 @@ import numpy as np import pytest -import pyzstd from numcodecs.zstd import Zstd +pyzstd = pytest.importorskip("pyzstd") + + test_data = [ b"Hello World!", np.arange(113).tobytes(), diff --git a/tests/test_shuffle.py b/tests/test_shuffle.py index 8035c558..848a8599 100644 --- a/tests/test_shuffle.py +++ b/tests/test_shuffle.py @@ -4,18 +4,15 @@ import numpy as np import pytest -try: - from numcodecs.shuffle import Shuffle -except ImportError: # pragma: no cover - pytest.skip("numcodecs.shuffle not available", allow_module_level=True) - - from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, + is_wasm, ) +Shuffle = pytest.importorskip("numcodecs.shuffle").Shuffle + codecs = [ Shuffle(), Shuffle(elementsize=0), @@ -87,6 +84,7 @@ def _decode_worker(enc): return compressor.decode(enc) +@pytest.mark.skipif(is_wasm, reason="WASM/Pyodide does not support multiprocessing") @pytest.mark.parametrize('pool', [Pool, ThreadPool]) def test_multiprocessing(pool): data = np.arange(1000000) diff --git a/tests/test_vlen_array.py b/tests/test_vlen_array.py index f25374a0..c44b8114 100644 --- a/tests/test_vlen_array.py +++ b/tests/test_vlen_array.py @@ -1,12 +1,6 @@ -import unittest - import numpy as np import pytest -try: - from numcodecs.vlen import VLenArray -except ImportError as e: # pragma: no cover - raise unittest.SkipTest("vlen-array not available") from e from tests.common import ( assert_array_items_equal, check_backwards_compatibility, @@ -15,6 +9,8 @@ check_repr, ) +VLenArray = pytest.importorskip("numcodecs.vlen").VLenArray + arrays = [ np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object), np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape( diff --git a/tests/test_vlen_bytes.py b/tests/test_vlen_bytes.py index facd0947..0320b6c1 100644 --- a/tests/test_vlen_bytes.py +++ b/tests/test_vlen_bytes.py @@ -1,12 +1,6 @@ -import unittest - import numpy as np import pytest -try: - from numcodecs.vlen import VLenBytes -except ImportError as e: # pragma: no cover - raise unittest.SkipTest("vlen-bytes not available") from e from tests.common import ( assert_array_items_equal, check_backwards_compatibility, @@ -16,6 +10,8 @@ greetings, ) +VLenBytes = pytest.importorskip("numcodecs.vlen").VLenBytes + greetings_bytes = [g.encode('utf-8') for g in greetings] diff --git a/tests/test_vlen_utf8.py b/tests/test_vlen_utf8.py index ee54793c..097c9b57 100644 --- a/tests/test_vlen_utf8.py +++ b/tests/test_vlen_utf8.py @@ -1,12 +1,6 @@ -import unittest - import numpy as np import pytest -try: - from numcodecs.vlen import VLenUTF8 -except ImportError as e: # pragma: no cover - raise unittest.SkipTest("vlen-utf8 not available") from e from tests.common import ( assert_array_items_equal, check_backwards_compatibility, @@ -16,6 +10,8 @@ greetings, ) +VLenUTF8 = pytest.importorskip("numcodecs.vlen").VLenUTF8 + arrays = [ np.array(['foo', 'bar', 'baz'] * 300, dtype=object), np.array(greetings * 100, dtype=object), diff --git a/tests/test_zstd.py b/tests/test_zstd.py index 09342088..bd57d763 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -153,9 +153,12 @@ def view_zstd_streaming_bytes(): def zstd_cli_available() -> bool: - return not subprocess.run( - ["zstd", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ).returncode + try: + return not subprocess.run( + ["zstd", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ).returncode + except OSError: + return False def test_multi_frame(): diff --git a/tools/ci/patches/0001-disable-multiprocessing-and-pthreads.patch b/tools/ci/patches/0001-disable-multiprocessing-and-pthreads.patch new file mode 100644 index 00000000..dcab2de0 --- /dev/null +++ b/tools/ci/patches/0001-disable-multiprocessing-and-pthreads.patch @@ -0,0 +1,34 @@ +This patch disables multiprocessing and pthread for blosc. This file +is adapted from and attributed to the Pyodide developers and can be +viewed at the upstream Pyodide repository at the following link: + +https://github.com/pyodide/pyodide/blob/d32e376013d8977b66c6aa828042b1fee8047aea/packages/numcodecs/patches/fixblosc.patch + + +diff --git a/c-blosc/blosc/blosc.h b/c-blosc/blosc/blosc.h +index 40857d0..8a1e969 100644 +--- a/c-blosc/blosc/blosc.h ++++ b/c-blosc/blosc/blosc.h +@@ -50,7 +50,7 @@ extern "C" { + ((INT_MAX - BLOSC_MAX_TYPESIZE * sizeof(int32_t)) / 3) + + /* The maximum number of threads (for some static arrays) */ +-#define BLOSC_MAX_THREADS 256 ++#define BLOSC_MAX_THREADS 1 + + /* Codes for shuffling (see blosc_compress) */ + #define BLOSC_NOSHUFFLE 0 /* no shuffle */ + + diff --git a/c-blosc/blosc/blosc.c b/c-blosc/blosc/blosc.c +index a5a5bd5..2a7797c 100644 +--- a/c-blosc/blosc/blosc.c ++++ b/c-blosc/blosc/blosc.c +@@ -2236,6 +2236,7 @@ void blosc_atfork_child(void) { + + void blosc_init(void) + { ++ g_initlib = 1; + /* Return if we are already initialized */ + if (g_initlib) return; + + diff --git a/tools/ci/patches/0002-add-missing-unistd-headers.patch b/tools/ci/patches/0002-add-missing-unistd-headers.patch new file mode 100644 index 00000000..7bbc88c1 --- /dev/null +++ b/tools/ci/patches/0002-add-missing-unistd-headers.patch @@ -0,0 +1,51 @@ +This patch adds missing headers in vendored zlib as done in numcodecs.js. This file +is attributed to the Pyodide developers and can be viewed at the upstream +Pyodide repository at the following link; + +https://github.com/pyodide/pyodide/blob/d32e376013d8977b66c6aa828042b1fee8047aea/packages/numcodecs/patches/fixzlib.patch + +This patch is applied in the c-blosc/internal-complibs/zlib-/ directory +in the .github/workflows/ci-emscripten.yaml workflow file. + +diff --git a/gzlib.c b/gzlib.c +index fae202e..80606a6 100644 +--- a/gzlib.c ++++ b/gzlib.c +@@ -2,7 +2,7 @@ + * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ +- ++#include + #include "gzguts.h" + + #if defined(_WIN32) && !defined(__BORLANDC__) + +diff --git a/gzread.c b/gzread.c +index bf4538e..afe6acd 100644 +--- a/gzread.c ++++ b/gzread.c +@@ -2,7 +2,7 @@ + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ +- ++#include + #include "gzguts.h" + + /* Local functions */ + + +diff --git a/gzwrite.c b/gzwrite.c +index aa767fb..a87676f 100644 +--- a/gzwrite.c ++++ b/gzwrite.c +@@ -2,7 +2,7 @@ + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ +- ++#include + #include "gzguts.h" + + /* Local functions */