Add Pyodide/WASM support in CI for numcodecs

.github/workflows/ci-emscripten.yaml

@@ -0,0 +1,31 @@
+name: Tests on Pyodide
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  build:
+    name: Build numcodecs WASM distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          # https://github.com/actions/checkout/issues/2041 isn't released yet
+          # fetch-tags: true # required for version resolution
+
+        # TODO: build for Pyodide 314.0.0 stable when released and added to cibuildwheel
+        # hopefully this will happen for us in sync with cibuildwheel v4.0.0 stable as well
+      - name: Build and test numcodecs for Pyodide
+        uses: pypa/cibuildwheel@54327ab9d35de03b359ac25c97de9417d94639c0 # v4.0.0rc1
+        env:
+          CIBW_ENABLE: pyodide-prerelease
+        with:
+          only: cp314-pyodide_wasm32

.github/workflows/ci.yaml

@@ -25,7 +25,9 @@ jobs:
         uses: actions/checkout@v5
         with:
           submodules: recursive
-          fetch-depth: 0 # required for version resolution
+          fetch-depth: 0
+          # https://github.com/actions/checkout/issues/2041 isn't released yet
+          # fetch-tags: true # required for version resolution
 
       - name: Set up Python
         uses: actions/setup-python@v6

.gitignore

@@ -47,6 +47,7 @@ coverage.xml
 *,cover
 .hypothesis/
 cover/
+.pytest_cache/
 
 
 # Translations
@@ -99,4 +100,4 @@ ENV/
 .pixi/*
 *.egg-info
 pixi.lock
-uv.lock
+uv.lock

meson.build

@@ -15,6 +15,18 @@ py = import('python').find_installation(pure: false)
 cc = meson.get_compiler('c')
 
 # NumPy include directory (needed for vlen extension)
-numpy_dep = dependency('numpy')
+# When cross-compiling for Emscripten/WASM, numpy-config is not available on
+# the target because of https://github.com/pyodide/pyodide-build/pull/21, so
+# we have tofall back to finding headers.
+if host_machine.cpu_family() == 'wasm32'
+  _numpy_inc = run_command(
+    find_program('python', native: true),
+    ['-c', 'import numpy; print(numpy.get_include())'],
+    check: true,
+  ).stdout().strip()
+  numpy_dep = declare_dependency(include_directories: include_directories(_numpy_inc))
+else
+  numpy_dep = dependency('numpy')
+endif
 
 subdir('src/numcodecs')

pyproject.toml

@@ -163,6 +163,23 @@ config-settings = "setup-args=-Davx2=disabled"
 select = "*-macosx_arm64"
 config-settings = {setup-args = ["-Davx2=disabled", "-Dsse2=disabled"]}
 
+[[tool.cibuildwheel.overrides]]
+select = "*-pyodide_wasm32"
+before-build = "patch -p1 < tools/ci/patches/0001-disable-multiprocessing-and-pthreads.patch && patch -p1 < tools/ci/patches/0002-add-missing-unistd-headers.patch -d c-blosc/internal-complibs/zlib-1.3.1/"
+config-settings = {setup-args = ["-Dsse2=disabled", "-Davx2=disabled"]}
+test-requires = [
+  "pytest",
+#   "pytest-cov",
+#   "pyzstd", # TODO: add back once Pyodide 314.0.0 stable includes compression.zstd builtin
+  "msgpack",
+  "crc32c",
+  "zfpy",
+  "zarr>=3",
+  "importlib_metadata",
+]
+# Override addopts over to speed up test suite and not run coverage
+test-command = "python -m pytest -p no:cacheprovider --override-ini=addopts= -svra {project}/tests"
+
 [tool.ruff]
 line-length = 100
 extend-exclude = ["c-blosc"]

src/numcodecs/blosc.pyx

@@ -85,6 +85,8 @@ def get_mutex():
             mutex = None
         except ImportError:
             mutex = None
+        except ModuleNotFoundError:
+            mutex = None
         _MUTEX = mutex
         _MUTEX_IS_INIT = True
     return _MUTEX

tests/common.py

@@ -1,6 +1,8 @@
 import array
 import json as _json
 import os
+import platform
+import sys
 import warnings
 from glob import glob
 
@@ -27,6 +29,8 @@
     'เฮลโลเวิลด์',
 ]
 
+is_wasm = (sys.platform == 'emscripten') or (platform.machine() in ['wasm32', 'wasm64'])
+
 
 def compare_arrays(arr, res, precision=None):
     # ensure numpy array with matching dtype

tests/test_blosc.py

@@ -4,23 +4,19 @@
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.blosc import Blosc
-
-    from numcodecs import blosc
-except ImportError:  # pragma: no cover
-    pytest.skip("numcodecs.blosc not available", allow_module_level=True)
-
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
     check_encode_decode,
     check_err_decode_object_buffer,
     check_err_encode_object_buffer,
     check_max_buffer_size,
+    is_wasm,
 )
 
+blosc = pytest.importorskip("numcodecs.blosc")
+Blosc = blosc.Blosc
+
 codecs = [
     Blosc(shuffle=Blosc.SHUFFLE),
     Blosc(clevel=0, shuffle=Blosc.SHUFFLE),
@@ -210,6 +206,7 @@ def _decode_worker(enc):
     return compressor.decode(enc)
 
 
+@pytest.mark.skipif(is_wasm, reason="WASM/Pyodide does not support multiprocessing")
 @pytest.mark.parametrize('pool', [Pool, ThreadPool])
 def test_multiprocessing(use_threads, pool):
     data = np.arange(1000000)

tests/test_entrypoints_backport.py

@@ -6,6 +6,7 @@
 import pytest
 
 import numcodecs.registry
+from tests.common import is_wasm
 
 importlib_spec = importlib.util.find_spec("importlib_metadata")
 if importlib_spec is None or importlib_spec.loader is None:  # pragma: no cover
@@ -29,6 +30,7 @@ def get_entrypoints_with_importlib_metadata_loaded():
     assert cls.codec_id == "test"
 
 
+@pytest.mark.skipif(is_wasm, reason="Spawning processes is not supported in Pyodide/WASM")
 def test_entrypoint_codec_with_importlib_metadata():
     p = Process(target=get_entrypoints_with_importlib_metadata_loaded)
     p.start()

tests/test_lz4.py

@@ -3,12 +3,6 @@
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.lz4 import LZ4
-except ImportError:  # pragma: no cover
-    pytest.skip("numcodecs.lz4 not available", allow_module_level=True)
-
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
@@ -19,6 +13,8 @@
     check_repr,
 )
 
+LZ4 = pytest.importorskip("numcodecs.lz4").LZ4
+
 codecs = [
     LZ4(),
     LZ4(acceleration=-1),

tests/test_lzma.py

@@ -1,18 +1,10 @@
 import itertools
-import unittest
 from types import ModuleType
 from typing import cast
 
 import numpy as np
 import pytest
 
-try:
-    # noinspection PyProtectedMember
-    from numcodecs.lzma import LZMA, _lzma
-except ImportError as e:  # pragma: no cover
-    raise unittest.SkipTest("LZMA not available") from e
-
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
@@ -22,6 +14,9 @@
     check_repr,
 )
 
+pytest.importorskip("lzma")
+from numcodecs.lzma import LZMA, _lzma
+
 _lzma = cast(ModuleType, _lzma)
 
 codecs = [

tests/test_msgpacks.py

@@ -1,14 +1,6 @@
-import unittest
-
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.msgpacks import MsgPack
-except ImportError as e:  # pragma: no cover
-    raise unittest.SkipTest("msgpack not available") from e
-
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
@@ -17,6 +9,8 @@
     greetings,
 )
 
+MsgPack = pytest.importorskip("numcodecs.msgpacks").MsgPack
+
 # object array with strings
 # object array with mix strings / nans
 # object array with mix of string, int, float

tests/test_pcodec.py

@@ -1,11 +1,6 @@
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.pcodec import PCodec
-except ImportError:  # pragma: no cover
-    pytest.skip("pcodec not available", allow_module_level=True)
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
@@ -15,6 +10,10 @@
     check_repr,
 )
 
+# pcodec is not installed in the Pyodide CI environment because Pyodide ships
+# pcodec>=1.0, which has an incompatible API with what we require, i.e., <0.4.
+PCodec = pytest.importorskip("numcodecs.pcodec").PCodec
+
 codecs = [
     PCodec(),
     PCodec(level=1),

tests/test_pyzstd.py

@@ -2,9 +2,11 @@
 
 import numpy as np
 import pytest
-import pyzstd
 from numcodecs.zstd import Zstd
 
+pyzstd = pytest.importorskip("pyzstd")
+
+
 test_data = [
     b"Hello World!",
     np.arange(113).tobytes(),

tests/test_shuffle.py

@@ -4,18 +4,15 @@
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.shuffle import Shuffle
-except ImportError:  # pragma: no cover
-    pytest.skip("numcodecs.shuffle not available", allow_module_level=True)
-
-
 from tests.common import (
     check_backwards_compatibility,
     check_config,
     check_encode_decode,
+    is_wasm,
 )
 
+Shuffle = pytest.importorskip("numcodecs.shuffle").Shuffle
+
 codecs = [
     Shuffle(),
     Shuffle(elementsize=0),
@@ -87,6 +84,7 @@ def _decode_worker(enc):
     return compressor.decode(enc)
 
 
+@pytest.mark.skipif(is_wasm, reason="WASM/Pyodide does not support multiprocessing")
 @pytest.mark.parametrize('pool', [Pool, ThreadPool])
 def test_multiprocessing(pool):
     data = np.arange(1000000)

tests/test_vlen_array.py

@@ -1,12 +1,6 @@
-import unittest
-
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.vlen import VLenArray
-except ImportError as e:  # pragma: no cover
-    raise unittest.SkipTest("vlen-array not available") from e
 from tests.common import (
     assert_array_items_equal,
     check_backwards_compatibility,
@@ -15,6 +9,8 @@
     check_repr,
 )
 
+VLenArray = pytest.importorskip("numcodecs.vlen").VLenArray
+
 arrays = [
     np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object),
     np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape(

tests/test_vlen_bytes.py

@@ -1,12 +1,6 @@
-import unittest
-
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.vlen import VLenBytes
-except ImportError as e:  # pragma: no cover
-    raise unittest.SkipTest("vlen-bytes not available") from e
 from tests.common import (
     assert_array_items_equal,
     check_backwards_compatibility,
@@ -16,6 +10,8 @@
     greetings,
 )
 
+VLenBytes = pytest.importorskip("numcodecs.vlen").VLenBytes
+
 greetings_bytes = [g.encode('utf-8') for g in greetings]
 
 

tests/test_vlen_utf8.py

@@ -1,12 +1,6 @@
-import unittest
-
 import numpy as np
 import pytest
 
-try:
-    from numcodecs.vlen import VLenUTF8
-except ImportError as e:  # pragma: no cover
-    raise unittest.SkipTest("vlen-utf8 not available") from e
 from tests.common import (
     assert_array_items_equal,
     check_backwards_compatibility,
@@ -16,6 +10,8 @@
     greetings,
 )
 
+VLenUTF8 = pytest.importorskip("numcodecs.vlen").VLenUTF8
+
 arrays = [
     np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
     np.array(greetings * 100, dtype=object),