-
Notifications
You must be signed in to change notification settings - Fork 78
Compile a Fortran-free WASM SciPy, with BLIS and semicolon-lapack #619
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
1800c4c
9781e96
62b0fd8
d0692e0
eccae9e
20c5ba2
4587296
2b29202
c086628
785a8ec
7b907c6
8d980f6
54da58b
c4738be
4c4b4e9
6581233
587e4a8
674db14
6fa67e2
d165a3b
a1c77a4
ce70e66
8f8d9f5
e2ef2a8
e3344fe
6e9ae63
b0356c3
ebf855b
7f32277
4a492ef
e26b839
073216d
a7dee16
8000053
1025d6f
3003336
87b3fdc
f1c913b
b41377f
6e70172
576103d
6ee5b8d
7ebbe15
01549da
178f440
fed27b9
7fe2d71
9845363
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,8 @@ dependencies: | |
| - swig | ||
| - make | ||
| - cmake<4 | ||
| - meson | ||
| - ninja | ||
| - texinfo | ||
| - autoconf | ||
| - automake | ||
|
|
||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are these patches upstreamed?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nope, not yet, but they live in https://github.com/pyodide/blis. I plan to do this soon-ish and send PRs to BLIS, but they span quite a lot of files, so I'll take it slowly :) I'm waiting for a response from the BLIS maintainers on flame/blis#491 first. |
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package: | ||
| name: libsemilapack | ||
| version: "0.01.3-pre" | ||
| tag: | ||
| - library | ||
| - shared_library | ||
| source: | ||
| url: https://github.com/ilayn/semicolon-lapack/archive/refs/tags/v0.01.3-pre.tar.gz | ||
| sha256: 5292396b9c2bffc1bf22dccad7acbf9f4fcb1edc0c45ed0f055d80f55e2a23a3 | ||
|
|
||
| build: | ||
| type: shared_library | ||
| script: | | ||
| export PKG_CONFIG_PATH="${WASM_LIBRARY_DIR}/lib/pkgconfig:${WASM_LIBRARY_DIR}/share/pkgconfig:${PKG_CONFIG_PATH}" | ||
|
|
||
| CC=emcc CXX=em++ AR=emar \ | ||
| meson setup builddir --cross-file "${MESON_CROSS_FILE}" \ | ||
| -Dblas=blis \ | ||
| -Dfabi_shim=true \ | ||
| -Dtests=false \ | ||
| --default-library=static \ | ||
| --prefix="${WASM_LIBRARY_DIR}" | ||
| ninja -C builddir | ||
| ninja -C builddir install | ||
|
|
||
| mkdir -p dist | ||
| emcc ${WASM_LIBRARY_DIR}/lib/libsemilapack.a \ | ||
| ${WASM_LIBRARY_DIR}/lib/libsemilapack_fortran.a \ | ||
| ${WASM_LIBRARY_DIR}/lib/libblis.so \ | ||
| ${SIDE_MODULE_LDFLAGS} \ | ||
| -o dist/libsemilapack.so | ||
|
|
||
| cp dist/libsemilapack.so ${WASM_LIBRARY_DIR}/lib/libsemilapack.so | ||
| requirements: | ||
| host: | ||
| - libblis | ||
| # TODO: explore whether libblis is needed at runtime, or if the CBLAS symbols | ||
| # can be resolved at build time and the libblis.so dependency dropped | ||
| run: | ||
| - libblis | ||
| about: | ||
| home: https://github.com/ilayn/semicolon-lapack | ||
| license: BSD-3-Clause | ||
| summary: LAPACK linear algebra routines implemented in C |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,13 +20,16 @@ package: | |
| source: | ||
| url: https://files.pythonhosted.org/packages/source/s/scipy/scipy-1.18.0.tar.gz | ||
| sha256: 67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378 | ||
|
|
||
| patches: | ||
| - patches/0001-gemm_-no-const.patch | ||
| - patches/0002-make-int-return-values.patch | ||
| - patches/0003-Fix-gees-calls.patch | ||
| - patches/0004-Remove-chla_transtype.patch | ||
| - patches/0005-Set-wrapper-return-type-to-int.patch | ||
| # TODO: figure out what to do with this patch. I realised that this patch is | ||
| # needed because the gees f2py wrapper is generated with the wrong signature. | ||
| # This comes up a lot in Schur. It passes two hidden Fortran string lengths | ||
| # for its char args (jobvs, sort) in the callstatement/callprotoargument. | ||
| # BLIS and semicolon-lapack (and the f2ced stack we had before, even) take no | ||
| # hidden lengths. | ||
| - patches/0001-Fix-gees-calls.patch | ||
| # Drop this patch with SciPy v1.19 | ||
| - patches/0002-Fix-signatures-for-getrf-getri-and-trlib-dlagtm.patch | ||
| build: | ||
| vendor-sharedlib: true | ||
| # NumPy 2.1 disabled visibility for symbols outside of extension modules | ||
|
|
@@ -55,67 +58,16 @@ build: | |
| unvendor-tests: true | ||
| # install-args=--tags=runtime,python-runtime,devel | ||
| # Disable when running tests, enable when a PR is ready, i.e., building for distribution. | ||
| # TODO before merging: update to SciPy 2.0 when it arrives and drop -D_without-fortran=true | ||
| backend-flags: | | ||
| build-dir=build | ||
| # IMPORTANT: Other locations important in scipy build process: | ||
| # There are two files built in the "capture" pass that need patching: | ||
| # _blas_subroutines.h, and _cython | ||
| # Scipy has a bunch of custom logic implemented in | ||
| # pyodide-build/pyodide_build/_f2c_fixes.py. | ||
| setup-args=-D_without-fortran=true | ||
| setup-args=-Dblas=blis | ||
| setup-args=-Dlapack=semicolon-lapack | ||
| script: | | ||
| set -x | ||
| git clone https://github.com/hoodmane/f2c.git --depth 1 | ||
| (cd f2c/src && cp makefile.u makefile && sed -i "s/gram.c:/gram.c1:/" makefile && make) | ||
| export F2C_PATH=$(pwd)/f2c/src/f2c | ||
| echo F2C_PATH: $F2C_PATH | ||
| export NPY_BLAS_LIBS="-I$WASM_LIBRARY_DIR/include $WASM_LIBRARY_DIR/lib/libopenblas.so" | ||
| export NPY_LAPACK_LIBS="-I$WASM_LIBRARY_DIR/include $WASM_LIBRARY_DIR/lib/libopenblas.so" | ||
| sed -i 's/void DQA/int DQA/g' scipy/integrate/__quadpack.h | ||
| # Change many functions that return void into functions that return int | ||
| find scipy -name "*.c*" -type f | xargs sed -i 's/extern void F_FUNC/extern int F_FUNC/g' | ||
| sed -i 's/void F_FUNC/int F_FUNC/g' scipy/odr/__odrpack.c | ||
| sed -i 's/^void/int/g' scipy/odr/odrpack.h | ||
| sed -i 's/^void/int/g' scipy/odr/__odrpack.c | ||
| # sed -i 's/void F_FUNC/int F_FUNC/g' scipy/linalg/_lapack_subroutines.h | ||
| sed -i 's/void/int/g' scipy/linalg/cython_blas_signatures.txt | ||
| sed -i 's/void/int/g' scipy/linalg/cython_lapack_signatures.txt | ||
| sed -i 's/void BLAS_FUNC/int BLAS_FUNC/g' scipy/linalg/src/_common_array_utils.h | ||
| sed -i 's/^void BLAS_FUNC/int BLAS_FUNC/g' scipy/linalg/src/_common_array_utils.hh | ||
| # Change fortran functions called in C code to return int instead of void | ||
| # This adhoc regex checks function names ending with _ such as `void zgetrs_(` | ||
| sed -i 's/void BLAS_FUNC/int BLAS_FUNC/g' scipy/sparse/linalg/_propack/PROPACK/src/include/blaslapack_declarations.h | ||
| sed -i 's/void ARNAUD_BLAS/int ARNAUD_BLAS/g' scipy/sparse/linalg/_eigen/arpack/arnaud/src/blaslapack_declarations.h | ||
| sed -i 's/void BLAS_FUNC/int BLAS_FUNC/g' scipy/integrate/src/blaslapack_declarations.h | ||
| sed -i 's/void[[:space:]]*BLAS_FUNC/int BLAS_FUNC/g' scipy/optimize/src/blaslapack_declarations.h | ||
| sed -i 's/^void/int/g' scipy/interpolate/src/_fitpackmodule.c | ||
| sed -i 's/void BLAS_FUNC/int BLAS_FUNC/g' scipy/interpolate/src/__fitpack.h | ||
| sed -i 's/extern void/extern int/g' scipy/sparse/linalg/_dsolve/SuperLU/SRC/*.{c,h} | ||
| sed -i 's/PUBLIC void/PUBLIC int/g' scipy/sparse/linalg/_dsolve/SuperLU/SRC/*.{c,h} | ||
| sed -i 's/^void/int/g' scipy/sparse/linalg/_dsolve/SuperLU/SRC/*.{c,h} | ||
| sed -i 's/^void/int/g' scipy/sparse/linalg/_dsolve/*.{c,h} | ||
| sed -i 's/void \(.\)print/int \1/g' scipy/sparse/linalg/_dsolve/SuperLU/SRC/*.{c,h} | ||
| sed -i 's/TYPE_GENERIC_FUNC(\(.*\), void)/TYPE_GENERIC_FUNC(\1, int)/g' scipy/sparse/linalg/_dsolve/_superluobject.h | ||
| sed -i 's/^void/int/g' scipy/optimize/_trlib/trlib_private.h | ||
| sed -i 's/^void/int/g' scipy/_build_utils/src/wrap_dummy_g77_abi.c | ||
| sed -i 's/, int)/)/g' scipy/optimize/_trlib/trlib_private.h | ||
| sed -i 's/, 1)/)/g' scipy/optimize/_trlib/trlib_private.h | ||
| sed -i 's/^void/int/g' scipy/spatial/qhull_misc.h | ||
| sed -i 's/, size_t)/)/g' scipy/spatial/qhull_misc.h | ||
| sed -i 's/,1)/)/g' scipy/spatial/qhull_misc.h | ||
| # Input error causes "duplicate symbol" linker errors. Empty out the file. | ||
| echo "" > scipy/sparse/linalg/_dsolve/SuperLU/SRC/input_error.c | ||
| export PKG_CONFIG_PATH="${WASM_LIBRARY_DIR}/lib/pkgconfig:${WASM_LIBRARY_DIR}/share/pkgconfig:${PKG_CONFIG_PATH}" | ||
| # https://github.com/mesonbuild/meson/blob/e542901af6e30865715d3c3c18f703910a096ec0/mesonbuild/backend/ninjabackend.py#L94 | ||
| # Prevent from using response file. The response file that meson generates is not compatible to pyodide-build | ||
|
|
@@ -133,15 +85,11 @@ build: | |
| requirements: | ||
| host: | ||
| - numpy | ||
| - libopenblas | ||
| - libblis | ||
| - libsemilapack | ||
| - libboost | ||
| run: | ||
| - numpy | ||
| executable: | ||
| - gfortran | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeeees |
||
| constraint: | ||
| # Getting: Error: Dynamic linking error: cannot resolve symbol pow_di | ||
| - meson < 1.10 | ||
| test: | ||
| imports: | ||
| - scipy | ||
|
|
@@ -165,7 +113,6 @@ test: | |
| - scipy.linalg.interpolative | ||
| - scipy.misc | ||
| - scipy.ndimage | ||
| - scipy.odr | ||
| - scipy.optimize | ||
| - scipy.signal | ||
| - scipy.signal.windows | ||
|
|
||
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| From af8b0fc9a2164c29cf179e71f6e139406474b89c Mon Sep 17 00:00:00 2001 | ||
| From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> | ||
| Date: Fri, 10 Jul 2026 14:22:09 +0300 | ||
| Subject: [PATCH 2/2] Correct `?getrf`/`?getri` return types and `trlib` `dlagtm` ABI for strict-signature toolchains | ||
|
|
||
| From https://github.com/scipy/scipy/pull/25592. Drop this with SciPy v1.19 | ||
|
|
||
| --- | ||
| scipy/linalg/src/_common_array_utils.hh | 16 ++++++++-------- | ||
| scipy/optimize/_trlib/trlib_private.h | 6 ++++-- | ||
| 2 files changed, 12 insertions(+), 10 deletions(-) | ||
|
|
||
| diff --git a/scipy/linalg/src/_common_array_utils.hh b/scipy/linalg/src/_common_array_utils.hh | ||
| index 7b42d4ec18..e1fe742322 100644 | ||
| --- a/scipy/linalg/src/_common_array_utils.hh | ||
| +++ b/scipy/linalg/src/_common_array_utils.hh | ||
| @@ -15,38 +15,38 @@ | ||
| extern "C" { | ||
|
|
||
| /* ?GETRF */ | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(sgetrf)(CBLAS_INT *m, CBLAS_INT *n, float a[], CBLAS_INT *lda, | ||
| CBLAS_INT ipiv[], CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(dgetrf)(CBLAS_INT *m, CBLAS_INT *n, double a[], CBLAS_INT *lda, | ||
| CBLAS_INT ipiv[], CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(cgetrf)(CBLAS_INT *m, CBLAS_INT *n, npy_complex64 a[], CBLAS_INT *lda, | ||
| CBLAS_INT ipiv[], CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(zgetrf)(CBLAS_INT *m, CBLAS_INT *n, npy_complex128 a[], CBLAS_INT *lda, | ||
| CBLAS_INT ipiv[], CBLAS_INT *info | ||
| ); | ||
|
|
||
|
|
||
| /* ?GETRI */ | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(sgetri)(CBLAS_INT *n, float a[], CBLAS_INT *lda, CBLAS_INT ipiv[], | ||
| float work[], CBLAS_INT *lwork, CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(dgetri)(CBLAS_INT *n, double a[], CBLAS_INT *lda, CBLAS_INT ipiv[], | ||
| double work[], CBLAS_INT *lwork, CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(cgetri)(CBLAS_INT *n, npy_complex64 a[], CBLAS_INT *lda, CBLAS_INT ipiv[], | ||
| npy_complex64 work[], CBLAS_INT *lwork, CBLAS_INT *info | ||
| ); | ||
| -CBLAS_INT | ||
| +void | ||
| BLAS_FUNC(zgetri)(CBLAS_INT *n, npy_complex128 a[], CBLAS_INT *lda, CBLAS_INT ipiv[], | ||
| npy_complex128 work[], CBLAS_INT *lwork, CBLAS_INT *info | ||
| ); | ||
| diff --git a/scipy/optimize/_trlib/trlib_private.h b/scipy/optimize/_trlib/trlib_private.h | ||
| index 92bfabe51d..06abcc72f2 100644 | ||
| --- a/scipy/optimize/_trlib/trlib_private.h | ||
| +++ b/scipy/optimize/_trlib/trlib_private.h | ||
| @@ -49,7 +49,9 @@ double BLAS_FUNC(ddot)(CBLAS_INT *n, double *x, CBLAS_INT *incx, double *y, CBLA | ||
| void BLAS_FUNC(dpttrf)(CBLAS_INT *n, double *d, double *e, CBLAS_INT *info); | ||
| void BLAS_FUNC(dpttrs)(CBLAS_INT *n, CBLAS_INT *nrhs, double *d, double *e, double *b, CBLAS_INT *ldb, CBLAS_INT *info); | ||
| void BLAS_FUNC(dptrfs)(CBLAS_INT *n, CBLAS_INT *nrhs, double *d, double *e, double *df, double *ef, double *b, CBLAS_INT *ldb, double *x, CBLAS_INT *ldx, double *ferr, double *berr, double *work, CBLAS_INT *info); | ||
| -void BLAS_FUNC(dlagtm)(char *trans, CBLAS_INT *n, CBLAS_INT *nrhs, double *alpha, double *dl, double *d, double *du, double *x, CBLAS_INT *ldx, double *beta, double *b, CBLAS_INT *ldb, int); | ||
| +// dlagtm is declared without a trailing hidden Fortran | ||
| +// string length for trans. See gh-25592. | ||
| +void BLAS_FUNC(dlagtm)(char *trans, CBLAS_INT *n, CBLAS_INT *nrhs, double *alpha, double *dl, double *d, double *du, double *x, CBLAS_INT *ldx, double *beta, double *b, CBLAS_INT *ldb); | ||
|
|
||
|
|
||
| static void trlib_daxpy(trlib_int_t *n, double *alpha, double *x, trlib_int_t *incx, double *y, trlib_int_t *incy) | ||
| @@ -108,7 +110,7 @@ static void trlib_dlagtm(char *trans, trlib_int_t *n, trlib_int_t *nrhs, double | ||
| trlib_int_t *ldx, double *beta, double *b, trlib_int_t *ldb) | ||
| { | ||
| CBLAS_INT n_ = *n, nrhs_ = *nrhs, ldb_ = *ldb, ldx_ = *ldx; | ||
| - BLAS_FUNC(dlagtm)(trans, &n_, &nrhs_, alpha, dl, d, du, x, &ldx_, beta, b, &ldb_, 1); | ||
| + BLAS_FUNC(dlagtm)(trans, &n_, &nrhs_, alpha, dl, d, du, x, &ldx_, beta, b, &ldb_); | ||
| } | ||
|
|
||
|
|
||
| -- | ||
| 2.50.1 (Apple Git-155) | ||
|
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why do we need host meson? doesn't scipy install it in the isolated environment?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is needed for semicolon-lapack, actually, which uses Meson as its build system. It's just that we haven't had any system-level library recipes so far that were using Meson, and this is the first.
If it were a Python package, then yes, it would have been picked up as a build-time dependency.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I see. Thanks for the clarafication!