diff --git a/.gitmodules b/.gitmodules index a15df7e..527905f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,7 @@ [submodule "third_party/cpython"] path = third_party/cpython url = https://github.com/python/cpython.git +[submodule "third_party/lib-python-numpy"] + path = third_party/lib-python-numpy + url = https://github.com/unikraft/lib-python-numpy.git + branch = staging diff --git a/build_musl_cpython.rs b/build_musl_cpython.rs index 07ce181..98bad20 100644 --- a/build_musl_cpython.rs +++ b/build_musl_cpython.rs @@ -30,7 +30,21 @@ fn main() { .unwrap_or_else(|| cpython_dir.join("build-submodule").into_os_string()), ); let python_lib = python_lib_dir.join("libpython3.11.a"); + let numpy_enabled = env::var_os("CARGO_FEATURE_NUMPY").is_some(); + let numpy_dir = repo_root.join("target/numpy-musl"); + let numpy_lib_dir = numpy_dir.join("lib"); + let numpy_lib = numpy_lib_dir.join("libnumpy_musl.a"); + let cxx_runtime_dir = PathBuf::from( + env::var_os("CXX_MUSL_PREFIX") + .unwrap_or_else(|| repo_root.join("target/cxx-musl").into_os_string()), + ); + let cxx_runtime_lib_dir = cxx_runtime_dir.join("lib"); let source = crate_dir.join("function.c"); + let datetime_source = repo_root.join("third_party/cpython/Modules/_datetimemodule.c"); + let math_source = repo_root.join("third_party/cpython/Modules/mathmodule.c"); + let struct_source = repo_root.join("third_party/cpython/Modules/_struct.c"); + let contextvars_source = repo_root.join("third_party/cpython/Modules/_contextvarsmodule.c"); + let binascii_source = repo_root.join("third_party/cpython/Modules/binascii.c"); assert!( musl_lib.is_file(), @@ -41,6 +55,13 @@ fn main() { python_include.join("Python.h").is_file() && python_lib.is_file(), "missing musl CPython; run `just musl_cpython`" ); + if numpy_enabled { + assert!( + numpy_lib.is_file(), + "missing {}; run `python3 scripts/build_numpy_musl.py --archive-only` first", + numpy_lib.display() + ); + } let cc = env::var_os("CC").unwrap_or_else(|| "cc".into()); let gcc_include = Command::new(&cc) @@ -54,48 +75,135 @@ fn main() { .to_owned(); let object = out_dir.join("function.o"); + let mut common_compile_args = vec![ + "-std=c11".into(), + "-O2".into(), + "-fPIC".into(), + "-ffreestanding".into(), + "-fno-stack-protector".into(), + "-fno-builtin".into(), + "-nostdinc".into(), + "-D_GNU_SOURCE".into(), + "-D_POSIX_C_SOURCE=200809L".into(), + ]; + if numpy_enabled { + common_compile_args.push("-DALLOY_ENABLE_NUMPY".into()); + } + common_compile_args.extend([ + "-isystem".into(), + musl_dir.join("include").into_os_string(), + "-I".into(), + repo_root.join("as_musl/include").into_os_string(), + "-isystem".into(), + python_config_include.clone().into_os_string(), + "-isystem".into(), + python_include.clone().into_os_string(), + "-isystem".into(), + python_include.join("internal").into_os_string(), + "-isystem".into(), + gcc_include.into(), + ]); + run(Command::new(&cc) - .arg("-std=c11") - .arg("-O2") - .arg("-fPIC") - .arg("-ffreestanding") - .arg("-fno-stack-protector") - .arg("-fno-builtin") - .arg("-nostdinc") - .arg("-D_GNU_SOURCE") - .arg("-D_POSIX_C_SOURCE=200809L") - .arg("-isystem") - .arg(musl_dir.join("include")) - .arg("-I") - .arg(repo_root.join("as_musl/include")) - .arg("-isystem") - .arg(&python_config_include) - .arg("-isystem") - .arg(&python_include) - .arg("-isystem") - .arg(gcc_include) + .args(&common_compile_args) .arg("-Dmain=alloy_c_main") .arg("-c") .arg(&source) .arg("-o") .arg(&object)); + let datetime_object = out_dir.join("_datetimemodule.o"); + let math_object = out_dir.join("mathmodule.o"); + let struct_object = out_dir.join("_struct.o"); + let contextvars_object = out_dir.join("_contextvarsmodule.o"); + let binascii_object = out_dir.join("binascii.o"); + if numpy_enabled { + run(Command::new(&cc) + .args(&common_compile_args) + .arg("-DPy_BUILD_CORE_MODULE") + .arg("-c") + .arg(&datetime_source) + .arg("-o") + .arg(&datetime_object)); + run(Command::new(&cc) + .args(&common_compile_args) + .arg("-DPy_BUILD_CORE_MODULE") + .arg("-c") + .arg(&math_source) + .arg("-o") + .arg(&math_object)); + run(Command::new(&cc) + .args(&common_compile_args) + .arg("-DPy_BUILD_CORE_MODULE") + .arg("-c") + .arg(&struct_source) + .arg("-o") + .arg(&struct_object)); + run(Command::new(&cc) + .args(&common_compile_args) + .arg("-DPy_BUILD_CORE_MODULE") + .arg("-c") + .arg(&contextvars_source) + .arg("-o") + .arg(&contextvars_object)); + run(Command::new(&cc) + .args(&common_compile_args) + .arg("-DPy_BUILD_CORE_MODULE") + .arg("-c") + .arg(&binascii_source) + .arg("-o") + .arg(&binascii_object)); + } + let app_archive = out_dir.join("liballoy_cpython_app.a"); let ar = env::var_os("AR").unwrap_or_else(|| "ar".into()); - run(Command::new(ar).arg("rcs").arg(&app_archive).arg(&object)); + let mut ar_command = Command::new(ar); + ar_command.arg("rcs").arg(&app_archive).arg(&object); + if numpy_enabled { + ar_command + .arg(&datetime_object) + .arg(&math_object) + .arg(&struct_object) + .arg(&contextvars_object) + .arg(&binascii_object); + } + run(&mut ar_command); println!("cargo:rerun-if-changed={}", source.display()); + println!("cargo:rerun-if-changed={}", datetime_source.display()); + println!("cargo:rerun-if-changed={}", math_source.display()); + println!("cargo:rerun-if-changed={}", struct_source.display()); + println!("cargo:rerun-if-changed={}", contextvars_source.display()); + println!("cargo:rerun-if-changed={}", binascii_source.display()); println!("cargo:rerun-if-changed={}", musl_lib.display()); println!("cargo:rerun-if-env-changed=PYTHON_INCLUDE"); println!("cargo:rerun-if-env-changed=PYTHON_CONFIG_INCLUDE"); println!("cargo:rerun-if-env-changed=PYTHON_LIB_DIR"); + println!("cargo:rerun-if-changed={}", numpy_lib.display()); println!("cargo:rustc-link-search=native={}", out_dir.display()); + if numpy_enabled { + println!("cargo:rustc-link-search=native={}", numpy_lib_dir.display()); + if cxx_runtime_lib_dir.is_dir() { + println!( + "cargo:rustc-link-search=native={}", + cxx_runtime_lib_dir.display() + ); + } + } println!("cargo:rustc-link-search=native={}", python_lib_dir.display()); println!( "cargo:rustc-link-search=native={}", musl_dir.join("lib").display() ); println!("cargo:rustc-link-lib=static=alloy_cpython_app"); + if numpy_enabled { + println!("cargo:rustc-link-lib=static=numpy_musl"); + for lib in ["c++", "c++abi", "unwind", "stdc++", "supc++", "gcc_eh"] { + if cxx_runtime_lib_dir.join(format!("lib{lib}.a")).is_file() { + println!("cargo:rustc-link-lib=static={lib}"); + } + } + } println!("cargo:rustc-link-lib=static=python3.11"); println!("cargo:rustc-link-lib=static=musl_alloy"); println!("cargo:rustc-link-arg=-Wl,-Bsymbolic-functions"); @@ -111,7 +219,13 @@ fn main() { let target_library = target_dir.join("libmusl_cpython.so"); match fs::symlink_metadata(&target_library) { - Ok(metadata) if metadata.file_type().is_symlink() => {} + Ok(metadata) if metadata.file_type().is_symlink() => { + if fs::read_link(&target_library).unwrap() != source_library { + fs::remove_file(&target_library).unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(&source_library, &target_library).unwrap(); + } + } Ok(_) => panic!("{} exists and is not a symlink", target_library.display()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => { #[cfg(unix)] diff --git a/isol_config/musl_cpython_numpy.json b/isol_config/musl_cpython_numpy.json new file mode 100644 index 0000000..f53f9f8 --- /dev/null +++ b/isol_config/musl_cpython_numpy.json @@ -0,0 +1,22 @@ +{ + "services": [ + ["sys", "libsys.so"], + ["time", "libtime.so"], + ["stdio", "libstdio.so"], + ["mm", "libmm.so"], + ["fdtab", "libfdtab.so"], + ["fatfs", "libfatfs.so"] + ], + "apps": [ + ["numpy_smoke", "libmusl_cpython.so"] + ], + "fs_image": "fs_images/fatfs_numpy.img", + "groups": [ + { + "list": ["numpy_smoke"], + "args": { + "pyfile_path": "/wasm_bench/numpy_smoke.py" + } + } + ] +} diff --git a/justfile b/justfile index 3086af3..373285a 100644 --- a/justfile +++ b/justfile @@ -16,6 +16,7 @@ mpk_flag := if enable_mpk == "1" { } else { "" } mpk_feature_flag := if mpk_flag == "" { "" } else { "--features " + mpk_flag } +musl_cpython_numpy_feature_flag := "--features " + if mpk_flag == "" { "numpy" } else { mpk_flag + ",numpy" } buffer_feature_flag := if enable_file_buffer == "1" { "--features file-based" } else { "" } @@ -139,6 +140,29 @@ musl_cpython: musl cargo build {{ release_flag }} {{ mpk_feature_flag }} --manifest-path user/musl_cpython/Cargo.toml for id in 0 1 2 3 4; do cp -L target/{{profile}}/libmusl_cpython.so target/{{profile}}/libmusl_cpython_${id}.so; done +cxx_musl: + python3 scripts/build_cxx_musl.py + +musl_cpython_numpy: musl cxx_musl + bash scripts/build_cpython_musl.sh + python3 scripts/build_numpy_musl.py --archive-only + cargo build {{ release_flag }} {{ musl_cpython_numpy_feature_flag }} --manifest-path user/musl_cpython/Cargo.toml + for id in 0 1 2 3 4; do cp -L target/{{profile}}/libmusl_cpython.so target/{{profile}}/libmusl_cpython_${id}.so; done + +musl_cpython_numpy_minimal: musl + bash scripts/build_cpython_musl.sh + python3 scripts/build_numpy_musl.py --minimal --archive-only + cargo build {{ release_flag }} {{ musl_cpython_numpy_feature_flag }} --manifest-path user/musl_cpython/Cargo.toml + for id in 0 1 2 3 4; do cp -L target/{{profile}}/libmusl_cpython.so target/{{profile}}/libmusl_cpython_${id}.so; done + +numpy_smoke: asvisor all_libos musl_cpython_numpy + NUMPY_MODE=full ./scripts/sync_numpy_workloads.sh + target/{{profile}}/asvisor --files isol_config/musl_cpython_numpy.json + +numpy_smoke_minimal: asvisor all_libos musl_cpython_numpy_minimal + NUMPY_MODE=minimal ./scripts/sync_numpy_workloads.sh + target/{{profile}}/asvisor --files isol_config/musl_cpython_numpy.json + all_musl_c: musl_wordcount musl_parallel_sort musl_long_chain musl_trans_data all_py_musl: musl_cpython diff --git a/scripts/build_cpython_musl.sh b/scripts/build_cpython_musl.sh index 592e962..9771206 100644 --- a/scripts/build_cpython_musl.sh +++ b/scripts/build_cpython_musl.sh @@ -7,6 +7,27 @@ TOOLCHAIN="$CACHE/toolchain" SOURCE="$ROOT/third_party/cpython" BUILD="$CACHE/build-submodule" CONFIG_SITE_FILE="$CACHE/config.site" +CONFIG_SIGNATURE_FILE="$CACHE/config.signature" +LOG_DIR="$ROOT/target/logs" +LOG_FILE="$LOG_DIR/build_cpython_musl.log" + +mkdir -p "$LOG_DIR" +: >"$LOG_FILE" + +log() { + printf '[%(%Y-%m-%dT%H:%M:%S%z)T] %s\n' -1 "$*" >>"$LOG_FILE" +} + +fail_with_log() { + echo "$*" >&2 + echo "see $LOG_FILE for details" >&2 + exit 1 +} + +run_logged() { + log "+ $*" + "$@" >>"$LOG_FILE" 2>&1 || fail_with_log "command failed: $*" +} resolve_build_python() { local candidate="" @@ -45,13 +66,22 @@ if [[ ! -f "$SOURCE/Include/Python.h" ]]; then exit 1 fi +rebuild_toolchain=0 if [[ ! -x "$TOOLCHAIN/bin/musl-gcc" ]]; then + rebuild_toolchain=1 +elif ! grep -Fq "$TOOLCHAIN/lib/musl-gcc.specs" "$TOOLCHAIN/bin/musl-gcc"; then + rebuild_toolchain=1 +fi + +if [[ "$rebuild_toolchain" == "1" ]]; then + echo "building musl CPython toolchain; log: $LOG_FILE" + rm -rf "$CACHE/musl-build" "$TOOLCHAIN" mkdir -p "$CACHE/musl-build" "$TOOLCHAIN" ( cd "$CACHE/musl-build" - "$ROOT/third_party/musl/configure" --prefix="$TOOLCHAIN" - make -j"${JOBS:-$(nproc)}" - make install + run_logged "$ROOT/third_party/musl/configure" --prefix="$TOOLCHAIN" + run_logged make -j"${JOBS:-$(nproc)}" + run_logged make install ) fi @@ -71,25 +101,64 @@ ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no EOF -rm -rf "$BUILD" -mkdir -p "$BUILD" -( - cd "$BUILD" - BUILD_PYTHON_PATH="$(resolve_build_python)" - PATH="$TOOLCHAIN/bin:$PATH" \ - CONFIG_SITE="$CONFIG_SITE_FILE" \ - CC="$TOOLCHAIN/bin/musl-gcc" \ - CFLAGS="$MUSL_CFLAGS" \ - "$SOURCE/configure" \ - --host=x86_64-linux-musl \ - --build=x86_64-pc-linux-gnu \ - --with-build-python="$BUILD_PYTHON_PATH" \ - --disable-shared \ - --disable-ipv6 \ - --without-ensurepip +BUILD_PYTHON_PATH="$(resolve_build_python)" +CONFIGURE_ARGS=( + --host=x86_64-linux-musl + --build=x86_64-pc-linux-gnu + --with-build-python="$BUILD_PYTHON_PATH" + --disable-shared + --disable-ipv6 + --without-ensurepip ) -PATH="$TOOLCHAIN/bin:$PATH" \ +NEW_CONFIG_SIGNATURE="$( + { + printf 'source=%s\n' "$SOURCE" + printf 'build_python=%s\n' "$BUILD_PYTHON_PATH" + printf 'cc=%s\n' "$TOOLCHAIN/bin/musl-gcc" + printf 'cflags=%s\n' "$MUSL_CFLAGS" + printf 'configure_args=' + printf '%q ' "${CONFIGURE_ARGS[@]}" + printf '\n' + printf 'config_site:\n' + cat "$CONFIG_SITE_FILE" + } +)" + +need_configure=0 +if [[ "${CLEAN_CPYTHON_MUSL:-0}" == "1" ]]; then + rm -rf "$BUILD" + need_configure=1 +elif [[ ! -x "$BUILD/config.status" ]]; then + need_configure=1 +elif [[ ! -f "$CONFIG_SIGNATURE_FILE" ]]; then + need_configure=1 +elif [[ "$(cat "$CONFIG_SIGNATURE_FILE")" != "$NEW_CONFIG_SIGNATURE" ]]; then + need_configure=1 +fi + +mkdir -p "$BUILD" + +echo "building musl CPython incrementally; log: $LOG_FILE" + +if [[ "$need_configure" == "1" ]]; then + log "configure required" + ( + cd "$BUILD" + log "+ $SOURCE/configure ${CONFIGURE_ARGS[*]}" + PATH="$TOOLCHAIN/bin:$PATH" \ + CONFIG_SITE="$CONFIG_SITE_FILE" \ + CC="$TOOLCHAIN/bin/musl-gcc" \ + CFLAGS="$MUSL_CFLAGS" \ + "$SOURCE/configure" "${CONFIGURE_ARGS[@]}" >>"$LOG_FILE" 2>&1 + ) || fail_with_log "CPython configure failed" + printf '%s\n' "$NEW_CONFIG_SIGNATURE" >"$CONFIG_SIGNATURE_FILE" +else + log "reuse existing CPython configure result" +fi + +run_logged env \ + PATH="$TOOLCHAIN/bin:$PATH" \ CONFIG_SITE="$CONFIG_SITE_FILE" \ make -C "$BUILD" -j"${JOBS:-$(nproc)}" libpython3.11.a echo "built $BUILD/libpython3.11.a" diff --git a/scripts/build_cxx_musl.py b/scripts/build_cxx_musl.py new file mode 100644 index 0000000..2f0b8ab --- /dev/null +++ b/scripts/build_cxx_musl.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +"""Prepare a musl-compatible C++ runtime for AlloyStack. + +NumPy contains C++ extension sources. Compiling them against AlloyStack's +musl headers cannot use the host glibc libstdc++ installation: those headers +are configured for glibc and pull in glibc-only locale/configuration symbols. + +This script installs or builds a small, explicit C++ runtime prefix at +target/cxx-musl. Downstream builds should consume only this prefix for C++ +headers and static libraries. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from pathlib import Path + + +TARGET_TRIPLE = "x86_64-linux-musl" +MUSLCC_ARCHIVE = f"{TARGET_TRIPLE}-cross.tgz" +MUSLCC_URL = f"https://musl.cc/{MUSLCC_ARCHIVE}" +MUSLCC_SHA512SUMS_URL = "https://musl.cc/SHA512SUMS" + + +def run(command: list[str], cwd: Path | None = None) -> None: + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=cwd, check=True) + + +def remove_path(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +def copy_or_link(src: Path, dst: Path, symlink: bool) -> None: + if dst.exists() or dst.is_symlink(): + remove_path(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + if symlink: + dst.symlink_to(src) + elif src.is_dir(): + shutil.copytree(src, dst) + else: + shutil.copy2(src, dst) + + +def download(url: str, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + return + print(f"downloading {url}", flush=True) + tmp = destination.with_suffix(destination.suffix + ".tmp") + if tmp.exists(): + tmp.unlink() + urllib.request.urlretrieve(url, tmp) + tmp.rename(destination) + + +def sha512(path: Path) -> str: + digest = hashlib.sha512() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_muslcc_archive(archive: Path, sums_file: Path) -> None: + expected = None + for line in sums_file.read_text().splitlines(): + parts = line.split() + if len(parts) >= 2 and Path(parts[-1]).name == archive.name: + expected = parts[0] + break + if expected is None: + raise SystemExit(f"missing {archive.name} entry in {sums_file}") + actual = sha512(archive) + if actual.lower() != expected.lower(): + raise SystemExit(f"SHA512 mismatch for {archive}: expected {expected}, got {actual}") + + +def safe_extract_tar(archive: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + root = destination.resolve() + with tarfile.open(archive) as tar: + for member in tar.getmembers(): + target = (destination / member.name).resolve() + if root != target and root not in target.parents: + raise SystemExit(f"unsafe tar entry outside destination: {member.name}") + tar.extractall(destination, filter="fully_trusted") + + +def candidate_lib_dirs(root: Path) -> list[Path]: + return [ + root / "lib", + root / "lib64", + root / TARGET_TRIPLE / "lib", + root / "usr" / "lib", + root / "usr" / "lib64", + root / "usr" / TARGET_TRIPLE / "lib", + ] + + +def find_existing(paths: list[Path]) -> Path | None: + for path in paths: + if path.exists(): + return path + return None + + +def detect_runtime(sysroot: Path) -> tuple[str, list[Path], Path, list[str]]: + libcxx_include = find_existing( + [ + sysroot / "include" / "c++" / "v1", + sysroot / "usr" / "include" / "c++" / "v1", + sysroot / TARGET_TRIPLE / "include" / "c++" / "v1", + ] + ) + lib_dirs = [path for path in candidate_lib_dirs(sysroot) if path.is_dir()] + + if libcxx_include is not None: + lib_dir = find_existing([path for path in lib_dirs if (path / "libc++.a").is_file()]) + if lib_dir is None: + raise SystemExit(f"found libc++ headers at {libcxx_include}, but no libc++.a") + libs = ["c++"] + for optional in ["c++abi", "unwind"]: + if (lib_dir / f"lib{optional}.a").is_file(): + libs.append(optional) + return "libc++", [libcxx_include], lib_dir, libs + + stdcxx_roots = [] + for base in [ + sysroot / "include" / "c++", + sysroot / "usr" / "include" / "c++", + sysroot / TARGET_TRIPLE / "include" / "c++", + ]: + if base.is_dir(): + stdcxx_roots.extend(path for path in base.iterdir() if path.is_dir()) + stdcxx_roots = sorted(stdcxx_roots, reverse=True) + for include_root in stdcxx_roots: + target_include = include_root / TARGET_TRIPLE + if not target_include.is_dir(): + # A host glibc libstdc++ install usually has + # x86_64-*-linux-gnu here. Reject it before it can be imported + # into target/cxx-musl. + continue + lib_dir = find_existing([path for path in lib_dirs if (path / "libstdc++.a").is_file()]) + if lib_dir is None: + continue + include_dirs = [include_root] + include_dirs.append(target_include) + backward = include_root / "backward" + if backward.is_dir(): + include_dirs.append(backward) + libs = ["stdc++"] + if (lib_dir / "libsupc++.a").is_file(): + libs.append("supc++") + if (lib_dir / "libgcc_eh.a").is_file(): + libs.append("gcc_eh") + return "libstdc++", include_dirs, lib_dir, libs + + raise SystemExit( + f"{sysroot} does not look like a musl C++ sysroot. Expected " + "include/c++/v1 + libc++.a, or include/c++/ + libstdc++.a." + ) + + +def write_env(prefix: Path, kind: str, include_dirs: list[Path], lib_dir: Path, libs: list[str]) -> None: + env_file = prefix / "cxx-runtime.env" + include_flags = " ".join(f"-isystem {path}" for path in include_dirs) + link_flags = " ".join(f"-L {prefix / 'lib'}" for _ in [0]) + " " + " ".join( + f"-l{lib}" for lib in libs + ) + env_file.write_text( + "\n".join( + [ + f"CXX_RUNTIME_KIND={kind}", + f"CXX_INCLUDE_FLAGS=-nostdinc++ {include_flags}", + f"CXX_LINK_FLAGS={link_flags.strip()}", + f"CXX_LINK_LIBS={' '.join(libs)}", + f"CXX_RUNTIME_SOURCE_LIB_DIR={lib_dir}", + "", + ] + ) + ) + + +def install_from_sysroot(repo_root: Path, sysroot: Path, prefix: Path, symlink: bool) -> None: + kind, include_dirs, lib_dir, libs = detect_runtime(sysroot) + if prefix.exists() and not prefix.is_dir(): + raise SystemExit(f"{prefix} exists and is not a directory") + (prefix / "include" / "c++").mkdir(parents=True, exist_ok=True) + (prefix / "lib").mkdir(parents=True, exist_ok=True) + (prefix / "bin").mkdir(parents=True, exist_ok=True) + + if kind == "libc++": + copy_or_link(include_dirs[0], prefix / "include" / "c++" / "v1", symlink) + else: + version = include_dirs[0].name + # GCC's target and backward include directories live below the version + # root. Import the version root once; do not create children under a + # symlinked destination. + copy_or_link(include_dirs[0], prefix / "include" / "c++" / version, symlink) + + for lib in libs: + src = lib_dir / f"lib{lib}.a" + if not src.is_file(): + raise SystemExit(f"missing {src}") + copy_or_link(src, prefix / "lib" / src.name, symlink) + + write_env(prefix, kind, [prefix / "include" / "c++" / "v1"] if kind == "libc++" else [ + prefix / "include" / "c++" / include_dirs[0].name, + *[ + prefix / "include" / "c++" / include_dirs[0].name / include_dir.name + for include_dir in include_dirs[1:] + ], + ], prefix / "lib", libs) + write_cxx_wrapper(repo_root, prefix) + check_prefix(repo_root, prefix) + print(f"installed {kind} runtime into {prefix}", flush=True) + + +def build_from_llvm(repo_root: Path, llvm_project: Path, prefix: Path, clean: bool) -> None: + runtimes = llvm_project / "runtimes" + if not (runtimes / "CMakeLists.txt").is_file(): + raise SystemExit(f"{llvm_project} is not an llvm-project checkout with runtimes/CMakeLists.txt") + + toolchain = repo_root / "target" / "cpython-musl" / "toolchain" + specs = toolchain / "lib" / "musl-gcc.specs" + if not specs.is_file(): + raise SystemExit("missing musl toolchain; run `bash scripts/build_cpython_musl.sh` first") + + build_dir = repo_root / "target" / "cxx-musl-build" + if clean and build_dir.exists(): + shutil.rmtree(build_dir) + build_dir.mkdir(parents=True, exist_ok=True) + + real_cc = os.environ.get("REALCC", shutil.which("gcc") or "cc") + real_cxx = os.environ.get("REALGXX", shutil.which("g++") or "c++") + common_flags = f"-O2 -fPIC -specs {specs}" + cxx_flags = ( + f"{common_flags} -nostdinc++ " + f"-I{llvm_project / 'libcxx' / 'include'} " + f"-I{llvm_project / 'libcxxabi' / 'include'} " + f"-I{llvm_project / 'libunwind' / 'include'}" + ) + + cmake = os.environ.get("CMAKE", "cmake") + build_tool = os.environ.get("CMAKE_BUILD_TOOL", "ninja") + run( + [ + cmake, + "-S", + str(runtimes), + "-B", + str(build_dir), + "-G", + "Ninja", + f"-DCMAKE_BUILD_TOOL={build_tool}", + f"-DCMAKE_INSTALL_PREFIX={prefix}", + "-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY", + f"-DCMAKE_C_COMPILER={real_cc}", + f"-DCMAKE_CXX_COMPILER={real_cxx}", + f"-DCMAKE_C_FLAGS={common_flags}", + f"-DCMAKE_CXX_FLAGS={cxx_flags}", + f"-DLLVM_ENABLE_RUNTIMES=libcxx;libcxxabi;libunwind", + "-DLIBCXX_ENABLE_SHARED=OFF", + "-DLIBCXX_ENABLE_STATIC=ON", + "-DLIBCXX_ENABLE_EXCEPTIONS=OFF", + "-DLIBCXX_ENABLE_RTTI=OFF", + "-DLIBCXX_ENABLE_FILESYSTEM=OFF", + "-DLIBCXX_ENABLE_LOCALIZATION=OFF", + "-DLIBCXX_ENABLE_WIDE_CHARACTERS=ON", + "-DLIBCXX_ENABLE_THREADS=ON", + "-DLIBCXX_HAS_MUSL_LIBC=ON", + "-DLIBCXXABI_ENABLE_SHARED=OFF", + "-DLIBCXXABI_ENABLE_STATIC=ON", + "-DLIBCXXABI_ENABLE_EXCEPTIONS=OFF", + "-DLIBCXXABI_USE_LLVM_UNWINDER=ON", + "-DLIBUNWIND_ENABLE_SHARED=OFF", + "-DLIBUNWIND_ENABLE_STATIC=ON", + ] + ) + run([cmake, "--build", str(build_dir), "--target", "install", "-j", os.environ.get("JOBS", str(os.cpu_count() or 1))]) + write_env(prefix, "libc++", [prefix / "include" / "c++" / "v1"], prefix / "lib", ["c++", "c++abi", "unwind"]) + write_cxx_wrapper(repo_root, prefix) + check_prefix(repo_root, prefix) + print(f"built libc++ runtime into {prefix}", flush=True) + + +def install_from_muslcc(repo_root: Path, prefix: Path, clean: bool) -> None: + cache = repo_root / "target" / "cxx-musl-downloads" + archive = cache / MUSLCC_ARCHIVE + sums = cache / "SHA512SUMS" + extract_root = repo_root / "target" / "cxx-musl-toolchain" + toolchain = extract_root / f"{TARGET_TRIPLE}-cross" + + if clean: + for path in [archive, sums, extract_root, prefix]: + if path.exists() or path.is_symlink(): + remove_path(path) + + download(MUSLCC_URL, archive) + download(MUSLCC_SHA512SUMS_URL, sums) + verify_muslcc_archive(archive, sums) + + toolchain_ready = ( + (toolchain / "bin" / f"{TARGET_TRIPLE}-g++").is_file() + and (toolchain / TARGET_TRIPLE / "include" / "c++").is_dir() + ) + if not toolchain_ready: + if extract_root.exists(): + shutil.rmtree(extract_root) + safe_extract_tar(archive, extract_root) + + install_from_sysroot(repo_root, toolchain, prefix, symlink=True) + + +def read_env(prefix: Path) -> dict[str, str]: + env_file = prefix / "cxx-runtime.env" + values: dict[str, str] = {} + if env_file.is_file(): + for line in env_file.read_text().splitlines(): + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + values[key] = value + return values + + +def include_flags(prefix: Path) -> list[str]: + env_values = read_env(prefix) + raw = env_values.get("CXX_INCLUDE_FLAGS") + if raw: + return raw.split() + if (prefix / "include" / "c++" / "v1").is_dir(): + return ["-nostdinc++", "-isystem", str(prefix / "include" / "c++" / "v1")] + roots = sorted((prefix / "include" / "c++").glob("*"), reverse=True) + for root in roots: + if not root.is_dir(): + continue + flags = ["-nostdinc++", "-isystem", str(root)] + target = root / TARGET_TRIPLE + if target.is_dir(): + flags.extend(["-isystem", str(target)]) + backward = root / "backward" + if backward.is_dir(): + flags.extend(["-isystem", str(backward)]) + return flags + raise SystemExit(f"missing C++ headers under {prefix}/include/c++") + + +def write_cxx_wrapper(repo_root: Path, prefix: Path) -> None: + wrapper = prefix / "bin" / "musl-g++" + specs = repo_root / "target" / "cpython-musl" / "toolchain" / "lib" / "musl-gcc.specs" + flags = " ".join(include_flags(prefix)) + wrapper.write_text( + "#!/bin/sh\n" + f'exec "${{REALGXX:-g++}}" "$@" -specs "{specs}" {flags} -L "{prefix / "lib"}"\n' + ) + wrapper.chmod(0o755) + + +def check_prefix(repo_root: Path, prefix: Path) -> None: + lib_dir = prefix / "lib" + if not lib_dir.is_dir(): + raise SystemExit(f"missing {lib_dir}") + if not any((lib_dir / name).is_file() for name in ["libc++.a", "libstdc++.a"]): + raise SystemExit(f"missing libc++.a or libstdc++.a in {lib_dir}") + + build_dir = repo_root / "target" / "cxx-musl-check" + build_dir.mkdir(parents=True, exist_ok=True) + source = build_dir / "check.cpp" + obj = build_dir / "check.o" + source.write_text( + "#include \n" + "#include \n" + "static std::array v = {1, 2};\n" + "int alloy_cxx_check(void) { return v[0] + (int)std::sqrt(4.0); }\n" + ) + cxx = os.environ.get("CXX", str(prefix / "bin" / "musl-g++")) + command = [cxx, *include_flags(prefix), "-O2", "-fPIC", "-c", str(source), "-o", str(obj)] + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode != 0: + sys.stderr.write("+ " + " ".join(command) + "\n") + sys.stderr.write(result.stdout[-4000:]) + sys.stderr.write(result.stderr[-4000:]) + raise SystemExit("C++ runtime preflight failed") + + +def probe_sysroots() -> list[Path]: + candidates = [ + Path("/usr") / TARGET_TRIPLE, + Path("/usr/local") / TARGET_TRIPLE, + Path("/opt") / TARGET_TRIPLE, + Path("/opt/musl"), + ] + gxx = shutil.which(f"{TARGET_TRIPLE}-g++") or shutil.which("musl-g++") + if gxx: + # Usually .../bin/x86_64-linux-musl-g++. + candidates.insert(0, Path(gxx).resolve().parents[1]) + return candidates + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--prefix", type=Path, default=None) + parser.add_argument("--from-sysroot", type=Path, default=None) + parser.add_argument("--llvm-project", type=Path, default=None) + parser.add_argument( + "--download-muslcc", + action=argparse.BooleanOptionalAction, + default=True, + help="download https://musl.cc/x86_64-linux-musl-cross.tgz if no local runtime is found", + ) + parser.add_argument("--check", action="store_true") + parser.add_argument("--clean", action="store_true") + parser.add_argument("--copy", action="store_true", help="copy files instead of symlinking for --from-sysroot") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + prefix = args.prefix or repo_root / "target" / "cxx-musl" + + if args.check: + check_prefix(repo_root, prefix) + print(f"ok: {prefix}", flush=True) + return 0 + + if args.llvm_project is not None: + build_from_llvm(repo_root, args.llvm_project.resolve(), prefix, args.clean) + return 0 + + sysroot = args.from_sysroot + if sysroot is None: + for candidate in probe_sysroots(): + try: + detect_runtime(candidate) + except SystemExit: + continue + sysroot = candidate + break + if sysroot is None: + if args.download_muslcc: + install_from_muslcc(repo_root, prefix, args.clean) + return 0 + raise SystemExit( + "no musl C++ runtime found. Provide one with " + "`python3 scripts/build_cxx_musl.py --from-sysroot /path/to/x86_64-linux-musl`, " + "or build LLVM runtimes with " + "`python3 scripts/build_cxx_musl.py --llvm-project third_party/llvm-project`." + ) + + install_from_sysroot(repo_root, sysroot.resolve(), prefix, symlink=not args.copy) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_musl.sh b/scripts/build_musl.sh index e6b9fd0..c558cd1 100644 --- a/scripts/build_musl.sh +++ b/scripts/build_musl.sh @@ -7,28 +7,50 @@ source_dir="$repo_root/target/musl-alloy-src" build_dir="$repo_root/target/musl-alloy-build" install_dir="$repo_root/target/musl-alloy" patch_dir="$repo_root/musl/patches" +log_dir="$repo_root/target/logs" +log_file="$log_dir/build_musl.log" + +mkdir -p "$log_dir" +: >"$log_file" + +log() { + printf '[%(%Y-%m-%dT%H:%M:%S%z)T] %s\n' -1 "$*" >>"$log_file" +} + +fail_with_log() { + echo "$*" >&2 + echo "see $log_file for details" >&2 + exit 1 +} + +run_logged() { + log "+ $*" + "$@" >>"$log_file" 2>&1 || fail_with_log "command failed: $*" +} if [[ ! -f "$upstream/VERSION" ]]; then echo "missing musl submodule; run: git submodule update --init third_party/musl" >&2 exit 1 fi +echo "building musl alloy; log: $log_file" + rm -rf "$source_dir" "$build_dir" "$install_dir" mkdir -p "$source_dir" "$build_dir" "$install_dir/lib" cp -a "$upstream/." "$source_dir/" for patch_file in "$patch_dir"/*.patch; do - patch -d "$source_dir" -p1 < "$patch_file" + run_logged patch -d "$source_dir" -p1 -i "$patch_file" done cd "$build_dir" -"$source_dir/configure" \ +run_logged "$source_dir/configure" \ --prefix="$install_dir" \ --syslibdir="$install_dir/lib" \ CC="${CC:-cc}" \ CFLAGS="-O2 -fPIC -ftls-model=global-dynamic" -make -s -j"${JOBS:-$(nproc)}" lib/libc-alloy.a install-headers +run_logged make -s -j"${JOBS:-$(nproc)}" lib/libc-alloy.a install-headers cp lib/libc-alloy.a "$install_dir/lib/libmusl_alloy.a" echo "built $install_dir/lib/libmusl_alloy.a" diff --git a/scripts/build_numpy_musl.py b/scripts/build_numpy_musl.py new file mode 100755 index 0000000..f23ad6d --- /dev/null +++ b/scripts/build_numpy_musl.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 +"""Build NumPy as built-in CPython extension modules for musl CPython. + +This follows the Unikraft lib-python-numpy approach: NumPy C extensions are +compiled into a static archive and imported through flat built-in module names +used by the importfix Python shims. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import datetime +import os +import re +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from pathlib import Path + + +NUMPY_VERSION = "1.25.0" +NUMPY_URL = ( + f"https://github.com/numpy/numpy/releases/download/" + f"v{NUMPY_VERSION}/numpy-{NUMPY_VERSION}.tar.gz" +) + +LOG_FILE: Path | None = None + + +def init_log(repo_root: Path) -> None: + global LOG_FILE + log_dir = repo_root / "target" / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + LOG_FILE = log_dir / "build_numpy_musl.log" + LOG_FILE.write_text( + f"[{datetime.datetime.now().isoformat(timespec='seconds')}] " + "start NumPy musl build\n" + ) + + +def log(message: str) -> None: + if LOG_FILE is None: + return + with LOG_FILE.open("a", encoding="utf-8") as fh: + fh.write(message) + if not message.endswith("\n"): + fh.write("\n") + +BUILTIN_MODULES = [ + ("numpy_core__multiarray_umath", "PyInit__multiarray_umath"), + ("numpy_core__multiarray_tests", "PyInit__multiarray_tests"), + ("numpy_core__operand_flag_tests", "PyInit__operand_flag_tests"), + ("numpy_core__rational_tests", "PyInit__rational_tests"), + ("numpy_core__simd", "PyInit__simd"), + ("numpy_core__struct_ufunc_tests", "PyInit__struct_ufunc_tests"), + ("numpy_core__umath_tests", "PyInit__umath_tests"), + ("numpy_fft__pocketfft_internal", "PyInit__pocketfft_internal"), + ("numpy_linalg_lapack_lite", "PyInit_lapack_lite"), + ("numpy_linalg__umath_linalg", "PyInit__umath_linalg"), + ("numpy_random_bit_generator", "PyInit_bit_generator"), + ("numpy_random__bounded_integers", "PyInit__bounded_integers"), + ("numpy_random__common", "PyInit__common"), + ("numpy_random__generator", "PyInit__generator"), + ("numpy_random__mt19937", "PyInit__mt19937"), + ("numpy_random_mtrand", "PyInit_mtrand"), + ("numpy_random__pcg64", "PyInit__pcg64"), + ("numpy_random__philox", "PyInit__philox"), + ("numpy_random__sfc64", "PyInit__sfc64"), +] + + +def run(command: list[str], cwd: Path | None = None) -> None: + log("+ " + " ".join(command)) + result = subprocess.run( + command, + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if result.stdout: + log(result.stdout) + if result.returncode != 0: + location = f"; see {LOG_FILE}" if LOG_FILE else "" + raise RuntimeError(f"command failed: {' '.join(command)}{location}") + + +def run_quiet(command: list[str], cwd: Path | None = None) -> None: + log("+ " + " ".join(command)) + result = subprocess.run( + command, + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.stdout: + log(result.stdout) + if result.stderr: + log(result.stderr) + if result.returncode != 0: + sys.stderr.write("+ " + " ".join(command) + "\n") + if LOG_FILE: + sys.stderr.write(f"see {LOG_FILE} for full compiler output\n") + else: + if result.stdout: + sys.stderr.write(result.stdout[-4000:]) + if result.stderr: + sys.stderr.write(result.stderr[-4000:]) + raise subprocess.CalledProcessError(result.returncode, command) + + +def output(command: list[str]) -> str: + return subprocess.check_output(command, text=True).strip() + + +def read_env_file(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.is_file(): + return values + for line in path.read_text().splitlines(): + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + values[key] = value + return values + + +def cxx_runtime_include_flags(repo_root: Path) -> list[str]: + prefix = Path(os.environ.get("CXX_MUSL_PREFIX", repo_root / "target" / "cxx-musl")) + env_values = read_env_file(prefix / "cxx-runtime.env") + raw_flags = env_values.get("CXX_INCLUDE_FLAGS") + if raw_flags: + return raw_flags.split() + + libcxx = prefix / "include" / "c++" / "v1" + if libcxx.is_dir(): + return ["-nostdinc++", "-isystem", str(libcxx)] + + include_root = prefix / "include" / "c++" + if include_root.is_dir(): + for version in sorted(include_root.iterdir(), reverse=True): + if not version.is_dir(): + continue + flags = ["-nostdinc++", "-isystem", str(version)] + target_dir = version / "x86_64-linux-musl" + if target_dir.is_dir(): + flags.extend(["-isystem", str(target_dir)]) + backward = version / "backward" + if backward.is_dir(): + flags.extend(["-isystem", str(backward)]) + return flags + + raise SystemExit( + "missing musl C++ runtime. Run `python3 scripts/build_cxx_musl.py` " + "or set CXX_MUSL_PREFIX to a prefix containing include/c++ and static " + "C++ runtime libraries." + ) + + +def compiler_supports(compiler: str, flags: list[str], language: str) -> bool: + command = [ + compiler, + *flags, + "-x", + language, + "-c", + os.devnull, + "-o", + os.devnull, + ] + return subprocess.run( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def ensure_numpy_source(repo_root: Path, clean: bool) -> Path: + cache = repo_root / "target" / "numpy-musl" + source_root = cache / "src" / f"numpy-{NUMPY_VERSION}" + archive = cache / "downloads" / f"numpy-{NUMPY_VERSION}.tar.gz" + patches = repo_root / "third_party" / "lib-python-numpy" / "patches" + + if clean and source_root.exists(): + shutil.rmtree(source_root) + + if source_root.exists(): + return source_root + + archive.parent.mkdir(parents=True, exist_ok=True) + if not archive.exists(): + print(f"downloading {NUMPY_URL}", flush=True) + urllib.request.urlretrieve(NUMPY_URL, archive) + + extract_dir = cache / "src" + extract_dir.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive) as tar: + tar.extractall(extract_dir) + + for patch_file in sorted(patches.glob("*.patch")): + run(["patch", "-d", str(source_root), "-p1", "-i", str(patch_file)]) + + return source_root + + +def parse_sources(repo_root: Path, source_root: Path) -> list[Path]: + makefile = repo_root / "third_party" / "lib-python-numpy" / "Makefile.uk" + generated = repo_root / "third_party" / "lib-python-numpy" / "generated" / "numpy" + source = source_root / "numpy" + sources: list[Path] = [] + + pattern = re.compile(r"LIBPYTHON_NUMPY_SRCS(?:-y|-\$\([^)]*\)) \+= (.+)$") + for line in makefile.read_text().splitlines(): + match = pattern.match(line.strip()) + if not match: + continue + entry = match.group(1) + if "OPENBLAS" in line: + continue + entry = entry.replace("$(LIBPYTHON_NUMPY_SRC)", str(source)) + entry = entry.replace("$(LIBPYTHON_NUMPY_BSRC)", str(generated)) + path = Path(entry) + if path.name == "arm64_exports.c": + continue + if path not in sources: + sources.append(path) + return sources + + +def file_flags(path: Path, compiler: str, language: str) -> list[str] | None: + name = path.name + sse41 = ["-msse", "-msse2", "-msse3", "-mssse3", "-msse4.1"] + sse42 = [*sse41, "-mpopcnt", "-msse4.2"] + avx2 = [*sse42, "-mavx", "-mf16c", "-mavx2"] + fma3_avx2 = [*avx2, "-mfma"] + avx512f = [*avx2, "-mavx512f", "-mno-mmx"] + avx512_skx = [*avx512f, "-mavx512cd", "-mavx512vl", "-mavx512bw", "-mavx512dq"] + avx512_icl = [ + *avx512_skx, + "-mavx512vnni", + "-mavx512ifma", + "-mavx512vbmi", + "-mavx512vbmi2", + "-mavx512bitalg", + "-mavx512vpopcntdq", + ] + avx512_spr = [*avx512_icl, "-mavx512fp16"] + + if "avx512_spr" in name: + flags = avx512_spr + elif "avx512_icl" in name: + flags = avx512_icl + elif "avx512_skx" in name: + flags = avx512_skx + elif "avx512f" in name: + flags = avx512f + elif "fma3.avx2" in name: + flags = fma3_avx2 + elif "avx2" in name: + flags = avx2 + elif "sse42" in name: + flags = sse42 + elif "sse41" in name: + flags = sse41 + else: + return [] + + if compiler_supports(compiler, flags, language): + return flags + print(f"skip unsupported SIMD source: {path.name}", flush=True) + return None + + +def object_path(build_dir: Path, source: Path) -> Path: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(source)) + return build_dir / "obj" / f"{safe}.o" + + +def compile_one( + source: Path, + obj: Path, + cc: str, + cxx: str, + cflags: list[str], + cxxflags: list[str], +) -> Path | None: + if obj.exists() and obj.stat().st_mtime >= source.stat().st_mtime: + return obj + language = "c++" if source.suffix in {".cpp", ".cc", ".cxx"} else "c" + compiler = cxx if language == "c++" else cc + simd_flags = file_flags(source, compiler, language) + if simd_flags is None: + return None + obj.parent.mkdir(parents=True, exist_ok=True) + flags = cxxflags if language == "c++" else cflags + run_quiet([compiler, *flags, *simd_flags, "-c", str(source), "-o", str(obj)]) + return obj + + +def check_cxx_preflight(cxx: str, cxxflags: list[str], build_dir: Path) -> None: + source = build_dir / "cxx_preflight.cpp" + obj = build_dir / "cxx_preflight.o" + source.write_text( + "#include \n" + "#include \n" + "#include \n" + "static std::array value = {1};\n" + "int alloy_numpy_cxx_preflight(void) { return value[0] + (int)std::sqrt(4.0); }\n" + ) + result = subprocess.run( + [cxx, *cxxflags, "-c", str(source), "-o", str(obj)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + log("+ " + " ".join([cxx, *cxxflags, "-c", str(source), "-o", str(obj)])) + if result.stdout: + log(result.stdout) + if result.stderr: + log(result.stderr) + if result.returncode != 0: + raise SystemExit( + "musl NumPy C-extension build needs a musl-compatible C++ " + "standard library. The current C++ toolchain cannot compile " + "/ against AlloyStack musl headers. Use " + "`python3 scripts/build_numpy_musl.py --minimal` for the smoke " + "test path, or provide a libc++/musl C++ toolchain via CXX." + ) + + +def generate_inittab(build_dir: Path) -> Path: + source = build_dir / "numpy_inittab.c" + lines = [ + "#include ", + "", + ] + for _, init in BUILTIN_MODULES: + lines.append(f"extern PyObject *{init}(void);") + lines.extend( + [ + "", + "int alloy_numpy_register(void)", + "{", + " struct alloy_numpy_module {", + " const char *name;", + " PyObject *(*init)(void);", + " };", + " static const struct alloy_numpy_module modules[] = {", + ] + ) + for name, init in BUILTIN_MODULES: + lines.append(f' {{"{name}", {init}}},') + lines.extend( + [ + " };", + " for (size_t i = 0; i < sizeof(modules) / sizeof(modules[0]); ++i) {", + " if (PyImport_AppendInittab(modules[i].name, modules[i].init) < 0)", + " return -1;", + " }", + " return 0;", + "}", + "", + ] + ) + source.write_text("\n".join(lines)) + return source + + +def build_archive(repo_root: Path, clean: bool) -> None: + third_party = repo_root / "third_party" / "lib-python-numpy" + if not (third_party / "Makefile.uk").is_file(): + raise SystemExit("missing third_party/lib-python-numpy; run git submodule update --init") + + print(f"building NumPy musl archive; log: {LOG_FILE}", flush=True) + source_root = ensure_numpy_source(repo_root, clean) + cache = repo_root / "target" / "numpy-musl" + build_dir = cache / "build" + lib_dir = cache / "lib" + include_dir = cache / "include" + if clean and build_dir.exists(): + shutil.rmtree(build_dir) + if clean and lib_dir.exists(): + shutil.rmtree(lib_dir) + build_dir.mkdir(parents=True, exist_ok=True) + lib_dir.mkdir(parents=True, exist_ok=True) + include_dir.mkdir(parents=True, exist_ok=True) + + cc = os.environ.get("CC", "cc") + cxx_runtime_prefix = Path(os.environ.get("CXX_MUSL_PREFIX", repo_root / "target" / "cxx-musl")) + cxx = os.environ.get("CXX", str(cxx_runtime_prefix / "bin" / "musl-g++")) + ar = os.environ.get("AR", "ar") + jobs = int(os.environ.get("JOBS", str(os.cpu_count() or 1))) + + musl_dir = repo_root / "target" / "musl-alloy" + cpython_dir = repo_root / "target" / "cpython-musl" + python_include = Path(os.environ.get("PYTHON_INCLUDE", repo_root / "third_party" / "cpython" / "Include")) + python_config_include = Path( + os.environ.get("PYTHON_CONFIG_INCLUDE", cpython_dir / "build-submodule") + ) + gcc_include = output([cc, "-print-file-name=include"]) + cxx_runtime_flags = cxx_runtime_include_flags(repo_root) + generated = third_party / "generated" / "numpy" + numpy_source = source_root / "numpy" + + common_flags = [ + "-O2", + "-fPIC", + "-ffreestanding", + "-fno-stack-protector", + "-fwrapv", + "-fasynchronous-unwind-tables", + "-nostdinc", + "-D_GNU_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE64_SOURCE=1", + "-D_LARGEFILE_SOURCE=1", + "-DDYNAMIC_ANNOTATIONS_ENABLED=1", + "-DHAVE_NPY_CONFIG_H=1", + "-DNPY_INTERNAL_BUILD=1", + "-isystem", + str(musl_dir / "include"), + "-I", + str(repo_root / "as_musl" / "include"), + "-isystem", + str(python_config_include), + "-isystem", + str(python_include), + "-isystem", + str(gcc_include), + "-I", + str(numpy_source / "core" / "include"), + "-I", + str(generated / "core" / "include"), + "-I", + str(generated / "distutils" / "include"), + ] + for rel in [ + "core", + "core/include", + "core/include/numpy", + "core/src", + "core/src/common", + "core/src/multiarray", + "core/src/npymath", + "core/src/npysort", + "core/src/_simd", + "core/src/umath", + "random", + ]: + common_flags.extend(["-I", str(generated / rel)]) + common_flags.extend(["-iquote", str(generated / rel)]) + common_flags.extend(["-I", str(numpy_source / rel)]) + + cflags = [*common_flags, "-std=c11"] + cxxflags = [*cxx_runtime_flags] + cxx_common_flags = [flag for flag in common_flags if flag != "-ffreestanding"] + cxxflags.extend([ + *cxx_common_flags, + "-std=c++17", + "-fno-exceptions", + "-fno-rtti", + "-fno-use-cxa-atexit", + ]) + check_cxx_preflight(cxx, cxxflags, build_dir) + + sources = [path for path in parse_sources(repo_root, source_root) if path.is_file()] + missing = [path for path in parse_sources(repo_root, source_root) if not path.is_file()] + if missing: + print(f"warning: {len(missing)} NumPy source files were not found", flush=True) + for path in missing[:10]: + print(f" missing: {path}", flush=True) + + objects: list[Path] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as executor: + futures = [ + executor.submit( + compile_one, + source, + object_path(build_dir, source), + cc, + cxx, + cflags, + cxxflags, + ) + for source in sources + ] + for future in concurrent.futures.as_completed(futures): + obj = future.result() + if obj is not None: + objects.append(obj) + + inittab = generate_inittab(build_dir) + inittab_obj = build_dir / "obj" / "numpy_inittab.o" + run([cc, *cflags, "-c", str(inittab), "-o", str(inittab_obj)]) + objects.append(inittab_obj) + + archive = lib_dir / "libnumpy_musl.a" + if archive.exists(): + archive.unlink() + run([ar, "rcs", str(archive), *map(str, sorted(objects))]) + + # Expose generated headers for downstream C extensions if needed later. + numpy_include = include_dir / "numpy" + if numpy_include.exists(): + shutil.rmtree(numpy_include) + shutil.copytree(generated / "core" / "include" / "numpy", numpy_include) + print(f"built {archive}", flush=True) + + +def build_minimal_archive(repo_root: Path, clean: bool) -> None: + print(f"building minimal NumPy musl archive; log: {LOG_FILE}", flush=True) + cache = repo_root / "target" / "numpy-musl" + build_dir = cache / "build-minimal" + lib_dir = cache / "lib" + if clean and build_dir.exists(): + shutil.rmtree(build_dir) + build_dir.mkdir(parents=True, exist_ok=True) + lib_dir.mkdir(parents=True, exist_ok=True) + + cc = os.environ.get("CC", "cc") + ar = os.environ.get("AR", "ar") + source = build_dir / "numpy_stub.c" + obj = build_dir / "numpy_stub.o" + archive = lib_dir / "libnumpy_musl.a" + source.write_text("int alloy_numpy_register(void) { return 0; }\n") + run([cc, "-O2", "-fPIC", "-c", str(source), "-o", str(obj)]) + if archive.exists(): + archive.unlink() + run([ar, "rcs", str(archive), str(obj)]) + print(f"built minimal {archive}", flush=True) + + +def copy_py_files(source: Path, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir(parents=True, exist_ok=True) + for path in source.rglob("*"): + rel = path.relative_to(source) + target = destination / rel + if path.is_dir(): + target.mkdir(exist_ok=True) + elif path.suffix == ".py": + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + +def disable_numpy_doc_initializers(numpy_root: Path) -> None: + for rel in [ + "core/_add_newdocs.py", + "core/_add_newdocs_scalars.py", + ]: + target = numpy_root / rel + if target.exists(): + target.write_text( + "# Disabled for AlloyStack musl smoke workloads.\n" + "# These modules only attach docstrings to NumPy objects.\n" + ) + + +def reduce_numpy_top_level_imports(numpy_root: Path) -> None: + init_py = numpy_root / "__init__.py" + if not init_py.is_file(): + return + text = init_py.read_text() + replacements = { + " from . import lib\n": " # AlloyStack smoke: skip eager import of lib\n", + " from .lib import *\n": " # AlloyStack smoke: skip eager import of lib symbols\n", + " from . import linalg\n": " # AlloyStack smoke: skip eager import of linalg\n", + " from . import fft\n": " # AlloyStack smoke: skip eager import of fft\n", + " from . import polynomial\n": " # AlloyStack smoke: skip eager import of polynomial\n", + " from . import random\n": " # AlloyStack smoke: skip eager import of random\n", + " from . import ctypeslib\n": " # AlloyStack smoke: skip eager import of ctypeslib\n", + " from . import ma\n": " # AlloyStack smoke: skip eager import of ma\n", + " from . import matrixlib as _mat\n": " # AlloyStack smoke: skip eager import of matrixlib\n", + " from .matrixlib import *\n": " # AlloyStack smoke: skip eager import of matrixlib symbols\n", + " core.getlimits._register_known_types()\n": ( + " # AlloyStack smoke: skip eager dtype limit registration\n" + ), + " _sanity_check()\n": " # AlloyStack smoke: skip NumPy import-time sanity check\n", + " use_hugepage = os.environ.get(\"NUMPY_MADVISE_HUGEPAGE\", None)\n": ( + " use_hugepage = 0\n" + ), + " core.multiarray._set_madvise_hugepage(use_hugepage)\n": ( + " # AlloyStack smoke: skip import-time madvise hugepage setup\n" + ), + " __all__.extend(lib.__all__)\n": " # AlloyStack smoke: lib was not eagerly imported\n", + " __all__.extend(_mat.__all__)\n": " # AlloyStack smoke: matrixlib was not eagerly imported\n", + " __all__.extend(['linalg', 'fft', 'random', 'ctypeslib', 'ma'])\n": ( + " # AlloyStack smoke: optional top-level packages were not eagerly imported\n" + ), + " __all__.remove('Arrayterator')\n": ( + " # AlloyStack smoke: Arrayterator is provided by numpy.lib, " + "which was not eagerly imported\n" + ), + " del Arrayterator\n": " # AlloyStack smoke: Arrayterator was not imported\n", + } + for old, new in replacements.items(): + text = text.replace(old, new) + init_py.write_text(text) + + +def prepare_rootfs(repo_root: Path, clean: bool) -> Path: + source_root = ensure_numpy_source(repo_root, clean) + cache = repo_root / "target" / "numpy-musl" + rootfs = Path(os.environ.get("NUMPY_MUSL_ROOTFS", cache / "rootfs")) + numpy_dst = rootfs / "Lib" / "site-packages" / "numpy" + numpy_src = source_root / "numpy" + generated = repo_root / "third_party" / "lib-python-numpy" / "generated" / "numpy" + importfix = repo_root / "third_party" / "lib-python-numpy" / "importfix" / "numpy" + + copy_py_files(numpy_src, numpy_dst) + shutil.copy2(generated / "__config__.py", numpy_dst / "__config__.py") + distutils_dst = numpy_dst / "distutils" + distutils_dst.mkdir(parents=True, exist_ok=True) + shutil.copy2(generated / "distutils" / "__config__.py", distutils_dst / "__config__.py") + shutil.copytree(importfix, numpy_dst, dirs_exist_ok=True) + disable_numpy_doc_initializers(numpy_dst) + reduce_numpy_top_level_imports(numpy_dst) + print(f"prepared {numpy_dst}", flush=True) + return rootfs + + +def prepare_minimal_rootfs(repo_root: Path, clean: bool) -> Path: + cache = repo_root / "target" / "numpy-musl" + rootfs = Path(os.environ.get("NUMPY_MUSL_ROOTFS", cache / "rootfs")) + source = repo_root / "user" / "python_workloads" / "numpy_minimal" / "numpy" + destination = rootfs / "Lib" / "site-packages" / "numpy" + if clean and destination.exists(): + shutil.rmtree(destination) + if destination.exists(): + shutil.rmtree(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(source, destination) + print(f"prepared minimal {destination}", flush=True) + return rootfs + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--clean", action="store_true") + parser.add_argument("--rootfs-only", action="store_true") + parser.add_argument("--archive-only", action="store_true") + parser.add_argument( + "--minimal", + action="store_true", + help="build the small pure-Python numpy-compatible smoke-test package", + ) + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + init_log(repo_root) + if args.minimal: + if not args.rootfs_only: + build_minimal_archive(repo_root, args.clean) + if not args.archive_only: + prepare_minimal_rootfs(repo_root, args.clean) + else: + if not args.rootfs_only: + build_archive(repo_root, args.clean) + if not args.archive_only: + prepare_rootfs(repo_root, args.clean) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/sync_numpy_workloads.sh b/scripts/sync_numpy_workloads.sh new file mode 100755 index 0000000..26c119d --- /dev/null +++ b/scripts/sync_numpy_workloads.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +workload_source="$repo_root/user/python_workloads/wasm_bench/numpy_smoke.py" +numpy_mode="${NUMPY_MODE:-minimal}" +if [[ "$numpy_mode" == "unikraft" ]]; then + numpy_mode="full" +fi +base_image="${BASE_FS_IMAGE:-$repo_root/fs_images/fatfs.img}" +image="${FS_IMAGE:-$repo_root/fs_images/fatfs_numpy.img}" +mount_dir="${FS_MOUNT_DIR:-$repo_root/image_content}" +rootfs="$repo_root/target/numpy-musl/rootfs-$numpy_mode" +numpy_source="$rootfs/Lib/site-packages/numpy" + +dir_has_entries() { + [[ -d "$1" ]] && find "$1" -mindepth 1 -maxdepth 1 -print -quit | grep -q . +} + +restore_rootfs_owner() { + if [[ "${EUID:-$(id -u)}" == "0" && -n "${SUDO_UID:-}" && -d "$rootfs" ]]; then + chown -R "$SUDO_UID:${SUDO_GID:-$SUDO_UID}" "$rootfs" + fi +} + +if [[ "$numpy_mode" == "minimal" ]]; then + NUMPY_MUSL_ROOTFS="$rootfs" python3 "$repo_root/scripts/build_numpy_musl.py" --minimal --rootfs-only +elif [[ "$numpy_mode" == "full" ]]; then + NUMPY_MUSL_ROOTFS="$rootfs" python3 "$repo_root/scripts/build_numpy_musl.py" --rootfs-only +else + echo "unknown NUMPY_MODE=$numpy_mode; expected minimal or full" >&2 + exit 1 +fi +restore_rootfs_owner + +if [[ ! -d "$numpy_source" ]]; then + echo "missing prepared numpy rootfs: $numpy_source" >&2 + exit 1 +fi + +if dir_has_entries "$mount_dir"; then + sudo mkdir -p "$mount_dir/Lib/site-packages" "$mount_dir/wasm_bench" + sudo rm -rf "$mount_dir/Lib/site-packages/numpy" + sudo cp -r "$numpy_source" "$mount_dir/Lib/site-packages/" + sudo cp "$workload_source" "$mount_dir/wasm_bench/numpy_smoke.py" + echo "synchronized NumPy workload and package files to mounted FAT directory $mount_dir" + sync + exit 0 +fi + +command -v mcopy >/dev/null || { + echo "mcopy is required to update the FAT image when $mount_dir is empty or unmounted" >&2 + exit 1 +} +command -v mmd >/dev/null || { + echo "mmd is required to update the FAT image when $mount_dir is empty or unmounted" >&2 + exit 1 +} + +if [[ ! -f "$image" || "${REBUILD_NUMPY_IMAGE:-0}" == "1" ]]; then + if [[ ! -f "$base_image" ]]; then + echo "missing base image: $base_image" >&2 + exit 1 + fi + mkdir -p "$(dirname -- "$image")" + cp "$base_image" "$image" +fi + +mmd -i "$image" "::/Lib" 2>/dev/null || true +mmd -i "$image" "::/Lib/site-packages" 2>/dev/null || true +mmd -i "$image" "::/wasm_bench" 2>/dev/null || true +if command -v mdeltree >/dev/null; then + mdeltree -i "$image" "::/Lib/site-packages/numpy" 2>/dev/null || true +fi +mcopy -s -o -i "$image" "$numpy_source" "::/Lib/site-packages/" +mcopy -o -i "$image" "$workload_source" "::/wasm_bench/numpy_smoke.py" + +echo "synchronized NumPy workload and package files to FAT image $image" diff --git a/third_party/lib-python-numpy b/third_party/lib-python-numpy new file mode 160000 index 0000000..4ebaac9 --- /dev/null +++ b/third_party/lib-python-numpy @@ -0,0 +1 @@ +Subproject commit 4ebaac9d49e26cd991f35550b98fa36953d9cd4f diff --git a/user/musl_cpython/Cargo.toml b/user/musl_cpython/Cargo.toml index 42b90ee..1cc2370 100644 --- a/user/musl_cpython/Cargo.toml +++ b/user/musl_cpython/Cargo.toml @@ -13,3 +13,4 @@ as_musl = { path = "../../as_musl" } [features] default = [] mpk = ["as_musl/mpk"] +numpy = [] diff --git a/user/musl_cpython/function.c b/user/musl_cpython/function.c index cd1bf6a..99c9f1c 100644 --- a/user/musl_cpython/function.c +++ b/user/musl_cpython/function.c @@ -204,6 +204,15 @@ PyMODINIT_FUNC PyInit_pyas(void) return PyModule_Create(&pyas_module); } +#ifdef ALLOY_ENABLE_NUMPY +int alloy_numpy_register(void); +PyMODINIT_FUNC PyInit__datetime(void); +PyMODINIT_FUNC PyInit_math(void); +PyMODINIT_FUNC PyInit__struct(void); +PyMODINIT_FUNC PyInit__contextvars(void); +PyMODINIT_FUNC PyInit_binascii(void); +#endif + static const char *find_arg(int argc, char **argv, const char *key) { size_t key_len = strlen(key); @@ -276,6 +285,32 @@ int main(int argc, char **argv) fprintf(stderr, "failed to register built-in pyas module\n"); goto init_failed; } +#ifdef ALLOY_ENABLE_NUMPY + if (PyImport_AppendInittab("math", PyInit_math) < 0) { + fprintf(stderr, "failed to register built-in math module\n"); + goto init_failed; + } + if (PyImport_AppendInittab("_struct", PyInit__struct) < 0) { + fprintf(stderr, "failed to register built-in _struct module\n"); + goto init_failed; + } + if (PyImport_AppendInittab("_contextvars", PyInit__contextvars) < 0) { + fprintf(stderr, "failed to register built-in _contextvars module\n"); + goto init_failed; + } + if (PyImport_AppendInittab("binascii", PyInit_binascii) < 0) { + fprintf(stderr, "failed to register built-in binascii module\n"); + goto init_failed; + } + if (PyImport_AppendInittab("_datetime", PyInit__datetime) < 0) { + fprintf(stderr, "failed to register built-in _datetime module\n"); + goto init_failed; + } + if (alloy_numpy_register() < 0) { + fprintf(stderr, "failed to register built-in numpy modules\n"); + goto init_failed; + } +#endif status = PyConfig_SetString(&config, &config.program_name, L"python"); if (report_status(status)) goto init_failed; @@ -283,6 +318,10 @@ int main(int argc, char **argv) if (report_status(status)) goto init_failed; status = PyWideStringList_Append(&config.module_search_paths, L"Lib"); + if (report_status(status)) + goto init_failed; + status = PyWideStringList_Append(&config.module_search_paths, + L"Lib/site-packages"); if (report_status(status)) goto init_failed; if (configure_argv(&config, script, argc, argv)) diff --git a/user/python_workloads/numpy_minimal/numpy/__init__.py b/user/python_workloads/numpy_minimal/numpy/__init__.py new file mode 100644 index 0000000..f047d40 --- /dev/null +++ b/user/python_workloads/numpy_minimal/numpy/__init__.py @@ -0,0 +1,98 @@ +__version__ = "minimal-0.1" + + +class _DType: + def __init__(self, name): + self.name = name + + def __repr__(self): + return f"dtype('{self.name}')" + + +int64 = _DType("int64") +float64 = _DType("float64") + + +class ndarray: + def __init__(self, data, shape=None, dtype=None): + self._data = list(data) + self.dtype = dtype + self.shape = tuple(shape) if shape is not None else (len(self._data),) + + def reshape(self, *shape): + if len(shape) == 1 and isinstance(shape[0], (tuple, list)): + shape = tuple(shape[0]) + size = 1 + for dim in shape: + size *= dim + if size != len(self._data): + raise ValueError("cannot reshape array of size {} into shape {}".format( + len(self._data), shape + )) + return ndarray(self._data, shape, self.dtype) + + def sum(self): + return sum(self._data) + + def tolist(self): + if len(self.shape) == 1: + return list(self._data) + if len(self.shape) == 2: + rows, cols = self.shape + return [ + self._data[row * cols:(row + 1) * cols] + for row in range(rows) + ] + return list(self._data) + + def __add__(self, other): + return ndarray([value + other for value in self._data], self.shape, self.dtype) + + def __radd__(self, other): + return self.__add__(other) + + def __iter__(self): + return iter(self.tolist()) + + def __len__(self): + return self.shape[0] + + def __repr__(self): + return "array({})".format(self.tolist()) + + +def array(data, dtype=None): + if isinstance(data, ndarray): + return ndarray(data._data, data.shape, dtype or data.dtype) + if data and isinstance(data[0], (list, tuple)): + rows = len(data) + cols = len(data[0]) + flat = [] + for row in data: + if len(row) != cols: + raise ValueError("ragged nested sequences are not supported") + flat.extend(row) + return ndarray(flat, (rows, cols), dtype) + return ndarray(data, dtype=dtype) + + +def arange(*args, dtype=None): + if len(args) == 1: + start, stop, step = 0, args[0], 1 + elif len(args) == 2: + start, stop = args + step = 1 + elif len(args) == 3: + start, stop, step = args + else: + raise TypeError("arange expected 1 to 3 positional arguments") + return ndarray(range(start, stop, step), dtype=dtype) + + +def sum(values): + if isinstance(values, ndarray): + return values.sum() + total = 0 + for value in values: + total += value + return total diff --git a/user/python_workloads/wasm_bench/numpy_smoke.py b/user/python_workloads/wasm_bench/numpy_smoke.py new file mode 100644 index 0000000..c44b5fe --- /dev/null +++ b/user/python_workloads/wasm_bench/numpy_smoke.py @@ -0,0 +1,46 @@ +import numpy as np +print(f"numpy v{np.__version__} imported", flush=True) + +def main(): + a = np.array([1, 2, 3, 4, 5]) + b = np.array([10, 20, 30, 40, 50]) + + print("a =", a) + print("b =", b) + + print("a + b =", a + b) + print("a * b =", a * b) + + matrix = np.array([ + [1, 2, 3], + [4, 5, 6] + ]) + + print("matrix:") + print(matrix) + + print("transpose:") + print(matrix.T) + + x = np.array([ + [1, 2], + [3, 4] + ]) + + y = np.array([ + [5, 6], + [7, 8] + ]) + + print("matrix multiplication:") + print(x @ y) + + data = np.array([1, 2, 3, 4, 5]) + + print("mean:", np.mean(data)) + print("sum:", np.sum(data)) + print("max:", np.max(data)) + return + +if __name__ == "__main__": + main()