diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48c3b524..d18454ae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[resolve]" --group test + pip install . --group test - name: Run tests run: | diff --git a/AGENTS.md b/AGENTS.md index 2ace4673..0c48bcc1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,8 @@ pyodide-build/ python -m venv .venv source .venv/bin/activate -# Install in editable mode with test and resolve dependencies -pip install -e ".[test,resolve]" +# Install in editable mode with test dependencies +pip install -e ".[test]" # Run unit tests (excludes integration tests) pytest pyodide_build -m "not integration" diff --git a/CHANGELOG.md b/CHANGELOG.md index 235dc610..96861a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.35.0] - 2026/06/XX + +### Removed + +- `pyodide build` no longer supports building from a URL or a PyPI requirement + specifier, building from a `requirements.txt` file (`-r`/`--requirements`), or + fetching and building dependencies (`--build-dependencies`, + `--output-lockfile`, `--skip-dependency`, `--skip-built-in-packages`, + `--compression-level`). It now only builds a local source directory (or the + current working directory), matching the behavior of `pypa/build`. + We have supported these features as PyPI didn't support Emscripten wheels, + but now that cross-compiled wheels can be uploaded to PyPI, each package + should be built independently and uploaded to PyPI. + ## [0.34.5] - 2026/05/20 ### Added diff --git a/pyodide_build/cli/build.py b/pyodide_build/cli/build.py index 0c834434..14ef4256 100644 --- a/pyodide_build/cli/build.py +++ b/pyodide_build/cli/build.py @@ -1,30 +1,18 @@ -import re -import shutil import sys -import tempfile from dataclasses import dataclass from pathlib import Path from typing import cast, get_args -from urllib.parse import urlparse import click -import requests from build import ConfigSettingsType from pyodide_build.build_env import ( ensure_emscripten, - get_pyodide_root, init_environment, ) from pyodide_build.common import default_xbuildenv_path from pyodide_build.logger import logger from pyodide_build.out_of_tree import build as _build -from pyodide_build.out_of_tree.pypi import ( - MissingOptionalDependencyError, - build_dependencies_for_wheel, - build_wheels_from_pypi_requirements, - fetch_pypi_package, -) from pyodide_build.pypabuild import parse_backend_flags from pyodide_build.spec import _BuildSpecExports, _ExportTypes @@ -52,18 +40,6 @@ def _convert_exports(exports: str) -> _BuildSpecExports: sys.exit(1) -def _download_url(url: str, output_directory: Path) -> str: - with requests.get(url, stream=True) as response: - urlpath = Path(urlparse(response.url).path) - if urlpath.suffix == ".gz": - urlpath = urlpath.with_suffix("") - file_name = urlpath.name - with open(output_directory / file_name, "wb") as f: - for chunk in response.iter_content(chunk_size=1 << 20): - f.write(chunk) - return file_name - - def _build_from_source( source_path: Path, output_directory: Path, @@ -79,156 +55,28 @@ def _build_from_source( ) -def _build_from_url( - package_url: str, - output_directory: Path, - args: BuildArgs, -) -> Path: - with tempfile.TemporaryDirectory() as tmpdir: - tmppath = Path(tmpdir) - filename = _download_url(package_url, tmppath) - if Path(filename).suffix == ".whl": - shutil.move(tmppath / filename, output_directory / filename) - return output_directory / filename - - builddir = tmppath / "build" - shutil.unpack_archive(tmppath / filename, builddir) - files = list(builddir.iterdir()) - if len(files) == 1 and files[0].is_dir(): - builddir = files[0] - return _build_from_source(builddir, output_directory, args) - - -def _build_from_pypi( - package: str, - output_directory: Path, - args: BuildArgs, -) -> Path: - with tempfile.TemporaryDirectory() as tmpdir: - srcdir = Path(tmpdir) - - try: - package_path = fetch_pypi_package(package, srcdir) - except MissingOptionalDependencyError as e: - print(str(e), file=sys.stderr) - sys.exit(1) - - if not package_path.is_dir(): - dest_file = output_directory / package_path.name - shutil.copyfile(str(package_path), dest_file) - print(f"Successfully fetched: {package_path.name}") - return dest_file - - return _build_from_source(srcdir, output_directory, args) - - -def _parse_requirements_file(requirements_txt: str) -> list[str]: - if not Path(requirements_txt).exists(): - raise RuntimeError(f"Couldn't find requirements text file {requirements_txt}") - - reqs = [] - with open(requirements_txt) as f: - raw_reqs = [x.strip() for x in f.readlines()] - - for line in raw_reqs: - comment_pos = line.find("#") - if comment_pos != -1: - line = line[:comment_pos].strip() - if not line: - continue - if line[0] == "-": - raise RuntimeError( - f"pyodide build only supports name-based PEP508 requirements. [{line}] will not work." - ) - if "@" in line: - raise RuntimeError( - f"pyodide build does not support URL based requirements. [{line}] will not work" - ) - reqs.append(line) - - return reqs - +def _resolve_source_dir(source_location: str) -> Path: + if not source_location: + return Path.cwd() -def _build_from_requirements( - requirements_txt: str, - output_directory: Path, - args: BuildArgs, - build_dependencies: bool, - skip_dependency_list: list[str], - output_lockfile: str, -) -> None: - reqs = _parse_requirements_file(requirements_txt) - try: - build_wheels_from_pypi_requirements( - reqs, - output_directory, - build_dependencies, - skip_dependency_list, - _convert_exports(args.exports), - args.config_settings, - isolation=args.isolation, - skip_dependency_check=args.skip_dependency_check, - output_lockfile=output_lockfile, + if "://" in source_location: + logger.stderr( + f"Building from a URL is no longer supported. " + f"Got '{source_location}'. Pass a local source directory instead, " + "or download a pre-built wheel from PyPI." ) - except MissingOptionalDependencyError as e: - print(str(e), file=sys.stderr) sys.exit(1) - except BaseException as e: - import traceback - - print("Failed building multiple wheels:", traceback.format_exc()) - raise e - -def _build_wheel_dependencies( - wheel: Path, - extras: list[str], - args: BuildArgs, - skip_dependency_list: list[str], - output_lockfile: str, - compression_level: int, -) -> None: - try: - build_dependencies_for_wheel( - wheel, - extras, - skip_dependency_list, - _convert_exports(args.exports), - args.config_settings, - isolation=args.isolation, - skip_dependency_check=args.skip_dependency_check, - output_lockfile=output_lockfile, - compression_level=compression_level, + source_path = Path(source_location) + if not source_path.is_dir(): + logger.stderr( + f"'{source_location}' is not a directory. Building from a PyPI " + "requirement specifier is no longer supported. Pass a local source " + "directory instead, or download a pre-built wheel from PyPI." ) - except MissingOptionalDependencyError as e: - print(str(e), file=sys.stderr) - wheel.unlink() sys.exit(1) - except BaseException as e: - import traceback - - print("Failed building dependencies for wheel:", traceback.format_exc()) - wheel.unlink() - raise e - - -def _detect_source_type(source_location: str) -> str: - if not source_location: - return "cwd" - if "://" in source_location: - return "url" - if Path(source_location).is_dir(): - return "directory" - if "/" not in source_location: - return "pypi" - raise RuntimeError(f"Couldn't determine source type for {source_location}") - -def _extract_extras(source_location: str) -> tuple[str, list[str]]: - extras = re.findall(r"\[(\w+)\]", source_location) - if extras: - source_location = source_location[: source_location.find("[")] - return source_location, extras + return source_path.resolve() DEFAULT_PATH = default_xbuildenv_path() @@ -249,13 +97,6 @@ def _extract_extras(source_location: str) -> tuple[str, list[str]]: default="", help="which directory should the output be placed into?", ) -@click.option( - "--requirements", - "-r", - "requirements_txt", - default="", - help="Build a list of package requirements from a requirements.txt file", -) @click.option( "--exports", default="requested", @@ -269,35 +110,6 @@ def _extract_extras(source_location: str) -> tuple[str, list[str]]: "or a comma-separated list of symbol names." ), ) -@click.option( - "--build-dependencies/--no-build-dependencies", - default=False, - help="Fetch dependencies from pypi and build them too.", -) -@click.option( - "--output-lockfile", - default="", - help="Output list of resolved dependencies to a file in requirements.txt format", -) -@click.option( - "--skip-dependency", - multiple=True, - help=( - "Skip building or resolving a single dependency, or a pyodide-lock.json file. " - "Use multiple times or provide a comma separated list to skip multiple dependencies." - ), -) -@click.option( - "--skip-built-in-packages/--no-skip-built-in-packages", - default=True, - help="Don't build dependencies that are built into the pyodide distribution.", -) -@click.option( - "--compression-level", - default=6, - show_default=True, - help="Compression level to use for the created zip file", -) @click.option( "--no-isolation", "-n", @@ -350,26 +162,19 @@ def main( ctx: click.Context, source_location: str, output_directory: str, - requirements_txt: str, exports: str, - build_dependencies: bool, - output_lockfile: str, - skip_dependency: tuple[str, ...], - skip_built_in_packages: bool, - compression_level: int, no_isolation: bool, skip_dependency_check: bool, config_setting: tuple[str, ...], xbuildenv_path: Path, skip_emscripten_install: bool, ) -> None: - """Use pypa/build to build a Python package from source, pypi or url. + """Use pypa/build to build a Python package from source. \b Arguments: - SOURCE_LOCATION: Build source, can be source folder, pypi version specification, - or url to a source dist archive or wheel file. If this is blank, it - will build the current directory. + SOURCE_LOCATION: Path to a local source folder to build. If this is + blank, it will build the current directory. """ init_environment(xbuildenv_path=xbuildenv_path) try: @@ -389,96 +194,5 @@ def main( skip_dependency_check=skip_dependency_check, ) - skip_dependency_list = list(skip_dependency) - if skip_built_in_packages: - package_lock_json = get_pyodide_root() / "dist" / "pyodide-lock.json" - skip_dependency_list.append(str(package_lock_json.absolute())) - - if requirements_txt: - _build_from_requirements( - requirements_txt, - outpath, - build_args, - build_dependencies, - skip_dependency_list, - output_lockfile, - ) - return - - source_location, extras = _extract_extras(source_location) - source_type = _detect_source_type(source_location) - - if source_type == "cwd": - wheel = _build_from_source(Path.cwd(), outpath, build_args) - elif source_type == "directory": - wheel = _build_from_source(Path(source_location).resolve(), outpath, build_args) - elif source_type == "url": - wheel = _build_from_url(source_location, outpath, build_args) - elif source_type == "pypi": - wheel = _build_from_pypi(source_location, outpath, build_args) - else: - raise RuntimeError(f"Unknown source type: {source_type}") - - if build_dependencies: - _build_wheel_dependencies( - wheel, - extras, - build_args, - skip_dependency_list, - output_lockfile, - compression_level, - ) - - -def pypi( - package: str, - output_directory: Path, - exports: str, - config_settings: ConfigSettingsType, - isolation: bool = True, - skip_dependency_check: bool = False, -) -> Path: - """Fetch a wheel from pypi, or build from source if none available.""" - args = BuildArgs( - exports=exports, - config_settings=config_settings, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - ) - return _build_from_pypi(package, output_directory, args) - - -def url( - package_url: str, - output_directory: Path, - exports: str, - config_settings: ConfigSettingsType, - isolation: bool = True, - skip_dependency_check: bool = False, -) -> Path: - """Fetch a wheel or build sdist from url.""" - args = BuildArgs( - exports=exports, - config_settings=config_settings, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - ) - return _build_from_url(package_url, output_directory, args) - - -def source( - source_location: Path, - output_directory: Path, - exports: str, - config_settings: ConfigSettingsType, - isolation: bool = True, - skip_dependency_check: bool = False, -) -> Path: - """Use pypa/build to build a Python package from source""" - args = BuildArgs( - exports=exports, - config_settings=config_settings, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - ) - return _build_from_source(source_location, output_directory, args) + source_dir = _resolve_source_dir(source_location) + _build_from_source(source_dir, outpath, build_args) diff --git a/pyodide_build/out_of_tree/pypi.py b/pyodide_build/out_of_tree/pypi.py deleted file mode 100644 index 49dac371..00000000 --- a/pyodide_build/out_of_tree/pypi.py +++ /dev/null @@ -1,526 +0,0 @@ -import io -import json -import os -import shutil -import sys -import tempfile -from collections.abc import Generator -from contextlib import contextmanager -from email.message import EmailMessage -from email.parser import BytesParser -from functools import cache -from io import BytesIO -from operator import attrgetter -from pathlib import Path -from platform import python_version -from typing import TYPE_CHECKING, Any, BinaryIO, cast -from urllib.parse import urlparse -from zipfile import ZipFile - -import requests -from build import ConfigSettingsType -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name -from packaging.version import Version - -if TYPE_CHECKING: - from resolvelib import BaseReporter, Resolver - from unearth.evaluator import TargetPython - from unearth.finder import PackageFinder - -try: - from resolvelib.providers import AbstractProvider as _AbstractProvider -except ImportError: - - class _AbstractProvider: # type: ignore[no-redef] - pass - - -from pyodide_build import build_env -from pyodide_build.common import repack_zip_archive -from pyodide_build.logger import logger -from pyodide_build.out_of_tree import build -from pyodide_build.spec import _BuildSpecExports - -_PYPI_INDEX = ["https://pypi.org/simple/"] -_PYPI_TRUSTED_HOSTS = ["pypi.org"] - - -class MissingOptionalDependencyError(RuntimeError): - pass - - -def _missing_resolve_extra_error() -> MissingOptionalDependencyError: - return MissingOptionalDependencyError( - "This functionality requires optional dependencies 'resolvelib' and 'unearth'. " - "Install them with `pip install pyodide-build[resolve]`" - ) - - -def _import_unearth() -> tuple[type["TargetPython"], type["PackageFinder"]]: - try: - from unearth.evaluator import TargetPython - from unearth.finder import PackageFinder - except ImportError as e: - raise _missing_resolve_extra_error() from e - return TargetPython, PackageFinder - - -def _import_resolvelib() -> tuple[type["BaseReporter"], type["Resolver"]]: - try: - from resolvelib import BaseReporter, Resolver - except ImportError as e: - raise _missing_resolve_extra_error() from e - return BaseReporter, Resolver - - -@contextmanager -def stream_redirected(to=os.devnull, stream=None): - """ - Context manager to redirect stdout or stderr. It does it with filenos and things rather than - just changing sys.stdout, so that output of subprocesses is also redirected. - """ - if stream is None: - stream = sys.stdout - try: - if not hasattr(stream, "fileno"): - yield - return - stream_fd = stream.fileno() - except io.UnsupportedOperation: - # in case we're already capturing to something that isn't really a file - # e.g. in pytest - yield - return - if isinstance(to, str): - to = open(to, "w") - with os.fdopen(os.dup(stream_fd), "wb") as copied: - stream.flush() - os.dup2(to.fileno(), stream_fd) # $ exec >&to - try: - yield stream # allow code to be run with the redirected stream - finally: - # restore stream to its previous value - # NOTE: dup2 makes stream_fd inheritable unconditionally - stream.flush() - os.dup2(copied.fileno(), stream_fd) # $ exec >&copied - to = None - - -def get_built_wheel(url, isolation=True, skip_dependency_check=False): - return _get_built_wheel_internal(url, isolation, skip_dependency_check)["path"] - - -@cache -def _get_built_wheel_internal(url, isolation=True, skip_dependency_check=False): - parsed_url = urlparse(url) - gz_name = Path(parsed_url.path).name - - cache_entry: dict[str, Any] = {} - build_dir = tempfile.TemporaryDirectory() - build_path = Path(build_dir.name) - - cache_entry["build_dir"] = build_dir - with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as f: - data = requests.get(url).content - f.write(data) - f.close() - shutil.unpack_archive(f.name, build_path) - os.unlink(f.name) - files = list(build_path.iterdir()) - if len(files) == 1 and files[0].is_dir(): - source_path = build_path / files[0] - else: - source_path = build_path - logger.info("Building wheel for %s...", gz_name) - with ( - tempfile.NamedTemporaryFile(mode="w+") as logfile, - stream_redirected(to=logfile, stream=sys.stdout), - stream_redirected(to=logfile, stream=sys.stderr), - ): - try: - wheel_path = build.run( - source_path, - build_path / "dist", - PyPIProvider.BUILD_EXPORTS, - PyPIProvider.BUILD_FLAGS, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - ) - except BaseException as e: - logger.error(" Failed\n Error is:") - logfile.seek(0) - logger.stderr(logfile.read()) - raise e - - logger.success("Success") - - cache_entry["path"] = wheel_path - return cache_entry - - -class Candidate: - def __init__(self, name, version, url=None, extras=None): - self.name = canonicalize_name(name) - self.version = version - self.url = url - self.extras = extras - - self._metadata = None - self._dependencies = None - - def __repr__(self): - if not self.extras: - return f"<{self.name}=={self.version}>" - return f"<{self.name}[{','.join(self.extras)}]=={self.version}>" - - @property - def metadata(self): - if self._metadata is None: - self._metadata = get_metadata_for_wheel(self.url) - return self._metadata - - @property - def requires_python(self): - return self.metadata.get("Requires-Python") - - def _get_dependencies(self): - deps = self.metadata.get_all("Requires-Dist", []) - extras = self.extras if self.extras else [""] - - for d in deps: - r = Requirement(d) - if r.marker is None: - yield r - else: - for e in extras: - if r.marker.evaluate({"extra": e}): - yield r - break - - @property - def dependencies(self): - if self._dependencies is None: - self._dependencies = list(self._get_dependencies()) - return self._dependencies - - -PYTHON_VERSION = Version(python_version()) - - -def get_target_python(): - TargetPython, _ = _import_unearth() - PYMAJOR = build_env.get_pyversion_major() - PYMINOR = build_env.get_pyversion_minor() - tp = TargetPython( - py_ver=(int(PYMAJOR), int(PYMINOR)), - platforms=[build_env.platform()], - abis=[f"cp{PYMAJOR}{PYMINOR}"], - ) - return tp - - -def get_project_from_pypi(package_name, extras): - """Return candidates created from the project name and extras.""" - _, PackageFinder = _import_unearth() - pf = PackageFinder( - index_urls=_PYPI_INDEX, - trusted_hosts=_PYPI_TRUSTED_HOSTS, - target_python=get_target_python(), - ) - matches = pf.find_all_packages(package_name) - for i in matches: - # TODO: ignore sourcedists if wheel for same version exists - yield Candidate(i.name, i.version, url=i.link.url, extras=extras) - - -def download_or_build_wheel( - url: str, - target_directory: Path, - compression_level: int = 6, - isolation: bool = True, - skip_dependency_check: bool = False, -) -> None: - parsed_url = urlparse(url) - if parsed_url.path.endswith("gz"): - wheel_file = get_built_wheel(url, isolation, skip_dependency_check) - shutil.copy(wheel_file, target_directory) - wheel_path = target_directory / wheel_file.name - elif parsed_url.path.endswith(".whl"): - wheel_path = target_directory / Path(parsed_url.path).name - with open(wheel_path, "wb") as f: - f.write(requests.get(url).content) - - repack_zip_archive(wheel_path, compression_level=compression_level) - - -def get_metadata_for_wheel(url): - parsed_url = urlparse(url) - if parsed_url.path.endswith("gz"): - wheel_file = get_built_wheel(url) - wheel_stream: BinaryIO = open(wheel_file, "rb") - elif parsed_url.path.endswith(".whl"): - data = requests.get(url).content - wheel_stream = BytesIO(data) - else: - raise RuntimeError(f"Distributions of this type are unsupported:{url}") - with ZipFile(wheel_stream) as z: - for n in z.namelist(): - if n.endswith(".dist-info/METADATA"): - p = BytesParser() - return p.parse(cast(BinaryIO, z.open(n)), headersonly=True) - - # If we didn't find the metadata, return an empty dict - return EmailMessage() - - -class PyPIProvider(_AbstractProvider): - BUILD_FLAGS: ConfigSettingsType = {} - BUILD_SKIP: list[str] = [] - BUILD_EXPORTS: _BuildSpecExports = [] - - def __init__(self, build_dependencies: bool): - self.build_dependencies = build_dependencies - - def identify(self, requirement_or_candidate): - base = canonicalize_name(requirement_or_candidate.name) - return base - - def get_extras_for(self, requirement_or_candidate): - # Extras is a set, which is not hashable - return tuple(sorted(requirement_or_candidate.extras)) - - def get_base_requirement(self, candidate): - return Requirement(f"{candidate.name}=={candidate.version}") - - def get_preference( - self, identifier, resolutions, candidates, information, backtrack_causes - ): - return sum(1 for _ in candidates[identifier]) - - def find_matches(self, identifier, requirements, incompatibilities): - requirements = list(requirements[identifier]) - - extra_requirements = {} - for r in requirements: - extra_requirements[tuple(r.extras)] = 1 - - bad_versions = {c.version for c in incompatibilities[identifier]} - - # Need to pass the extras to the search, so they - # are added to the candidate at creation - we - # treat candidates as immutable once created. - for extra_tuple in extra_requirements.keys(): - extras = set(extra_tuple) - - candidates = ( - candidate - for candidate in get_project_from_pypi(identifier, extras) - if candidate.version not in bad_versions - and all(candidate.version in r.specifier for r in requirements) - ) - - return sorted(candidates, key=attrgetter("version"), reverse=True) - - def is_satisfied_by(self, requirement, candidate): - if canonicalize_name(requirement.name) != candidate.name: - return False - return candidate.version in requirement.specifier - - def get_dependencies(self, candidate): - deps = [] - if self.build_dependencies: - for d in candidate.dependencies: - if d.name not in PyPIProvider.BUILD_SKIP: - deps.append(d) - if candidate.extras: - # add the base package as a dependency too, so we can avoid conflicts between same package - # but with different extras - req = self.get_base_requirement(candidate) - deps.append(req) - return deps - - -def _get_json_package_list(fname: Path) -> Generator[str, None, None]: - json_data = json.load(fname.open()) - if "packages" in json_data: - # pyodide-lock.json format - yield from json_data["packages"].keys() - else: - # jupyterlite all.json format - for k in json_data.keys(): - if "releases" in json_data[k]: - yield k - - -def _parse_skip_list(skip_dependency: list[str]) -> None: - PyPIProvider.BUILD_SKIP = [] - for skip in skip_dependency: - split_deps = skip.split(",") - for dep in split_deps: - if dep.endswith(".json"): - # a pyodide json file - # or a jupyterlite json file - # skip all packages in it - PyPIProvider.BUILD_SKIP.extend(_get_json_package_list(Path(dep))) - else: - PyPIProvider.BUILD_SKIP.append(dep) - - -def _resolve_and_build( - deps: list[str], - target_folder: Path, - build_dependencies: bool, - extras: list[str], - output_lockfile: str | None, - isolation: bool = True, - skip_dependency_check: bool = False, - compression_level: int = 6, -) -> None: - BaseReporter, Resolver = _import_resolvelib() - requirements = [] - - target_env = { - "python_version": build_env.get_pyversion_major_minor(), - "sys_platform": build_env.platform().split("_")[0], - "extra": ",".join(extras), - } - - for d in deps: - r = Requirement(d) - if (r.name not in PyPIProvider.BUILD_SKIP) and ( - (not r.marker) or r.marker.evaluate(target_env) - ): - requirements.append(r) - - # Create the (reusable) resolver. - provider = PyPIProvider(build_dependencies=build_dependencies) - reporter = BaseReporter() - resolver = Resolver(provider, reporter) - - # Kick off the resolution process, and get the final result. - result = resolver.resolve(requirements) - target_folder.mkdir(parents=True, exist_ok=True) - version_file = None - if output_lockfile is not None and len(output_lockfile) > 0: - version_file = open(output_lockfile, "w") - for x in result.mapping.values(): - download_or_build_wheel(x.url, target_folder, compression_level) - if len(x.extras) > 0: - extratxt = "[" + ",".join(x.extras) + "]" - else: - extratxt = "" - if version_file: - version_file.write(f"{x.name}{extratxt}=={x.version}\n") - if version_file: - version_file.close() - - -def build_wheels_from_pypi_requirements( - reqs: list[str], - target_folder: Path, - build_dependencies: bool, - skip_dependency: list[str], - exports: _BuildSpecExports, - config_settings: ConfigSettingsType, - isolation: bool = True, - skip_dependency_check: bool = False, - output_lockfile: str | None = None, - compression_level: int = 6, -) -> None: - """ - Given a list of package requirements, build or fetch them. If build_dependencies is true, then - package dependencies will be built or fetched also. - """ - _parse_skip_list(skip_dependency) - PyPIProvider.BUILD_EXPORTS = exports - PyPIProvider.BUILD_FLAGS = config_settings - _resolve_and_build( - reqs, - target_folder, - build_dependencies, - extras=[], - output_lockfile=output_lockfile, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - compression_level=compression_level, - ) - - -def build_dependencies_for_wheel( - wheel: Path, - extras: list[str], - skip_dependency: list[str], - exports: _BuildSpecExports, - config_settings: ConfigSettingsType, - isolation: bool = True, - skip_dependency_check: bool = False, - output_lockfile: str | None = None, - compression_level: int = 6, -) -> None: - """Extract dependencies from this wheel and build pypi dependencies - for each one in ./dist/ - - n.b. because dependency resolution may need to backtrack, this - is potentially quite slow in the case that one needs to build an - sdist in order to discover dependencies of a candidate sub-dependency. - """ - metadata = None - _parse_skip_list(skip_dependency) - - PyPIProvider.BUILD_EXPORTS = exports - PyPIProvider.BUILD_FLAGS = config_settings - with ZipFile(wheel) as z: - for n in z.namelist(): - if n.endswith(".dist-info/METADATA"): - p = BytesParser() - metadata = p.parse(cast(BinaryIO, z.open(n)), headersonly=True) - if metadata is None: - raise RuntimeError(f"Can't find package metadata in {wheel}") - - deps: list[str] = metadata.get_all("Requires-Dist", []) - metadata.get("version") - _resolve_and_build( - deps, - wheel.parent, - build_dependencies=True, - extras=extras, - output_lockfile=output_lockfile, - isolation=isolation, - skip_dependency_check=skip_dependency_check, - compression_level=compression_level, - ) - # add the current wheel to the package-versions.txt - if output_lockfile is not None and len(output_lockfile) > 0: - with open(output_lockfile, "a") as version_txt: - name = metadata.get("Name") - version = metadata.get("Version") - if extras: - extratxt = "[" + ",".join(extras) + "]" - else: - extratxt = "" - version_txt.write(f"{name}{extratxt}=={version}\n") - - -def fetch_pypi_package(package_spec: str, destdir: Path) -> Path: - _, PackageFinder = _import_unearth() - pf = PackageFinder( - index_urls=_PYPI_INDEX, - trusted_hosts=_PYPI_TRUSTED_HOSTS, - target_python=get_target_python(), - ) - match = pf.find_best_match(package_spec) - if match.best is None: - if len(match.candidates) != 0: - error = f"""Can't find version matching {package_spec} -versions found: -""" - for c in match.candidates: - error += " " + str(c.version) + "\t" - raise RuntimeError(error) - else: - raise RuntimeError(f"Can't find package: {package_spec}") - with tempfile.TemporaryDirectory() as download_dir: - return pf.download_and_unpack( - link=match.best.link, location=destdir, download_dir=download_dir - ) diff --git a/pyodide_build/tests/test_cli.py b/pyodide_build/tests/test_cli.py index a2d87e0a..eafe62ec 100644 --- a/pyodide_build/tests/test_cli.py +++ b/pyodide_build/tests/test_cli.py @@ -505,13 +505,6 @@ def my_get_config_vars(*args): def test_build_exports(monkeypatch, dummy_xbuildenv): - def download_url_shim(url, tmppath): - (tmppath / "build").mkdir() - return "blah" - - def unpack_archive_shim(*args): - pass - exports_ = None def run_shim( @@ -524,10 +517,9 @@ def run_shim( ): nonlocal exports_ exports_ = exports + return output_directory / "package-1.0.0-py3-none-any.whl" monkeypatch.setattr(build, "ensure_emscripten", lambda skip_install=False: None) - monkeypatch.setattr(build, "_download_url", download_url_shim) - monkeypatch.setattr(shutil, "unpack_archive", unpack_archive_shim) monkeypatch.setattr(out_of_tree_build, "run", run_shim) def run(*args): diff --git a/pyodide_build/tests/test_pypi.py b/pyodide_build/tests/test_pypi.py deleted file mode 100644 index 0007a367..00000000 --- a/pyodide_build/tests/test_pypi.py +++ /dev/null @@ -1,366 +0,0 @@ -import re -import subprocess -import sys -import tempfile -from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from textwrap import dedent -from threading import Event, Thread -from typing import Any - -import pytest -from click.testing import CliRunner - -pytest.importorskip("resolvelib") -pytest.importorskip("unearth") - -from pyodide_build.cli import build - -runner = CliRunner() - - -def _make_fake_package( - root: Path, name: str, ver: str, requires: list[str], wheel: bool -) -> None: - canonical_name = re.sub("[_.]", "-", name) - module_name = re.sub("-", "_", name) - packageDir = root / canonical_name - packageDir.mkdir(exist_ok=True) - with tempfile.TemporaryDirectory() as td: - build_path = Path(td) - src_path = build_path / "src" / module_name - src_path.mkdir(exist_ok=True, parents=True) - with open(build_path / "pyproject.toml", "w") as cf: - requirements = [] - extras_requirements: dict[str, list[str]] = {} - for x in requires: - extras = [] - if x.find(";") != -1: - requirement, marker_part = x.split(";") - extras = re.findall(r"extra\s*==\s*[\"'](\w+)[\"']", marker_part) - else: - requirement = x - if len(extras) > 0: - for match in extras: - if match not in extras_requirements: - extras_requirements[match] = [] - extras_requirements[match].append(requirement) - else: - requirements.append(requirement) - extras_requirements_text = "" - for key, value in extras_requirements.items(): - extras_requirements_text += f"{key} = [\n" - for r in value: - extras_requirements_text += f"'{r}',\n" - extras_requirements_text += "]\n" - template = dedent( - """ - [project] - name = "{name}" - version = "{version}" - authors = [{{name = "Your Name", email = "you@yourdomain.com"}}] - description = "Example project {name}" - readme = "README.md" - requires-python = ">=3.8" - classifiers = [ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - ] - dependencies = {requirements} - - [build-system] - build-backend = "setuptools.build_meta" - requires = ["setuptools >= 65.0","wheel","cython >= 0.29.0"] - - [project.optional-dependencies] - {optional_deps_text} - """ - ) - config_str = template.format( - name=canonical_name, - version=ver, - requirements=str(requirements), - optional_deps_text=extras_requirements_text, - ) - cf.write(config_str) - with open(build_path / "README.md", "w") as rf: - rf.write("\n") - if wheel: - with open(src_path / "__init__.py", "w") as f: - f.write(f'print("Hello from {name} module")\n') - subprocess.run( - [ - sys.executable, - "-m", - "build", - "--wheel", - build_path, - "--outdir", - packageDir, - ], - check=True, - ) - else: - # make cython sdist - # i.e. create pyc + setup.cfg (needs cython) in folder and run python -m build - with open(src_path / "__init__.py", "w") as f: - f.write("from .compiled_mod import *") - with open(src_path / "compiled_mod.pyx", "w") as f: - f.write(f'print("Hello from compiled module {name}")') - with open(build_path / "setup.py", "w") as sf: - sf.write( - f""" -from setuptools import setup -from Cython.Build import cythonize -setup(ext_modules=cythonize("src/{module_name}/*.pyx",language_level=3)) -""" - ) - with open(build_path / "MANIFEST.in", "w") as mf: - mf.write("global-include *.pyx\n") - subprocess.run( - [ - sys.executable, - "-m", - "build", - "--sdist", - build_path, - "--outdir", - packageDir, - ], - check=True, - ) - - -# module scope fixture that makes a fake pypi -@pytest.fixture(scope="module") -def fake_pypi_server(): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - simple_root = root / "simple" - if not simple_root.exists(): - simple_root.mkdir(exist_ok=True, parents=True) - # top package resolves_package that should resolve okay - nb: - # this package depends on micropip which is in pyodide already - # and should not be rebuilt - _make_fake_package( - simple_root, - "resolves-package", - "1.0.0", - ["pkg-a", "pkg-b[docs]", "micropip"], - True, - ) - _make_fake_package(simple_root, "pkg-a", "1.0.0", ["pkg-c"], True) - _make_fake_package( - simple_root, "pkg-b", "1.0.0", ['pkg-d==2.0.0;extra=="docs"'], False - ) - _make_fake_package(simple_root, "pkg-c", "1.0.0", [], True) - _make_fake_package(simple_root, "pkg-c", "2.0.0", [], True) - _make_fake_package(simple_root, "pkg-d", "1.0.0", [], False) - _make_fake_package(simple_root, "pkg-d", "2.0.0", [], False) - - # top package doesn't resolve package that requires - _make_fake_package( - simple_root, - "fails_package", - "1.0.0", - ["pkg_a", "pkg_b[docs]", "pkg_d==1.0.0"], - True, - ) - - # spawn webserver - def server_thread( - root_path: Path, server_evt: Event, ret_values: list[Any] - ) -> None: - class PathRequestHandler(SimpleHTTPRequestHandler): - def __init__(self, *args, **argv): - argv["directory"] = root_path.resolve() - super().__init__(*args, **argv) - - server = ThreadingHTTPServer( - ("127.0.0.1", 0), RequestHandlerClass=PathRequestHandler - ) - ret_values.append(server) - server_evt.set() - server.serve_forever(poll_interval=0.05) - - server_evt = Event() - ret_values: list[ThreadingHTTPServer] = [] - running_thread = Thread( - target=server_thread, - kwargs={ - "root_path": root, - "server_evt": server_evt, - "ret_values": ret_values, - }, - ) - running_thread.start() - server_evt.wait() - # now ret_values[0] should be server - server = ret_values[0] - addr = f"http://{server.server_address[0]}:{server.server_address[1]}/simple" # type: ignore[str-bytes-safe] - - yield (addr, f"{server.server_address[0]}:{server.server_address[1]}") # type: ignore[str-bytes-safe] - # cleanup - server.shutdown() - - -# fixture to redirect a single test to use fake pypi in resolution -@pytest.fixture -def fake_pypi_url(fake_pypi_server): - import pyodide_build.out_of_tree.pypi - - pypi_old = pyodide_build.out_of_tree.pypi._PYPI_INDEX - pyodide_build.out_of_tree.pypi._PYPI_TRUSTED_HOSTS = [fake_pypi_server[1]] - pyodide_build.out_of_tree.pypi._PYPI_INDEX = [fake_pypi_server[0]] - yield fake_pypi_server[0] - pyodide_build.out_of_tree.pypi._PYPI_INDEX = pypi_old - - -def test_fetch_or_build_pypi(dummy_xbuildenv, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - # one pure-python package (doesn't need building) and one sdist package (needs building) - pkgs = ["pytest-pyodide", "pycryptodome==3.15.0"] - - for p in pkgs: - result = runner.invoke( - build.main, - [p], - ) - assert result.exit_code == 0, result.output - - built_wheels = set(output_dir.glob("*.whl")) - assert len(built_wheels) == len(pkgs) - - -def test_fetch_or_build_pypi_with_deps_and_extras(dummy_xbuildenv, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - # one pure-python package (doesn't need building) which depends on one sdist package (needs building) - pkgs = ["eth-hash[pycryptodome]==0.5.1", "safe-pysha3 (>=1.0.0)"] - - for p in pkgs: - result = runner.invoke( - build.main, - [p, "--build-dependencies"], - ) - assert result.exit_code == 0, result.output - - built_wheels = set(output_dir.glob("*.whl")) - assert len(built_wheels) == 3 - - -def test_fake_pypi_succeed(dummy_xbuildenv, fake_pypi_url, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - # build package that resolves right - result = runner.invoke( - build.main, - ["resolves-package", "--build-dependencies"], - ) - - assert result.exit_code == 0, str(result.output) + str(result) - - built_wheels = set(output_dir.glob("*.whl")) - assert len(built_wheels) == 5 - # make sure built in package micropip is not rebuilt - assert len(set(output_dir.glob("micropip*.whl"))) == 0 - - -def test_fake_pypi_resolve_fail(dummy_xbuildenv, fake_pypi_url, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - - # build package that resolves right - result = runner.invoke( - build.main, - ["fails-package", "--build-dependencies"], - ) - - # this should fail and should not build any wheels - assert result.exit_code != 0, result.output - built_wheels = set(output_dir.glob("*.whl")) - assert len(built_wheels) == 0 - - -def test_fake_pypi_extras_build(dummy_xbuildenv, fake_pypi_url, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - # build package that resolves right - result = runner.invoke( - build.main, - ["pkg-b[docs]", "--build-dependencies"], - ) - - # this should work - assert result.exit_code == 0, result.output - built_wheels = set(output_dir.glob("*.whl")) - assert len(built_wheels) == 2 - - -def test_fake_pypi_repeatable_build(dummy_xbuildenv, fake_pypi_url, mock_emscripten): - output_dir = dummy_xbuildenv / "dist" - - # build package that resolves right - # override a dependency version and build - # pkg-a - with open("requirements.txt", "w") as req_file: - req_file.write( - """ -# Whole line comment -pkg-c~=1.0.0 # end of line comment -pkg-a - """ - ) - - result = runner.invoke( - build.main, - [ - "-r", - "requirements.txt", - "--build-dependencies", - "--output-lockfile", - "lockfile.txt", - ], - ) - # this should work - assert result.exit_code == 0, result.output - built_wheels = list(output_dir.glob("*.whl")) - assert len(built_wheels) == 2, result.output - - # should have built version 1.0.0 of pkg-c - for x in built_wheels: - if x.name.startswith("pkg_c"): - assert x.name.find("1.0.0") != -1, x.name - x.unlink() - - # rebuild from package-versions lockfile and - # check it outputs the same version number - result = runner.invoke( - build.main, - ["-r", "lockfile.txt"], - ) - - # should still have built 1.0.0 of pkg-c - built_wheels = list(output_dir.glob("*.whl")) - for x in built_wheels: - if x.name.startswith("pkg_c"): - assert x.name.find("1.0.0") != -1, x.name - - assert len(built_wheels) == 2, result.output - - -def test_bad_requirements_text(dummy_xbuildenv, mock_emscripten): - # test 1 - error on URL location in requirements - # test 2 - error on advanced options - # test 3 - error on editable install of package - bad_lines = [" pkg-c@http://www.pkg-c.org", " -r bob.txt", " -e pkg-c"] - for line in bad_lines: - with open("requirements.txt", "w") as req_file: - req_file.write(line + "\n") - - result = runner.invoke( - build.main, - ["-r", "requirements.txt"], - ) - assert result.exit_code != 0 and line.strip() in str(result) diff --git a/pyproject.toml b/pyproject.toml index add5d6ff..d15723c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,10 +54,6 @@ config = "pyodide_build.cli.config:app" xbuildenv = "pyodide_build.cli.xbuildenv:app" [project.optional-dependencies] -resolve = [ - "resolvelib", - "unearth~=0.6", -] uv = [ "build[uv]>=1.4,<1.6,!=1.4.4", ] @@ -125,14 +121,11 @@ strict_equality = true [[tool.mypy.overrides]] module = [ "loky", - "resolvelib", "ruamel.yaml", "termcolor", "test", "tomli", "tomllib", - "unearth", - "unearth.*", "virtualenv>=20.31.2", "auditwheel_emscripten.*" ]