diff --git a/README.md b/README.md index d3c0d92b..5d2f22e1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ SPDX-License-Identifier: CC-BY-SA-4.0 This repository is home to various command line tools and Python libraries that aim to help with software supply chain challenges: - [`sbomnix`](#generate-sbom) is a utility that generates SBOMs given a [Nix](https://nixos.org/) flake reference or store path. - [`nixgraph`](./doc/nixgraph.md) helps query and visualize dependency graphs for [Nix](https://nixos.org/) packages. -- [`nixmeta`](./doc/nixmeta.md) summarizes nixpkgs meta-attributes from the given nixpkgs version. - [`vulnxscan`](./doc/vulnxscan.md) is a vulnerability scanner demonstrating the usage of SBOMs in running vulnerability scans. - [`repology_cli`](./doc/repology_cli.md) and [`repology_cve`](./doc/repology_cli.md#repology-cve-search) are command line clients to [repology.org](https://repology.org/). - [`nix_outdated`](./doc/nix_outdated.md) is a utility that finds outdated nix dependencies for given out path, listing the outdated packages in priority order based on how many other packages depend on the given outdated package. @@ -71,7 +70,10 @@ $ cd sbomnix $ nix develop ``` -The devshell adds all CLI entry points (`sbomnix`, `nixgraph`, `nixmeta`, `vulnxscan`, `repology_cli`, `repology_cve`, `nix_outdated`, `provenance`) to `PATH`. They run against the local source tree, so any edits are picked up immediately without reinstalling. +The devshell adds all CLI entry points (`sbomnix`, `nixgraph`, +`vulnxscan`, `repology_cli`, `repology_cve`, `nix_outdated`, +`provenance`) to `PATH`. They run against the local source tree, so any +edits are picked up immediately without reinstalling. All tools support a consistent verbosity flag: no flag or `--verbose=0` shows INFO output, `-v` or `--verbose=1` enables VERBOSE progress @@ -103,7 +105,7 @@ $ sbomnix github:NixOS/nixpkgs?ref=nixos-unstable#wget ``` #### Generate SBOM Based on Derivation File or Out-path -Flake references are the recommended target for `sbomnix`. When the target is a flake reference, `sbomnix` can resolve the nixpkgs version used to build the package and enrich the SBOM with metadata such as descriptions, licenses, maintainers, and homepage links. When the target is a store path, there is no information about which nixpkgs version produced it, so metadata enrichment is skipped by default; see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). +Flake references are the recommended target for `sbomnix`. When the target is a flake reference, `sbomnix` can resolve the nixpkgs version used to build the package and enrich the SBOM with metadata such as descriptions, licenses, maintainers, and homepage links. When the target is a store path, there is no information about which nixpkgs version produced it, so metadata enrichment is skipped; see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). By default `sbomnix` scans the given target and generates an SBOM including the runtime dependencies. Notice: determining the target runtime dependencies in Nix requires building the target. @@ -135,7 +137,7 @@ $ sbomnix github:NixOS/nixpkgs/nixos-unstable#wget --buildtime ```bash $ sbomnix /path/to/result ``` -Note: store paths carry no record of which nixpkgs version produced them, so nixpkgs metadata enrichment is skipped by default. Pass `--meta-nixpkgs` to supply a nixpkgs source explicitly, or see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). +Note: store paths carry no record of which nixpkgs version produced them, so nixpkgs metadata enrichment is skipped. Use a flakeref target when nixpkgs metadata is required. #### Nixpkgs Metadata Source Selection `sbomnix` enriches packages with nixpkgs metadata, such as descriptions, @@ -147,19 +149,18 @@ toplevel flakerefs are handled through the selected NixOS package set, so overlays, package overrides, nixpkgs config, and system-specific package-set changes can be represented. -Store-path targets skip nixpkgs metadata by default; pass `--meta-nixpkgs` to -choose the source explicitly. - -`--meta-nixpkgs ` scans an explicit nixpkgs source. -`--meta-nixpkgs nix-path` scans the `nixpkgs=` entry from `NIX_PATH` as an -explicit opt-in source. `--exclude-meta` disables this enrichment and cannot be -combined with `--meta-nixpkgs`. +Store-path targets skip nixpkgs metadata because the store path does not +identify the nixpkgs source that produced it. `--exclude-meta` disables metadata +enrichment. CycloneDX and SPDX outputs record the selected metadata source in document metadata, including fields such as `nixpkgs:metadata_source_method`, `nixpkgs:path`, `nixpkgs:rev`, `nixpkgs:flakeref`, `nixpkgs:version`, and `nixpkgs:message`. +See [sbomnix metadata enrichment](./doc/sbomnix_metadata.md) for a short +overview of source selection, component matching, and metadata caching. + #### Visualize Package Dependencies `sbomnix` uses structured Nix JSON to find package dependencies where available. `nixgraph` can also be used as a stand-alone tool for visualizing diff --git a/doc/nixmeta.md b/doc/nixmeta.md deleted file mode 100644 index 5f4b38e6..00000000 --- a/doc/nixmeta.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# Getting Started -To get started, follow the [Getting Started](../README.md#getting-started) section from the main [README](../README.md). - -As an example, to run the [`nixmeta`](../src/nixmeta/main.py) from the `tiiuae/sbomnix` repository: -```bash -# '--' signifies the end of argument list for `nix`. -# '--help' is the first argument to `nixmeta` -$ nix run github:tiiuae/sbomnix#nixmeta -- --help -``` - -# nixmeta -[`nixmeta`](../src/nixmeta/main.py) is a command line tool to summarize nixpkgs meta-attributes from the given nixpkgs version. The output is written to a csv file. Nixpkgs version is specified with [`flakeref`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake#flake-references). As an example, `--flakeref=github:NixOS/nixpkgs?ref=master` would make `nixmeta` output the meta-attributes from the nixpkgs version in the [master](https://github.com/NixOS/nixpkgs/tree/master) branch. Similarly, `--flakeref=github:NixOS/nixpkgs?ref=release-23.11` would output the meta-attributes from the nixpkgs version in the [release-23.11](https://github.com/NixOS/nixpkgs/tree/release-23.11) branch. Note that `--flakeref` does not necessarily have to reference `github:NixOS/nixpkgs` but any flakeref or even `NIX_ENV` environment variable can be used to specify the nixpkgs version. As an example, `--flakeref=github:tiiuae/sbomnix` would make `nixmeta` output the meta-attributes from the nixpkgs version [pinned by the sbomnix flake](https://github.com/tiiuae/sbomnix/blob/c243db5272fb01c4d97cbbb01a095ae514cd2dcb/flake.lock#L68) in its default branch. - -As an example, below command outputs nixpkgs meta-attributes from the nixpkgs version pinned by flake `github:NixOS/nixpkgs?ref=master`: - -```bash -$ nixmeta --flakeref=github:NixOS/nixpkgs?ref=master -INFO Finding meta-info for nixpkgs pinned in flake: github:NixOS/nixpkgs?ref=master -INFO Wrote: /home/foo/sbomnix-fork/nixmeta.csv -``` - -Output summarizes the meta-attributes of all the target nixpkgs packages enumerated by `nix-env --query --available`. -For each package, the output includes the following details: - -```bash -$ head -n2 nixmeta.csv | csvlook -| name | pname | version | meta_homepage | meta_unfree | meta_license_short | meta_license_spdxid | meta_maintainers_email | -| ---------- | ----- | ------- | -------------------- | ----------- | -------------------------------- | -------------------------------------- | ---------------------- | -| 0ad-0.0.26 | 0ad | 0.0.26 | https://play0ad.com/ | False | gpl2;lgpl21;mit;cc-by-sa-30;zlib | GPL-2.0;LGPL-2.1;MIT;CC-BY-SA-3.0;Zlib | nixpkgs@cvpetegem.be | - -``` diff --git a/doc/sbomnix_metadata.md b/doc/sbomnix_metadata.md new file mode 100644 index 00000000..2e460c31 --- /dev/null +++ b/doc/sbomnix_metadata.md @@ -0,0 +1,53 @@ + + +# sbomnix metadata enrichment + +`sbomnix` can enrich SBOM components with nixpkgs package metadata such as +descriptions, licenses, maintainers, and homepage links. Metadata enrichment is +best-effort: an SBOM can still be generated when metadata is unavailable or when +no metadata candidate matches a component exactly. + +## Metadata source selection + +For flakeref targets, `sbomnix` selects a nixpkgs metadata source from the +target context. For NixOS toplevel flakerefs, it uses the evaluated +`nixosConfigurations..pkgs` package set, so overlays, package overrides, +nixpkgs config, and system-specific settings can be represented. + +For other flakeref targets, `sbomnix` uses the target flake lock graph to find +the pinned nixpkgs source. Store-path targets skip nixpkgs metadata because a +store path does not identify which nixpkgs source produced it. + +## Component matching + +Component names, pnames, and versions are used as lookup hints to find possible +nixpkgs derivations. Those names are not trusted as the final match. The +metadata helper returns derivation and output identities, and `sbomnix` accepts +metadata only when the returned `drvPath` or `outPath` matches an SBOM component +exactly. + +This keeps metadata enrichment tied to the actual components in the SBOM while +still allowing targeted lookups in package sets. + +## Flake input packages + +Some targets include packages that come from flake inputs rather than the +primary nixpkgs package set. After scanning the selected package set, `sbomnix` +may scan package roots exported by the target flake inputs for components that +remain unmatched. + +## Caching and diagnostics + +Package metadata lookups are cached by metadata source identity, lookup set, +lookup mode, and helper implementation fingerprint. With `-v`, `sbomnix` logs +the selected metadata source, lookup counts, and whether package metadata came +from a cache hit or a fresh scan. + +CycloneDX and SPDX outputs record the selected metadata source in document +metadata, including fields such as `nixpkgs:metadata_source_method`, +`nixpkgs:path`, `nixpkgs:rev`, `nixpkgs:flakeref`, `nixpkgs:version`, and +`nixpkgs:message`. diff --git a/nix/apps.nix b/nix/apps.nix index 0a3b44ff..9e65844e 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -29,9 +29,6 @@ # nix run .#nixgraph nixgraph = mkApp "${sbomnix}/bin/nixgraph" "Visualize nix package dependencies"; - # nix run .#nixmeta - nixmeta = mkApp "${sbomnix}/bin/nixmeta" "Summarize nixpkgs meta-attributes"; - # nix run .#vulnxscan vulnxscan = mkApp "${sbomnix}/bin/vulnxscan" "Scan nix artifacts or SBOMs for vulnerabilities"; diff --git a/nix/packages.nix b/nix/packages.nix index 220da821..2b9f2ee9 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -103,7 +103,6 @@ ++ [ (mkDevEntry "sbomnix" "sbomnix.main") (mkDevEntry "nixgraph" "nixgraph.main") - (mkDevEntry "nixmeta" "nixmeta.main") (mkDevEntry "nix_outdated" "nixupdate.nix_outdated") (mkDevEntry "vulnxscan" "vulnxscan.vulnxscan_cli") (mkDevEntry "repology_cli" "repology.repology_cli") diff --git a/pyproject.toml b/pyproject.toml index 2edebfa4..5d98d771 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ Homepage = "https://github.com/tiiuae/sbomnix" [project.scripts] sbomnix = "sbomnix.main:main" nixgraph = "nixgraph.main:main" -nixmeta = "nixmeta.main:main" nix_outdated = "nixupdate.nix_outdated:main" vulnxscan = "vulnxscan.vulnxscan_cli:main" repology_cli = "repology.repology_cli:main" @@ -59,6 +58,9 @@ version = { file = ["VERSION"] } [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +sbomnix = ["*.nix"] + [tool.ruff] line-length = 88 target-version = "py310" @@ -87,7 +89,6 @@ select = [ known-first-party = [ "common", "nixgraph", - "nixmeta", "nixupdate", "provenance", "repology", diff --git a/src/common/flakeref.py b/src/common/flakeref.py index e3dfc5ea..4a9a1ec7 100644 --- a/src/common/flakeref.py +++ b/src/common/flakeref.py @@ -7,6 +7,7 @@ import logging import pathlib import re +import time from common.errors import FlakeRefRealisationError, FlakeRefResolutionError from common.log import LOG, LOG_VERBOSE @@ -65,21 +66,29 @@ def try_resolve_flakeref( # noqa: PLR0913 else: log.log(LOG_VERBOSE, "Evaluating '%s'", flakeref) cmd = nix_cmd("eval", "--raw", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix eval failed for '%s' after %.3fs", flakeref, elapsed) if looks_like_flakeref: raise FlakeRefResolutionError(flakeref, ret.stderr if ret else "") log.debug("not a flakeref: '%s'", flakeref) return None nixpath = ret.stdout.strip() + log.log(LOG_VERBOSE, "Evaluated '%s' in %.3fs", flakeref, elapsed) log.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) if not force_realise: return nixpath log.info("Realising flakeref '%s'", flakeref) cmd = nix_cmd("build", "--no-link", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix build failed for '%s' after %.3fs", flakeref, elapsed) raise FlakeRefRealisationError(flakeref, ret.stderr if ret else "") + log.log(LOG_VERBOSE, "Realised flakeref '%s' in %.3fs", flakeref, elapsed) return nixpath @@ -92,8 +101,15 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): """Return the derivation path for a flakeref target.""" log.info("Evaluating flakeref '%s'", flakeref) cmd = nix_cmd("derivation", "show", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug( + "nix derivation show failed for flakeref '%s' after %.3fs", + flakeref, + elapsed, + ) raise FlakeRefResolutionError(flakeref, ret.stderr if ret else "") drv_paths = parse_nix_derivation_show(ret.stdout) drv_path = next(iter(drv_paths), "") @@ -102,6 +118,12 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): flakeref, "nix derivation show returned no derivation path", ) + log.log( + LOG_VERBOSE, + "Resolved flakeref derivation in %.3fs to '%s'", + elapsed, + drv_path, + ) log.debug("flakeref='%s' maps to derivation='%s'", flakeref, drv_path) return drv_path @@ -109,15 +131,6 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): def _force_realise_flakeref(flakeref, *, impure, exec_cmd_fn, log): """Return the realized output path for a runtime flakeref target.""" log.info("Evaluating flakeref '%s'", flakeref) - nixpath = _realised_flakeref_path( - flakeref, - impure=impure, - exec_cmd_fn=exec_cmd_fn, - ) - if nixpath: - log.debug("flakeref='%s' maps to realised path='%s'", flakeref, nixpath) - return nixpath - return _build_flakeref_path( flakeref, impure=impure, @@ -136,8 +149,11 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): flakeref, impure=impure, ) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix build failed for flakeref '%s' after %.3fs", flakeref, elapsed) raise FlakeRefRealisationError(flakeref, ret.stderr if ret else "") nixpath = _first_output_path(ret.stdout) if not nixpath: @@ -145,19 +161,16 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): flakeref, "nix build returned no output path", ) + log.log( + LOG_VERBOSE, + "Resolved flakeref to built path '%s' in %.3fs", + nixpath, + elapsed, + ) log.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) return nixpath -def _realised_flakeref_path(flakeref, *, impure, exec_cmd_fn): - """Return the already-realised output path for ``flakeref`` when available.""" - cmd = nix_cmd("path-info", flakeref, impure=impure) - ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) - if ret is None or ret.returncode != 0: - return None - return _first_output_path(ret.stdout) - - def parse_nixos_configuration_ref( flakeref: str, *, diff --git a/src/common/proc.py b/src/common/proc.py index d676ac91..a63cbba8 100644 --- a/src/common/proc.py +++ b/src/common/proc.py @@ -8,6 +8,7 @@ import os import shlex import subprocess +import time from collections.abc import Callable, Sequence from shutil import which from typing import IO, Literal, overload @@ -20,6 +21,18 @@ ExecCmdFn = Callable[..., ExecCmdResult] +def _stream_summary(value: str | None) -> str: + if value is None: + return "not captured" + return f"{len(value)} byte(s), {value.count(chr(10))} line(s)" + + +def _stream_preview(value: str, limit: int = 4000) -> str: + if len(value) <= limit: + return value.rstrip() + return f"{value[:limit].rstrip()}\n... truncated {len(value) - limit} byte(s)" + + @overload def exec_cmd( cmd: Sequence[CommandPart], @@ -62,6 +75,7 @@ def exec_cmd( raise TypeError("cmd must be an argv sequence, not a string-like value") argv = [os.fspath(part) for part in cmd] command_str = shlex.join(argv) + started = time.perf_counter() LOG.debug("Running: %s", command_str) try: if stdout: @@ -73,8 +87,26 @@ def exec_cmd( encoding="utf-8", check=True, ) + LOG.debug( + "Finished in %.3fs (rc=%s, stdout=%s, stderr=%s): %s", + time.perf_counter() - started, + ret.returncode, + _stream_summary(ret.stdout), + _stream_summary(ret.stderr), + command_str, + ) + if ret.stderr: + LOG.debug("Command stderr:\n%s", _stream_preview(ret.stderr)) return ret except subprocess.CalledProcessError as error: + LOG.debug( + "Failed in %.3fs (rc=%s, stdout=%s, stderr=%s): %s", + time.perf_counter() - started, + error.returncode, + _stream_summary(error.stdout), + _stream_summary(error.stderr), + command_str, + ) if log_error: LOG.error( "Error running shell command:\n cmd: '%s'\n stdout: %s\n stderr: %s", diff --git a/src/nixmeta/__init__.py b/src/nixmeta/__init__.py deleted file mode 100644 index 5cc74e16..00000000 --- a/src/nixmeta/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 diff --git a/src/nixmeta/main.py b/src/nixmeta/main.py deleted file mode 100755 index 1f07db47..00000000 --- a/src/nixmeta/main.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# SPDX-License-Identifier: Apache-2.0 - -"""Python script for summarizing nixpkgs meta-attributes""" - -import argparse -import pathlib - -from common.cli_args import add_verbose_argument, add_version_argument -from common.errors import SbomnixError -from common.log import LOG, set_log_verbosity -from common.proc import exit_unless_command_exists -from nixmeta.scanner import NixMetaScanner - -################################################################################ - - -def _getargs(args=None): - """Parse command line arguments""" - desc = ( - "Summarize nixpkgs meta-attributes from the given nixpkgs version " - "to a csv output file." - ) - epil = "Example: nixmeta --flakeref=github:NixOS/nixpkgs?ref=master" - parser = argparse.ArgumentParser(description=desc, epilog=epil) - helps = ( - "Flake reference specifying the location of the flake " - "from which the pinned nixpkgs target version is read. " - "The default value is the " - "current nixpkgs version in its 'nixos-unstable' branch. " - "For more details, see: " - "https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake" - "#flake-references and " - "https://nixos.wiki/wiki/Nix_channels " - "(default: --flakeref=github:NixOS/nixpkgs?ref=nixos-unstable)." - ) - parser.add_argument( - "-f", - "--flakeref", - help=helps, - type=str, - default="github:NixOS/nixpkgs?ref=nixos-unstable", - ) - helps = "Path to output file (default: --out=nixmeta.csv)." - parser.add_argument( - "-o", - "--out", - help=helps, - type=pathlib.Path, - default="nixmeta.csv", - ) - helps = ( - "Append to output file - removing duplicate entries - instead of " - "completely overwriting possible earlier output file." - ) - parser.add_argument( - "-a", - "--append", - help=helps, - action="store_true", - ) - add_version_argument(parser) - add_verbose_argument(parser) - return parser.parse_args(args) - - -############################################################################### - - -def main(): - """main entry point""" - args = _getargs() - set_log_verbosity(args.verbose) - try: - _run(args) - except SbomnixError as error: - LOG.fatal("%s", error) - raise SystemExit(1) from error - - -def _run(args): - # Fail early if the following commands are not in PATH - exit_unless_command_exists("nix") - exit_unless_command_exists("nix-env") - # Scan metadata from the flakeref pinned nixpkgs - LOG.info("Scanning nixpkgs metadata for '%s'", args.flakeref) - scanner = NixMetaScanner() - scanner.scan(args.flakeref) - # Output to csv file - scanner.to_csv(args.out, args.append) - - -################################################################################ - -if __name__ == "__main__": - main() - -################################################################################ diff --git a/src/nixmeta/metadata_json.py b/src/nixmeta/metadata_json.py deleted file mode 100644 index c5b22e44..00000000 --- a/src/nixmeta/metadata_json.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Helpers for flattening nix-env metadata JSON.""" - -import json - -import pandas as pd - -from common import columns as cols -from common.log import LOG - - -def parse_meta_entry(meta, key): - """Flatten nested metadata values for a single key into a string.""" - items = [] - if isinstance(meta, dict): - items.extend([parse_meta_entry(meta.get(key, ""), key)]) - elif isinstance(meta, list): - items.extend([parse_meta_entry(item, key) for item in meta]) - else: - return str(meta) - return ";".join(list(filter(None, items))) - - -def parse_json_metadata(json_filename, *, log=LOG): - """Parse package metadata from a ``nix-env --json`` output file.""" - with open(json_filename, "r", encoding="utf-8") as inf: - log.debug('Loading meta-info from "%s"', json_filename) - json_dict = json.loads(inf.read()) - dict_selected = {} - setcol = dict_selected.setdefault - for pkg in json_dict.values(): - setcol(cols.NAME, []).append(pkg.get("name", "")) - setcol("pname", []).append(pkg.get("pname", "")) - setcol(cols.VERSION, []).append(pkg.get("version", "")) - meta = pkg.get("meta", {}) - setcol("meta_homepage", []).append(parse_meta_entry(meta, key="homepage")) - setcol("meta_unfree", []).append(meta.get("unfree", "")) - setcol("meta_description", []).append(meta.get("description", "")) - setcol("meta_position", []).append(meta.get("position", "")) - meta_license = meta.get("license", {}) - setcol("meta_license_short", []).append( - parse_meta_entry(meta_license, key="shortName") - ) - setcol("meta_license_spdxid", []).append( - parse_meta_entry(meta_license, key="spdxId") - ) - meta_maintainers = meta.get("maintainers", {}) - setcol("meta_maintainers_email", []).append( - parse_meta_entry(meta_maintainers, key="email") - ) - return pd.DataFrame(dict_selected).astype(str) diff --git a/src/nixmeta/scanner.py b/src/nixmeta/scanner.py deleted file mode 100755 index d8990168..00000000 --- a/src/nixmeta/scanner.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# SPDX-License-Identifier: Apache-2.0 - -"""Summarize nixpkgs meta-attributes""" - -import pathlib -import subprocess -from tempfile import NamedTemporaryFile - -import pandas as pd - -from common.df import df_from_csv_file, df_to_csv_file -from common.log import LOG, LOG_SPAM -from common.proc import exec_cmd, nix_cmd -from nixmeta.flake_metadata import get_flake_metadata, nixref_to_nixpkgs_path -from nixmeta.metadata_json import parse_json_metadata - -############################################################################### - - -def _run_nix_env_metadata(cmd, stdout): - """Run nix-env metadata scan while keeping successful eval warnings quiet.""" - ret = subprocess.run( - cmd, - encoding="utf-8", - check=True, - stdout=stdout, - stderr=subprocess.PIPE, - ) - if ret.stderr: - LOG.debug("nix-env metadata stderr:\n%s", ret.stderr.strip()) - - -class NixMetaScanner: - """Scan nixpkgs meta-info""" - - def __init__(self): - self.df_meta = None - - def scan(self, nixref): - """ - Scan nixpkgs meta-info using nixpkgs version pinned in nixref; - nixref can be a nix store path, flakeref or dynamical attribute set. - """ - nixpkgs_path = nixref_to_nixpkgs_path( - nixref, - get_flake_metadata_fn=lambda flakeref: get_flake_metadata( - flakeref, - exec_cmd_fn=exec_cmd, - nix_cmd_fn=nix_cmd, - log=LOG, - ), - log=LOG, - log_spam=LOG_SPAM, - ) - if not nixpkgs_path: - # try format which is understood by nix-env: - # https://ianthehenry.com/posts/how-to-learn-nix/chipping-away-at-flakes/ - # ownpkgs-nix-env.nix: - # { ... }: - # (builtins.getFlake "/tmp/ownpkgs-special-unstable"). - # outputs.packages.${builtins.currentSystem} - # and execute - # NIX_PATH="nixpkgs=/tmp/ownpkgs-special-unstable/ownpkgs-nix-env.nix" - # sbomnix /nix/store/outputpath-for-ownpkgs-special-unstable-flake-output - nixpkgs_path = pathlib.Path(nixref) - self.scan_path(nixpkgs_path) - - def scan_path(self, nixpkgs_path): - """Scan nixpkgs meta-info using an already resolved nixpkgs path.""" - nixpkgs_path = pathlib.Path(nixpkgs_path) - if not nixpkgs_path.exists(): - LOG.warning("Nixpkgs not in nix store: %s", nixpkgs_path.as_posix()) - return - LOG.debug("nixpkgs: %s", nixpkgs_path) - self._read_nixpkgs_meta(nixpkgs_path) - - def scan_expression(self, expression, *, impure=False): - """Scan nixpkgs meta-info using an expression returning a package set.""" - prefix = "nixmeta_expr_" - suffix = ".nix" - with NamedTemporaryFile( - mode="w", - delete=True, - encoding="utf-8", - prefix=prefix, - suffix=suffix, - ) as f: - f.write(expression) - f.flush() - self._read_nixpkgs_meta( - pathlib.Path(f.name), - enable_flakes=True, - impure=impure, - ) - - def to_csv(self, csv_path, append=False): - """Export meta-info to a csv file""" - csv_path = pathlib.Path(csv_path) - if append and csv_path.exists(): - df = df_from_csv_file(csv_path) - self.df_meta = pd.concat([self.df_meta, df], ignore_index=True) - self._drop_duplicates() - if self.df_meta is None or self.df_meta.empty: - LOG.info("Nothing to output") - return - csv_path.parent.mkdir(parents=True, exist_ok=True) - df_to_csv_file(self.df_meta, csv_path.absolute().as_posix()) - - def to_df(self): - """Return meta-info as dataframe""" - return self.df_meta - - def _read_nixpkgs_meta( - self, - nixpkgs_path, - *, - enable_flakes=False, - impure=False, - ): - prefix = "nixmeta_" - suffix = ".json" - with NamedTemporaryFile(delete=True, prefix=prefix, suffix=suffix) as f: - LOG.info("Reading nixpkgs metadata from '%s'", nixpkgs_path.as_posix()) - cmd = [ - "nix-env", - "-qa", - "--meta", - "--json", - "-f", - f"{nixpkgs_path.as_posix()}", - ] - if enable_flakes: - cmd.extend(["--option", "experimental-features", "nix-command flakes"]) - if impure: - cmd.append("--impure") - cmd.extend(["--arg", "config", "{allowAliases=false;}"]) - _run_nix_env_metadata(cmd, stdout=f) - LOG.debug("Generated meta.json: %s", f.name) - LOG.info("Parsing nixpkgs metadata") - self.df_meta = parse_json_metadata(f.name, log=LOG) - self._drop_duplicates() - - def _drop_duplicates(self): - if self.df_meta is None or self.df_meta.empty: - return - self.df_meta = self.df_meta.astype(str) - self.df_meta.fillna("", inplace=True) - uids = [ - "name", - "version", - "meta_license_short", - "meta_license_spdxid", - "meta_homepage", - ] - self.df_meta.sort_values(by=uids, inplace=True) - self.df_meta.drop_duplicates(subset=uids, keep="last", inplace=True) - - -############################################################################### diff --git a/src/sbomnix/artifacts.py b/src/sbomnix/artifacts.py new file mode 100644 index 00000000..536c4913 --- /dev/null +++ b/src/sbomnix/artifacts.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Helpers for recognizing file artifacts in Nix build closures.""" + +import re + +_PATCH_OR_DIFF_RE = r"[.](?:patch|diff)(?:[.][a-zA-Z0-9]+)?" +_SOURCE_ARCHIVE_RE = ( + r"[.]tar(?:[.][a-zA-Z0-9]+)?|" + r"[.](?:tgz|tbz2?|txz|zip|whl|gem|cabal|crate)" +) +_CONFIG_OR_DATA_RE = ( + r"[.](?:" + r"conf|cfg|ini|json|toml|ya?ml|lock|sum|mod|list|rules|preset|" + r"desktop|service|socket|target|timer|mount|path|slice|network|" + r"link|netdev|nmconnection|pam|pc|xml|crt|pem|cer|der|dat|" + r"asc|sig|txt|md|rst" + r")" +) +_FONT_OR_MEDIA_RE = ( + r"[.](?:" + r"bdf|pcf|otf|ttf|ttc|woff2?|pfa|pfb|" + r"mp3|mp4|mov|mkv|mxf|wav|ogg|amv|dmg|png|svg|gif|jpe?g|webp" + r")" + r"(?:[.](?:gz|xz|bz2|zst))?" +) +_SOURCE_FILE_RE = ( + r"[.](?:" + r"sh|bash|fish|py|pl|pm|lua|cgi|c|cc|cpp|h|hpp|java|js|ts|" + r"go|rs|hs|m4|in|cmake|mk|proto|css|scss|html" + r")" +) +_GENERATED_CARGO_ARTIFACT_RE = re.compile( + r"^cargo-(?:src|package)-.+-[0-9].*$", + re.IGNORECASE, +) +_NON_PACKAGE_ARTIFACT_NAME_RE = re.compile( + rf".*(?:{_PATCH_OR_DIFF_RE}|{_SOURCE_ARCHIVE_RE}|" + rf"{_CONFIG_OR_DATA_RE}|{_FONT_OR_MEDIA_RE}|{_SOURCE_FILE_RE})(?:[?].*)?$", + re.IGNORECASE, +) + + +def is_non_package_artifact_name(name): + """Return True when a store-path name looks like a file, not a package.""" + if not isinstance(name, str) or not name: + return False + return ( + "?" in name + or _GENERATED_CARGO_ARTIFACT_RE.match(name) is not None + or _NON_PACKAGE_ARTIFACT_NAME_RE.match(name) is not None + ) diff --git a/src/sbomnix/builder.py b/src/sbomnix/builder.py index 3a954ace..fd36ea31 100644 --- a/src/sbomnix/builder.py +++ b/src/sbomnix/builder.py @@ -7,6 +7,7 @@ """SBOM builder orchestration.""" import logging +import time import uuid from dataclasses import dataclass from typing import Any @@ -47,6 +48,9 @@ # Namespace UUID (a UUIDv4) for stable UUIDv5 identifiers. # See RFC9562, *6.6. Namespace ID Usage and Allocation*. SBOMNIX_UUID_NAMESPACE = uuid.UUID("136af32e-0d0e-48bc-912c-31b26af294b9") +_PACKAGE_META_PRIVATE_COLUMNS = [ + cols.NAME, +] @dataclass(frozen=True) @@ -85,7 +89,6 @@ def __init__( # noqa: PLR0913, PLR0917 depth=None, flakeref=None, original_ref=None, - meta_nixpkgs=None, impure=False, include_meta=True, include_vulns=False, @@ -94,6 +97,14 @@ def __init__( # noqa: PLR0913, PLR0917 # self.uid specifies the attribute that identifies SBOM components. # See the column names in # self.df_sbomdb (sbom.csv) for a list of all components' attributes. + started = time.perf_counter() + LOG.verbose( + "Initializing SBOM builder for '%s' (buildtime=%s, meta=%s, cpe=%s)", + nix_path, + buildtime, + include_meta, + include_cpe, + ) self.uid = cols.STORE_PATH self.nix_path = nix_path self.buildtime = buildtime @@ -110,7 +121,6 @@ def __init__( # noqa: PLR0913, PLR0917 self.dependency_index = None self.flakeref = flakeref self.original_ref = original_ref - self.meta_nixpkgs = meta_nixpkgs self.impure = impure self.meta = None # "disabled" records explicit opt-out; "none" means auto-selection @@ -134,12 +144,30 @@ def __init__( # noqa: PLR0913, PLR0917 self.sbom_type = "runtime_and_buildtime" if not self.buildtime: self.sbom_type = "runtime_only" + LOG.verbose( + "Initialized SBOM builder in %.3fs", + time.perf_counter() - started, + ) def _resolve_target_deriver(self, nix_path): + started = time.perf_counter() + LOG.verbose("Resolving target deriver for '%s'", nix_path) if self.buildtime: - return require_deriver(nix_path) + drv_path = require_deriver(nix_path) + LOG.verbose( + "Resolved build-time target deriver in %.3fs: %s", + time.perf_counter() - started, + drv_path, + ) + return drv_path try: - return find_deriver(nix_path) + drv_path = find_deriver(nix_path) + LOG.verbose( + "Resolved runtime target deriver in %.3fs: %s", + time.perf_counter() - started, + drv_path, + ) + return drv_path except SbomnixError: raise except RuntimeError: @@ -148,6 +176,10 @@ def _resolve_target_deriver(self, nix_path): nix_path, exc_info=True, ) + LOG.verbose( + "No loadable runtime target deriver found in %.3fs", + time.perf_counter() - started, + ) return None def _load_structured_closure(self, nix_path): @@ -170,7 +202,14 @@ def _load_recursive_buildtime_closure(self): """Load build-time dependencies from recursive derivation JSON.""" if self.target_deriver is None: raise MissingNixDeriverError(self.nix_path) + started = time.perf_counter() + LOG.verbose("Loading build-time closure for '%s'", self.target_deriver) derivations, drv_infos = load_recursive(self.target_deriver) + LOG.verbose( + "Loaded %d recursive derivation record(s) in %.3fs", + len(drv_infos), + time.perf_counter() - started, + ) df_deps = derivation_dependencies_df(drv_infos) if self.depth: df_deps = self._filter_dependencies_to_depth( @@ -178,6 +217,11 @@ def _load_recursive_buildtime_closure(self): self.target_deriver, self.depth, ) + LOG.verbose( + "Loaded build-time closure with %d dependency edge(s) in %.3fs", + len(df_deps), + time.perf_counter() - started, + ) return StructuredClosure( df_deps=df_deps, recursive_buildtime_derivations=derivations, @@ -185,7 +229,15 @@ def _load_recursive_buildtime_closure(self): def _load_runtime_path_info_closure(self, nix_path): """Load runtime dependencies from structured path-info JSON.""" + started = time.perf_counter() + LOG.verbose("Loading runtime closure for '%s'", nix_path) runtime_closure = load_runtime_closure(nix_path) + LOG.verbose( + "Loaded raw runtime closure with %d dependency edge(s) and %d deriver(s) in %.3fs", + len(runtime_closure.df_deps), + len(runtime_closure.output_paths_by_drv), + time.perf_counter() - started, + ) output_paths_by_load_path = _runtime_output_paths_by_load_path( runtime_closure.output_paths_by_drv ) @@ -207,6 +259,11 @@ def _load_runtime_path_info_closure(self, nix_path): nix_path, self.depth, ) + LOG.verbose( + "Loaded runtime closure with %d dependency edge(s) in %.3fs", + len(df_deps), + time.perf_counter() - started, + ) return StructuredClosure( df_deps=df_deps, runtime_output_paths_by_load_path=output_paths_by_load_path, @@ -237,7 +294,9 @@ def _filter_dependencies_to_depth( def _init_components(self, include_meta): """Initialize the SBOM component dataframe.""" + started = time.perf_counter() paths = self._sbom_component_paths() + LOG.verbose("Initializing SBOM components from %d path(s)", len(paths)) # Populate store based on the dependencies if self._recursive_buildtime_derivations is not None: self.df_sbomdb = recursive_derivations_to_dataframe( @@ -250,6 +309,11 @@ def _init_components(self, include_meta): else: # _load_structured_closure always selects exactly one metadata source. raise AssertionError("Structured dependency metadata was not initialized") + LOG.verbose( + "Initialized %d SBOM component(s) in %.3fs", + len(self.df_sbomdb), + time.perf_counter() - started, + ) # Join with meta information if include_meta: self._join_meta() @@ -259,6 +323,11 @@ def _init_components(self, include_meta): self.df_sbomdb.sort_values(by=[cols.NAME, self.uid], inplace=True) self.df_sbomdb_outputs_exploded = self.df_sbomdb.explode(cols.OUTPUTS) self._init_dependency_index() + LOG.verbose( + "Prepared SBOM database with %d component(s) in %.3fs", + len(self.df_sbomdb), + time.perf_counter() - started, + ) def _sbom_component_paths(self): if self.df_deps is None or self.df_deps.empty: @@ -309,15 +378,21 @@ def _join_meta(self): """Join component rows with nixpkgs metadata.""" if self.df_sbomdb is None: raise AssertionError("SBOM component metadata was not initialized") + started = time.perf_counter() self.meta = Meta() - df_meta, source = self.meta.get_nixpkgs_meta_with_source( + df_meta, source = self.meta.get_package_meta_with_source( + components=self.df_sbomdb, + buildtime=self.buildtime, target_path=self.nix_path, flakeref=self.flakeref, original_ref=self.original_ref, - explicit_nixpkgs=self.meta_nixpkgs, impure=self.impure, ) self.nixpkgs_meta_source = source + LOG.verbose( + "Resolved nixpkgs metadata candidates in %.3fs", + time.perf_counter() - started, + ) if df_meta is None or df_meta.empty: if source.message: LOG.info("%s", source.message) @@ -334,14 +409,25 @@ def _join_meta(self): return if is_debug_enabled(): df_to_csv_file(df_meta, "meta.csv") - # Join based on package name including the version number + # Join only metadata that matched component drvPath/outPath exactly. + df_meta = df_meta.drop( + columns=[ + column + for column in _PACKAGE_META_PRIVATE_COLUMNS + if column in df_meta.columns + ] + ) self.df_sbomdb = self.df_sbomdb.merge( df_meta, how="left", - left_on=[cols.NAME], - right_on=[cols.NAME], + on=[cols.STORE_PATH], suffixes=("", "_meta"), ) + LOG.verbose("Joined nixpkgs metadata for %d component(s)", len(df_meta)) + LOG.verbose( + "Joined nixpkgs metadata in %.3fs", + time.perf_counter() - started, + ) def lookup_dependencies(self, drv, uid=cols.STORE_PATH): """Return indexed dependency values for one SBOM component.""" diff --git a/src/sbomnix/dfcache.py b/src/sbomnix/dfcache.py index 65c98a9f..c91deed2 100644 --- a/src/sbomnix/dfcache.py +++ b/src/sbomnix/dfcache.py @@ -28,6 +28,12 @@ def set(self, key, value, ttl=None): dfcache = DataFrameDiskCache(cache_dir_path=dfcache_dir()) return _cache_local_set(dfcache, key, value, ttl=ttl) + def get_many(self, keys): + """Return multiple dataframe cache entries using one cache connection.""" + with self.dflock: + dfcache = DataFrameDiskCache(cache_dir_path=dfcache_dir()) + return {key: dfcache.get(key) for key in keys} + def __getattr__(self, name): def wrap(*a, **k): with self.dflock: diff --git a/src/nixmeta/flake_metadata.py b/src/sbomnix/flake_metadata.py similarity index 82% rename from src/nixmeta/flake_metadata.py rename to src/sbomnix/flake_metadata.py index 30b74346..451ff314 100644 --- a/src/nixmeta/flake_metadata.py +++ b/src/sbomnix/flake_metadata.py @@ -12,6 +12,40 @@ from common.proc import exec_cmd, nix_cmd +def normalize_current_flake_shorthand(flakeref): + """Return ``.#attr`` for valid current-flake shorthand ``#attr``.""" + flakeref = str(flakeref or "") + flake, separator, attr = flakeref.partition("#") + if separator and not flake: + return f".{separator}{attr}" + return flakeref + + +def normalize_local_flake_ref(flake): + """Return an absolute local flake ref, leaving registry and URL refs intact.""" + if flake.startswith("path:"): + path_text, separator, query = flake.removeprefix("path:").partition("?") + path = pathlib.Path(path_text).expanduser() + if not path.is_absolute(): + path_text = path.resolve().as_posix() + return f"path:{path_text}{separator}{query}" + if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or ""): + return flake + path = pathlib.Path(flake).expanduser() + if path.exists() or flake.startswith((".", "/", "~")): + return path.resolve().as_posix() + return flake + + +def normalize_flakeref_for_nix_eval(flakeref): + """Normalize current-flake shorthand and local paths before Nix evaluation.""" + flakeref = normalize_current_flake_shorthand(flakeref) + flake, separator, attr = flakeref.partition("#") + if not flake: + return flakeref + return f"{normalize_local_flake_ref(flake)}{separator}{attr}" + + def get_flake_metadata(flakeref, *, exec_cmd_fn=exec_cmd, nix_cmd_fn=nix_cmd, log=LOG): """Return ``nix flake metadata`` JSON for the given flakeref.""" if flakeref.startswith("nixpkgs="): @@ -169,6 +203,7 @@ def nixref_to_nixpkgs_path( """Return the nix store path of the nixpkgs pinned by ``flakeref``.""" if not flakeref: return None + flakeref = normalize_current_flake_shorthand(flakeref) log.info("Resolving nixpkgs path for '%s'", flakeref) log.debug("Finding meta-info for nixpkgs pinned in nixref: %s", flakeref) match = re.match(r"([^#]+)#", flakeref) diff --git a/src/sbomnix/main.py b/src/sbomnix/main.py index 4d2b2afd..4398a91b 100755 --- a/src/sbomnix/main.py +++ b/src/sbomnix/main.py @@ -7,6 +7,7 @@ """Python script that generates SBOMs from nix packages""" import argparse +import time from common.cli_args import add_verbose_argument, add_version_argument, check_positive from common.errors import SbomnixError @@ -50,12 +51,6 @@ def getargs(args=None): parser.add_argument( "--exclude-meta", help=helps, action="store_true", default=False ) - helps = ( - "Nixpkgs source used for metadata enrichment. Accepts a nixpkgs " - "flakeref, a nixpkgs source path, or nix-path. Overrides automatic " - "metadata-source detection." - ) - parser.add_argument("--meta-nixpkgs", help=helps, metavar="META_NIXPKGS") helps = "Exclude using heuristics-based CPE matches in the output" parser.add_argument( "--exclude-cpe-matching", help=helps, action="store_true", default=False @@ -89,33 +84,59 @@ def main(): def _run(args): - if args.exclude_meta and args.meta_nixpkgs: - raise SbomnixError("--exclude-meta cannot be used with --meta-nixpkgs") + started = time.perf_counter() + resolve_started = time.perf_counter() target = resolve_nix_target( args.NIXREF, buildtime=args.buildtime, impure=args.impure ) + LOG.verbose( + "Resolved CLI target in %.3fs: path='%s', flakeref='%s'", + time.perf_counter() - resolve_started, + target.path, + target.flakeref, + ) LOG.info("Generating SBOM for target '%s'", target.path) + build_started = time.perf_counter() sbom = SbomBuilder( nix_path=target.path, buildtime=args.buildtime, depth=args.depth, flakeref=target.flakeref, original_ref=target.original_ref, - meta_nixpkgs=args.meta_nixpkgs, impure=args.impure, include_meta=not args.exclude_meta, include_vulns=args.include_vulns, include_cpe=not args.exclude_cpe_matching, ) + LOG.verbose( + "Constructed SBOM model in %.3fs", + time.perf_counter() - build_started, + ) if args.cdx: + export_started = time.perf_counter() cdx = sbom.to_cdx_data() if args.include_vulns: sbom.enrich_cdx_with_vulnerabilities(cdx) sbom.write_json(args.cdx, cdx, printinfo=True) + LOG.verbose( + "Wrote CycloneDX output in %.3fs", + time.perf_counter() - export_started, + ) if args.spdx: + export_started = time.perf_counter() sbom.to_spdx(args.spdx) + LOG.verbose( + "Wrote SPDX output in %.3fs", + time.perf_counter() - export_started, + ) if args.csv: + export_started = time.perf_counter() sbom.to_csv(args.csv) + LOG.verbose( + "Wrote CSV output in %.3fs", + time.perf_counter() - export_started, + ) + LOG.verbose("Completed sbomnix run in %.3fs", time.perf_counter() - started) ################################################################################ diff --git a/src/sbomnix/meta.py b/src/sbomnix/meta.py index 01cefb9d..393eddb7 100644 --- a/src/sbomnix/meta.py +++ b/src/sbomnix/meta.py @@ -4,33 +4,41 @@ """Cache and scan nixpkgs meta information.""" +import re +import time from dataclasses import replace +import pandas as pd from filelock import FileLock +from common import columns as cols from common.log import LOG -from nixmeta.scanner import NixMetaScanner from sbomnix.cache_paths import meta_lock_path from sbomnix.dfcache import LockedDfCache from sbomnix.meta_source import ( - META_NIXPKGS_NIX_PATH, - SCAN_EXCEPTIONS, NixpkgsMetaSource, NixpkgsMetaSourceResolver, - classify_meta_nixpkgs, +) +from sbomnix.package_meta import ( + PACKAGE_META_METHOD, + match_package_metadata_to_components, + package_meta_cache_fingerprint, + package_meta_candidate_key, + package_meta_lookup_keys_for_components, + try_scan_package_meta, ) ############################################################################### -# Update locally generated nixpkgs meta-info every 30 days or when local cache -# is cleaned. -_NIXMETA_NIXPKGS_TTL = 60 * 60 * 24 * 30 +# Update generated package metadata every 30 days or when local cache is cleaned. +_PACKAGE_META_TTL = 60 * 60 * 24 * 30 +_PACKAGE_META_FLAKE_INPUT_LOOKUP_LIMIT = 500 +_PACKAGE_META_CACHE_REF_COLUMN = "_package_meta_cache_ref" +_PACKAGE_META_LOOKUP_ID_COLUMN = "_package_meta_lookup_id" __all__ = [ - "META_NIXPKGS_NIX_PATH", "Meta", "NixpkgsMetaSource", - "classify_meta_nixpkgs", ] @@ -42,33 +50,142 @@ def __init__(self): self.cache = LockedDfCache() self.source_resolver = NixpkgsMetaSourceResolver() - def get_nixpkgs_meta(self, nixref=None): - """ - Return nixpkgs meta pinned in `nixref`. `nixref` can point to a - nix store path or flake reference. If nixref is None, attempt to - read the nixpkgs store path from NIX_PATH environment variable. - """ - source = self.source_resolver.resolve_default_source(nixref) - return self._scan_source(source) - - def get_nixpkgs_meta_with_source( + def get_package_meta_with_source( # noqa: PLR0913 self, *, + components, + buildtime, target_path=None, flakeref=None, original_ref=None, - explicit_nixpkgs=None, impure=False, ): - """Return nixpkgs metadata and selected metadata source.""" + """Return exact component metadata and selected metadata source.""" + started = time.perf_counter() source = self._resolve_source( target_path=target_path, flakeref=flakeref, original_ref=original_ref, - explicit_nixpkgs=explicit_nixpkgs, impure=impure, ) - return self._scan_source_with_source(source) + LOG.verbose( + "Resolved nixpkgs metadata source method '%s' in %.3fs", + source.method, + time.perf_counter() - started, + ) + component_count = 0 if components is None else len(components) + LOG.verbose("Resolving package metadata for %d component(s)", component_count) + out_source = replace( + source, + method=f"{source.method}+{PACKAGE_META_METHOD}", + ) + if not source.path and not source.expression and not flakeref: + return None, source + scan_flakeref = _package_scan_flakeref(source, flakeref) + lookup_keys = package_meta_lookup_keys_for_components( + components, + target_path=target_path, + flakeref=scan_flakeref, + ) + if not lookup_keys: + return None, replace( + out_source, + message="No scannable package names supplied. Skipping nixpkgs metadata.", + ) + LOG.verbose( + "Looking up nixpkgs metadata for %d package name(s)", len(lookup_keys) + ) + df_candidates = self._scan_package_source( + source, + lookup_keys, + flakeref=scan_flakeref, + impure=impure, + ) + if df_candidates is None: + return None, replace( + out_source, + message="package_meta.nix scan failed. Skipping nixpkgs metadata.", + ) + df = match_package_metadata_to_components( + components, + df_candidates, + buildtime=buildtime, + ) + if scan_flakeref: + df = self._add_flake_input_package_meta( + components, + df, + source, + flakeref=scan_flakeref, + buildtime=buildtime, + impure=impure, + ) + if df.empty: + return None, replace( + out_source, + message=( + "No package metadata matched component derivation " + "or output paths. Skipping nixpkgs metadata." + ), + ) + LOG.verbose( + "Matched nixpkgs metadata for %d component(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df, out_source + + def _add_flake_input_package_meta( # noqa: PLR0913 + self, + components, + df, + source, + *, + flakeref, + buildtime, + impure, + ): + unmatched = _components_without_package_matches(components, df) + if unmatched is None or unmatched.empty: + return df + + lookup_keys = package_meta_lookup_keys_for_components(unmatched) + if not lookup_keys: + return df + if len(lookup_keys) > _PACKAGE_META_FLAKE_INPUT_LOOKUP_LIMIT: + LOG.debug( + "Skipping flake-input metadata scan for %d unmatched lookup(s)", + len(lookup_keys), + ) + return df + LOG.verbose( + "Looking up flake input metadata for %d unmatched package name(s)", + len(lookup_keys), + ) + + started = time.perf_counter() + df_candidates = self._scan_package_source( + source, + lookup_keys, + flakeref=flakeref, + input_roots_only=True, + impure=impure, + ) + df_input = match_package_metadata_to_components( + unmatched, + df_candidates, + buildtime=buildtime, + ) + LOG.verbose( + "Matched flake input metadata for %d component(s) in %.3fs", + len(df_input), + time.perf_counter() - started, + ) + if df_input.empty: + return df + if df.empty: + return df_input + return pd.concat([df, df_input], ignore_index=True) def _resolve_source( self, @@ -76,14 +193,8 @@ def _resolve_source( target_path=None, flakeref=None, original_ref=None, - explicit_nixpkgs=None, impure=False, ): - if explicit_nixpkgs: - return self.source_resolver.resolve_meta_nixpkgs_option( - explicit_nixpkgs, - target_path=target_path, - ) if flakeref: source = self.source_resolver.resolve_flakeref_target_source( flakeref, @@ -91,104 +202,370 @@ def _resolve_source( ) if source is not None: return source - return self.source_resolver.resolve_flakeref_lock_source(flakeref) + return self.source_resolver.resolve_flakeref_lock_source( + flakeref, + impure=impure, + ) return self.source_resolver.path_target_without_source( target_path=target_path, original_ref=original_ref, ) - def _scan_source(self, source): - df, _source = self._scan_source_with_source(source) - return df - - def _scan_source_with_source(self, source): - if not source.path: - return None, source - if source.expression: - LOG.debug("Scanning meta-info using nix expression for: %s", source.path) - df = self._scan_expression( - source.expression, - cache_key=source.expression_cache_key, - impure=source.expression_impure, - ) - if df is not None and not df.empty: - return df, source - LOG.warning( - "Failed scanning evaluated package set: %s", - source.path, + def _scan_package_source( + self, + source, + lookup_keys, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + scan_impure = impure or source.expression_impure + lock_started = time.perf_counter() + LOG.debug("Acquiring package metadata cache lock: %s", self.lock.lock_file) + with self.lock: + LOG.verbose( + "Acquired package metadata cache lock in %.3fs", + time.perf_counter() - lock_started, ) - return None, replace( + cache_scope = self._package_cache_scope( source, - message=( - "Evaluated package-set metadata scan failed. " - "Skipping nixpkgs metadata." - ), + flakeref=flakeref, + input_roots_only=input_roots_only, + impure=scan_impure, ) - LOG.debug("Scanning meta-info using nixpkgs path: %s", source.path) - return self._scan(source.path), source - - def _scan_expression(self, expression, *, cache_key=None, impure=False): - if cache_key is None: - with self.lock: - LOG.debug("cache disabled, scanning expression") - df = self._try_scan_expression(expression, impure=impure) - if df is None or df.empty: - LOG.warning("Failed scanning uncached nixmeta expression") - return None - return df - cache_key = f"expr:{cache_key}" - with self.lock: - df = self.cache.get(cache_key) - if df is not None and not df.empty: - LOG.debug("found from cache: %s", cache_key) - return df - LOG.debug("cache miss, scanning expression: %s", cache_key) - df = self._try_scan_expression(expression, impure=impure) - if df is None or df.empty: - LOG.warning("Failed scanning nixmeta expression: %s", cache_key) + exact_cache_key = None + df_cached = pd.DataFrame() + missing_lookup_keys = list(lookup_keys) + hit_count = 0 + if cache_scope is not None: + exact_cache_key = self._package_scan_cache_key( + cache_scope, + lookup_keys, + ) + df = self.cache.get(exact_cache_key) + if df is not None: + LOG.verbose( + "Package metadata cache hit for %d package name(s)", + len(lookup_keys), + ) + LOG.debug( + "found exact package metadata cache entry: %s", exact_cache_key + ) + return df + + ( + df_cached, + missing_lookup_keys, + hit_count, + ) = self._cached_package_metadata( + cache_scope, + lookup_keys, + ) + if not missing_lookup_keys: + LOG.verbose( + "Package metadata cache hit for %d package name(s)", + len(lookup_keys), + ) + LOG.debug( + "found package metadata cache references: %s", cache_scope + ) + return df_cached + if hit_count: + LOG.verbose( + "Package metadata cache partial hit for %d/%d package name(s); " + "scanning %d", + hit_count, + len(lookup_keys), + len(missing_lookup_keys), + ) + LOG.debug( + "partial package metadata cache hit: %s", + cache_scope, + ) + else: + LOG.verbose( + "Package metadata cache miss for %d package name(s); scanning", + len(lookup_keys), + ) + LOG.debug("cache miss, scanning package metadata: %s", cache_scope) + else: + LOG.verbose( + "Package metadata cache disabled for %d package name(s); scanning", + len(lookup_keys), + ) + LOG.debug( + "cache disabled, scanning package metadata for %d lookup(s)", + len(lookup_keys), + ) + scan_started = time.perf_counter() + df = try_scan_package_meta( + missing_lookup_keys, + flakeref=_package_scan_flakeref(source, flakeref), + nixpkgs_path=source.path, + pkgs_expression=source.expression, + input_roots_only=input_roots_only, + impure=scan_impure, + ) + row_count = "failed" if df is None else len(df) + LOG.verbose( + "Package metadata scan returned %s candidate row(s) in %.3fs", + row_count, + time.perf_counter() - scan_started, + ) + if df is None: return None - self.cache.set(key=cache_key, value=df, ttl=_NIXMETA_NIXPKGS_TTL) - return df + result = _concat_package_metadata_frames([df_cached, df]) + if cache_scope is not None: + self._store_package_metadata( + cache_scope, + missing_lookup_keys, + df, + ) + if hit_count and exact_cache_key is not None: + cache_started = time.perf_counter() + self.cache.set( + key=exact_cache_key, + value=result, + ttl=_PACKAGE_META_TTL, + ) + LOG.debug( + "Stored combined package metadata cache entry in %.3fs: %s", + time.perf_counter() - cache_started, + exact_cache_key, + ) + return result + + def _cached_package_metadata(self, cache_scope, lookup_keys): + lookup_cache_items = [ + (self._package_lookup_cache_id(lookup_key), lookup_key) + for lookup_key in lookup_keys + ] + df_index = self.cache.get(self._package_lookup_index_cache_key(cache_scope)) + ref_key_by_lookup_id = _package_lookup_index_refs(df_index) + missing_lookup_keys = [] + ref_keys = set() + hit_count = 0 + + for lookup_id, lookup_key in lookup_cache_items: + ref_key = ref_key_by_lookup_id.get(lookup_id) + if not ref_key: + missing_lookup_keys.append(lookup_key) + continue + ref_keys.add(ref_key) + + cached_ref_frames = _cache_get_many(self.cache, sorted(ref_keys)) + ref_frames = [] + seen_ref_keys = set() + for lookup_id, lookup_key in lookup_cache_items: + ref_key = ref_key_by_lookup_id.get(lookup_id) + if not ref_key: + continue + df_ref = cached_ref_frames.get(ref_key) + if df_ref is None: + missing_lookup_keys.append(lookup_key) + continue + hit_count += 1 + if ref_key in seen_ref_keys: + continue + seen_ref_keys.add(ref_key) + ref_frames.append(df_ref) + + return ( + _concat_package_metadata_frames(ref_frames), + missing_lookup_keys, + hit_count, + ) + + def _store_package_metadata(self, cache_scope, lookup_keys, df): + if not lookup_keys: + return + cache_started = time.perf_counter() + scan_cache_key = self._package_scan_cache_key(cache_scope, lookup_keys) + self.cache.set(key=scan_cache_key, value=df, ttl=_PACKAGE_META_TTL) + self._store_package_lookup_index(cache_scope, lookup_keys, scan_cache_key) + LOG.debug( + "Stored package metadata cache group for %d lookup(s) in %.3fs: %s", + len(lookup_keys), + time.perf_counter() - cache_started, + scan_cache_key, + ) + + def _store_package_lookup_index(self, cache_scope, lookup_keys, scan_cache_key): + index_cache_key = self._package_lookup_index_cache_key(cache_scope) + df_index = self.cache.get(index_cache_key) + df_new = pd.DataFrame( + [ + { + _PACKAGE_META_LOOKUP_ID_COLUMN: self._package_lookup_cache_id( + lookup_key + ), + _PACKAGE_META_CACHE_REF_COLUMN: scan_cache_key, + } + for lookup_key in lookup_keys + ] + ) + df_index = _merge_package_lookup_index(df_index, df_new) + self.cache.set(key=index_cache_key, value=df_index, ttl=_PACKAGE_META_TTL) @staticmethod - def _try_scan_expression(expression, *, impure=False): - try: - scanner = NixMetaScanner() - scanner.scan_expression(expression, impure=impure) - return scanner.to_df() - except SCAN_EXCEPTIONS: - LOG.debug("Failed scanning nixmeta expression", exc_info=True) + def _package_cache_scope( + source, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + if impure: + return None + if source.expression: + source_key = source.expression_cache_key + else: + source_key = source.path or source.flakeref + if not source_key: return None + flakeref_key = _package_cache_flake_key( + source, + flakeref, + require_stable=not source.expression, + ) + if flakeref_key is None: + return None + mode = "flake-inputs-only" if input_roots_only else "base" + return ( + f"package-meta:{source_key}:flake:{flakeref_key}:mode:{mode}:" + f"fingerprint:{package_meta_cache_fingerprint()}" + ) - def _scan(self, nixpkgs_path): - # In case sbomnix is run concurrently, we want to make sure there's - # only one instance of NixMetaScanner.scan_path() running at a time. - # The reason is, NixMetaScanner.scan_path() potentially invokes - # `nix-env -qa --meta --json -f /path/to/nixpkgs` which is very - # memory intensive. The locking needs to happen here (and not in - # NixMetaScanner) because this object caches the nixmeta info. - # First scan generates the cache, after which the consecutive scans - # will read the scan results from the cache, not having to run - # the nix-env command again, making the consecutive scans relatively - # fast and light-weight. - with self.lock: - df = self.cache.get(nixpkgs_path) - if df is not None and not df.empty: - LOG.debug("found from cache: %s", nixpkgs_path) - return df - LOG.debug("cache miss, scanning: %s", nixpkgs_path) - scanner = NixMetaScanner() - scanner.scan_path(nixpkgs_path) - df = scanner.to_df() - if df is None or df.empty: - LOG.warning("Failed scanning nixmeta: %s", nixpkgs_path) - return None - # Cache requires some TTL, so we set it to some value here. - # Although, we could as well store it indefinitely as it should - # not change given the same key (nixpkgs store path). - self.cache.set(key=nixpkgs_path, value=df, ttl=_NIXMETA_NIXPKGS_TTL) - return df + @classmethod + def _package_cache_key( + cls, + source, + lookup_keys, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + cache_scope = cls._package_cache_scope( + source, + flakeref=flakeref, + input_roots_only=input_roots_only, + impure=impure, + ) + if cache_scope is None: + return None + return cls._package_scan_cache_key(cache_scope, lookup_keys) + + @staticmethod + def _package_scan_cache_key(cache_scope, lookup_keys): + return f"{cache_scope}:lookups:{package_meta_candidate_key(lookup_keys)}" + @staticmethod + def _package_lookup_index_cache_key(cache_scope): + return f"{cache_scope}:lookup-index" -############################################################################### + @staticmethod + def _package_lookup_cache_id(lookup_key): + return package_meta_candidate_key([lookup_key]) + + +def _stable_flakeref_for_package_cache(flakeref): + flake, _separator, _attr = str(flakeref or "").partition("#") + return ( + flake.startswith("/nix/store/") + or flake.startswith("path:/nix/store/") + or re.search(r"(?:[?&])(?:narHash|rev)=", flake) is not None + ) + + +def _components_without_package_matches(components, package_meta): + if components is None or components.empty: + return components + if package_meta is None or package_meta.empty: + return components + matched = set(package_meta[cols.STORE_PATH].astype(str)) + return components[~components[cols.STORE_PATH].astype(str).isin(matched)] + + +def _package_scan_flakeref(source, flakeref): + return source.flakeref_cache_key or flakeref or source.flakeref + + +def _package_cache_flakeref_key(source, flakeref, *, require_stable): + if source.flakeref_cache_key: + return source.flakeref_cache_key + flakeref_key = flakeref or source.flakeref or "" + if ( + not flakeref_key + or not require_stable + or _stable_flakeref_for_package_cache(flakeref_key) + ): + return flakeref_key + return None + + +def _package_cache_flake_key(source, flakeref, *, require_stable): + flakeref_key = _package_cache_flakeref_key( + source, + flakeref, + require_stable=require_stable, + ) + if flakeref_key is None: + return None + flake, _separator, _attr = str(flakeref_key).partition("#") + return flake + + +def _package_lookup_index_refs(df): + if ( + df is None + or df.empty + or _PACKAGE_META_LOOKUP_ID_COLUMN not in df.columns + or _PACKAGE_META_CACHE_REF_COLUMN not in df.columns + ): + return {} + refs = {} + df = df[[_PACKAGE_META_LOOKUP_ID_COLUMN, _PACKAGE_META_CACHE_REF_COLUMN]].dropna() + for row in df.itertuples(index=False): + lookup_id = str(row[0] or "") + ref_key = str(row[1] or "") + if lookup_id and ref_key: + refs[lookup_id] = ref_key + return refs + + +def _merge_package_lookup_index(df_index, df_new): + frames = [ + frame for frame in (df_index, df_new) if frame is not None and not frame.empty + ] + if not frames: + return pd.DataFrame( + { + _PACKAGE_META_LOOKUP_ID_COLUMN: pd.Series(dtype="object"), + _PACKAGE_META_CACHE_REF_COLUMN: pd.Series(dtype="object"), + } + ) + return ( + pd.concat(frames, ignore_index=True) + .dropna(subset=[_PACKAGE_META_LOOKUP_ID_COLUMN, _PACKAGE_META_CACHE_REF_COLUMN]) + .drop_duplicates(subset=[_PACKAGE_META_LOOKUP_ID_COLUMN], keep="last") + .reset_index(drop=True) + ) + + +def _cache_get_many(cache, keys): + if not keys: + return {} + get_many = getattr(cache, "get_many", None) + if get_many is not None: + return get_many(keys) + return {key: cache.get(key) for key in keys} + + +def _concat_package_metadata_frames(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames, ignore_index=True).drop_duplicates(ignore_index=True) diff --git a/src/sbomnix/meta_source.py b/src/sbomnix/meta_source.py index 0d507a20..897e0304 100644 --- a/src/sbomnix/meta_source.py +++ b/src/sbomnix/meta_source.py @@ -2,17 +2,14 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Resolve nixpkgs metadata sources from target context and CLI options.""" +"""Resolve nixpkgs metadata sources from target context.""" import json -import os import pathlib import re from dataclasses import dataclass, replace -from subprocess import CalledProcessError from urllib.parse import urlencode -from common.errors import SbomnixError from common.flakeref import ( NIXOS_CONFIGURATION_TOPLEVEL_SUFFIX, parse_nixos_configuration_ref, @@ -20,14 +17,11 @@ ) from common.log import LOG from common.proc import exec_cmd, nix_cmd -from nixmeta.scanner import nixref_to_nixpkgs_path - -META_NIXPKGS_NIX_PATH = "nix-path" - -RESERVED_META_NIXPKGS_MODES = frozenset({META_NIXPKGS_NIX_PATH}) - -SCAN_EXCEPTIONS = (KeyError, OSError, CalledProcessError, TypeError, ValueError) -_NIXREF_RESOLUTION_EXCEPTIONS = (AttributeError, *SCAN_EXCEPTIONS) +from sbomnix.flake_metadata import ( + nixref_to_nixpkgs_path, + normalize_current_flake_shorthand, + normalize_local_flake_ref, +) @dataclass(frozen=True) @@ -37,6 +31,7 @@ class NixpkgsMetaSource: method: str path: str | None = None flakeref: str | None = None + flakeref_cache_key: str | None = None rev: str | None = None version: str | None = None message: str | None = None @@ -45,13 +40,6 @@ class NixpkgsMetaSource: expression_impure: bool = False -def classify_meta_nixpkgs(value): - """Classify a --meta-nixpkgs value as a reserved mode or explicit source.""" - if value in RESERVED_META_NIXPKGS_MODES: - return value - return "explicit" - - def read_nixpkgs_version(nixpkgs_path): """Read nixpkgs version from a source path if available.""" try: @@ -76,6 +64,16 @@ def nixpkgs_meta_source_with_path(source): return replace(source, version=read_nixpkgs_version(source.path)) +def log_nixpkgs_meta_source(source): + """Log a concise summary of a selected nixpkgs metadata source.""" + parts = [f"method '{source.method}'"] + if source.path: + parts.append(f"path '{source.path}'") + if source.flakeref: + parts.append(f"flakeref '{source.flakeref}'") + LOG.verbose("Selected nixpkgs metadata source: %s", ", ".join(parts)) + + class NixpkgsMetaSourceResolver: """Resolve a nixpkgs metadata source without scanning metadata.""" @@ -90,45 +88,38 @@ def path_target_without_source(target_path=None, original_ref=None): return NixpkgsMetaSource( method="none", message=( - "No nixpkgs metadata source was provided for store-path target. " - "Skipping nixpkgs metadata. Re-run with " - "--meta-nixpkgs to include metadata." + "No nixpkgs metadata source is available for store-path targets. " + "Skipping nixpkgs metadata. Use a flakeref target instead." ), ) - def resolve_meta_nixpkgs_option(self, meta_nixpkgs, *, target_path=None): - """Resolve an explicit --meta-nixpkgs source or reserved mode.""" - LOG.debug( - "Resolving explicit nixpkgs metadata source for target path=%s", - target_path, - ) - mode = classify_meta_nixpkgs(meta_nixpkgs) - if mode == META_NIXPKGS_NIX_PATH: - return self.resolve_nix_path_source( - message="NIX_PATH metadata source may not match the target", - required=True, - ) - return self.resolve_explicit_source(meta_nixpkgs) - def resolve_flakeref_target_source(self, flakeref, *, impure=False): """Resolve target-specific nixpkgs metadata for known flakeref outputs.""" + flakeref = normalize_current_flake_shorthand(flakeref) parsed = self._parse_nixos_toplevel_flakeref(flakeref) if not parsed: return None + LOG.info("Resolving nixpkgs path for '%s'", flakeref) flake, name = parsed name_attr = quote_nix_attr_segment(name) pkgs_path_ref = f"{flake}#nixosConfigurations.{name_attr}.pkgs.path" + LOG.info("Evaluating nixpkgs path for '%s'", pkgs_path_ref) pkgs_path = self._nix_eval_raw(pkgs_path_ref, impure=impure) if pkgs_path: expression_flake = self._flake_ref_for_expression( flake, impure=impure, ) - return nixpkgs_meta_source_with_path( + flakeref_cache_key = self._stable_flakeref_cache_key( + flakeref, + impure=impure, + ) + source = nixpkgs_meta_source_with_path( NixpkgsMetaSource( method="flakeref-target", path=pkgs_path, flakeref=pkgs_path_ref, + flakeref_cache_key=flakeref_cache_key, message="Scanning evaluated NixOS package set from flakeref", expression=self._nixos_pkgs_expression(expression_flake, name), expression_cache_key=self._nixos_pkgs_expression_cache_key( @@ -139,6 +130,8 @@ def resolve_flakeref_target_source(self, flakeref, *, impure=False): expression_impure=impure, ), ) + log_nixpkgs_meta_source(source) + return source return self._nixos_toplevel_without_source() @@ -148,8 +141,7 @@ def _nixos_toplevel_without_source(): method="none", message=( "Failed resolving target-specific nixpkgs metadata source from " - "NixOS configuration flakeref. Skipping nixpkgs metadata. Re-run " - "with --meta-nixpkgs to include metadata." + "NixOS configuration flakeref. Skipping nixpkgs metadata." ), ) @@ -174,23 +166,23 @@ def _nixos_pkgs_expression(flake, name): def _flake_ref_for_expression(self, flake, *, impure=False): if self._flake_ref_has_stable_lock(flake): return flake - if self._should_lock_flake_ref_for_expression(flake): + if self._should_lock_flake_ref(flake): locked_ref = self._locked_flake_ref_from_metadata(flake, impure=impure) if locked_ref: return locked_ref - return self._normalize_local_flake_ref_for_expression(flake) + return normalize_local_flake_ref(flake) @staticmethod def _flake_ref_has_stable_lock(flake): return re.search(r"(?:[?&])(?:narHash|rev)=", flake) is not None @classmethod - def _should_lock_flake_ref_for_expression(cls, flake): + def _should_lock_flake_ref(cls, flake): if cls._flake_ref_has_stable_lock(flake): return False if cls._is_existing_local_flake_ref(flake): return True - return re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or "") is not None + return bool(flake) @staticmethod def _is_existing_local_flake_ref(flake): @@ -223,6 +215,19 @@ def _locked_flake_ref_from_metadata(flake, *, impure=False): query["dir"] = locked_dir return f"path:{source_path}?{urlencode(query, safe='/')}" + @classmethod + def _stable_flakeref_cache_key(cls, flakeref, *, impure=False): + flakeref = normalize_current_flake_shorthand(flakeref) + flake, separator, attr = str(flakeref or "").partition("#") + if not flake: + return None + stable_flake = cls._stable_flake_ref_for_cache(flake) + if not stable_flake and cls._should_lock_flake_ref(flake): + stable_flake = cls._locked_flake_ref_from_metadata(flake, impure=impure) + if not stable_flake: + return None + return f"{stable_flake}{separator}{attr}" + @staticmethod def _nix_flake_metadata(flake, *, impure=False): LOG.debug("Reading flake metadata for nixpkgs metadata expression: %s", flake) @@ -241,33 +246,18 @@ def _nix_flake_metadata(flake, *, impure=False): LOG.debug("Failed parsing flake metadata for expression: %s", flake) return None - @staticmethod - def _normalize_local_flake_ref_for_expression(flake): - if flake.startswith("path:"): - path_text, separator, query = flake.removeprefix("path:").partition("?") - path = pathlib.Path(path_text).expanduser() - if not path.is_absolute(): - path_text = path.resolve().as_posix() - return f"path:{path_text}{separator}{query}" - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or ""): - return flake - path = pathlib.Path(flake).expanduser() - if path.exists() or flake.startswith((".", "/", "~")): - return path.resolve().as_posix() - return flake - @classmethod def _nixos_pkgs_expression_cache_key(cls, flake, name, *, impure=False): if impure: return None - stable_ref = cls._stable_flake_ref_for_expression_cache(flake) + stable_ref = cls._stable_flake_ref_for_cache(flake) if not stable_ref: return None cache_parts = json.dumps([stable_ref, name], separators=(",", ":")) return f"nixos-pkgs:{cache_parts}" @staticmethod - def _stable_flake_ref_for_expression_cache(flake): + def _stable_flake_ref_for_cache(flake): if flake.startswith("path:/nix/store/"): return flake if flake.startswith("/nix/store/"): @@ -292,103 +282,24 @@ def _nix_eval_raw(flakeref, *, impure=False): return None return ret.stdout.strip() or None - def resolve_explicit_source(self, meta_nixpkgs): - """Resolve an explicit --meta-nixpkgs path or flakeref.""" - path = pathlib.Path(meta_nixpkgs) - if path.exists(): - resolved_path = path.resolve() - if is_nix_store_path(resolved_path): - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=resolved_path.as_posix(), - ), - ) - nixpath = self._try_normalize_mutable_path(resolved_path) - if nixpath: - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=nixpath.as_posix(), - flakeref=resolved_path.as_posix(), - ), - ) - raise SbomnixError( - "Explicit --meta-nixpkgs path must resolve to an immutable " - f"/nix/store source before scanning: '{meta_nixpkgs}'" - ) - try: - nixpath = nixref_to_nixpkgs_path(meta_nixpkgs) - except _NIXREF_RESOLUTION_EXCEPTIONS as error: - raise SbomnixError( - f"Failed resolving --meta-nixpkgs source: '{meta_nixpkgs}'" - ) from error - if not nixpath: - raise SbomnixError( - f"Failed resolving --meta-nixpkgs source: '{meta_nixpkgs}'" - ) - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=nixpath.as_posix(), - flakeref=meta_nixpkgs, - ), - ) - - @staticmethod - def _try_normalize_mutable_path(path): - try: - nixpath = nixref_to_nixpkgs_path(path.as_posix()) - except _NIXREF_RESOLUTION_EXCEPTIONS: - LOG.debug( - "Failed normalizing mutable nixpkgs path: %s", - path.as_posix(), - exc_info=True, - ) - return None - if nixpath and is_nix_store_path(nixpath): - return nixpath - return None - - def resolve_flakeref_lock_source(self, nixref): + def resolve_flakeref_lock_source(self, nixref, *, impure=False): """Return the nixpkgs source selected by a flakeref lock graph.""" if nixref: - LOG.debug("Reading nixpkgs path from nixref: %s", nixref) + nixref = normalize_current_flake_shorthand(nixref) nixpath = nixref_to_nixpkgs_path(nixref) + flakeref_cache_key = self._stable_flakeref_cache_key( + nixref, + impure=impure, + ) if nixpath: - return nixpkgs_meta_source_with_path( + source = nixpkgs_meta_source_with_path( NixpkgsMetaSource( method="flakeref-lock", path=nixpath.as_posix(), flakeref=nixref, + flakeref_cache_key=flakeref_cache_key, ), ) - return NixpkgsMetaSource(method="none") - - def resolve_default_source(self, nixref=None): - """Return the metadata source for the older direct Meta API.""" - if nixref: - return self.resolve_flakeref_lock_source(nixref) - if "NIX_PATH" in os.environ: - return self.resolve_nix_path_source() - return NixpkgsMetaSource(method="none") - - def resolve_nix_path_source(self, *, message=None, required=False): - """Return the nixpkgs source referenced by NIX_PATH.""" - LOG.debug("Reading nixpkgs path from NIX_PATH environment") - nix_path = os.environ.get("NIX_PATH", "") - m_nixpkgs = re.search(r"(?:^|:)nixpkgs=([^:]+)", nix_path) - if m_nixpkgs: - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="nix-path", - path=m_nixpkgs.group(1), - message=message, - ), - ) - if required: - raise SbomnixError( - "NIX_PATH does not contain a nixpkgs= entry required by " - "--meta-nixpkgs nix-path" - ) + log_nixpkgs_meta_source(source) + return source return NixpkgsMetaSource(method="none") diff --git a/src/sbomnix/package_meta.nix b/src/sbomnix/package_meta.nix new file mode 100644 index 00000000..035ae75c --- /dev/null +++ b/src/sbomnix/package_meta.nix @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +# Resolve package metadata for Python-generated attr lookup tiers. The helper +# scans selected package roots or flake input package roots, then returns metadata +# plus derivation identity. Python accepts only rows whose drvPath/outPath +# matches an SBOM component exactly. + +{ + request ? null, + pkgs ? null, +}: + +let + requestText = if request != null then request else throw "Missing mandatory argument: 'request'"; + req = builtins.fromJSON requestText; + flakeref = req.flakeref or null; + split = if flakeref == null then null else builtins.match "^([^#]+)#(.*)$" flakeref; + flakeRef = if split != null then builtins.elemAt split 0 else flakeref; + flake = if flakeRef == null then null else builtins.getFlake flakeRef; + inherit (req) system; + nixpkgsPath = req.nixpkgsPath or null; + lookupKeys = req.lookupKeys or [ ]; + inputRootsOnly = req.inputRootsOnly or false; + + optionals = cond: values: if cond then values else [ ]; + concatMap = f: values: builtins.concatLists (builtins.map f values); + isAttrs = builtins.isAttrs; + isDerivation = value: isAttrs value && (value.type or null) == "derivation"; + hasPrefix = + prefix: str: + let + pl = builtins.stringLength prefix; + sl = builtins.stringLength str; + in + pl <= sl && builtins.substring 0 pl str == prefix; + findFirst = + pred: default: values: + if values == [ ] then + default + else if pred (builtins.head values) then + builtins.head values + else + findFirst pred default (builtins.tail values); + hasMatches = values: builtins.length values > 0; + firstMatchingCandidates = candidateLists: findFirst hasMatches [ ] candidateLists; + + safeAttr = + attrs: attr: + let + res = builtins.tryEval ( + if isAttrs attrs && builtins.hasAttr attr attrs then builtins.getAttr attr attrs else null + ); + in + if res.success then res.value else null; + + attrByPath = + path: attrs: + if path == [ ] then + attrs + else + let + value = safeAttr attrs (builtins.head path); + in + if value == null then null else attrByPath (builtins.tail path) value; + + importedPkgs = + if pkgs != null then + null + else if nixpkgsPath == null || nixpkgsPath == "" then + null + else + import nixpkgsPath { + inherit system; + config.allowAliases = false; + }; + + rootPackages = attrByPath [ + "packages" + system + ] flake; + rootLegacyPackages = attrByPath [ + "legacyPackages" + system + ] flake; + + uniqueNames = + values: + builtins.attrNames ( + builtins.listToAttrs ( + builtins.map (name: { + inherit name; + value = true; + }) values + ) + ); + + packageSetNames = uniqueNames ( + builtins.filter (name: name != "") ( + concatMap ( + lookup: concatMap (tier: concatMap (candidate: candidate.packageSets) tier) lookup.candidateTiers + ) lookupKeys + ) + ); + + mkRoot = + attrSet: + let + nestedByName = builtins.listToAttrs ( + builtins.filter (entry: isAttrs entry.value) ( + builtins.map (name: { + inherit name; + value = safeAttr attrSet name; + }) packageSetNames + ) + ); + in + { + base = attrSet; + inherit nestedByName; + }; + + validRoot = root: isAttrs root.base; + flakeInputs = attrByPath [ "inputs" ] flake; + + inputRoots = + if flake == null || !isAttrs flakeInputs then + [ ] + else + concatMap ( + inputName: + let + input = safeAttr flakeInputs inputName; + packages = attrByPath [ + "packages" + system + ] input; + legacyPackages = attrByPath [ + "legacyPackages" + system + ] input; + in + optionals (isAttrs packages) [ + (mkRoot packages) + ] + ++ optionals (isAttrs legacyPackages) [ + (mkRoot legacyPackages) + ] + ) (builtins.attrNames flakeInputs); + + explicitPkgsRoots = optionals (isAttrs pkgs) [ + (mkRoot pkgs) + ]; + flakePackageRoots = builtins.filter validRoot ( + optionals (isAttrs rootPackages) [ + (mkRoot rootPackages) + ] + ++ optionals (isAttrs rootLegacyPackages) [ + (mkRoot rootLegacyPackages) + ] + ); + importedPkgsRoots = optionals (isAttrs importedPkgs) [ + (mkRoot importedPkgs) + ]; + + baseRoots = builtins.filter validRoot (explicitPkgsRoots ++ flakePackageRoots ++ importedPkgsRoots); + roots = if inputRootsOnly then builtins.filter validRoot inputRoots else baseRoots; + + packageSetByName = root: name: if name == "" then root.base else root.nestedByName.${name} or null; + packageSetsByNames = + root: names: + builtins.filter (packageSet: packageSet != null) (builtins.map (packageSetByName root) names); + + lookupAttr = + attr: packageSet: + let + res = builtins.tryEval ( + let + candidate = safeAttr packageSet attr; + in + if attr != "" && candidate != null && isDerivation candidate then { drv = candidate; } else null + ); + in + if res.success && res.value != null then [ res.value ] else [ ]; + + isSimpleOutputName = output: builtins.match "[a-zA-Z0-9][a-zA-Z0-9_+-]*" output != null; + + candidateMatchesSplitOutputName = + inputName: candidate: + let + inherit (candidate) drv; + drvName = drv.name or null; + prefix = if drvName == null then null else "${drvName}-"; + suffix = + if prefix != null && hasPrefix prefix inputName then + builtins.substring (builtins.stringLength prefix) ( + builtins.stringLength inputName - builtins.stringLength prefix + ) inputName + else + null; + outputs = builtins.filter (output: output != "out") (drv.outputs or [ "out" ]); + in + suffix != null && suffix != "" && isSimpleOutputName suffix && builtins.elem suffix outputs; + + canonicalCandidatesByIdentity = + inputName: candidates: + let + exactNameMatches = builtins.filter ( + candidate: (candidate.drv.name or null) == inputName + ) candidates; + splitOutputMatches = builtins.filter ( + candidate: candidateMatchesSplitOutputName inputName candidate + ) candidates; + in + if hasMatches exactNameMatches then + exactNameMatches + else if hasMatches splitOutputMatches then + splitOutputMatches + else + [ ]; + + candidateMatchesExpectedPnameOrName = + inputName: expectedPname: candidate: + (candidate.drv.name or null) == inputName || (candidate.drv.pname or null) == expectedPname; + + candidatePlausible = + inputName: expectedPname: expectedVersion: candidate: + let + inherit (candidate) drv; + drvName = drv.name or null; + drvPname = drv.pname or null; + drvVersion = drv.version or null; + versionMatches = expectedVersion == "" || drvVersion == expectedVersion; + in + drvName == inputName + || candidateMatchesSplitOutputName inputName candidate + || (expectedPname != "" && drvPname == expectedPname && versionMatches); + + narrowPlausible = + inputName: expectedPname: expectedVersion: candidates: + let + plausible = builtins.filter (candidatePlausible inputName expectedPname expectedVersion) candidates; + in + if hasMatches plausible then plausible else [ ]; + + lookupCandidate = + root: lookup: candidate: + let + inputName = lookup.name or ""; + expectedPname = lookup.pname or ""; + matches = concatMap (lookupAttr candidate.attr) (packageSetsByNames root candidate.packageSets); + in + if candidate.requirePnameOrName or false then + builtins.filter (candidateMatchesExpectedPnameOrName inputName expectedPname) matches + else + matches; + + lookupTier = + root: lookup: tier: + concatMap (lookupCandidate root lookup) tier; + + findByLookup = + root: lookup: + let + name = lookup.name or ""; + pname = lookup.pname or ""; + version = lookup.version or ""; + tiers = lookup.candidateTiers; + tierCandidates = builtins.map (lookupTier root lookup) tiers; + canonicalCandidateLists = builtins.map (canonicalCandidatesByIdentity name) tierCandidates; + canonicalCandidates = + if lookup.collectAllCanonical or false then + builtins.concatLists canonicalCandidateLists + else + firstMatchingCandidates canonicalCandidateLists; + plausibleCandidates = firstMatchingCandidates ( + builtins.map (narrowPlausible name pname version) tierCandidates + ); + in + # Prefer matches whose derivation/output identity explains the input + # component name. Pname/version matches are a fallback for packages whose + # attr name cannot be inferred exactly. + if hasMatches canonicalCandidates then canonicalCandidates else plausibleCandidates; + + filteredMeta = meta: { + description = meta.description or null; + homepage = meta.homepage or null; + unfree = meta.unfree or false; + position = meta.position or null; + license = meta.license or { }; + maintainers = builtins.map (maintainer: { email = maintainer.email or ""; }) ( + builtins.filter isAttrs (meta.maintainers or [ ]) + ); + }; + + drvOutputs = + drv: if drv ? outputs then builtins.map (output: drv.${output}) drv.outputs else [ drv ]; + + field = + fallbackDrv: drv: name: default: + let + value = safeAttr drv name; + fallbackValue = safeAttr fallbackDrv name; + in + if value != null then + value + else if fallbackValue != null then + fallbackValue + else + default; + + fields = fallbackDrv: drv: { + path = field fallbackDrv drv "outPath" null; + drvPath = field fallbackDrv drv "drvPath" null; + name = field fallbackDrv drv "name" ""; + pname = field fallbackDrv drv "pname" ""; + version = field fallbackDrv drv "version" ""; + meta = filteredMeta (field fallbackDrv drv "meta" { }); + }; + + rowsForCandidate = + candidate: + let + rawRows = builtins.map (fields candidate.drv) (drvOutputs candidate.drv); + forced = builtins.tryEval (builtins.deepSeq rawRows rawRows); + in + if forced.success then forced.value else [ ]; + + rowsForLookupInRoot = + root: lookup: + let + candidatesTry = builtins.tryEval (findByLookup root lookup); + candidates = if candidatesTry.success then candidatesTry.value else [ ]; + in + concatMap rowsForCandidate candidates; + + rowsForLookup = lookup: concatMap (root: rowsForLookupInRoot root lookup) roots; + +in +{ + derivations = concatMap rowsForLookup lookupKeys; +} diff --git a/src/sbomnix/package_meta.py b/src/sbomnix/package_meta.py new file mode 100644 index 00000000..0695f848 --- /dev/null +++ b/src/sbomnix/package_meta.py @@ -0,0 +1,949 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Package metadata lookup from package sets and flake package outputs. + +Python sends compact SBOM component lookup keys to the Nix helper. The helper +does package-set specific name lookup and returns derivation identity; this +module accepts metadata only when the returned drvPath/outPath matches a +component exactly. +""" + +import functools +import hashlib +import json +import pathlib +import platform +import re +import shutil +import subprocess +import time +from tempfile import TemporaryDirectory + +import pandas as pd + +from common import columns as cols +from common.log import LOG +from common.proc import exec_cmd, nix_cmd +from sbomnix.artifacts import is_non_package_artifact_name +from sbomnix.flake_metadata import normalize_flakeref_for_nix_eval + +PACKAGE_META_METHOD = "package-meta-nix" +META_OUTPUT_PATH = "meta_output_path" + +_META_DERIVATION_PATH = "_meta_derivation_path" +_RICH_META_COLUMNS = ( + "meta_homepage", + "meta_description", + "meta_license_short", + "meta_license_spdxid", +) +_PNAME_PATTERN = ( + r"[a-zA-Z0-9][a-zA-Z0-9_+.]*" + r"(?:-[a-zA-Z][a-zA-Z0-9_+.]*)*" +) +_STD_PNAME_RE = re.compile( + rf"^(?P{_PNAME_PATTERN})-[0-9].*$", +) +_PYTHON_NAME_RE = re.compile( + rf"^python(?P[23])\.(?P[0-9]+)-(?P{_PNAME_PATTERN})-[0-9].*" +) +_PERL3_NAME_RE = re.compile( + rf"^perl[0-9]+\.[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*" +) +_PERL_NAME_RE = re.compile(rf"^perl[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*") +_RUBY_NAME_RE = re.compile(rf"^ruby[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*") +_BARE_NAME_RE = re.compile(r"^(?P[a-zA-Z0-9][a-zA-Z0-9_+.-]*)") +_QT_KDE_NAME_RE = re.compile(r"^(qt|kde|kf|plasma|kirigami|kwayland|kwin|qqc2).*") +_K_PREFIXED_PNAME_RE = re.compile(r"^k[a-z0-9].*") +_SUFFIX_RE = re.compile(r"^(?P.*)-[a-zA-Z][a-zA-Z0-9_+]*$") +_WEBKIT_ABI_RE = re.compile(r"^[^+]*\+abi=([0-9]+)\.([0-9]+).*") +_HASKELL_VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+){2,4}$") +_BASE_PACKAGE_SET = "" +_EXPLICIT_ATTR_REWRITES = { + "bash": "bashNonInteractive", + "gcc": "gcc-unwrapped", + "libwww-perl": "LWP", + "nss-cacert": "cacert", + "util-linux-minimal": "util-linuxMinimal", +} + + +def package_meta_nix_path(): + """Return the packaged Nix helper used for package metadata evaluation.""" + return pathlib.Path(__file__).with_name("package_meta.nix") + + +def nix_system(): + """Return the host Nix system string used for package-set lookup.""" + machine = platform.machine() + machine = {"AMD64": "x86_64", "arm64": "aarch64"}.get(machine, machine) + system = platform.system().lower() + system = {"darwin": "darwin", "linux": "linux"}.get(system, system) + return f"{machine}-{system}" + + +def _eval_package_meta_request(request, *, pkgs_expression=None, impure=False): + request_json = json.dumps(request, sort_keys=True) + LOG.debug( + "Evaluating package metadata request for system=%s lookup_count=%d input_roots_only=%s json_bytes=%d", + request.get("system"), + len(request.get("lookupKeys") or []), + request.get("inputRootsOnly"), + len(request_json), + ) + if len(request_json) <= 30_000: + return _eval_request( + f"request = {_nix_string_literal(request_json)};", + pkgs_expression=pkgs_expression, + impure=impure, + ) + + # Pure Nix eval can read files relative to its --file entrypoint, but not + # arbitrary absolute temp paths passed through --apply. + with TemporaryDirectory( + prefix="sbomnix_package_meta_", + ) as tmpdir: + tmpdir_path = pathlib.Path(tmpdir) + (tmpdir_path / "request.json").write_text(request_json, encoding="utf-8") + shutil.copyfile(package_meta_nix_path(), tmpdir_path / "package_meta.nix") + eval_file = tmpdir_path / "eval.nix" + eval_file.write_text( + _request_file_eval_expression(pkgs_expression), + encoding="utf-8", + ) + return _eval_file( + eval_file.as_posix(), + impure=impure, + ) + + +def _request_file_eval_expression(pkgs_expression): + pkgs_arg = f"\n pkgs = {pkgs_expression};" if pkgs_expression else "" + return ( + "import ./package_meta.nix {\n" + " request = builtins.readFile ./request.json;" + f"{pkgs_arg}\n" + "}\n" + ) + + +def _nix_string_literal(value): + return json.dumps(value).replace("${", r"\${") + + +def _eval_request(request_arg, *, pkgs_expression=None, impure=False): + started = time.perf_counter() + ret = exec_cmd( + _eval_command( + _apply_expression(request_arg, pkgs_expression), + impure, + ), + log_error=False, + ) + df = parse_package_metadata(ret.stdout) + LOG.verbose( + "Evaluated inline package metadata request with %d row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df + + +def _eval_file(eval_file, *, impure=False): + started = time.perf_counter() + ret = exec_cmd( + _eval_file_command(eval_file, impure), + log_error=False, + ) + df = parse_package_metadata(ret.stdout) + LOG.verbose( + "Evaluated file-backed package metadata request with %d row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df + + +def _eval_command(apply_expression, impure): + return nix_cmd( + "eval", + "--json", + "--file", + package_meta_nix_path().as_posix(), + "--apply", + apply_expression, + impure=impure, + ) + + +def _eval_file_command(eval_file, impure): + return nix_cmd( + "eval", + "--json", + "--file", + eval_file, + impure=impure, + ) + + +def _apply_expression(request_arg, pkgs_expression): + pkgs_arg = f" pkgs = {pkgs_expression};" if pkgs_expression else "" + return f"f: f {{ {request_arg}{pkgs_arg} }}" + + +def parse_package_metadata(json_text): + """Parse metadata JSON emitted by package_meta.nix.""" + data = json.loads(json_text) + derivations = data.get("derivations", []) + if not isinstance(derivations, list): + derivations = [] + + rows = [] + for drv in derivations: + if not isinstance(drv, dict): + continue + row = _metadata_row(drv) + if row[cols.STORE_PATH] or row[META_OUTPUT_PATH]: + rows.append(row) + df = pd.DataFrame.from_records(rows) + if df.empty: + return df + df = df.astype(str) + df.fillna("", inplace=True) + df.drop_duplicates( + subset=[cols.STORE_PATH, META_OUTPUT_PATH], + keep="first", + inplace=True, + ) + return df + + +def _metadata_row(drv): + meta = drv.get("meta", {}) + if not isinstance(meta, dict): + meta = {} + meta_license = meta.get("license", {}) + meta_maintainers = meta.get("maintainers", {}) + return { + cols.STORE_PATH: drv.get("drvPath", "") or "", + META_OUTPUT_PATH: drv.get("path", "") or "", + cols.NAME: drv.get("name", "") or "", + cols.PNAME: drv.get("pname", "") or "", + cols.VERSION: drv.get("version", "") or "", + "meta_homepage": _parse_optional_meta_entry(meta, key="homepage"), + "meta_unfree": meta.get("unfree", ""), + "meta_description": meta.get("description", "") or "", + "meta_position": meta.get("position", "") or "", + "meta_license_short": _parse_optional_meta_entry(meta_license, key="shortName"), + "meta_license_spdxid": _parse_optional_meta_entry(meta_license, key="spdxId"), + "meta_maintainers_email": _parse_optional_meta_entry( + meta_maintainers, + key="email", + ), + } + + +def _parse_optional_meta_entry(meta, key): + if meta is None: + return "" + if isinstance(meta, dict): + return _parse_optional_meta_entry(meta.get(key, ""), key) + if isinstance(meta, list): + return ";".join( + filter(None, (_parse_optional_meta_entry(item, key) for item in meta)) + ) + return str(meta) + + +def package_meta_lookup_keys_for_components( + df_components, + *, + target_path=None, + flakeref=None, +): + """Return compact Nix metadata lookup keys for the given component rows.""" + target_attr = _flake_package_attr(flakeref) + keys = {} + for component in _records(df_components): + if _is_non_package_artifact(component): + continue + lookup = _lookup_key_for_component(component) + if lookup is None: + continue + if ( + target_attr + and lookup["version"] + and _component_matches_target_path(component, target_path) + ): + lookup["candidateAttrs"] = [target_attr] + key = ( + lookup["name"], + lookup["pname"], + lookup["version"], + lookup.get("system", ""), + tuple(lookup.get("candidateAttrs", [])), + ) + keys[key] = lookup + return [keys[key] for key in sorted(keys)] + + +def _lookup_key_for_component(component): + name = _clean_lookup_value(component.get(cols.NAME, "")) + pname = _clean_lookup_value(component.get(cols.PNAME, "")) + version = _clean_lookup_value(component.get(cols.VERSION, "")) + system = _clean_lookup_value(component.get("system", "")) + if not name: + if not pname: + return None + name = f"{pname}-{version}" if version else pname + if not pname: + pname = _pname_from_package_name(name) + lookup = { + "name": name, + "pname": pname, + "version": version, + } + if system: + lookup["system"] = system + return lookup + + +def _clean_lookup_value(value): + if value is None: + return "" + if pd.isna(value): + return "" + return str(value).strip() + + +def _component_matches_target_path(component, target_path): + target_path = _clean_lookup_value(target_path) + if not target_path: + return False + if _clean_lookup_value(component.get(cols.STORE_PATH, "")) == target_path: + return True + if _clean_lookup_value(component.get("out", "")) == target_path: + return True + outputs = component.get(cols.OUTPUTS, []) + if isinstance(outputs, str): + outputs = [outputs] + elif not isinstance(outputs, (list, tuple, set)): + return False + return target_path in {_clean_lookup_value(output) for output in outputs} + + +def _flake_package_attr(flakeref): + _flake, separator, attr = str(flakeref or "").partition("#") + if not separator: + return "" + parts = attr.split(".") + if len(parts) == 1: + return parts[0] + if ( + len(parts) < 3 + or parts[0] not in {"packages", "legacyPackages"} + or not _is_nix_system_name(parts[1]) + ): + return "" + return parts[2] + + +def _clean_candidate_attrs(attrs): + if isinstance(attrs, str): + attrs = [attrs] + cleaned = [] + seen = set() + for raw_attr in attrs or []: + attr = _clean_lookup_value(raw_attr) + if not attr or attr in seen: + continue + seen.add(attr) + cleaned.append(attr) + return cleaned + + +def _normalized_lookup_keys(lookup_keys): + if isinstance(lookup_keys, dict): + lookup_keys = lookup_keys.get("lookupKeys", []) + normalized = [] + seen = set() + for lookup in lookup_keys or []: + if not isinstance(lookup, dict): + continue + name = _clean_lookup_value(lookup.get("name", "")) + pname = _clean_lookup_value(lookup.get("pname", "")) + version = _clean_lookup_value(lookup.get("version", "")) + system = _clean_lookup_value(lookup.get("system", "")) + if not name: + continue + if not pname: + pname = _pname_from_package_name(name) + candidate_attrs = _clean_candidate_attrs(lookup.get("candidateAttrs", [])) + key = (name, pname, version, system, tuple(candidate_attrs)) + if key in seen: + continue + seen.add(key) + normalized_lookup = {"name": name, "pname": pname, "version": version} + if system: + normalized_lookup["system"] = system + if candidate_attrs: + normalized_lookup["candidateAttrs"] = candidate_attrs + normalized.append(normalized_lookup) + return sorted( + normalized, + key=lambda item: ( + item["name"], + item["pname"], + item["version"], + item.get("system", ""), + tuple(item.get("candidateAttrs", [])), + ), + ) + + +def _request_system_groups(flakeref, lookup_keys): + fallback_system = _system_from_flakeref(flakeref) or nix_system() + groups = {} + for lookup in lookup_keys: + system = _clean_lookup_value(lookup.get("system", "")) + if not _is_nix_system_name(system): + system = fallback_system + groups.setdefault(system, []).append(lookup) + return [(system, groups[system]) for system in sorted(groups)] + + +def _is_nix_system_name(system): + return re.match(r"^[A-Za-z0-9_]+-[A-Za-z0-9_]+$", system or "") is not None + + +def _system_from_flakeref(flakeref): + _flake, separator, attr = (flakeref or "").partition("#") + if not separator or not attr: + return "" + attr_parts = attr.split(".") + if ( + len(attr_parts) >= 3 + and attr_parts[0] in {"packages", "legacyPackages"} + and _is_nix_system_name(attr_parts[1]) + ): + return attr_parts[1] + return "" + + +def _request_lookup_keys(lookup_keys): + return [_lookup_key_with_candidate_tiers(lookup) for lookup in lookup_keys] + + +def _lookup_key_with_candidate_tiers(lookup): + name = lookup["name"] + version = lookup["version"] + parsed_pname = _pname_from_package_name(name) + pname = ( + parsed_pname + if _is_prefixed_language_name(name) or (not version and lookup["pname"] == name) + else lookup["pname"] or parsed_pname + ) + pname = pname or "" + key = { + "name": name, + "pname": pname, + "version": version, + "candidateTiers": _candidate_tiers( + name, + pname, + version, + candidate_attrs=lookup.get("candidateAttrs", []), + ), + } + if _is_haskell_package_name(name, pname, version): + key["collectAllCanonical"] = True + return key + + +def _candidate_tiers(name, pname, version="", *, candidate_attrs=None): + if not pname: + return [] + + # Tiers are searched in order by package_meta.nix, so keep broad rewrites + # after higher-confidence package-set and exact-name candidates. + specs = _package_set_hints_for_lookup(name) + rewrite_specs = ( + specs + if any(spec != _BASE_PACKAGE_SET for spec in specs) + else [_BASE_PACKAGE_SET] + ) + strip_specs = ( + specs if _is_python_name(name) or _is_ruby_name(name) else [_BASE_PACKAGE_SET] + ) + p1 = _strip_suffix(pname) + p2 = _strip_suffix(p1) if p1 else None + + tiers = [] + _add_tier( + tiers, + [ + _candidate(attr, [_BASE_PACKAGE_SET]) + for attr in _clean_candidate_attrs(candidate_attrs) + ], + ) + _add_tier(tiers, [_candidate(pname, specs), _kde_exact_candidate(name, pname)]) + _add_tier(tiers, _perl_candidates(name, pname)) + _add_tier(tiers, _low_loss_rewrite_candidates(pname, rewrite_specs)) + _add_tier(tiers, [_candidate(p1, strip_specs)]) + _add_tier(tiers, _low_loss_rewrite_candidates(p1, strip_specs)) + _add_tier(tiers, [_candidate(p2, strip_specs)]) + _add_tier(tiers, _low_loss_rewrite_candidates(p2, strip_specs)) + _add_tier(tiers, [_cxx_candidate(pname)]) + _add_tier(tiers, [_gtk_candidate(pname)]) + _add_tier(tiers, [_webkit_candidate(name)]) + _add_tier( + tiers, [_candidate(pname.lower(), specs) if pname.lower() != pname else None] + ) + _add_tier(tiers, [_leading_digit_candidate(pname, specs)]) + _add_tier(tiers, [_no_dash_candidate(pname, specs)]) + _add_tier(tiers, [_candidate(f"{pname}{n}", specs) for n in range(1, 5)]) + _add_tier(tiers, _underscore_version_candidates(name, pname, specs)) + _add_tier(tiers, [_dot_dash_candidate(pname, specs)]) + _add_tier(tiers, [_plus_candidate(pname, specs)]) + _add_tier(tiers, _haskell_candidates(name, pname, version)) + return tiers + + +def _add_tier(tiers, candidates): + tier = [] + seen = set() + for candidate in candidates: + if not candidate: + continue + key = ( + candidate["attr"], + tuple(candidate["packageSets"]), + candidate.get("requirePnameOrName", False), + ) + if key in seen: + continue + seen.add(key) + tier.append(candidate) + if tier: + tiers.append(tier) + + +def _candidate(attr, package_sets, *, require_pname_or_name=False): + if not attr: + return None + candidate = { + "attr": attr, + "packageSets": list(package_sets), + } + if require_pname_or_name: + candidate["requirePnameOrName"] = True + return candidate + + +def _package_set_hints_for_lookup(name): + package_sets = _component_package_set_hints(name) + package_sets.append(_BASE_PACKAGE_SET) + return _unique_package_sets(package_sets) + + +def _component_package_set_hints(name): + if _is_python_name(name): + return _python_package_sets_for_name(name) + if _is_perl_name(name): + return ["perlPackages"] + if _is_ruby_name(name): + return ["rubyPackages"] + if _is_qt_kde_name(name): + return ["qt6Packages", "qt6", "kdePackages"] + return [] + + +def _unique_package_sets(package_sets): + unique = [] + seen = set() + for package_set in package_sets: + if package_set in seen: + continue + seen.add(package_set) + unique.append(package_set) + return unique + + +def _python_package_sets_for_name(name): + match = _PYTHON_NAME_RE.match(name) + if not match: + return ["python3Packages"] + compact = f"{match.group('major')}{match.group('minor')}" + if compact.startswith("3"): + return [f"python{compact}Packages", "python3Packages"] + return [f"python{compact}Packages"] + + +def _is_prefixed_language_name(name): + return _is_python_name(name) or _is_perl_name(name) or _is_ruby_name(name) + + +def _is_python_name(name): + return _PYTHON_NAME_RE.match(name) is not None + + +def _is_perl_name(name): + return ( + _PERL3_NAME_RE.match(name) is not None or _PERL_NAME_RE.match(name) is not None + ) + + +def _is_ruby_name(name): + return _RUBY_NAME_RE.match(name) is not None + + +def _is_qt_kde_name(name): + return _QT_KDE_NAME_RE.match(name) is not None + + +def _kde_exact_candidate(name, pname): + if ( + _is_prefixed_language_name(name) + or _is_qt_kde_name(name) + or _K_PREFIXED_PNAME_RE.match(pname or "") is None + ): + return None + return _candidate(pname, ["kdePackages"]) + + +def _haskell_candidates(name, pname, version): + if _is_haskell_package_name(name, pname, version): + return [_candidate(pname, ["haskellPackages"])] + return [] + + +def _is_haskell_package_name(name, pname, version): + if ( + not name + or not pname + or not version + or _is_prefixed_language_name(name) + or _is_qt_kde_name(name) + or is_non_package_artifact_name(name) + ): + return False + if not _HASKELL_VERSION_RE.match(version): + return False + return name == f"{pname}-{version}" + + +def _pname_from_package_name(value): + for regex in ( + _PYTHON_NAME_RE, + _PERL3_NAME_RE, + _PERL_NAME_RE, + _RUBY_NAME_RE, + _STD_PNAME_RE, + _BARE_NAME_RE, + ): + match = regex.match(str(value or "")) + if match: + return match.group("pname") + return "" + + +def _perl_candidates(name, pname): + if not _is_perl_name(name): + return [] + explicit = _EXPLICIT_ATTR_REWRITES.get(pname) + no_dash = pname.replace("-", "") + return [ + _candidate(explicit, ["perlPackages"]), + _candidate(no_dash, ["perlPackages"]) if no_dash != pname else None, + ] + + +def _low_loss_rewrite_candidates(pname, specs): + if not pname: + return [] + return [ + _rewritten_candidate(pname, _EXPLICIT_ATTR_REWRITES.get(pname), specs), + _rewritten_candidate(pname, pname.replace("-", "_"), specs), + _rewritten_candidate(pname, _dash_to_camel_case(pname), specs), + ] + + +def _rewritten_candidate(pname, rewritten, specs): + if not rewritten or rewritten == pname: + return None + return _candidate(rewritten, specs, require_pname_or_name=True) + + +def _strip_suffix(pname): + if not pname: + return None + match = _SUFFIX_RE.match(pname) + if match: + return match.group("pname") + return None + + +def _dash_to_camel_case(pname): + if not pname or "-" not in pname: + return None + first, *rest = pname.split("-") + return first + "".join(_uppercase_ascii_first(part) for part in rest) + + +def _uppercase_ascii_first(value): + if not value: + return "" + return value[:1].upper() + value[1:] + + +def _cxx_candidate(pname): + if pname.endswith("++"): + return _candidate(f"{pname.removesuffix('++')}xx", [_BASE_PACKAGE_SET]) + return None + + +def _gtk_candidate(pname): + attrs = { + "gtk+": "gtk2", + "gtk+3": "gtk3", + } + return _candidate(attrs.get(pname), [_BASE_PACKAGE_SET]) + + +def _webkit_candidate(name): + match = _WEBKIT_ABI_RE.match(name) + if not match: + return None + return _candidate( + f"webkitgtk_{match.group(1)}_{match.group(2)}", [_BASE_PACKAGE_SET] + ) + + +def _leading_digit_candidate(pname, specs): + if re.match(r"^[0-9].*", pname): + return _candidate(f"_{pname}", specs) + return None + + +def _no_dash_candidate(pname, specs): + no_dash = pname.replace("-", "") + if no_dash != pname: + return _candidate(no_dash, specs) + return None + + +def _underscore_version_candidates(name, pname, specs): + major_minor = re.match(rf"{re.escape(pname)}-([0-9]+)\.([0-9]+).*", name) + major = re.match(rf"{re.escape(pname)}-([0-9]+).*", name) + if major_minor: + attrs = [ + f"{pname}_{major_minor.group(1)}_{major_minor.group(2)}", + f"{pname}_{major_minor.group(1)}", + ] + elif major: + attrs = [f"{pname}_{major.group(1)}"] + else: + attrs = [] + return [_candidate(attr, specs) for attr in attrs] + + +def _dot_dash_candidate(pname, specs): + dot_dash = pname.replace(".", "-") + if dot_dash != pname: + return _candidate(dot_dash, specs) + return None + + +def _plus_candidate(pname, specs): + plus_name = pname.replace("+", "plus") + if plus_name != pname: + return _candidate(plus_name, specs) + return None + + +def _records(df): + if df is None or df.empty: + return [] + return df.to_dict("records") + + +def _is_non_package_artifact(component): + return any( + is_non_package_artifact_name(str(component.get(column, "") or "")) + for column in (cols.NAME, cols.PNAME) + ) + + +def package_meta_candidate_key(candidates): + """Return a short stable digest for a candidate request.""" + data = json.dumps(candidates, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(data.encode()).hexdigest()[:16] + + +@functools.cache +def package_meta_cache_fingerprint(): + """Return a short hash for the package metadata lookup implementation.""" + try: + paths = [ + pathlib.Path(__file__), + package_meta_nix_path(), + pathlib.Path(__file__).with_name("artifacts.py"), + ] + h = hashlib.sha256() + for path in paths: + h.update(path.read_bytes()) + h.update(b"\0") + return h.hexdigest()[:16] + except OSError as error: + LOG.warning("Packaged package metadata helper is unavailable: %s", error) + return "missing" + + +def match_package_metadata_to_components(components, metadata, *, buildtime): + """Return metadata rows that match component drvPath/outPath exactly.""" + if components is None or components.empty or metadata is None or metadata.empty: + return pd.DataFrame() + + matches = [] + if buildtime: + matches.append(_drv_path_matches(components, metadata, priority=0)) + matches.append(_output_path_matches(components, metadata, priority=1)) + else: + matches.append(_output_path_matches(components, metadata, priority=0)) + matches.append(_drv_path_matches(components, metadata, priority=1)) + return _best_component_matches(matches) + + +def _output_path_matches(components, metadata, *, priority): + output_components = components[[cols.STORE_PATH, cols.OUTPUTS]].copy() + output_components = output_components.explode(cols.OUTPUTS) + output_components.rename( + columns={ + cols.STORE_PATH: "_component_store_path", + cols.OUTPUTS: META_OUTPUT_PATH, + }, + inplace=True, + ) + output_components = output_components[ + output_components[META_OUTPUT_PATH].astype(bool) + ] + meta = metadata.rename(columns={cols.STORE_PATH: _META_DERIVATION_PATH}) + matched = output_components.merge(meta, how="inner", on=META_OUTPUT_PATH) + if matched.empty: + return matched + matched[cols.STORE_PATH] = matched["_component_store_path"] + matched["_meta_match_priority"] = priority + return matched + + +def _drv_path_matches(components, metadata, *, priority): + component_paths = components[[cols.STORE_PATH]].copy() + meta = metadata.rename(columns={cols.STORE_PATH: _META_DERIVATION_PATH}) + matched = component_paths.merge( + meta, + how="inner", + left_on=cols.STORE_PATH, + right_on=_META_DERIVATION_PATH, + ) + if matched.empty: + return matched + matched["_meta_match_priority"] = priority + return matched + + +def _best_component_matches(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + df = pd.concat(frames, ignore_index=True) + df["_meta_richness"] = _metadata_richness(df) + df.sort_values( + by=[cols.STORE_PATH, "_meta_match_priority", "_meta_richness"], + ascending=[True, True, False], + inplace=True, + ) + df.drop_duplicates(subset=[cols.STORE_PATH], keep="first", inplace=True) + drop_columns = [ + "_component_store_path", + _META_DERIVATION_PATH, + META_OUTPUT_PATH, + "_meta_match_priority", + "_meta_richness", + ] + df.drop(columns=[column for column in drop_columns if column in df], inplace=True) + return df + + +def _metadata_richness(df): + existing = [column for column in _RICH_META_COLUMNS if column in df] + if not existing: + return 0 + return df[existing].replace("", pd.NA).notna().sum(axis=1) + + +def _concat_metadata_frames(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames, ignore_index=True) + + +def try_scan_package_meta( # noqa: PLR0913 + lookup_keys, + *, + flakeref=None, + nixpkgs_path=None, + pkgs_expression=None, + input_roots_only=False, + impure=False, +): + """Return package metadata dataframe, or None on failure.""" + try: + started = time.perf_counter() + lookup_keys = _normalized_lookup_keys(lookup_keys) + if not lookup_keys: + return pd.DataFrame() + flakeref = normalize_flakeref_for_nix_eval(flakeref) + LOG.debug("Reading package metadata for flakeref: %s", flakeref) + frames = [] + for request_system, system_lookup_keys in _request_system_groups( + flakeref, lookup_keys + ): + LOG.verbose( + "Evaluating package metadata for %d lookup(s) on system '%s'", + len(system_lookup_keys), + request_system, + ) + request = { + "flakeref": flakeref or None, + "system": request_system, + "nixpkgsPath": str(nixpkgs_path) if nixpkgs_path else None, + "lookupKeys": _request_lookup_keys(system_lookup_keys), + "inputRootsOnly": bool(input_roots_only), + } + frames.append( + _eval_package_meta_request( + request, + pkgs_expression=pkgs_expression, + impure=impure, + ) + ) + df = _concat_metadata_frames(frames) + LOG.verbose( + "Read package metadata with %d candidate row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df + except ( + FileNotFoundError, + KeyError, + OSError, + subprocess.CalledProcessError, + TypeError, + ValueError, + ): + LOG.debug("Failed reading package metadata", exc_info=True) + return None diff --git a/tests/integration/test_nixmeta_cli.py b/tests/integration/test_nixmeta_cli.py deleted file mode 100644 index 18dd9fb7..00000000 --- a/tests/integration/test_nixmeta_cli.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""CLI integration tests for nixmeta.""" - -import pytest - -from common.df import df_from_csv_file -from tests.testpaths import NIXMETA, RESOURCES_DIR - - -def test_nixmeta_help(_run_python_script): - """Test nixmeta command line argument: '-h'.""" - _run_python_script([NIXMETA, "-h"]) - - -@pytest.mark.slow -def test_nixmeta_sbomnix_flakeref(_run_python_script, test_work_dir): - """Test nixmeta with a small package-set path.""" - out_path = test_work_dir / "nixmeta.csv" - package_set = RESOURCES_DIR / "nixmeta-package-set.nix" - _run_python_script( - [ - NIXMETA, - "--out", - out_path.as_posix(), - "--flakeref", - package_set, - ] - ) - assert out_path.exists() - df_meta = df_from_csv_file(out_path) - assert df_meta is not None - assert set(df_meta["name"]) == { - "sbomnix-meta-first-1.0", - "sbomnix-meta-second-2.0", - } diff --git a/tests/resources/nixmeta-package-set.nix b/tests/resources/nixmeta-package-set.nix deleted file mode 100644 index dcbfe30a..00000000 --- a/tests/resources/nixmeta-package-set.nix +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -{ ... }: - -let - mkPackage = - { - name, - pname, - version, - description, - homepage, - }: - builtins.derivation { - inherit name pname version; - system = builtins.currentSystem; - builder = "/bin/sh"; - args = [ - "-c" - "echo ${name} > $out" - ]; - meta = { - inherit description homepage; - license = { - shortName = "Apache-2.0"; - spdxId = "Apache-2.0"; - }; - }; - }; -in -{ - first = mkPackage { - name = "sbomnix-meta-first-1.0"; - pname = "sbomnix-meta-first"; - version = "1.0"; - description = "First sbomnix metadata fixture package"; - homepage = "https://example.test/sbomnix-meta-first"; - }; - - second = mkPackage { - name = "sbomnix-meta-second-2.0"; - pname = "sbomnix-meta-second"; - version = "2.0"; - description = "Second sbomnix metadata fixture package"; - homepage = "https://example.test/sbomnix-meta-second"; - }; -} diff --git a/tests/test_builder_runtime.py b/tests/test_builder_runtime.py index 0cf93684..f939344f 100644 --- a/tests/test_builder_runtime.py +++ b/tests/test_builder_runtime.py @@ -13,6 +13,7 @@ from sbomnix import builder as sbomnix_builder from sbomnix.builder import SbomBuilder from sbomnix.closure import dependency_rows_to_dataframe +from sbomnix.meta import NixpkgsMetaSource from sbomnix.runtime import RuntimeClosure TARGET_PATH = "/nix/store/11111111111111111111111111111111-target-1.0" @@ -257,3 +258,55 @@ def test_target_component_ref_rejects_missing_runtime_target_metadata(): with pytest.raises(MissingNixDerivationMetadataError, match=TARGET_PATH): builder._resolve_target_component_ref() + + +def test_join_meta_uses_package_component_identity(monkeypatch): + class FakeMeta: + def get_package_meta_with_source(self, **kwargs): + assert kwargs["components"].equals(builder.df_sbomdb) + return ( + pd.DataFrame( + [ + { + cols.STORE_PATH: TARGET_DERIVER, + cols.NAME: "target-from-meta", + cols.PNAME: "target-pname-from-meta", + cols.VERSION: "1.0-from-meta", + "meta_description": "package target metadata", + } + ] + ), + NixpkgsMetaSource(method="package-meta-nix"), + ) + + monkeypatch.setattr(sbomnix_builder, "Meta", FakeMeta) + builder = _builder_double() + builder.flakeref = "nixpkgs#target" + builder.original_ref = "nixpkgs#target" + builder.impure = False + builder.df_sbomdb = pd.DataFrame( + [ + { + cols.STORE_PATH: TARGET_DERIVER, + cols.NAME: "target", + cols.PNAME: "target", + cols.VERSION: "1.0", + cols.OUTPUTS: [TARGET_PATH], + }, + { + cols.STORE_PATH: "/nix/store/other.drv", + cols.NAME: "target", + cols.PNAME: "target", + cols.VERSION: "1.0", + cols.OUTPUTS: ["/nix/store/other"], + }, + ] + ) + + builder._join_meta() + + assert builder.df_sbomdb["meta_description"].isna().to_list() == [False, True] + assert builder.df_sbomdb["meta_description"].iloc[0] == "package target metadata" + assert builder.df_sbomdb["pname_meta"].iloc[0] == "target-pname-from-meta" + assert builder.df_sbomdb["version_meta"].iloc[0] == "1.0-from-meta" + assert "name_meta" not in builder.df_sbomdb.columns diff --git a/tests/test_cli_conventions.py b/tests/test_cli_conventions.py index 31c6a167..cc7c3536 100644 --- a/tests/test_cli_conventions.py +++ b/tests/test_cli_conventions.py @@ -13,7 +13,6 @@ from common.pkgmeta import _dev_version, get_py_pkg_version from nixgraph import main as nixgraph_main -from nixmeta import main as nixmeta_main from nixupdate import nix_outdated from provenance import main as provenance_main from repology import repology_cli, repology_cve @@ -31,7 +30,6 @@ def _stringify(value): CLI_ARG_CASES = [ (sbomnix_main.getargs, [".#pkg"]), (nixgraph_main.getargs, [".#pkg"]), - (nixmeta_main._getargs, []), (nix_outdated.getargs, [".#pkg"]), (vulnxscan_cli.getargs, [".#pkg"]), (osv_cli.getargs, ["sbom.json"]), @@ -49,7 +47,6 @@ def _stringify(value): [ sbomnix_main.getargs, nixgraph_main.getargs, - nixmeta_main._getargs, nix_outdated.getargs, vulnxscan_cli.getargs, osv_cli.getargs, @@ -112,7 +109,6 @@ def test_cli_verbose_level_two_forms_match(getargs, base_argv, verbose_argv): ("getargs", "argv", "expected_out"), [ (nixgraph_main.getargs, ["-o", "graph.dot", ".#pkg"], "graph.dot"), - (nixmeta_main._getargs, ["-o", "meta.csv"], "meta.csv"), (nix_outdated.getargs, ["-o", "nix_outdated.csv", ".#pkg"], "nix_outdated.csv"), (vulnxscan_cli.getargs, ["-o", "vulns.csv", ".#pkg"], "vulns.csv"), (osv_cli.getargs, ["-o", "osv.csv", "sbom.json"], "osv.csv"), diff --git a/tests/test_cli_error_boundaries.py b/tests/test_cli_error_boundaries.py index f071ed36..b7f23047 100644 --- a/tests/test_cli_error_boundaries.py +++ b/tests/test_cli_error_boundaries.py @@ -11,7 +11,6 @@ from common.errors import SbomnixError from nixgraph import main as nixgraph_main -from nixmeta import main as nixmeta_main from nixupdate import nix_outdated from provenance import main as provenance_main from vulnxscan import osv as osv_cli @@ -109,17 +108,6 @@ def test_osv_invalid_sbom_exits_nonzero(tmp_path, monkeypatch): ), "resolve_nix_target", ), - ( - nixmeta_main, - SimpleNamespace( - flakeref="github:NixOS/nixpkgs?ref=nixos-unstable", - out="nixmeta.csv", - append=False, - verbose=0, - ), - lambda monkeypatch: None, - "exit_unless_command_exists", - ), ( provenance_main, SimpleNamespace( diff --git a/tests/test_flakeref_resolution.py b/tests/test_flakeref_resolution.py index f55f1ef0..91fd4b1a 100644 --- a/tests/test_flakeref_resolution.py +++ b/tests/test_flakeref_resolution.py @@ -79,8 +79,6 @@ def test_try_resolve_flakeref_uses_argv_lists(): def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) return SimpleNamespace(stdout="/nix/store/resolved\n", stderr="", returncode=0) resolved = try_resolve_flakeref( @@ -92,19 +90,6 @@ def fake_exec_cmd(cmd, **kwargs): assert resolved == "/nix/store/resolved" assert calls == [ - ( - [ - "nix", - "path-info", - "/tmp/my flake#pkg", - "--extra-experimental-features", - "flakes", - "--extra-experimental-features", - "nix-command", - "--impure", - ], - {"raise_on_error": False, "return_error": True, "log_error": False}, - ), ( [ "nix", @@ -123,12 +108,12 @@ def fake_exec_cmd(cmd, **kwargs): ] -def test_try_resolve_flakeref_reuses_realised_path_info(): +def test_try_resolve_flakeref_realises_with_build_print_out_paths(): calls = [] def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - assert cmd[1] == "path-info" + assert cmd[1] == "build" return SimpleNamespace( stdout="/nix/store/00000000000000000000000000000000-resolved\n", stderr="", @@ -146,7 +131,9 @@ def fake_exec_cmd(cmd, **kwargs): ( [ "nix", - "path-info", + "build", + "--no-link", + "--print-out-paths", ".#hello", "--extra-experimental-features", "flakes", @@ -163,12 +150,6 @@ def test_try_resolve_flakeref_rejects_invalid_explicit_output_selector(): def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - if cmd[1] == "path-info": - return SimpleNamespace( - stdout="", - stderr="does not have output", - returncode=1, - ) if cmd[1] == "eval": raise AssertionError("invalid output selectors must not use eval fallback") return SimpleNamespace( @@ -187,7 +168,7 @@ def fake_exec_cmd(cmd, **kwargs): exec_cmd_fn=fake_exec_cmd, ) - assert [cmd[1] for cmd, _kwargs in calls] == ["path-info", "build"] + assert [cmd[1] for cmd, _kwargs in calls] == ["build"] def test_try_resolve_flakeref_can_return_derivation_path(): @@ -280,8 +261,7 @@ def fake_exec_cmd(_cmd, **_kwargs): def test_try_resolve_flakeref_raises_on_failed_force_realise(): def fake_exec_cmd(cmd, **_kwargs): - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) + assert cmd[1] == "build" return SimpleNamespace(stdout="", stderr="build failed", returncode=1) with pytest.raises(FlakeRefRealisationError, match="build failed"): @@ -294,8 +274,7 @@ def fake_exec_cmd(cmd, **_kwargs): def test_try_resolve_flakeref_raises_when_force_realise_prints_no_path(): def fake_exec_cmd(cmd, **_kwargs): - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) + assert cmd[1] == "build" return SimpleNamespace(stdout="\n", stderr="", returncode=0) with pytest.raises(FlakeRefRealisationError, match="returned no output path"): diff --git a/tests/test_nix_cli_argv.py b/tests/test_nix_cli_argv.py index 748421d9..a89d6df2 100644 --- a/tests/test_nix_cli_argv.py +++ b/tests/test_nix_cli_argv.py @@ -10,11 +10,9 @@ import pytest -from common.proc import exec_cmd, nix_cmd -from nixmeta import flake_metadata +from common.proc import exec_cmd from nixupdate import nix_outdated from sbomnix import derivers as sbomnix_derivers -from sbomnix.meta import Meta def test_exec_cmd_rejects_string_commands(): @@ -138,54 +136,6 @@ def fake_exec_cmd(cmd, **kwargs): ] -def test_get_flake_metadata_uses_argv_list(): - calls = [] - - def fake_exec_cmd(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - meta = flake_metadata.get_flake_metadata( - "/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - nix_cmd_fn=nix_cmd, - ) - - assert meta == {"path": "/nix/store/nixpkgs"} - assert calls == [ - ( - [ - "nix", - "flake", - "metadata", - "/tmp/my flake", - "--json", - "--extra-experimental-features", - "flakes", - "--extra-experimental-features", - "nix-command", - ], - {"raise_on_error": False, "return_error": True, "log_error": False}, - ) - ] - - -def test_get_flake_metadata_strips_nixpkgs_prefix_without_splitting_spaces(): - calls = [] - - def fake_exec_cmd(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - flake_metadata.get_flake_metadata( - "nixpkgs=/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - nix_cmd_fn=nix_cmd, - ) - - assert calls[0][0][3] == "/tmp/my flake" - - def test_run_nix_visualize_uses_argv_list(tmp_path, monkeypatch): calls = [] output_path = tmp_path / "graph output.csv" @@ -226,15 +176,3 @@ def fake_exec_cmd(cmd, **kwargs): {}, ) ] - - -def test_meta_reads_nix_path_entry_with_spaces(monkeypatch): - scanned = [] - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other:nixpkgs=/tmp/my flake") - monkeypatch.setattr(Meta, "_scan", lambda self, path: scanned.append(path) or path) - - resolved = Meta().get_nixpkgs_meta() - - assert resolved == "/tmp/my flake" - assert scanned == ["/tmp/my flake"] diff --git a/tests/test_nixmeta_parsing.py b/tests/test_nixmeta_parsing.py deleted file mode 100644 index 1a569b98..00000000 --- a/tests/test_nixmeta_parsing.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixmeta parsing helpers.""" - -import json - -from nixmeta import metadata_json - - -def test_parse_json_metadata_flattens_nested_fields(tmp_path): - json_path = tmp_path / "meta.json" - json_path.write_text( - json.dumps( - { - "hello": { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta": { - "homepage": ["https://example.invalid/hello"], - "unfree": False, - "description": "GNU hello", - "position": "pkgs/tools/misc/hello/default.nix:1", - "license": [ - {"shortName": "GPLv3+", "spdxId": "GPL-3.0-or-later"} - ], - "maintainers": [ - {"email": "maintainer@example.invalid"}, - ], - }, - } - } - ), - encoding="utf-8", - ) - - df = metadata_json.parse_json_metadata(json_path) - - assert df.to_dict(orient="records") == [ - { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta_homepage": "https://example.invalid/hello", - "meta_unfree": "False", - "meta_description": "GNU hello", - "meta_position": "pkgs/tools/misc/hello/default.nix:1", - "meta_license_short": "GPLv3+", - "meta_license_spdxid": "GPL-3.0-or-later", - "meta_maintainers_email": "maintainer@example.invalid", - } - ] diff --git a/tests/test_nixmeta_progress.py b/tests/test_nixmeta_progress.py deleted file mode 100644 index 77590e8e..00000000 --- a/tests/test_nixmeta_progress.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixmeta progress logging.""" - -import json -import subprocess -from types import SimpleNamespace - -from nixmeta import flake_metadata -from nixmeta import main as nixmeta_main -from nixmeta import scanner as nixmeta_scanner - - -class CapturingLogger: - def __init__(self): - self.records = [] - - def debug(self, msg, *args): - self.records.append(("debug", msg, args)) - - def info(self, msg, *args): - self.records.append(("info", msg, args)) - - def warning(self, msg, *args): - self.records.append(("warning", msg, args)) - - def fatal(self, msg, *args): - self.records.append(("fatal", msg, args)) - - def log(self, level, msg, *args): - self.records.append(("log", level, msg, args)) - - -def test_nixmeta_main_logs_scan_start(monkeypatch): - args = SimpleNamespace( - flakeref="github:NixOS/nixpkgs?ref=nixos-unstable", - out="nixmeta.csv", - append=False, - ) - logger = CapturingLogger() - events = [] - - class FakeScanner: - def scan(self, flakeref): - events.append(("scan", flakeref)) - - def to_csv(self, out, append): - events.append(("to_csv", out, append)) - - monkeypatch.setattr(nixmeta_main, "LOG", logger) - monkeypatch.setattr( - nixmeta_main, - "exit_unless_command_exists", - lambda command: events.append(("command", command)), - ) - monkeypatch.setattr(nixmeta_main, "NixMetaScanner", FakeScanner) - - nixmeta_main._run(args) - - assert ( - "info", - "Scanning nixpkgs metadata for '%s'", - ("github:NixOS/nixpkgs?ref=nixos-unstable",), - ) in logger.records - assert events == [ - ("command", "nix"), - ("command", "nix-env"), - ("scan", "github:NixOS/nixpkgs?ref=nixos-unstable"), - ("to_csv", "nixmeta.csv", False), - ] - - -def test_get_flake_metadata_logs_metadata_read(): - logger = CapturingLogger() - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - meta = flake_metadata.get_flake_metadata( - "nixpkgs=/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - log=logger, - ) - - assert meta == {"path": "/nix/store/nixpkgs"} - assert ( - "info", - "Reading flake metadata for '%s'", - ("/tmp/my flake",), - ) in logger.records - - -def test_get_nixpkgs_flakeref_uses_root_nixpkgs_input_with_renamed_node(): - meta_json = { - "locks": { - "root": "root", - "nodes": { - "root": {"inputs": {"nixpkgs": "nixpkgs_3"}}, - "nixpkgs": { - "locked": { - "type": "github", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "wrong", - } - }, - "nixpkgs_3": { - "locked": { - "type": "github", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "right", - } - }, - }, - } - } - - assert ( - flake_metadata.get_nixpkgs_flakeref(meta_json) - == "github:NixOS/nixpkgs?rev=right" - ) - - -def test_nixmeta_scanner_logs_nix_env_progress(tmp_path, monkeypatch): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - logger = CapturingLogger() - commands = [] - - def fake_run_nix_env_metadata(_cmd, stdout): - commands.append(_cmd) - stdout.write( - json.dumps( - { - "hello": { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta": { - "homepage": "https://example.invalid/hello", - "unfree": False, - "description": "GNU hello", - "position": "pkgs/tools/misc/hello/default.nix:1", - "license": { - "shortName": "GPLv3+", - "spdxId": "GPL-3.0-or-later", - }, - "maintainers": { - "email": "maintainer@example.invalid", - }, - }, - } - } - ).encode("utf-8") - ) - stdout.flush() - - monkeypatch.setattr(nixmeta_scanner, "LOG", logger) - monkeypatch.setattr( - nixmeta_scanner, - "nixref_to_nixpkgs_path", - lambda *_args, **_kwargs: nixpkgs_path, - ) - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan("github:NixOS/nixpkgs?ref=nixos-unstable") - - assert ( - "info", - "Reading nixpkgs metadata from '%s'", - (nixpkgs_path.as_posix(),), - ) in logger.records - assert ("info", "Parsing nixpkgs metadata", ()) in logger.records - assert commands - assert scanner.to_df() is not None - - -def test_run_nix_env_metadata_captures_successful_stderr(monkeypatch, tmp_path): - calls = [] - logger = CapturingLogger() - - def fake_run(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stderr="warning: noisy eval\n") - - monkeypatch.setattr(nixmeta_scanner.subprocess, "run", fake_run) - monkeypatch.setattr(nixmeta_scanner, "LOG", logger) - out_path = tmp_path / "meta.json" - - with out_path.open("w", encoding="utf-8") as out: - nixmeta_scanner._run_nix_env_metadata(["nix-env"], stdout=out) - - assert calls - assert calls[0][1]["stderr"] is subprocess.PIPE - assert calls[0][1]["stdout"].name == out_path.as_posix() - assert ( - "debug", - "nix-env metadata stderr:\n%s", - ("warning: noisy eval",), - ) in logger.records - - -def test_nixmeta_scanner_tolerates_empty_metadata_json(tmp_path, monkeypatch): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - - def fake_run_nix_env_metadata(_cmd, stdout): - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_path(nixpkgs_path) - - assert scanner.to_df() is not None - assert scanner.to_df().empty - - -def test_nixmeta_expression_scan_enables_flakes(monkeypatch): - commands = [] - - def fake_run_nix_env_metadata(cmd, stdout): - commands.append(cmd) - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_expression('builtins.getFlake "github:NixOS/nixpkgs"') - - assert commands - assert [ - "--option", - "experimental-features", - "nix-command flakes", - ] in [commands[0][idx : idx + 3] for idx in range(len(commands[0]) - 2)] - - -def test_nixmeta_expression_scan_honors_impure(monkeypatch): - commands = [] - - def fake_run_nix_env_metadata(cmd, stdout): - commands.append(cmd) - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_expression( - 'builtins.getFlake "path:/tmp/local-flake"', - impure=True, - ) - - assert commands - assert "--impure" in commands[0] diff --git a/tests/test_nixmeta_source.py b/tests/test_nixmeta_source.py deleted file mode 100644 index 388a0f47..00000000 --- a/tests/test_nixmeta_source.py +++ /dev/null @@ -1,788 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixpkgs metadata source selection.""" - -import json -import pathlib -from types import SimpleNamespace - -import pytest - -from common.errors import SbomnixError -from sbomnix import meta as sbomnix_meta -from sbomnix import meta_source as sbomnix_meta_source - - -def test_classify_meta_nixpkgs_reserved_modes_before_explicit_source(): - assert ( - sbomnix_meta.classify_meta_nixpkgs(sbomnix_meta.META_NIXPKGS_NIX_PATH) - == sbomnix_meta.META_NIXPKGS_NIX_PATH - ) - assert sbomnix_meta.classify_meta_nixpkgs("/nix/store/source") == "explicit" - - -def test_get_nixpkgs_meta_with_source_records_flakeref_lock(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - (nixpkgs_path / "lib").mkdir(parents=True) - (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: nixpkgs_path, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=".#target", - original_ref=".#target", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="flakeref-lock", - path=nixpkgs_path.as_posix(), - flakeref=".#target", - version="25.11", - ) - - -def test_get_nixpkgs_meta_with_source_records_opt_in_nix_path(monkeypatch): - scanned = [] - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other:nixpkgs=/tmp/my flake") - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=None, - original_ref="/nix/store/target", - explicit_nixpkgs=sbomnix_meta.META_NIXPKGS_NIX_PATH, - ) - - assert df_meta == "df" - assert scanned == ["/tmp/my flake"] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="nix-path", - path="/tmp/my flake", - message="NIX_PATH metadata source may not match the target", - ) - - -def test_explicit_nix_path_source_requires_nixpkgs_entry(monkeypatch): - def fail_if_scanned(self, path): - raise AssertionError(f"nix-path scan should not run: {path}") - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other") - monkeypatch.setattr(sbomnix_meta.Meta, "_scan", fail_if_scanned) - - with pytest.raises(SbomnixError, match="NIX_PATH.*nixpkgs="): - sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=None, - original_ref="/nix/store/target", - explicit_nixpkgs=sbomnix_meta.META_NIXPKGS_NIX_PATH, - ) - - -def test_path_target_without_source_skips_nix_path_metadata(monkeypatch): - def fail_if_scanned(self, path): - raise AssertionError(f"path-target scan should be skipped: {path}") - - monkeypatch.setenv("NIX_PATH", "nixpkgs=/tmp/nixpkgs") - monkeypatch.setattr(sbomnix_meta.Meta, "_scan", fail_if_scanned) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/resolved-target", - flakeref=None, - original_ref="./result", - ) - - assert df_meta is None - assert source.method == "none" - assert source.path is None - assert "store-path target" in source.message - assert "./result" not in source.message - assert "--meta-nixpkgs" in source.message - - -def test_explicit_store_path_source_records_explicit_method(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - (nixpkgs_path / "lib").mkdir(parents=True) - (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - monkeypatch.setattr( - sbomnix_meta_source, - "is_nix_store_path", - lambda path: path.as_posix() == nixpkgs_path.as_posix(), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=nixpkgs_path.as_posix(), - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=nixpkgs_path.as_posix(), - version="25.11", - ) - - -def test_explicit_flakeref_source_resolves_nixpkgs_path(monkeypatch): - nixpkgs_path = "/nix/store/abc-source" - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: pathlib.Path(nixpkgs_path), - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs="github:NixOS/nixpkgs?rev=abc", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=nixpkgs_path, - flakeref="github:NixOS/nixpkgs?rev=abc", - ) - - -def test_mutable_explicit_path_is_normalized_before_scanning(monkeypatch, tmp_path): - mutable_path = tmp_path / "nixpkgs-checkout" - mutable_path.mkdir() - store_path = pathlib.Path("/nix/store/normalized-source") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda nixref: store_path if nixref == mutable_path.as_posix() else None, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=mutable_path.as_posix(), - ) - - assert df_meta == "df" - assert scanned == [store_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=store_path.as_posix(), - flakeref=mutable_path.as_posix(), - ) - - -def test_mutable_explicit_path_is_rejected_if_not_cache_safe(monkeypatch, tmp_path): - mutable_path = tmp_path / "nixpkgs-checkout" - mutable_path.mkdir() - - monkeypatch.setattr( - sbomnix_meta_source, "nixref_to_nixpkgs_path", lambda _nixref: None - ) - - with pytest.raises(SbomnixError, match="immutable /nix/store source"): - sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=mutable_path.as_posix(), - ) - - -def test_nixos_toplevel_flakeref_prefers_configuration_pkgs_path( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: (_ for _ in ()).throw( - AssertionError("lock-node lookup should not run") - ), - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is fake_df - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'] - ] - assert source.method == "flakeref-target" - assert source.path == nixpkgs_path.as_posix() - assert source.flakeref == '/flake#nixosConfigurations."host".pkgs.path' - assert source.message == "Scanning evaluated NixOS package set from flakeref" - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - None, - False, - ) - ] - - -def test_nixos_toplevel_expression_locks_relative_flake_refs( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - source_path = "/nix/store/root-source" - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '.#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - if cmd == ["nix", "flake", "metadata", ".", "--json"]: - return SimpleNamespace( - stdout=json.dumps( - { - "path": source_path, - "locked": {"narHash": "sha256-abc"}, - } - ), - returncode=0, - ) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=".#nixosConfigurations.host.config.system.build.toplevel", - original_ref=".#nixosConfigurations.host.config.system.build.toplevel", - ) - - locked_ref = f"path:{source_path}?narHash=sha256-abc" - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert source.flakeref == '.#nixosConfigurations."host".pkgs.path' - assert calls == [ - ["nix", "eval", "--raw", '.#nixosConfigurations."host".pkgs.path'], - ["nix", "flake", "metadata", ".", "--json"], - ] - cache_key = "nixos-pkgs:" + json.dumps( - [locked_ref, "host"], - separators=(",", ":"), - ) - assert expressions == [ - ( - f'let\n flake = builtins.getFlake "{locked_ref}";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_expression_preserves_locked_subflake_dir( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - source_path = "/nix/store/root-source" - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - 'path:.?dir=sub/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - if cmd == ["nix", "flake", "metadata", "path:.?dir=sub/flake", "--json"]: - return SimpleNamespace( - stdout=json.dumps( - { - "path": source_path, - "locked": { - "narHash": "sha256-abc", - "dir": "sub/flake", - }, - } - ), - returncode=0, - ) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - "path:.?dir=sub/flake#nixosConfigurations.host.config.system.build.toplevel" - ), - original_ref=( - "path:.?dir=sub/flake#nixosConfigurations.host.config.system.build.toplevel" - ), - ) - - locked_ref = f"path:{source_path}?narHash=sha256-abc&dir=sub/flake" - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert calls == [ - [ - "nix", - "eval", - "--raw", - 'path:.?dir=sub/flake#nixosConfigurations."host".pkgs.path', - ], - ["nix", "flake", "metadata", "path:.?dir=sub/flake", "--json"], - ] - cache_key = "nixos-pkgs:" + json.dumps( - [locked_ref, "host"], - separators=(",", ":"), - ) - assert expressions == [ - ( - f'let\n flake = builtins.getFlake "{locked_ref}";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_flakeref_handles_quoted_configuration_names( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host.example.com".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - '/flake#nixosConfigurations."host.example.com".config.system.build.toplevel' - ), - original_ref=( - '/flake#nixosConfigurations."host.example.com".config.system.build.toplevel' - ), - ) - - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert source.flakeref == ( - '/flake#nixosConfigurations."host.example.com".pkgs.path' - ) - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host.example.com".pkgs\n', - None, - False, - ) - ] - - -def test_nixos_toplevel_flakeref_metadata_eval_honors_impure(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - impure=True, - ) - - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert calls == [ - [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - "--impure", - ] - ] - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - None, - True, - ) - ] - assert source.expression_cache_key is None - assert source.expression_impure is True - - -def test_nixos_toplevel_expression_cache_uses_only_stable_refs(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - "github:example/flake?rev=abc" - '#nixosConfigurations."host:8080".config.system.build.toplevel' - ), - original_ref=( - "github:example/flake?rev=abc" - '#nixosConfigurations."host:8080".config.system.build.toplevel' - ), - ) - - cache_key = "nixos-pkgs:" + json.dumps( - ["github:example/flake?rev=abc", "host:8080"], - separators=(",", ":"), - ) - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "github:example/flake?rev=abc";\n' - "in\n" - ' flake.nixosConfigurations."host:8080".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_expression_scan_failure_skips_metadata( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - scanned = [] - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda *_args, **_kwargs: None, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert scanned == [] - assert source.method == "flakeref-target" - assert source.expression is not None - assert source.path == nixpkgs_path.as_posix() - assert source.message == ( - "Evaluated package-set metadata scan failed. Skipping nixpkgs metadata." - ) - - -def test_nixos_toplevel_flakeref_without_pkgs_path_returns_message(monkeypatch): - calls = [] - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout="", stderr="missing", returncode=1) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'], - ] - assert source.method == "none" - assert source.path is None - assert "NixOS configuration flakeref" in source.message - assert "--meta-nixpkgs" in source.message - - -def test_nixos_toplevel_flakeref_without_pkgs_returns_message( - monkeypatch, -): - calls = [] - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - return SimpleNamespace(stdout="", stderr="missing", returncode=1) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: None, - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'], - ] - assert source.method == "none" - assert source.path is None - assert "NixOS configuration flakeref" in source.message - assert "--meta-nixpkgs" in source.message - - -def test_plain_nixos_configuration_attrset_is_not_target_inferred( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "lock-source" - nixpkgs_path.mkdir() - scanned = [] - - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: nixpkgs_path, - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host", - original_ref="/flake#nixosConfigurations.host", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source.method == "flakeref-lock" - - -def test_meta_scan_uses_already_resolved_scanner_path(monkeypatch): - calls = [] - fake_df = SimpleNamespace(empty=False) - - class FakeScanner: - """Scanner stand-in that records normalized scan paths.""" - - def scan(self, path): - raise AssertionError(f"scan should not resolve path again: {path}") - - def scan_path(self, path): - calls.append(path) - - def to_df(self): - return fake_df - - meta = sbomnix_meta.Meta() - monkeypatch.setattr(meta.cache, "get", lambda _key: None) - monkeypatch.setattr(meta.cache, "set", lambda **_kwargs: None) - monkeypatch.setattr(sbomnix_meta, "NixMetaScanner", FakeScanner) - - assert meta._scan("/nix/store/source") is fake_df - assert calls == ["/nix/store/source"] diff --git a/tests/test_nixmeta_source_export.py b/tests/test_nixmeta_source_export.py deleted file mode 100644 index 7b2c315a..00000000 --- a/tests/test_nixmeta_source_export.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for SBOM-level nixpkgs metadata source export.""" - -import uuid - -import pandas as pd - -from sbomnix.builder import SbomBuilder -from sbomnix.meta import NixpkgsMetaSource - - -def _make_minimal_sbom(): - sbomdb = object.__new__(SbomBuilder) - sbomdb.uid = "store_path" - sbomdb.nix_path = "/nix/store/target" - sbomdb.buildtime = False - sbomdb.target_deriver = "/nix/store/target.drv" - sbomdb.target_component_ref = "/nix/store/target.drv" - sbomdb.depth = None - sbomdb.uuid = uuid.uuid4() - sbomdb.sbom_type = "runtime_only" - sbomdb.nixpkgs_meta_source = NixpkgsMetaSource( - method="flakeref-target", - path="/nix/store/source", - rev="1234", - flakeref=".#target", - version="25.11", - message="base nixpkgs source metadata", - ) - sbomdb.df_sbomdb = pd.DataFrame( - [ - { - "store_path": "/nix/store/target.drv", - "pname": "target", - "name": "target", - "version": "1.0", - "outputs": ["/nix/store/target"], - "out": "/nix/store/target", - "purl": "", - "cpe": "", - "urls": "", - "patches": "", - } - ] - ) - return sbomdb - - -def test_cdx_document_records_nixpkgs_metadata_source(monkeypatch): - sbomdb = _make_minimal_sbom() - monkeypatch.setattr( - SbomBuilder, "lookup_dependencies", lambda *_args, **_kwargs: None - ) - - cdx = sbomdb.to_cdx_data() - - properties = {prop["name"]: prop["value"] for prop in cdx["metadata"]["properties"]} - assert properties["nixpkgs:metadata_source_method"] == "flakeref-target" - assert properties["nixpkgs:path"] == "/nix/store/source" - assert properties["nixpkgs:rev"] == "1234" - assert properties["nixpkgs:flakeref"] == ".#target" - assert properties["nixpkgs:version"] == "25.11" - assert properties["nixpkgs:message"] == "base nixpkgs source metadata" - - -def test_spdx_document_records_nixpkgs_metadata_source(monkeypatch): - sbomdb = _make_minimal_sbom() - monkeypatch.setattr( - SbomBuilder, "lookup_dependencies", lambda *_args, **_kwargs: None - ) - - spdx = sbomdb.to_spdx_data() - - assert "included dependencies: 'runtime_only'" in spdx["comment"] - assert ( - "nixpkgs metadata source: metadata_source_method=flakeref-target" - in spdx["comment"] - ) - assert "path=/nix/store/source" in spdx["comment"] - assert "rev=1234" in spdx["comment"] - assert "message=base nixpkgs source metadata" in spdx["comment"] - assert "warning=" not in spdx["comment"] diff --git a/tests/test_package_meta.py b/tests/test_package_meta.py new file mode 100644 index 00000000..3d656b51 --- /dev/null +++ b/tests/test_package_meta.py @@ -0,0 +1,1013 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Focused tests for package metadata lookup and identity matching.""" + +import json +import shutil +from dataclasses import replace + +import pandas as pd +import pytest + +from common import columns as cols +from sbomnix import flake_metadata, package_meta +from sbomnix import meta as meta_module +from sbomnix import meta_source as meta_source_module +from sbomnix.meta import Meta, NixpkgsMetaSource +from sbomnix.meta_source import NixpkgsMetaSourceResolver + + +def test_package_meta_lookup_keys_stay_compact(): + df_components = pd.DataFrame( + [ + { + cols.NAME: "python3.13-requests-2.32.5", + cols.PNAME: "python3.13-requests", + cols.VERSION: "2.32.5", + }, + { + cols.NAME: "libcap-ng-0.8.5", + cols.PNAME: "libcap-ng", + cols.VERSION: "0.8.5", + }, + ] + ) + + lookup_keys = package_meta.package_meta_lookup_keys_for_components(df_components) + + assert lookup_keys == [ + { + "name": "libcap-ng-0.8.5", + "pname": "libcap-ng", + "version": "0.8.5", + }, + { + "name": "python3.13-requests-2.32.5", + "pname": "python3.13-requests", + "version": "2.32.5", + }, + ] + + +def test_package_meta_lookup_keys_keep_dotted_package_names(): + df_components = pd.DataFrame( + [ + { + cols.NAME: "python3.13-zope.interface-7.2", + cols.PNAME: "zope.interface", + cols.VERSION: "7.2", + }, + { + cols.NAME: "source.json", + cols.PNAME: "source.json", + cols.VERSION: "", + }, + ] + ) + + lookup_keys = package_meta.package_meta_lookup_keys_for_components(df_components) + + assert lookup_keys == [ + { + "name": "python3.13-zope.interface-7.2", + "pname": "zope.interface", + "version": "7.2", + } + ] + + +def test_short_request_escapes_nix_antiquotation(monkeypatch): + captured = {} + + def fake_eval_request(request_arg, **kwargs): + captured["request_arg"] = request_arg + return pd.DataFrame() + + monkeypatch.setattr(package_meta, "_eval_request", fake_eval_request) + + package_meta._eval_package_meta_request( + {"flakeref": "flake#${bad}", "lookupKeys": []} + ) + + assert r"\${bad}" in captured["request_arg"] + + +def test_large_request_uses_wrapper_file_for_pure_eval(monkeypatch): + captured = {} + + def fake_eval_file(eval_file, **kwargs): + eval_path = package_meta.pathlib.Path(eval_file) + captured["request_exists"] = (eval_path.parent / "request.json").is_file() + captured["eval"] = eval_path.read_text(encoding="utf-8") + return pd.DataFrame() + + monkeypatch.setattr(package_meta, "_eval_file", fake_eval_file) + + package_meta._eval_package_meta_request({"lookupKeys": [{"name": "x" * 30_000}]}) + + assert captured["request_exists"] is True + assert "builtins.readFile ./request.json" in captured["eval"] + assert "requestFile" not in captured["eval"] + + +def test_try_scan_package_meta_normalizes_current_flake_shorthand( + monkeypatch, tmp_path +): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta( + [{"name": "hello-1.0", "pname": "hello", "version": "1.0"}], + flakeref="#hello", + ) + + assert captured["request"]["flakeref"] == f"{tmp_path.as_posix()}#hello" + assert "flakePackageAttrPath" not in captured["request"] + + +@pytest.mark.parametrize( + ("flakeref", "lookup_keys", "expected_system"), + [ + ( + "flake#packages.aarch64-linux.foo", + [{"name": "foo-1.0", "pname": "foo", "version": "1.0"}], + "aarch64-linux", + ), + ( + "flake#default", + [ + { + "name": "foo-1.0", + "pname": "foo", + "version": "1.0", + "system": "aarch64-linux", + } + ], + "aarch64-linux", + ), + ], +) +def test_try_scan_package_meta_uses_target_system( + monkeypatch, flakeref, lookup_keys, expected_system +): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta(lookup_keys, flakeref=flakeref) + + assert captured["request"]["system"] == expected_system + + +def test_try_scan_package_meta_splits_mixed_system_requests(monkeypatch): + captured = [] + + def fake_eval_package_meta_request(request, **kwargs): + captured.append(request) + return pd.DataFrame( + [ + { + cols.STORE_PATH: f"/nix/store/{request['system']}.drv", + package_meta.META_OUTPUT_PATH: "", + } + ] + ) + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + df = package_meta.try_scan_package_meta( + [ + { + "name": "pkg-x86-1.0", + "pname": "pkg-x86", + "version": "1.0", + "system": "x86_64-linux", + }, + { + "name": "source-patch", + "pname": "source-patch", + "version": "", + "system": "builtin", + }, + { + "name": "pkg-aarch-1.0", + "pname": "pkg-aarch", + "version": "1.0", + "system": "aarch64-linux", + }, + ], + flakeref="flake#default", + ) + + fallback_system = package_meta.nix_system() + expected_names_by_system = { + "aarch64-linux": ["pkg-aarch-1.0"], + "x86_64-linux": ["pkg-x86-1.0"], + } + expected_names_by_system.setdefault(fallback_system, []).append("source-patch") + names_by_system = { + request["system"]: [lookup["name"] for lookup in request["lookupKeys"]] + for request in captured + } + + assert names_by_system == expected_names_by_system + assert sorted(df[cols.STORE_PATH].to_list()) == [ + f"/nix/store/{system}.drv" for system in sorted(expected_names_by_system) + ] + + +def test_try_scan_package_meta_request_omits_direct_flake_attr_path(monkeypatch): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta( + [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}], + flakeref="flake#expensive-target", + ) + + assert "flakePackageAttrPath" not in captured["request"] + + +def test_request_lookup_keys_expand_package_set_candidates(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "python3.13-requests-2.32.5", + "pname": "python3.13-requests", + "version": "2.32.5", + } + ] + )[0] + + assert lookup["pname"] == "requests" + assert lookup["candidateTiers"][0] == [ + { + "attr": "requests", + "packageSets": ["python313Packages", "python3Packages", ""], + } + ] + + +def test_request_lookup_keys_use_haskell_package_set_hint(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "vector-0.13.2.0", + "pname": "vector", + "version": "0.13.2.0", + } + ] + )[0] + + assert lookup["collectAllCanonical"] is True + assert lookup["candidateTiers"][-1] == [ + { + "attr": "vector", + "packageSets": ["haskellPackages"], + } + ] + + +def test_request_lookup_keys_probe_parsed_name_for_fallback_pname(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "foo-1.0", + "pname": "foo-1.0", + "version": "", + } + ] + )[0] + + assert lookup["pname"] == "foo" + assert lookup["candidateTiers"][0] == [ + { + "attr": "foo", + "packageSets": [""], + } + ] + + +def test_request_lookup_keys_keep_qt_hint_before_haskell_shape(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "qtbase-6.10.2", + "pname": "qtbase", + "version": "6.10.2", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "qtbase", + "packageSets": ["qt6Packages", "qt6", "kdePackages", ""], + } + ] + assert all( + "haskellPackages" not in candidate["packageSets"] + for tier in lookup["candidateTiers"] + for candidate in tier + ) + assert "collectAllCanonical" not in lookup + + +def test_request_lookup_keys_probe_kde_packages_for_k_prefixed_frameworks(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "kconfig-6.17.0", + "pname": "kconfig", + "version": "6.17.0", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "kconfig", + "packageSets": [""], + }, + { + "attr": "kconfig", + "packageSets": ["kdePackages"], + }, + ] + assert all( + "kdePackages" not in candidate["packageSets"] + for tier in lookup["candidateTiers"][1:] + for candidate in tier + ) + + +def test_request_lookup_keys_keep_haskell_hint_for_k_prefixed_packages(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "kan-extensions-5.2.7", + "pname": "kan-extensions", + "version": "5.2.7", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "kan-extensions", + "packageSets": [""], + }, + { + "attr": "kan-extensions", + "packageSets": ["kdePackages"], + }, + ] + assert lookup["candidateTiers"][-1] == [ + { + "attr": "kan-extensions", + "packageSets": ["haskellPackages"], + } + ] + assert lookup["collectAllCanonical"] is True + + +@pytest.mark.parametrize( + ("buildtime", "expected_description"), + [ + (False, "runtime match"), + (True, "buildtime match"), + ], +) +def test_metadata_requires_exact_component_identity(buildtime, expected_description): + components = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/component-real.drv", + cols.OUTPUTS: ["/nix/store/component-real-out"], + } + ] + ) + metadata = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/wrong.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/component-real-out", + "meta_description": "runtime match", + }, + { + cols.STORE_PATH: "/nix/store/component-real.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/other-output", + "meta_description": "buildtime match", + }, + ] + ) + + matched = package_meta.match_package_metadata_to_components( + components, + metadata, + buildtime=buildtime, + ) + + assert matched.to_dict("records")[0]["meta_description"] == expected_description + assert package_meta.META_OUTPUT_PATH not in matched.columns + + +def test_buildtime_metadata_can_fall_back_to_exact_output_identity(): + components = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/component-real.drv", + cols.OUTPUTS: ["/nix/store/component-real-out"], + } + ] + ) + metadata = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/metadata-drv.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/component-real-out", + "meta_description": "output identity match", + } + ] + ) + + matched = package_meta.match_package_metadata_to_components( + components, + metadata, + buildtime=True, + ) + + assert matched.to_dict("records")[0]["meta_description"] == ( + "output identity match" + ) + assert matched.to_dict("records")[0][cols.STORE_PATH] == ( + "/nix/store/component-real.drv" + ) + + +def test_parse_package_metadata_records_identity_fields(): + df = package_meta.parse_package_metadata( + json.dumps( + { + "derivations": [ + { + "drvPath": "/nix/store/pkg.drv", + "path": "/nix/store/pkg-out", + "name": "pkg-1.0", + "pname": "pkg", + "version": "1.0", + "meta": { + "description": "Package", + "license": [{"spdxId": "MIT", "shortName": "MIT"}], + "maintainers": [{"email": "maintainer@example.invalid"}], + }, + } + ] + } + ) + ) + + record = df.to_dict("records")[0] + assert record[cols.STORE_PATH] == "/nix/store/pkg.drv" + assert record[package_meta.META_OUTPUT_PATH] == "/nix/store/pkg-out" + assert record["meta_description"] == "Package" + assert record["meta_license_spdxid"] == "MIT" + + +def test_package_meta_scans_flake_package_roots_and_preserves_output_metadata(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + dep_dir = tmp_path / "dep" + mk_shadow = """ +{ system, description }: +(derivation { + name = "shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; +}) // { + pname = "shadow"; + version = "1.0"; + meta.description = description; +} +""" + package_set = """{ system, config ? { } }: { + shadow = import ./mk-shadow.nix { inherit system; description = "imported"; }; +} +""" + flake = """ +{ + inputs.dep.url = "path:__DEP__"; + + outputs = { self, dep }: + let + system = "__SYSTEM__"; + in + { + packages.${system} = { + shadow = import ./mk-shadow.nix { + inherit system; + description = "flake-root"; + }; + }; + legacyPackages.${system}.default = throw "unrelated legacy default failed"; + }; +} +""".replace("__DEP__", dep_dir.as_posix()).replace("__SYSTEM__", system) + dep_flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system}."input-shadow" = (derivation { + name = "input-shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; + }) // { + pname = "input-shadow"; + version = "1.0"; + meta.description = "flake-input"; + }; + }; +} +""".replace("__SYSTEM__", system) + + package_set_path = tmp_path / "package-set.nix" + package_set_path.write_text(package_set, encoding="utf-8") + (tmp_path / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + dep_dir.mkdir() + (dep_dir / "flake.nix").write_text(dep_flake, encoding="utf-8") + (dep_dir / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + (flake_dir / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + + df = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow", "version": "1.0"}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["flake-root", "flake-root"] + assert df[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df[cols.VERSION].to_list() == ["1.0", "1.0"] + + df_fallback = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow-1.0", "version": ""}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df_fallback is not None + assert df_fallback["meta_description"].to_list() == ["flake-root", "flake-root"] + assert df_fallback[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df_fallback[cols.VERSION].to_list() == ["1.0", "1.0"] + + df_input = package_meta.try_scan_package_meta( + [ + { + "name": "input-shadow-1.0", + "pname": "input-shadow", + "version": "1.0", + } + ], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + input_roots_only=True, + impure=True, + ) + + assert df_input is not None + assert df_input["meta_description"].to_list() == ["flake-input", "flake-input"] + assert df_input[cols.PNAME].to_list() == ["input-shadow", "input-shadow"] + assert df_input[cols.VERSION].to_list() == ["1.0", "1.0"] + + +def test_package_meta_scans_target_flake_attr_when_name_differs(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system}.doc = (derivation { + name = "ghaf-docs-0.1.0"; + inherit system; + builder = "/bin/sh"; + args = [ "-c" "echo docs > $out" ]; + }) // { + pname = "ghaf-docs"; + version = "0.1.0"; + meta.description = "target flake package attr"; + }; + }; +} +""".replace("__SYSTEM__", system) + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + + components = pd.DataFrame( + [ + { + cols.NAME: "ghaf-docs-0.1.0", + cols.PNAME: "ghaf-docs", + cols.VERSION: "0.1.0", + "system": system, + cols.STORE_PATH: "/nix/store/placeholder.drv", + cols.OUTPUTS: ["/nix/store/placeholder-out"], + } + ] + ) + + for flakeref in ( + f"{flake_dir.as_posix()}#packages.{system}.doc", + f"{flake_dir.as_posix()}#doc", + ): + lookup_keys = package_meta.package_meta_lookup_keys_for_components( + components, + target_path="/nix/store/placeholder-out", + flakeref=flakeref, + ) + + assert lookup_keys[0]["candidateAttrs"] == ["doc"] + + df = package_meta.try_scan_package_meta( + lookup_keys, + flakeref=flakeref, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["target flake package attr"] + assert df[cols.NAME].to_list() == ["ghaf-docs-0.1.0"] + assert df[cols.PNAME].to_list() == ["ghaf-docs"] + + +def test_package_meta_skips_target_flake_attr_for_versionless_artifact(): + system = "x86_64-linux" + lookup_keys = package_meta.package_meta_lookup_keys_for_components( + pd.DataFrame( + [ + { + cols.NAME: "ghaf-host-disko-images", + cols.PNAME: "ghaf-host-disko-images", + cols.VERSION: "", + "system": system, + cols.STORE_PATH: "/nix/store/placeholder.drv", + cols.OUTPUTS: ["/nix/store/placeholder-out"], + } + ] + ), + target_path="/nix/store/placeholder-out", + flakeref=f"/flake#packages.{system}.lenovo-x1-carbon-gen11-debug", + ) + + assert lookup_keys == [ + { + "name": "ghaf-host-disko-images", + "pname": "ghaf-host-disko-images", + "version": "", + "system": system, + } + ] + + +def test_package_meta_ignores_broken_optional_flake_package_roots(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + mk_shadow = """ +{ system, description }: +(derivation { + name = "shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; +}) // { + pname = "shadow"; + version = "1.0"; + meta.description = description; +} +""" + package_set = """{ system, config ? { } }: { + shadow = import ./mk-shadow.nix { inherit system; description = "imported"; }; +} +""" + flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system} = throw "broken package root"; + legacyPackages.${system} = throw "broken legacy package root"; + }; +} +""".replace("__SYSTEM__", system) + + package_set_path = tmp_path / "package-set.nix" + package_set_path.write_text(package_set, encoding="utf-8") + (tmp_path / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + + df = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow", "version": "1.0"}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["imported", "imported"] + assert df[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df[cols.VERSION].to_list() == ["1.0", "1.0"] + + +def test_expression_source_cache_requires_expression_identity(): + lookup_keys = [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}] + source = NixpkgsMetaSource( + method="flakeref-target", + path="/nix/store/source", + flakeref="flake#nixosConfigurations.host.pkgs.path", + expression='builtins.getFlake "flake"', + ) + + assert Meta._package_cache_key(source, lookup_keys) is None + + cache_key = Meta._package_cache_key( + replace( + source, + expression_cache_key='nixos-pkgs:["path:/nix/store/flake","host"]', + ), + lookup_keys, + ) + + assert cache_key is not None + assert "nixos-pkgs:" in cache_key + assert "/nix/store/source" not in cache_key + + +def test_mutable_flakeref_cache_uses_locked_identity(): + source = NixpkgsMetaSource(method="flakeref-lock", path="/nix/store/nixpkgs") + stable_source = replace( + source, + flakeref_cache_key="path:/nix/store/source?narHash=sha256-test#default", + ) + lookup_keys = [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}] + + assert ( + Meta._package_cache_key( + source, + lookup_keys, + flakeref="./flake#default", + ) + is None + ) + + cache_key = Meta._package_cache_key( + stable_source, + lookup_keys, + flakeref="./flake#default", + ) + assert cache_key is not None + assert "path:/nix/store/source?narHash=sha256-test" in cache_key + assert "#default" not in cache_key + assert "./flake#default" not in cache_key + assert ( + meta_module._package_scan_flakeref( + stable_source, + "./flake#default", + ) + == "path:/nix/store/source?narHash=sha256-test#default" + ) + + +def test_package_metadata_cache_reuses_overlapping_lookup_sets(monkeypatch): + class NullLock: + lock_file = "test-lock" + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc, _traceback): + return False + + class FakeDfCache: + def __init__(self): + self.values = {} + self.set_calls = [] + + def get(self, key): + return self.values.get(key) + + def get_many(self, keys): + return {key: self.values.get(key) for key in keys} + + def set(self, key, value, ttl=None): + del ttl + self.set_calls.append(key) + self.values[key] = value.copy(deep=True) + + lookup_a = {"name": "alpha-1.0", "pname": "alpha", "version": "1.0"} + lookup_b = {"name": "beta-1.0", "pname": "beta", "version": "1.0"} + lookup_c = {"name": "gamma-1.0", "pname": "gamma", "version": "1.0"} + scan_calls = [] + + def fake_scan_package_meta(lookup_keys, **_kwargs): + scan_calls.append([lookup_key["name"] for lookup_key in lookup_keys]) + return pd.DataFrame( + [ + { + cols.NAME: lookup_key["name"], + cols.PNAME: lookup_key["pname"], + cols.VERSION: lookup_key["version"], + } + for lookup_key in lookup_keys + ] + ) + + monkeypatch.setattr( + meta_module, + "try_scan_package_meta", + fake_scan_package_meta, + ) + meta = Meta() + meta.lock = NullLock() + meta.cache = FakeDfCache() + source_alpha = NixpkgsMetaSource( + method="flakeref-lock", + path="/nix/store/nixpkgs", + flakeref_cache_key=( + "path:/nix/store/source?narHash=sha256-test#packages.x86_64-linux.alpha" + ), + ) + source_beta = NixpkgsMetaSource( + method="flakeref-lock", + path="/nix/store/nixpkgs", + flakeref_cache_key=( + "path:/nix/store/source?narHash=sha256-test#packages.x86_64-linux.beta" + ), + ) + + meta._scan_package_source( + source_alpha, + [lookup_a, lookup_b], + flakeref="./flake#packages.x86_64-linux.alpha", + ) + df_second = meta._scan_package_source( + source_beta, + [lookup_b, lookup_c], + flakeref="./flake#packages.x86_64-linux.beta", + ) + df_third = meta._scan_package_source( + source_beta, + [lookup_b, lookup_c], + flakeref="./flake#packages.x86_64-linux.beta", + ) + + assert scan_calls == [["alpha-1.0", "beta-1.0"], ["gamma-1.0"]] + assert sorted(df_second[cols.NAME].unique()) == [ + "alpha-1.0", + "beta-1.0", + "gamma-1.0", + ] + assert sorted(df_third[cols.NAME].unique()) == [ + "alpha-1.0", + "beta-1.0", + "gamma-1.0", + ] + assert len(meta.cache.set_calls) == 5 + assert not any(":lookup:" in key for key in meta.cache.set_calls) + assert sum(":lookup-index" in key for key in meta.cache.set_calls) == 2 + + +def test_registry_flakeref_cache_key_uses_locked_identity(monkeypatch): + def fake_locked_ref(flake, *, impure=False): + assert flake == "nixpkgs" + assert impure is False + return "path:/nix/store/registry-source?narHash=sha256-registry" + + monkeypatch.setattr( + NixpkgsMetaSourceResolver, + "_locked_flake_ref_from_metadata", + staticmethod(fake_locked_ref), + ) + + assert ( + NixpkgsMetaSourceResolver._stable_flakeref_cache_key("nixpkgs#firefox") + == "path:/nix/store/registry-source?narHash=sha256-registry#firefox" + ) + + +def test_current_flake_shorthand_source_resolution_uses_dot(monkeypatch, tmp_path): + nixpkgs_path = tmp_path / "nixpkgs" + (nixpkgs_path / "lib").mkdir(parents=True) + (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") + resolved_refs = [] + locked_refs = [] + + def fake_nixref_to_nixpkgs_path(nixref): + resolved_refs.append(nixref) + return nixpkgs_path + + def fake_locked_ref(flake, *, impure=False): + locked_refs.append((flake, impure)) + return "path:/nix/store/current-flake?narHash=sha256-current" + + monkeypatch.setattr( + meta_source_module, + "nixref_to_nixpkgs_path", + fake_nixref_to_nixpkgs_path, + ) + monkeypatch.setattr( + NixpkgsMetaSourceResolver, + "_locked_flake_ref_from_metadata", + staticmethod(fake_locked_ref), + ) + + source = NixpkgsMetaSourceResolver().resolve_flakeref_lock_source("#hello") + + assert resolved_refs == [".#hello"] + assert locked_refs == [(".", False)] + assert source.method == "flakeref-lock" + assert source.path == nixpkgs_path.as_posix() + assert source.flakeref == ".#hello" + assert source.flakeref_cache_key == ( + "path:/nix/store/current-flake?narHash=sha256-current#hello" + ) + assert source.version == "25.11" + + +def test_nixref_to_nixpkgs_path_normalizes_current_flake_shorthand(): + metadata_refs = [] + + def fake_get_flake_metadata(flakeref): + metadata_refs.append(flakeref) + return { + "path": "/nix/store/current-nixpkgs", + "description": "A collection of packages for the Nix package manager", + } + + nixpkgs_path = flake_metadata.nixref_to_nixpkgs_path( + "#hello", + get_flake_metadata_fn=fake_get_flake_metadata, + ) + + assert metadata_refs == ["."] + assert nixpkgs_path.as_posix() == "/nix/store/current-nixpkgs" diff --git a/tests/test_sbom_vuln_enrichment.py b/tests/test_sbom_vuln_enrichment.py index 344b6cef..543b496a 100644 --- a/tests/test_sbom_vuln_enrichment.py +++ b/tests/test_sbom_vuln_enrichment.py @@ -12,7 +12,6 @@ import pandas as pd import pytest -from common.errors import SbomnixError from sbomnix import cli_utils as sbomnix_cli_utils from sbomnix import main as sbomnix_main from sbomnix import vuln_enrichment as sbomnix_vuln_enrichment @@ -26,42 +25,13 @@ def __init__(self): def info(self, msg, *args): self.records.append(("info", msg, args)) + def verbose(self, _msg, *_args): + pass + def fatal(self, msg, *args): self.records.append(("fatal", msg, args)) -def test_sbomnix_getargs_accepts_meta_nixpkgs(): - args = sbomnix_main.getargs( - [ - "/nix/store/target", - "--meta-nixpkgs", - "nix-path", - ] - ) - - assert args.meta_nixpkgs == "nix-path" - - -def test_sbomnix_run_rejects_exclude_meta_with_meta_nixpkgs(): - args = SimpleNamespace( - NIXREF="/nix/store/target", - buildtime=False, - depth=None, - verbose=0, - include_vulns=False, - exclude_meta=True, - meta_nixpkgs="nix-path", - exclude_cpe_matching=False, - csv=None, - cdx=None, - spdx=None, - impure=True, - ) - - with pytest.raises(SbomnixError, match="--exclude-meta"): - sbomnix_main._run(args) - - def test_sbomnix_main_enriches_cdx_explicitly_when_include_vulns_is_set(monkeypatch): args = SimpleNamespace( NIXREF=".#target", @@ -70,7 +40,6 @@ def test_sbomnix_main_enriches_cdx_explicitly_when_include_vulns_is_set(monkeypa verbose=0, include_vulns=True, exclude_meta=False, - meta_nixpkgs=None, exclude_cpe_matching=False, csv=None, cdx="sbom.cdx.json", @@ -123,7 +92,6 @@ def to_csv(self, _path): "depth": None, "flakeref": ".#target", "original_ref": None, - "meta_nixpkgs": None, "impure": True, "include_meta": True, "include_vulns": True, @@ -149,7 +117,6 @@ def test_sbomnix_main_logs_generation_before_initializing_builder(monkeypatch): verbose=0, include_vulns=False, exclude_meta=False, - meta_nixpkgs=None, exclude_cpe_matching=False, csv=None, cdx=None, @@ -188,7 +155,6 @@ def __init__(self, **kwargs): "depth": None, "flakeref": ".#target", "original_ref": None, - "meta_nixpkgs": None, "impure": False, "include_meta": True, "include_vulns": False, diff --git a/tests/testpaths.py b/tests/testpaths.py index e6e17e18..c2a3dedd 100644 --- a/tests/testpaths.py +++ b/tests/testpaths.py @@ -18,7 +18,6 @@ SBOMNIX = SRCDIR / "sbomnix" / "main.py" NIXGRAPH = SRCDIR / "nixgraph" / "main.py" -NIXMETA = SRCDIR / "nixmeta" / "main.py" PROVENANCE = SRCDIR / "provenance" / "main.py" NIX_OUTDATED = SRCDIR / "nixupdate" / "nix_outdated.py" VULNXSCAN = SRCDIR / "vulnxscan" / "vulnxscan_cli.py"