From 5086836bc77b17129eeb30244f558693bb403761 Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Wed, 20 May 2026 09:05:09 +0300 Subject: [PATCH 1/7] sbomnix: realise flake targets with nix build Signed-off-by: Henri Rosten --- src/common/flakeref.py | 18 --------------- tests/test_flakeref_resolution.py | 37 +++++++------------------------ 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/src/common/flakeref.py b/src/common/flakeref.py index e3dfc5e..7307194 100644 --- a/src/common/flakeref.py +++ b/src/common/flakeref.py @@ -109,15 +109,6 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): def _force_realise_flakeref(flakeref, *, impure, exec_cmd_fn, log): """Return the realized output path for a runtime flakeref target.""" log.info("Evaluating flakeref '%s'", flakeref) - nixpath = _realised_flakeref_path( - flakeref, - impure=impure, - exec_cmd_fn=exec_cmd_fn, - ) - if nixpath: - log.debug("flakeref='%s' maps to realised path='%s'", flakeref, nixpath) - return nixpath - return _build_flakeref_path( flakeref, impure=impure, @@ -149,15 +140,6 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): return nixpath -def _realised_flakeref_path(flakeref, *, impure, exec_cmd_fn): - """Return the already-realised output path for ``flakeref`` when available.""" - cmd = nix_cmd("path-info", flakeref, impure=impure) - ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) - if ret is None or ret.returncode != 0: - return None - return _first_output_path(ret.stdout) - - def parse_nixos_configuration_ref( flakeref: str, *, diff --git a/tests/test_flakeref_resolution.py b/tests/test_flakeref_resolution.py index f55f1ef..91fd4b1 100644 --- a/tests/test_flakeref_resolution.py +++ b/tests/test_flakeref_resolution.py @@ -79,8 +79,6 @@ def test_try_resolve_flakeref_uses_argv_lists(): def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) return SimpleNamespace(stdout="/nix/store/resolved\n", stderr="", returncode=0) resolved = try_resolve_flakeref( @@ -92,19 +90,6 @@ def fake_exec_cmd(cmd, **kwargs): assert resolved == "/nix/store/resolved" assert calls == [ - ( - [ - "nix", - "path-info", - "/tmp/my flake#pkg", - "--extra-experimental-features", - "flakes", - "--extra-experimental-features", - "nix-command", - "--impure", - ], - {"raise_on_error": False, "return_error": True, "log_error": False}, - ), ( [ "nix", @@ -123,12 +108,12 @@ def fake_exec_cmd(cmd, **kwargs): ] -def test_try_resolve_flakeref_reuses_realised_path_info(): +def test_try_resolve_flakeref_realises_with_build_print_out_paths(): calls = [] def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - assert cmd[1] == "path-info" + assert cmd[1] == "build" return SimpleNamespace( stdout="/nix/store/00000000000000000000000000000000-resolved\n", stderr="", @@ -146,7 +131,9 @@ def fake_exec_cmd(cmd, **kwargs): ( [ "nix", - "path-info", + "build", + "--no-link", + "--print-out-paths", ".#hello", "--extra-experimental-features", "flakes", @@ -163,12 +150,6 @@ def test_try_resolve_flakeref_rejects_invalid_explicit_output_selector(): def fake_exec_cmd(cmd, **kwargs): calls.append((cmd, kwargs)) - if cmd[1] == "path-info": - return SimpleNamespace( - stdout="", - stderr="does not have output", - returncode=1, - ) if cmd[1] == "eval": raise AssertionError("invalid output selectors must not use eval fallback") return SimpleNamespace( @@ -187,7 +168,7 @@ def fake_exec_cmd(cmd, **kwargs): exec_cmd_fn=fake_exec_cmd, ) - assert [cmd[1] for cmd, _kwargs in calls] == ["path-info", "build"] + assert [cmd[1] for cmd, _kwargs in calls] == ["build"] def test_try_resolve_flakeref_can_return_derivation_path(): @@ -280,8 +261,7 @@ def fake_exec_cmd(_cmd, **_kwargs): def test_try_resolve_flakeref_raises_on_failed_force_realise(): def fake_exec_cmd(cmd, **_kwargs): - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) + assert cmd[1] == "build" return SimpleNamespace(stdout="", stderr="build failed", returncode=1) with pytest.raises(FlakeRefRealisationError, match="build failed"): @@ -294,8 +274,7 @@ def fake_exec_cmd(cmd, **_kwargs): def test_try_resolve_flakeref_raises_when_force_realise_prints_no_path(): def fake_exec_cmd(cmd, **_kwargs): - if cmd[1] == "path-info": - return SimpleNamespace(stdout="", stderr="not realised", returncode=1) + assert cmd[1] == "build" return SimpleNamespace(stdout="\n", stderr="", returncode=0) with pytest.raises(FlakeRefRealisationError, match="returned no output path"): From 5cf10264f5f72625b3b077c58d4af22df0ca261b Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Tue, 19 May 2026 13:53:40 +0300 Subject: [PATCH 2/7] sbomnix: add flake metadata helpers Move flake metadata resolution helpers into sbomnix so the SBOM metadata pipeline no longer depends on the nixmeta package. The helper also centralizes local flake reference normalization for current-flake shorthand and path-based flake references. Signed-off-by: Henri Rosten --- src/sbomnix/flake_metadata.py | 225 ++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 src/sbomnix/flake_metadata.py diff --git a/src/sbomnix/flake_metadata.py b/src/sbomnix/flake_metadata.py new file mode 100644 index 0000000..451ff31 --- /dev/null +++ b/src/sbomnix/flake_metadata.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Helpers for resolving nixpkgs metadata from flakerefs.""" + +import json +import pathlib +import re + +from common.log import LOG, LOG_SPAM +from common.proc import exec_cmd, nix_cmd + + +def normalize_current_flake_shorthand(flakeref): + """Return ``.#attr`` for valid current-flake shorthand ``#attr``.""" + flakeref = str(flakeref or "") + flake, separator, attr = flakeref.partition("#") + if separator and not flake: + return f".{separator}{attr}" + return flakeref + + +def normalize_local_flake_ref(flake): + """Return an absolute local flake ref, leaving registry and URL refs intact.""" + if flake.startswith("path:"): + path_text, separator, query = flake.removeprefix("path:").partition("?") + path = pathlib.Path(path_text).expanduser() + if not path.is_absolute(): + path_text = path.resolve().as_posix() + return f"path:{path_text}{separator}{query}" + if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or ""): + return flake + path = pathlib.Path(flake).expanduser() + if path.exists() or flake.startswith((".", "/", "~")): + return path.resolve().as_posix() + return flake + + +def normalize_flakeref_for_nix_eval(flakeref): + """Normalize current-flake shorthand and local paths before Nix evaluation.""" + flakeref = normalize_current_flake_shorthand(flakeref) + flake, separator, attr = flakeref.partition("#") + if not flake: + return flakeref + return f"{normalize_local_flake_ref(flake)}{separator}{attr}" + + +def get_flake_metadata(flakeref, *, exec_cmd_fn=exec_cmd, nix_cmd_fn=nix_cmd, log=LOG): + """Return ``nix flake metadata`` JSON for the given flakeref.""" + if flakeref.startswith("nixpkgs="): + flakeref = flakeref.removeprefix("nixpkgs=") + log.info("Reading flake metadata for '%s'", flakeref) + cmd = nix_cmd_fn("flake", "metadata", flakeref, "--json") + ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + if ret is None or ret.returncode != 0: + log.warning("Failed reading flake metadata: %s", flakeref) + return None + meta_json = json.loads(ret.stdout) + log.log(LOG_SPAM, meta_json) + return meta_json + + +def is_nixpkgs_metadata(meta_json): + """Return true if the given metadata describes nixpkgs.""" + try: + if ( + "path" in meta_json + and "description" in meta_json + and meta_json["description"] + == "A collection of packages for the Nix package manager" + ): + return True + if ( + "path" in meta_json + and meta_json["locked"]["owner"] == "NixOS" + and meta_json["locked"]["repo"] == "nixpkgs" + ): + return True + except (KeyError, TypeError): + return False + return False + + +def _locked_obj_is_nixpkgs(node_name, locked_obj): + try: + if locked_obj.get("repo") == "nixpkgs": + return True + if node_name.startswith("nixpkgs") and locked_obj.get("type") == "path": + return True + except AttributeError: + return False + return False + + +def _input_node_names(value): + if isinstance(value, str): + return [value] + if isinstance(value, list) and value and isinstance(value[-1], str): + # Lock-file override chains store the resolved input node as the last item. + return [value[-1]] + return [] + + +def _get_flake_nixpkgs_obj(meta_json): + try: + nodes = meta_json["locks"]["nodes"] + root_name = meta_json["locks"]["root"] + root_inputs = nodes[root_name].get("inputs", {}) + except (KeyError, TypeError, AttributeError): + return None + + for node_name in _input_node_names(root_inputs.get("nixpkgs")): + try: + return nodes[node_name]["locked"] + except (KeyError, TypeError): + continue + + candidates = [] + for node_name, node in nodes.items(): + try: + locked_obj = node["locked"] + except (KeyError, TypeError): + continue + if _locked_obj_is_nixpkgs(node_name, locked_obj): + candidates.append(locked_obj) + if len(candidates) == 1: + return candidates[0] + return None + + +def _get_flake_nixpkgs_val(meta_json, key): + nixpkgs_obj = _get_flake_nixpkgs_obj(meta_json) + if nixpkgs_obj is None: + return None + try: + return nixpkgs_obj[key] + except (KeyError, TypeError): + return None + + +def _get_nixpkgs_flakeref_github(meta_json, *, log=LOG): + owner = _get_flake_nixpkgs_val(meta_json, "owner") + repo = _get_flake_nixpkgs_val(meta_json, "repo") + rev = _get_flake_nixpkgs_val(meta_json, "rev") + if None in [owner, repo, rev]: + log.debug( + "owner, repo, or rev not found: %s", + _get_flake_nixpkgs_obj(meta_json), + ) + return None + return f"github:{owner}/{repo}?rev={rev}" + + +def _get_nixpkgs_flakeref_git(meta_json, *, log=LOG): + url = _get_flake_nixpkgs_val(meta_json, "url") + rev = _get_flake_nixpkgs_val(meta_json, "rev") + ref = _get_flake_nixpkgs_val(meta_json, "ref") + if None in [url, rev, ref]: + log.debug("url, rev, or ref not found: %s", _get_flake_nixpkgs_obj(meta_json)) + return None + return f"git+{url}?ref={ref}&rev={rev}" + + +def _get_nixpkgs_flakeref_path(meta_json, *, log=LOG): + path = _get_flake_nixpkgs_val(meta_json, "path") + if path is None: + log.debug("path not found: %s", _get_flake_nixpkgs_obj(meta_json)) + return None + return f"path:{path}" + + +def _get_nixpkgs_flakeref_tarball(meta_json, *, log=LOG): + url = _get_flake_nixpkgs_val(meta_json, "url") + if url is None: + log.debug("url not found: %s", _get_flake_nixpkgs_obj(meta_json)) + return None + return f"{url}" + + +def get_nixpkgs_flakeref(meta_json, *, log=LOG): + """Given flake metadata, return the locked nixpkgs flakeref.""" + locked_type = _get_flake_nixpkgs_val(meta_json, "type") + if locked_type == "github": + return _get_nixpkgs_flakeref_github(meta_json, log=log) + if locked_type == "git": + return _get_nixpkgs_flakeref_git(meta_json, log=log) + if locked_type == "path": + return _get_nixpkgs_flakeref_path(meta_json, log=log) + if locked_type == "tarball": + return _get_nixpkgs_flakeref_tarball(meta_json, log=log) + log.debug("Unsupported nixpkgs locked type: %s", locked_type) + return None + + +def nixref_to_nixpkgs_path( + flakeref, + *, + get_flake_metadata_fn=get_flake_metadata, + log=LOG, + log_spam=LOG_SPAM, +): + """Return the nix store path of the nixpkgs pinned by ``flakeref``.""" + if not flakeref: + return None + flakeref = normalize_current_flake_shorthand(flakeref) + log.info("Resolving nixpkgs path for '%s'", flakeref) + log.debug("Finding meta-info for nixpkgs pinned in nixref: %s", flakeref) + match = re.match(r"([^#]+)#", flakeref) + if match: + flakeref = match.group(1) + log.debug("Stripped target specifier: %s", flakeref) + meta_json = get_flake_metadata_fn(flakeref) + if not is_nixpkgs_metadata(meta_json): + log.debug("non-nixpkgs flakeref: %s", flakeref) + nixpkgs_flakeref = get_nixpkgs_flakeref(meta_json, log=log) + if not nixpkgs_flakeref: + log.warning("Failed parsing locked nixpkgs: %s", flakeref) + return None + log.log(log_spam, "using nixpkgs_flakeref: %s", nixpkgs_flakeref) + meta_json = get_flake_metadata_fn(nixpkgs_flakeref) + if not is_nixpkgs_metadata(meta_json): + log.warning("Failed reading nixpkgs metadata: %s", flakeref) + return None + return pathlib.Path(meta_json["path"]).absolute() From b682c67fce8cb4a5a863d232c940d104e017d46a Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Tue, 19 May 2026 13:53:40 +0300 Subject: [PATCH 3/7] sbomnix: add package metadata lookup helper Add the Python and Nix helpers used to evaluate package metadata from package roots and flake package outputs. The lookup request stays compact and the result includes derivation and output identities so callers can reject metadata that does not belong to an SBOM component. Signed-off-by: Henri Rosten --- pyproject.toml | 3 + src/sbomnix/artifacts.py | 54 +++ src/sbomnix/package_meta.nix | 343 +++++++++++++ src/sbomnix/package_meta.py | 914 +++++++++++++++++++++++++++++++++++ 4 files changed, 1314 insertions(+) create mode 100644 src/sbomnix/artifacts.py create mode 100644 src/sbomnix/package_meta.nix create mode 100644 src/sbomnix/package_meta.py diff --git a/pyproject.toml b/pyproject.toml index 2edebfa..cc7de47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,9 @@ version = { file = ["VERSION"] } [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +sbomnix = ["*.nix"] + [tool.ruff] line-length = 88 target-version = "py310" diff --git a/src/sbomnix/artifacts.py b/src/sbomnix/artifacts.py new file mode 100644 index 0000000..536c491 --- /dev/null +++ b/src/sbomnix/artifacts.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Helpers for recognizing file artifacts in Nix build closures.""" + +import re + +_PATCH_OR_DIFF_RE = r"[.](?:patch|diff)(?:[.][a-zA-Z0-9]+)?" +_SOURCE_ARCHIVE_RE = ( + r"[.]tar(?:[.][a-zA-Z0-9]+)?|" + r"[.](?:tgz|tbz2?|txz|zip|whl|gem|cabal|crate)" +) +_CONFIG_OR_DATA_RE = ( + r"[.](?:" + r"conf|cfg|ini|json|toml|ya?ml|lock|sum|mod|list|rules|preset|" + r"desktop|service|socket|target|timer|mount|path|slice|network|" + r"link|netdev|nmconnection|pam|pc|xml|crt|pem|cer|der|dat|" + r"asc|sig|txt|md|rst" + r")" +) +_FONT_OR_MEDIA_RE = ( + r"[.](?:" + r"bdf|pcf|otf|ttf|ttc|woff2?|pfa|pfb|" + r"mp3|mp4|mov|mkv|mxf|wav|ogg|amv|dmg|png|svg|gif|jpe?g|webp" + r")" + r"(?:[.](?:gz|xz|bz2|zst))?" +) +_SOURCE_FILE_RE = ( + r"[.](?:" + r"sh|bash|fish|py|pl|pm|lua|cgi|c|cc|cpp|h|hpp|java|js|ts|" + r"go|rs|hs|m4|in|cmake|mk|proto|css|scss|html" + r")" +) +_GENERATED_CARGO_ARTIFACT_RE = re.compile( + r"^cargo-(?:src|package)-.+-[0-9].*$", + re.IGNORECASE, +) +_NON_PACKAGE_ARTIFACT_NAME_RE = re.compile( + rf".*(?:{_PATCH_OR_DIFF_RE}|{_SOURCE_ARCHIVE_RE}|" + rf"{_CONFIG_OR_DATA_RE}|{_FONT_OR_MEDIA_RE}|{_SOURCE_FILE_RE})(?:[?].*)?$", + re.IGNORECASE, +) + + +def is_non_package_artifact_name(name): + """Return True when a store-path name looks like a file, not a package.""" + if not isinstance(name, str) or not name: + return False + return ( + "?" in name + or _GENERATED_CARGO_ARTIFACT_RE.match(name) is not None + or _NON_PACKAGE_ARTIFACT_NAME_RE.match(name) is not None + ) diff --git a/src/sbomnix/package_meta.nix b/src/sbomnix/package_meta.nix new file mode 100644 index 0000000..035ae75 --- /dev/null +++ b/src/sbomnix/package_meta.nix @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +# Resolve package metadata for Python-generated attr lookup tiers. The helper +# scans selected package roots or flake input package roots, then returns metadata +# plus derivation identity. Python accepts only rows whose drvPath/outPath +# matches an SBOM component exactly. + +{ + request ? null, + pkgs ? null, +}: + +let + requestText = if request != null then request else throw "Missing mandatory argument: 'request'"; + req = builtins.fromJSON requestText; + flakeref = req.flakeref or null; + split = if flakeref == null then null else builtins.match "^([^#]+)#(.*)$" flakeref; + flakeRef = if split != null then builtins.elemAt split 0 else flakeref; + flake = if flakeRef == null then null else builtins.getFlake flakeRef; + inherit (req) system; + nixpkgsPath = req.nixpkgsPath or null; + lookupKeys = req.lookupKeys or [ ]; + inputRootsOnly = req.inputRootsOnly or false; + + optionals = cond: values: if cond then values else [ ]; + concatMap = f: values: builtins.concatLists (builtins.map f values); + isAttrs = builtins.isAttrs; + isDerivation = value: isAttrs value && (value.type or null) == "derivation"; + hasPrefix = + prefix: str: + let + pl = builtins.stringLength prefix; + sl = builtins.stringLength str; + in + pl <= sl && builtins.substring 0 pl str == prefix; + findFirst = + pred: default: values: + if values == [ ] then + default + else if pred (builtins.head values) then + builtins.head values + else + findFirst pred default (builtins.tail values); + hasMatches = values: builtins.length values > 0; + firstMatchingCandidates = candidateLists: findFirst hasMatches [ ] candidateLists; + + safeAttr = + attrs: attr: + let + res = builtins.tryEval ( + if isAttrs attrs && builtins.hasAttr attr attrs then builtins.getAttr attr attrs else null + ); + in + if res.success then res.value else null; + + attrByPath = + path: attrs: + if path == [ ] then + attrs + else + let + value = safeAttr attrs (builtins.head path); + in + if value == null then null else attrByPath (builtins.tail path) value; + + importedPkgs = + if pkgs != null then + null + else if nixpkgsPath == null || nixpkgsPath == "" then + null + else + import nixpkgsPath { + inherit system; + config.allowAliases = false; + }; + + rootPackages = attrByPath [ + "packages" + system + ] flake; + rootLegacyPackages = attrByPath [ + "legacyPackages" + system + ] flake; + + uniqueNames = + values: + builtins.attrNames ( + builtins.listToAttrs ( + builtins.map (name: { + inherit name; + value = true; + }) values + ) + ); + + packageSetNames = uniqueNames ( + builtins.filter (name: name != "") ( + concatMap ( + lookup: concatMap (tier: concatMap (candidate: candidate.packageSets) tier) lookup.candidateTiers + ) lookupKeys + ) + ); + + mkRoot = + attrSet: + let + nestedByName = builtins.listToAttrs ( + builtins.filter (entry: isAttrs entry.value) ( + builtins.map (name: { + inherit name; + value = safeAttr attrSet name; + }) packageSetNames + ) + ); + in + { + base = attrSet; + inherit nestedByName; + }; + + validRoot = root: isAttrs root.base; + flakeInputs = attrByPath [ "inputs" ] flake; + + inputRoots = + if flake == null || !isAttrs flakeInputs then + [ ] + else + concatMap ( + inputName: + let + input = safeAttr flakeInputs inputName; + packages = attrByPath [ + "packages" + system + ] input; + legacyPackages = attrByPath [ + "legacyPackages" + system + ] input; + in + optionals (isAttrs packages) [ + (mkRoot packages) + ] + ++ optionals (isAttrs legacyPackages) [ + (mkRoot legacyPackages) + ] + ) (builtins.attrNames flakeInputs); + + explicitPkgsRoots = optionals (isAttrs pkgs) [ + (mkRoot pkgs) + ]; + flakePackageRoots = builtins.filter validRoot ( + optionals (isAttrs rootPackages) [ + (mkRoot rootPackages) + ] + ++ optionals (isAttrs rootLegacyPackages) [ + (mkRoot rootLegacyPackages) + ] + ); + importedPkgsRoots = optionals (isAttrs importedPkgs) [ + (mkRoot importedPkgs) + ]; + + baseRoots = builtins.filter validRoot (explicitPkgsRoots ++ flakePackageRoots ++ importedPkgsRoots); + roots = if inputRootsOnly then builtins.filter validRoot inputRoots else baseRoots; + + packageSetByName = root: name: if name == "" then root.base else root.nestedByName.${name} or null; + packageSetsByNames = + root: names: + builtins.filter (packageSet: packageSet != null) (builtins.map (packageSetByName root) names); + + lookupAttr = + attr: packageSet: + let + res = builtins.tryEval ( + let + candidate = safeAttr packageSet attr; + in + if attr != "" && candidate != null && isDerivation candidate then { drv = candidate; } else null + ); + in + if res.success && res.value != null then [ res.value ] else [ ]; + + isSimpleOutputName = output: builtins.match "[a-zA-Z0-9][a-zA-Z0-9_+-]*" output != null; + + candidateMatchesSplitOutputName = + inputName: candidate: + let + inherit (candidate) drv; + drvName = drv.name or null; + prefix = if drvName == null then null else "${drvName}-"; + suffix = + if prefix != null && hasPrefix prefix inputName then + builtins.substring (builtins.stringLength prefix) ( + builtins.stringLength inputName - builtins.stringLength prefix + ) inputName + else + null; + outputs = builtins.filter (output: output != "out") (drv.outputs or [ "out" ]); + in + suffix != null && suffix != "" && isSimpleOutputName suffix && builtins.elem suffix outputs; + + canonicalCandidatesByIdentity = + inputName: candidates: + let + exactNameMatches = builtins.filter ( + candidate: (candidate.drv.name or null) == inputName + ) candidates; + splitOutputMatches = builtins.filter ( + candidate: candidateMatchesSplitOutputName inputName candidate + ) candidates; + in + if hasMatches exactNameMatches then + exactNameMatches + else if hasMatches splitOutputMatches then + splitOutputMatches + else + [ ]; + + candidateMatchesExpectedPnameOrName = + inputName: expectedPname: candidate: + (candidate.drv.name or null) == inputName || (candidate.drv.pname or null) == expectedPname; + + candidatePlausible = + inputName: expectedPname: expectedVersion: candidate: + let + inherit (candidate) drv; + drvName = drv.name or null; + drvPname = drv.pname or null; + drvVersion = drv.version or null; + versionMatches = expectedVersion == "" || drvVersion == expectedVersion; + in + drvName == inputName + || candidateMatchesSplitOutputName inputName candidate + || (expectedPname != "" && drvPname == expectedPname && versionMatches); + + narrowPlausible = + inputName: expectedPname: expectedVersion: candidates: + let + plausible = builtins.filter (candidatePlausible inputName expectedPname expectedVersion) candidates; + in + if hasMatches plausible then plausible else [ ]; + + lookupCandidate = + root: lookup: candidate: + let + inputName = lookup.name or ""; + expectedPname = lookup.pname or ""; + matches = concatMap (lookupAttr candidate.attr) (packageSetsByNames root candidate.packageSets); + in + if candidate.requirePnameOrName or false then + builtins.filter (candidateMatchesExpectedPnameOrName inputName expectedPname) matches + else + matches; + + lookupTier = + root: lookup: tier: + concatMap (lookupCandidate root lookup) tier; + + findByLookup = + root: lookup: + let + name = lookup.name or ""; + pname = lookup.pname or ""; + version = lookup.version or ""; + tiers = lookup.candidateTiers; + tierCandidates = builtins.map (lookupTier root lookup) tiers; + canonicalCandidateLists = builtins.map (canonicalCandidatesByIdentity name) tierCandidates; + canonicalCandidates = + if lookup.collectAllCanonical or false then + builtins.concatLists canonicalCandidateLists + else + firstMatchingCandidates canonicalCandidateLists; + plausibleCandidates = firstMatchingCandidates ( + builtins.map (narrowPlausible name pname version) tierCandidates + ); + in + # Prefer matches whose derivation/output identity explains the input + # component name. Pname/version matches are a fallback for packages whose + # attr name cannot be inferred exactly. + if hasMatches canonicalCandidates then canonicalCandidates else plausibleCandidates; + + filteredMeta = meta: { + description = meta.description or null; + homepage = meta.homepage or null; + unfree = meta.unfree or false; + position = meta.position or null; + license = meta.license or { }; + maintainers = builtins.map (maintainer: { email = maintainer.email or ""; }) ( + builtins.filter isAttrs (meta.maintainers or [ ]) + ); + }; + + drvOutputs = + drv: if drv ? outputs then builtins.map (output: drv.${output}) drv.outputs else [ drv ]; + + field = + fallbackDrv: drv: name: default: + let + value = safeAttr drv name; + fallbackValue = safeAttr fallbackDrv name; + in + if value != null then + value + else if fallbackValue != null then + fallbackValue + else + default; + + fields = fallbackDrv: drv: { + path = field fallbackDrv drv "outPath" null; + drvPath = field fallbackDrv drv "drvPath" null; + name = field fallbackDrv drv "name" ""; + pname = field fallbackDrv drv "pname" ""; + version = field fallbackDrv drv "version" ""; + meta = filteredMeta (field fallbackDrv drv "meta" { }); + }; + + rowsForCandidate = + candidate: + let + rawRows = builtins.map (fields candidate.drv) (drvOutputs candidate.drv); + forced = builtins.tryEval (builtins.deepSeq rawRows rawRows); + in + if forced.success then forced.value else [ ]; + + rowsForLookupInRoot = + root: lookup: + let + candidatesTry = builtins.tryEval (findByLookup root lookup); + candidates = if candidatesTry.success then candidatesTry.value else [ ]; + in + concatMap rowsForCandidate candidates; + + rowsForLookup = lookup: concatMap (root: rowsForLookupInRoot root lookup) roots; + +in +{ + derivations = concatMap rowsForLookup lookupKeys; +} diff --git a/src/sbomnix/package_meta.py b/src/sbomnix/package_meta.py new file mode 100644 index 0000000..a68da76 --- /dev/null +++ b/src/sbomnix/package_meta.py @@ -0,0 +1,914 @@ +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Package metadata lookup from package sets and flake package outputs. + +Python sends compact SBOM component lookup keys to the Nix helper. The helper +does package-set specific name lookup and returns derivation identity; this +module accepts metadata only when the returned drvPath/outPath matches a +component exactly. +""" + +import functools +import hashlib +import json +import pathlib +import platform +import re +import shutil +import subprocess +from tempfile import TemporaryDirectory + +import pandas as pd + +from common import columns as cols +from common.log import LOG +from common.proc import exec_cmd, nix_cmd +from sbomnix.artifacts import is_non_package_artifact_name +from sbomnix.flake_metadata import normalize_flakeref_for_nix_eval + +PACKAGE_META_METHOD = "package-meta-nix" +META_OUTPUT_PATH = "meta_output_path" + +_META_DERIVATION_PATH = "_meta_derivation_path" +_RICH_META_COLUMNS = ( + "meta_homepage", + "meta_description", + "meta_license_short", + "meta_license_spdxid", +) +_PNAME_PATTERN = ( + r"[a-zA-Z0-9][a-zA-Z0-9_+.]*" + r"(?:-[a-zA-Z][a-zA-Z0-9_+.]*)*" +) +_STD_PNAME_RE = re.compile( + rf"^(?P{_PNAME_PATTERN})-[0-9].*$", +) +_PYTHON_NAME_RE = re.compile( + rf"^python(?P[23])\.(?P[0-9]+)-(?P{_PNAME_PATTERN})-[0-9].*" +) +_PERL3_NAME_RE = re.compile( + rf"^perl[0-9]+\.[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*" +) +_PERL_NAME_RE = re.compile(rf"^perl[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*") +_RUBY_NAME_RE = re.compile(rf"^ruby[0-9]+\.[0-9]+-(?P{_PNAME_PATTERN})-[0-9].*") +_BARE_NAME_RE = re.compile(r"^(?P[a-zA-Z0-9][a-zA-Z0-9_+.-]*)") +_QT_KDE_NAME_RE = re.compile(r"^(qt|kde|kf|plasma|kirigami|kwayland|kwin|qqc2).*") +_K_PREFIXED_PNAME_RE = re.compile(r"^k[a-z0-9].*") +_SUFFIX_RE = re.compile(r"^(?P.*)-[a-zA-Z][a-zA-Z0-9_+]*$") +_WEBKIT_ABI_RE = re.compile(r"^[^+]*\+abi=([0-9]+)\.([0-9]+).*") +_HASKELL_VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+){2,4}$") +_BASE_PACKAGE_SET = "" +_EXPLICIT_ATTR_REWRITES = { + "bash": "bashNonInteractive", + "gcc": "gcc-unwrapped", + "libwww-perl": "LWP", + "nss-cacert": "cacert", + "util-linux-minimal": "util-linuxMinimal", +} + + +def package_meta_nix_path(): + """Return the packaged Nix helper used for package metadata evaluation.""" + return pathlib.Path(__file__).with_name("package_meta.nix") + + +def nix_system(): + """Return the host Nix system string used for package-set lookup.""" + machine = platform.machine() + machine = {"AMD64": "x86_64", "arm64": "aarch64"}.get(machine, machine) + system = platform.system().lower() + system = {"darwin": "darwin", "linux": "linux"}.get(system, system) + return f"{machine}-{system}" + + +def _eval_package_meta_request(request, *, pkgs_expression=None, impure=False): + request_json = json.dumps(request, sort_keys=True) + if len(request_json) <= 30_000: + return _eval_request( + f"request = {_nix_string_literal(request_json)};", + pkgs_expression=pkgs_expression, + impure=impure, + ) + + # Pure Nix eval can read files relative to its --file entrypoint, but not + # arbitrary absolute temp paths passed through --apply. + with TemporaryDirectory( + prefix="sbomnix_package_meta_", + ) as tmpdir: + tmpdir_path = pathlib.Path(tmpdir) + (tmpdir_path / "request.json").write_text(request_json, encoding="utf-8") + shutil.copyfile(package_meta_nix_path(), tmpdir_path / "package_meta.nix") + eval_file = tmpdir_path / "eval.nix" + eval_file.write_text( + _request_file_eval_expression(pkgs_expression), + encoding="utf-8", + ) + return _eval_file( + eval_file.as_posix(), + impure=impure, + ) + + +def _request_file_eval_expression(pkgs_expression): + pkgs_arg = f"\n pkgs = {pkgs_expression};" if pkgs_expression else "" + return ( + "import ./package_meta.nix {\n" + " request = builtins.readFile ./request.json;" + f"{pkgs_arg}\n" + "}\n" + ) + + +def _nix_string_literal(value): + return json.dumps(value).replace("${", r"\${") + + +def _eval_request(request_arg, *, pkgs_expression=None, impure=False): + ret = exec_cmd( + _eval_command( + _apply_expression(request_arg, pkgs_expression), + impure, + ), + log_error=False, + ) + return parse_package_metadata(ret.stdout) + + +def _eval_file(eval_file, *, impure=False): + ret = exec_cmd( + _eval_file_command(eval_file, impure), + log_error=False, + ) + return parse_package_metadata(ret.stdout) + + +def _eval_command(apply_expression, impure): + return nix_cmd( + "eval", + "--json", + "--file", + package_meta_nix_path().as_posix(), + "--apply", + apply_expression, + impure=impure, + ) + + +def _eval_file_command(eval_file, impure): + return nix_cmd( + "eval", + "--json", + "--file", + eval_file, + impure=impure, + ) + + +def _apply_expression(request_arg, pkgs_expression): + pkgs_arg = f" pkgs = {pkgs_expression};" if pkgs_expression else "" + return f"f: f {{ {request_arg}{pkgs_arg} }}" + + +def parse_package_metadata(json_text): + """Parse metadata JSON emitted by package_meta.nix.""" + data = json.loads(json_text) + derivations = data.get("derivations", []) + if not isinstance(derivations, list): + derivations = [] + + rows = [] + for drv in derivations: + if not isinstance(drv, dict): + continue + row = _metadata_row(drv) + if row[cols.STORE_PATH] or row[META_OUTPUT_PATH]: + rows.append(row) + df = pd.DataFrame.from_records(rows) + if df.empty: + return df + df = df.astype(str) + df.fillna("", inplace=True) + df.drop_duplicates( + subset=[cols.STORE_PATH, META_OUTPUT_PATH], + keep="first", + inplace=True, + ) + return df + + +def _metadata_row(drv): + meta = drv.get("meta", {}) + if not isinstance(meta, dict): + meta = {} + meta_license = meta.get("license", {}) + meta_maintainers = meta.get("maintainers", {}) + return { + cols.STORE_PATH: drv.get("drvPath", "") or "", + META_OUTPUT_PATH: drv.get("path", "") or "", + cols.NAME: drv.get("name", "") or "", + cols.PNAME: drv.get("pname", "") or "", + cols.VERSION: drv.get("version", "") or "", + "meta_homepage": _parse_optional_meta_entry(meta, key="homepage"), + "meta_unfree": meta.get("unfree", ""), + "meta_description": meta.get("description", "") or "", + "meta_position": meta.get("position", "") or "", + "meta_license_short": _parse_optional_meta_entry(meta_license, key="shortName"), + "meta_license_spdxid": _parse_optional_meta_entry(meta_license, key="spdxId"), + "meta_maintainers_email": _parse_optional_meta_entry( + meta_maintainers, + key="email", + ), + } + + +def _parse_optional_meta_entry(meta, key): + if meta is None: + return "" + if isinstance(meta, dict): + return _parse_optional_meta_entry(meta.get(key, ""), key) + if isinstance(meta, list): + return ";".join( + filter(None, (_parse_optional_meta_entry(item, key) for item in meta)) + ) + return str(meta) + + +def package_meta_lookup_keys_for_components( + df_components, + *, + target_path=None, + flakeref=None, +): + """Return compact Nix metadata lookup keys for the given component rows.""" + target_attr = _flake_package_attr(flakeref) + keys = {} + for component in _records(df_components): + if _is_non_package_artifact(component): + continue + lookup = _lookup_key_for_component(component) + if lookup is None: + continue + if target_attr and _component_matches_target_path(component, target_path): + lookup["candidateAttrs"] = [target_attr] + key = ( + lookup["name"], + lookup["pname"], + lookup["version"], + lookup.get("system", ""), + tuple(lookup.get("candidateAttrs", [])), + ) + keys[key] = lookup + return [keys[key] for key in sorted(keys)] + + +def _lookup_key_for_component(component): + name = _clean_lookup_value(component.get(cols.NAME, "")) + pname = _clean_lookup_value(component.get(cols.PNAME, "")) + version = _clean_lookup_value(component.get(cols.VERSION, "")) + system = _clean_lookup_value(component.get("system", "")) + if not name: + if not pname: + return None + name = f"{pname}-{version}" if version else pname + if not pname: + pname = _pname_from_package_name(name) + lookup = { + "name": name, + "pname": pname, + "version": version, + } + if system: + lookup["system"] = system + return lookup + + +def _clean_lookup_value(value): + if value is None: + return "" + if pd.isna(value): + return "" + return str(value).strip() + + +def _component_matches_target_path(component, target_path): + target_path = _clean_lookup_value(target_path) + if not target_path: + return False + if _clean_lookup_value(component.get(cols.STORE_PATH, "")) == target_path: + return True + if _clean_lookup_value(component.get("out", "")) == target_path: + return True + outputs = component.get(cols.OUTPUTS, []) + if isinstance(outputs, str): + outputs = [outputs] + elif not isinstance(outputs, (list, tuple, set)): + return False + return target_path in {_clean_lookup_value(output) for output in outputs} + + +def _flake_package_attr(flakeref): + _flake, separator, attr = str(flakeref or "").partition("#") + if not separator: + return "" + parts = attr.split(".") + if ( + len(parts) < 3 + or parts[0] not in {"packages", "legacyPackages"} + or not _is_nix_system_name(parts[1]) + ): + return "" + return parts[2] + + +def _clean_candidate_attrs(attrs): + if isinstance(attrs, str): + attrs = [attrs] + cleaned = [] + seen = set() + for raw_attr in attrs or []: + attr = _clean_lookup_value(raw_attr) + if not attr or attr in seen: + continue + seen.add(attr) + cleaned.append(attr) + return cleaned + + +def _normalized_lookup_keys(lookup_keys): + if isinstance(lookup_keys, dict): + lookup_keys = lookup_keys.get("lookupKeys", []) + normalized = [] + seen = set() + for lookup in lookup_keys or []: + if not isinstance(lookup, dict): + continue + name = _clean_lookup_value(lookup.get("name", "")) + pname = _clean_lookup_value(lookup.get("pname", "")) + version = _clean_lookup_value(lookup.get("version", "")) + system = _clean_lookup_value(lookup.get("system", "")) + if not name: + continue + if not pname: + pname = _pname_from_package_name(name) + candidate_attrs = _clean_candidate_attrs(lookup.get("candidateAttrs", [])) + key = (name, pname, version, system, tuple(candidate_attrs)) + if key in seen: + continue + seen.add(key) + normalized_lookup = {"name": name, "pname": pname, "version": version} + if system: + normalized_lookup["system"] = system + if candidate_attrs: + normalized_lookup["candidateAttrs"] = candidate_attrs + normalized.append(normalized_lookup) + return sorted( + normalized, + key=lambda item: ( + item["name"], + item["pname"], + item["version"], + item.get("system", ""), + tuple(item.get("candidateAttrs", [])), + ), + ) + + +def _request_system_groups(flakeref, lookup_keys): + fallback_system = _system_from_flakeref(flakeref) or nix_system() + groups = {} + for lookup in lookup_keys: + system = _clean_lookup_value(lookup.get("system", "")) + if not _is_nix_system_name(system): + system = fallback_system + groups.setdefault(system, []).append(lookup) + return [(system, groups[system]) for system in sorted(groups)] + + +def _is_nix_system_name(system): + return re.match(r"^[A-Za-z0-9_]+-[A-Za-z0-9_]+$", system or "") is not None + + +def _system_from_flakeref(flakeref): + _flake, separator, attr = (flakeref or "").partition("#") + if not separator or not attr: + return "" + attr_parts = attr.split(".") + if ( + len(attr_parts) >= 3 + and attr_parts[0] in {"packages", "legacyPackages"} + and _is_nix_system_name(attr_parts[1]) + ): + return attr_parts[1] + return "" + + +def _request_lookup_keys(lookup_keys): + return [_lookup_key_with_candidate_tiers(lookup) for lookup in lookup_keys] + + +def _lookup_key_with_candidate_tiers(lookup): + name = lookup["name"] + version = lookup["version"] + parsed_pname = _pname_from_package_name(name) + pname = ( + parsed_pname + if _is_prefixed_language_name(name) or (not version and lookup["pname"] == name) + else lookup["pname"] or parsed_pname + ) + pname = pname or "" + key = { + "name": name, + "pname": pname, + "version": version, + "candidateTiers": _candidate_tiers( + name, + pname, + version, + candidate_attrs=lookup.get("candidateAttrs", []), + ), + } + if _is_haskell_package_name(name, pname, version): + key["collectAllCanonical"] = True + return key + + +def _candidate_tiers(name, pname, version="", *, candidate_attrs=None): + if not pname: + return [] + + # Tiers are searched in order by package_meta.nix, so keep broad rewrites + # after higher-confidence package-set and exact-name candidates. + specs = _package_set_hints_for_lookup(name) + rewrite_specs = ( + specs + if any(spec != _BASE_PACKAGE_SET for spec in specs) + else [_BASE_PACKAGE_SET] + ) + strip_specs = ( + specs if _is_python_name(name) or _is_ruby_name(name) else [_BASE_PACKAGE_SET] + ) + p1 = _strip_suffix(pname) + p2 = _strip_suffix(p1) if p1 else None + + tiers = [] + _add_tier( + tiers, + [ + _candidate(attr, [_BASE_PACKAGE_SET]) + for attr in _clean_candidate_attrs(candidate_attrs) + ], + ) + _add_tier(tiers, [_candidate(pname, specs), _kde_exact_candidate(name, pname)]) + _add_tier(tiers, _perl_candidates(name, pname)) + _add_tier(tiers, _low_loss_rewrite_candidates(pname, rewrite_specs)) + _add_tier(tiers, [_candidate(p1, strip_specs)]) + _add_tier(tiers, _low_loss_rewrite_candidates(p1, strip_specs)) + _add_tier(tiers, [_candidate(p2, strip_specs)]) + _add_tier(tiers, _low_loss_rewrite_candidates(p2, strip_specs)) + _add_tier(tiers, [_cxx_candidate(pname)]) + _add_tier(tiers, [_gtk_candidate(pname)]) + _add_tier(tiers, [_webkit_candidate(name)]) + _add_tier( + tiers, [_candidate(pname.lower(), specs) if pname.lower() != pname else None] + ) + _add_tier(tiers, [_leading_digit_candidate(pname, specs)]) + _add_tier(tiers, [_no_dash_candidate(pname, specs)]) + _add_tier(tiers, [_candidate(f"{pname}{n}", specs) for n in range(1, 5)]) + _add_tier(tiers, _underscore_version_candidates(name, pname, specs)) + _add_tier(tiers, [_dot_dash_candidate(pname, specs)]) + _add_tier(tiers, [_plus_candidate(pname, specs)]) + _add_tier(tiers, _haskell_candidates(name, pname, version)) + return tiers + + +def _add_tier(tiers, candidates): + tier = [] + seen = set() + for candidate in candidates: + if not candidate: + continue + key = ( + candidate["attr"], + tuple(candidate["packageSets"]), + candidate.get("requirePnameOrName", False), + ) + if key in seen: + continue + seen.add(key) + tier.append(candidate) + if tier: + tiers.append(tier) + + +def _candidate(attr, package_sets, *, require_pname_or_name=False): + if not attr: + return None + candidate = { + "attr": attr, + "packageSets": list(package_sets), + } + if require_pname_or_name: + candidate["requirePnameOrName"] = True + return candidate + + +def _package_set_hints_for_lookup(name): + package_sets = _component_package_set_hints(name) + package_sets.append(_BASE_PACKAGE_SET) + return _unique_package_sets(package_sets) + + +def _component_package_set_hints(name): + if _is_python_name(name): + return _python_package_sets_for_name(name) + if _is_perl_name(name): + return ["perlPackages"] + if _is_ruby_name(name): + return ["rubyPackages"] + if _is_qt_kde_name(name): + return ["qt6Packages", "qt6", "kdePackages"] + return [] + + +def _unique_package_sets(package_sets): + unique = [] + seen = set() + for package_set in package_sets: + if package_set in seen: + continue + seen.add(package_set) + unique.append(package_set) + return unique + + +def _python_package_sets_for_name(name): + match = _PYTHON_NAME_RE.match(name) + if not match: + return ["python3Packages"] + compact = f"{match.group('major')}{match.group('minor')}" + if compact.startswith("3"): + return [f"python{compact}Packages", "python3Packages"] + return [f"python{compact}Packages"] + + +def _is_prefixed_language_name(name): + return _is_python_name(name) or _is_perl_name(name) or _is_ruby_name(name) + + +def _is_python_name(name): + return _PYTHON_NAME_RE.match(name) is not None + + +def _is_perl_name(name): + return ( + _PERL3_NAME_RE.match(name) is not None or _PERL_NAME_RE.match(name) is not None + ) + + +def _is_ruby_name(name): + return _RUBY_NAME_RE.match(name) is not None + + +def _is_qt_kde_name(name): + return _QT_KDE_NAME_RE.match(name) is not None + + +def _kde_exact_candidate(name, pname): + if ( + _is_prefixed_language_name(name) + or _is_qt_kde_name(name) + or _K_PREFIXED_PNAME_RE.match(pname or "") is None + ): + return None + return _candidate(pname, ["kdePackages"]) + + +def _haskell_candidates(name, pname, version): + if _is_haskell_package_name(name, pname, version): + return [_candidate(pname, ["haskellPackages"])] + return [] + + +def _is_haskell_package_name(name, pname, version): + if ( + not name + or not pname + or not version + or _is_prefixed_language_name(name) + or _is_qt_kde_name(name) + or is_non_package_artifact_name(name) + ): + return False + if not _HASKELL_VERSION_RE.match(version): + return False + return name == f"{pname}-{version}" + + +def _pname_from_package_name(value): + for regex in ( + _PYTHON_NAME_RE, + _PERL3_NAME_RE, + _PERL_NAME_RE, + _RUBY_NAME_RE, + _STD_PNAME_RE, + _BARE_NAME_RE, + ): + match = regex.match(str(value or "")) + if match: + return match.group("pname") + return "" + + +def _perl_candidates(name, pname): + if not _is_perl_name(name): + return [] + explicit = _EXPLICIT_ATTR_REWRITES.get(pname) + no_dash = pname.replace("-", "") + return [ + _candidate(explicit, ["perlPackages"]), + _candidate(no_dash, ["perlPackages"]) if no_dash != pname else None, + ] + + +def _low_loss_rewrite_candidates(pname, specs): + if not pname: + return [] + return [ + _rewritten_candidate(pname, _EXPLICIT_ATTR_REWRITES.get(pname), specs), + _rewritten_candidate(pname, pname.replace("-", "_"), specs), + _rewritten_candidate(pname, _dash_to_camel_case(pname), specs), + ] + + +def _rewritten_candidate(pname, rewritten, specs): + if not rewritten or rewritten == pname: + return None + return _candidate(rewritten, specs, require_pname_or_name=True) + + +def _strip_suffix(pname): + if not pname: + return None + match = _SUFFIX_RE.match(pname) + if match: + return match.group("pname") + return None + + +def _dash_to_camel_case(pname): + if not pname or "-" not in pname: + return None + first, *rest = pname.split("-") + return first + "".join(_uppercase_ascii_first(part) for part in rest) + + +def _uppercase_ascii_first(value): + if not value: + return "" + return value[:1].upper() + value[1:] + + +def _cxx_candidate(pname): + if pname.endswith("++"): + return _candidate(f"{pname.removesuffix('++')}xx", [_BASE_PACKAGE_SET]) + return None + + +def _gtk_candidate(pname): + attrs = { + "gtk+": "gtk2", + "gtk+3": "gtk3", + } + return _candidate(attrs.get(pname), [_BASE_PACKAGE_SET]) + + +def _webkit_candidate(name): + match = _WEBKIT_ABI_RE.match(name) + if not match: + return None + return _candidate( + f"webkitgtk_{match.group(1)}_{match.group(2)}", [_BASE_PACKAGE_SET] + ) + + +def _leading_digit_candidate(pname, specs): + if re.match(r"^[0-9].*", pname): + return _candidate(f"_{pname}", specs) + return None + + +def _no_dash_candidate(pname, specs): + no_dash = pname.replace("-", "") + if no_dash != pname: + return _candidate(no_dash, specs) + return None + + +def _underscore_version_candidates(name, pname, specs): + major_minor = re.match(rf"{re.escape(pname)}-([0-9]+)\.([0-9]+).*", name) + major = re.match(rf"{re.escape(pname)}-([0-9]+).*", name) + if major_minor: + attrs = [ + f"{pname}_{major_minor.group(1)}_{major_minor.group(2)}", + f"{pname}_{major_minor.group(1)}", + ] + elif major: + attrs = [f"{pname}_{major.group(1)}"] + else: + attrs = [] + return [_candidate(attr, specs) for attr in attrs] + + +def _dot_dash_candidate(pname, specs): + dot_dash = pname.replace(".", "-") + if dot_dash != pname: + return _candidate(dot_dash, specs) + return None + + +def _plus_candidate(pname, specs): + plus_name = pname.replace("+", "plus") + if plus_name != pname: + return _candidate(plus_name, specs) + return None + + +def _records(df): + if df is None or df.empty: + return [] + return df.to_dict("records") + + +def _is_non_package_artifact(component): + return any( + is_non_package_artifact_name(str(component.get(column, "") or "")) + for column in (cols.NAME, cols.PNAME) + ) + + +def package_meta_candidate_key(candidates): + """Return a short stable digest for a candidate request.""" + data = json.dumps(candidates, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(data.encode()).hexdigest()[:16] + + +@functools.cache +def package_meta_cache_fingerprint(): + """Return a short hash for the package metadata lookup implementation.""" + try: + paths = [ + pathlib.Path(__file__), + package_meta_nix_path(), + pathlib.Path(__file__).with_name("artifacts.py"), + ] + h = hashlib.sha256() + for path in paths: + h.update(path.read_bytes()) + h.update(b"\0") + return h.hexdigest()[:16] + except OSError as error: + LOG.warning("Packaged package metadata helper is unavailable: %s", error) + return "missing" + + +def match_package_metadata_to_components(components, metadata, *, buildtime): + """Return metadata rows that match component drvPath/outPath exactly.""" + if components is None or components.empty or metadata is None or metadata.empty: + return pd.DataFrame() + + matches = [] + if buildtime: + matches.append(_drv_path_matches(components, metadata, priority=0)) + matches.append(_output_path_matches(components, metadata, priority=1)) + else: + matches.append(_output_path_matches(components, metadata, priority=0)) + matches.append(_drv_path_matches(components, metadata, priority=1)) + return _best_component_matches(matches) + + +def _output_path_matches(components, metadata, *, priority): + output_components = components[[cols.STORE_PATH, cols.OUTPUTS]].copy() + output_components = output_components.explode(cols.OUTPUTS) + output_components.rename( + columns={ + cols.STORE_PATH: "_component_store_path", + cols.OUTPUTS: META_OUTPUT_PATH, + }, + inplace=True, + ) + output_components = output_components[ + output_components[META_OUTPUT_PATH].astype(bool) + ] + meta = metadata.rename(columns={cols.STORE_PATH: _META_DERIVATION_PATH}) + matched = output_components.merge(meta, how="inner", on=META_OUTPUT_PATH) + if matched.empty: + return matched + matched[cols.STORE_PATH] = matched["_component_store_path"] + matched["_meta_match_priority"] = priority + return matched + + +def _drv_path_matches(components, metadata, *, priority): + component_paths = components[[cols.STORE_PATH]].copy() + meta = metadata.rename(columns={cols.STORE_PATH: _META_DERIVATION_PATH}) + matched = component_paths.merge( + meta, + how="inner", + left_on=cols.STORE_PATH, + right_on=_META_DERIVATION_PATH, + ) + if matched.empty: + return matched + matched["_meta_match_priority"] = priority + return matched + + +def _best_component_matches(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + df = pd.concat(frames, ignore_index=True) + df["_meta_richness"] = _metadata_richness(df) + df.sort_values( + by=[cols.STORE_PATH, "_meta_match_priority", "_meta_richness"], + ascending=[True, True, False], + inplace=True, + ) + df.drop_duplicates(subset=[cols.STORE_PATH], keep="first", inplace=True) + drop_columns = [ + "_component_store_path", + _META_DERIVATION_PATH, + META_OUTPUT_PATH, + "_meta_match_priority", + "_meta_richness", + ] + df.drop(columns=[column for column in drop_columns if column in df], inplace=True) + return df + + +def _metadata_richness(df): + existing = [column for column in _RICH_META_COLUMNS if column in df] + if not existing: + return 0 + return df[existing].replace("", pd.NA).notna().sum(axis=1) + + +def _concat_metadata_frames(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames, ignore_index=True) + + +def try_scan_package_meta( # noqa: PLR0913 + lookup_keys, + *, + flakeref=None, + nixpkgs_path=None, + pkgs_expression=None, + input_roots_only=False, + impure=False, +): + """Return package metadata dataframe, or None on failure.""" + try: + lookup_keys = _normalized_lookup_keys(lookup_keys) + if not lookup_keys: + return pd.DataFrame() + flakeref = normalize_flakeref_for_nix_eval(flakeref) + LOG.debug("Reading package metadata for flakeref: %s", flakeref) + frames = [] + for request_system, system_lookup_keys in _request_system_groups( + flakeref, lookup_keys + ): + LOG.verbose( + "Evaluating package metadata for %d lookup(s) on system '%s'", + len(system_lookup_keys), + request_system, + ) + request = { + "flakeref": flakeref or None, + "system": request_system, + "nixpkgsPath": str(nixpkgs_path) if nixpkgs_path else None, + "lookupKeys": _request_lookup_keys(system_lookup_keys), + "inputRootsOnly": bool(input_roots_only), + } + frames.append( + _eval_package_meta_request( + request, + pkgs_expression=pkgs_expression, + impure=impure, + ) + ) + return _concat_metadata_frames(frames) + except ( + FileNotFoundError, + KeyError, + OSError, + subprocess.CalledProcessError, + TypeError, + ValueError, + ): + LOG.debug("Failed reading package metadata", exc_info=True) + return None From 65585e34570b1764c2300ffcaef138f636c98031 Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Tue, 19 May 2026 13:53:40 +0300 Subject: [PATCH 4/7] sbomnix: resolve package metadata by component identity Replace the old name-based nixpkgs metadata join with an exact component identity match against derivation and output paths. Cache package-root scans by stable source identity and lookup set, and scan flake input package roots only for components left unmatched by the primary source. Signed-off-by: Henri Rosten --- src/common/flakeref.py | 1 + src/sbomnix/builder.py | 31 +- src/sbomnix/dfcache.py | 6 + src/sbomnix/meta.py | 565 ++++++++++++++---- src/sbomnix/meta_source.py | 82 ++- src/sbomnix/package_meta.py | 8 +- tests/test_builder_runtime.py | 53 ++ tests/test_nix_cli_argv.py | 64 +-- tests/test_package_meta.py | 1013 +++++++++++++++++++++++++++++++++ 9 files changed, 1622 insertions(+), 201 deletions(-) create mode 100644 tests/test_package_meta.py diff --git a/src/common/flakeref.py b/src/common/flakeref.py index 7307194..9732599 100644 --- a/src/common/flakeref.py +++ b/src/common/flakeref.py @@ -136,6 +136,7 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): flakeref, "nix build returned no output path", ) + log.log(LOG_VERBOSE, "Resolved flakeref to built path '%s'", nixpath) log.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) return nixpath diff --git a/src/sbomnix/builder.py b/src/sbomnix/builder.py index 3a954ac..bde9c65 100644 --- a/src/sbomnix/builder.py +++ b/src/sbomnix/builder.py @@ -47,6 +47,9 @@ # Namespace UUID (a UUIDv4) for stable UUIDv5 identifiers. # See RFC9562, *6.6. Namespace ID Usage and Allocation*. SBOMNIX_UUID_NAMESPACE = uuid.UUID("136af32e-0d0e-48bc-912c-31b26af294b9") +_PACKAGE_META_PRIVATE_COLUMNS = [ + cols.NAME, +] @dataclass(frozen=True) @@ -170,6 +173,7 @@ def _load_recursive_buildtime_closure(self): """Load build-time dependencies from recursive derivation JSON.""" if self.target_deriver is None: raise MissingNixDeriverError(self.nix_path) + LOG.verbose("Loading build-time closure for '%s'", self.target_deriver) derivations, drv_infos = load_recursive(self.target_deriver) df_deps = derivation_dependencies_df(drv_infos) if self.depth: @@ -178,6 +182,9 @@ def _load_recursive_buildtime_closure(self): self.target_deriver, self.depth, ) + LOG.verbose( + "Loaded build-time closure with %d dependency edge(s)", len(df_deps) + ) return StructuredClosure( df_deps=df_deps, recursive_buildtime_derivations=derivations, @@ -185,6 +192,7 @@ def _load_recursive_buildtime_closure(self): def _load_runtime_path_info_closure(self, nix_path): """Load runtime dependencies from structured path-info JSON.""" + LOG.verbose("Loading runtime closure for '%s'", nix_path) runtime_closure = load_runtime_closure(nix_path) output_paths_by_load_path = _runtime_output_paths_by_load_path( runtime_closure.output_paths_by_drv @@ -207,6 +215,7 @@ def _load_runtime_path_info_closure(self, nix_path): nix_path, self.depth, ) + LOG.verbose("Loaded runtime closure with %d dependency edge(s)", len(df_deps)) return StructuredClosure( df_deps=df_deps, runtime_output_paths_by_load_path=output_paths_by_load_path, @@ -250,6 +259,7 @@ def _init_components(self, include_meta): else: # _load_structured_closure always selects exactly one metadata source. raise AssertionError("Structured dependency metadata was not initialized") + LOG.verbose("Initialized %d SBOM component(s)", len(self.df_sbomdb)) # Join with meta information if include_meta: self._join_meta() @@ -259,6 +269,10 @@ def _init_components(self, include_meta): self.df_sbomdb.sort_values(by=[cols.NAME, self.uid], inplace=True) self.df_sbomdb_outputs_exploded = self.df_sbomdb.explode(cols.OUTPUTS) self._init_dependency_index() + LOG.verbose( + "Prepared SBOM database with %d component(s)", + len(self.df_sbomdb), + ) def _sbom_component_paths(self): if self.df_deps is None or self.df_deps.empty: @@ -310,7 +324,9 @@ def _join_meta(self): if self.df_sbomdb is None: raise AssertionError("SBOM component metadata was not initialized") self.meta = Meta() - df_meta, source = self.meta.get_nixpkgs_meta_with_source( + df_meta, source = self.meta.get_package_meta_with_source( + components=self.df_sbomdb, + buildtime=self.buildtime, target_path=self.nix_path, flakeref=self.flakeref, original_ref=self.original_ref, @@ -334,14 +350,21 @@ def _join_meta(self): return if is_debug_enabled(): df_to_csv_file(df_meta, "meta.csv") - # Join based on package name including the version number + # Join only metadata that matched component drvPath/outPath exactly. + df_meta = df_meta.drop( + columns=[ + column + for column in _PACKAGE_META_PRIVATE_COLUMNS + if column in df_meta.columns + ] + ) self.df_sbomdb = self.df_sbomdb.merge( df_meta, how="left", - left_on=[cols.NAME], - right_on=[cols.NAME], + on=[cols.STORE_PATH], suffixes=("", "_meta"), ) + LOG.verbose("Joined nixpkgs metadata for %d component(s)", len(df_meta)) def lookup_dependencies(self, drv, uid=cols.STORE_PATH): """Return indexed dependency values for one SBOM component.""" diff --git a/src/sbomnix/dfcache.py b/src/sbomnix/dfcache.py index 65c98a9..c91deed 100644 --- a/src/sbomnix/dfcache.py +++ b/src/sbomnix/dfcache.py @@ -28,6 +28,12 @@ def set(self, key, value, ttl=None): dfcache = DataFrameDiskCache(cache_dir_path=dfcache_dir()) return _cache_local_set(dfcache, key, value, ttl=ttl) + def get_many(self, keys): + """Return multiple dataframe cache entries using one cache connection.""" + with self.dflock: + dfcache = DataFrameDiskCache(cache_dir_path=dfcache_dir()) + return {key: dfcache.get(key) for key in keys} + def __getattr__(self, name): def wrap(*a, **k): with self.dflock: diff --git a/src/sbomnix/meta.py b/src/sbomnix/meta.py index 01cefb9..7a1acaf 100644 --- a/src/sbomnix/meta.py +++ b/src/sbomnix/meta.py @@ -4,33 +4,40 @@ """Cache and scan nixpkgs meta information.""" +import re from dataclasses import replace +import pandas as pd from filelock import FileLock +from common import columns as cols from common.log import LOG -from nixmeta.scanner import NixMetaScanner from sbomnix.cache_paths import meta_lock_path from sbomnix.dfcache import LockedDfCache from sbomnix.meta_source import ( - META_NIXPKGS_NIX_PATH, - SCAN_EXCEPTIONS, NixpkgsMetaSource, NixpkgsMetaSourceResolver, - classify_meta_nixpkgs, +) +from sbomnix.package_meta import ( + PACKAGE_META_METHOD, + match_package_metadata_to_components, + package_meta_cache_fingerprint, + package_meta_candidate_key, + package_meta_lookup_keys_for_components, + try_scan_package_meta, ) ############################################################################### -# Update locally generated nixpkgs meta-info every 30 days or when local cache -# is cleaned. -_NIXMETA_NIXPKGS_TTL = 60 * 60 * 24 * 30 +# Update generated package metadata every 30 days or when local cache is cleaned. +_PACKAGE_META_TTL = 60 * 60 * 24 * 30 +_PACKAGE_META_FLAKE_INPUT_LOOKUP_LIMIT = 500 +_PACKAGE_META_CACHE_REF_COLUMN = "_package_meta_cache_ref" +_PACKAGE_META_LOOKUP_ID_COLUMN = "_package_meta_lookup_id" __all__ = [ - "META_NIXPKGS_NIX_PATH", "Meta", "NixpkgsMetaSource", - "classify_meta_nixpkgs", ] @@ -42,25 +49,18 @@ def __init__(self): self.cache = LockedDfCache() self.source_resolver = NixpkgsMetaSourceResolver() - def get_nixpkgs_meta(self, nixref=None): - """ - Return nixpkgs meta pinned in `nixref`. `nixref` can point to a - nix store path or flake reference. If nixref is None, attempt to - read the nixpkgs store path from NIX_PATH environment variable. - """ - source = self.source_resolver.resolve_default_source(nixref) - return self._scan_source(source) - - def get_nixpkgs_meta_with_source( + def get_package_meta_with_source( # noqa: PLR0913 self, *, + components, + buildtime, target_path=None, flakeref=None, original_ref=None, explicit_nixpkgs=None, impure=False, ): - """Return nixpkgs metadata and selected metadata source.""" + """Return exact component metadata and selected metadata source.""" source = self._resolve_source( target_path=target_path, flakeref=flakeref, @@ -68,7 +68,109 @@ def get_nixpkgs_meta_with_source( explicit_nixpkgs=explicit_nixpkgs, impure=impure, ) - return self._scan_source_with_source(source) + component_count = 0 if components is None else len(components) + LOG.verbose("Resolving package metadata for %d component(s)", component_count) + out_source = replace( + source, + method=f"{source.method}+{PACKAGE_META_METHOD}", + ) + if not source.path and not source.expression and not flakeref: + return None, source + scan_flakeref = _package_scan_flakeref(source, flakeref) + lookup_keys = package_meta_lookup_keys_for_components( + components, + target_path=target_path, + flakeref=scan_flakeref, + ) + if not lookup_keys: + return None, replace( + out_source, + message="No scannable package names supplied. Skipping nixpkgs metadata.", + ) + LOG.verbose( + "Looking up nixpkgs metadata for %d package name(s)", len(lookup_keys) + ) + df_candidates = self._scan_package_source( + source, + lookup_keys, + flakeref=scan_flakeref, + impure=impure, + ) + if df_candidates is None: + return None, replace( + out_source, + message="package_meta.nix scan failed. Skipping nixpkgs metadata.", + ) + df = match_package_metadata_to_components( + components, + df_candidates, + buildtime=buildtime, + ) + if scan_flakeref: + df = self._add_flake_input_package_meta( + components, + df, + source, + flakeref=scan_flakeref, + buildtime=buildtime, + impure=impure, + ) + if df.empty: + return None, replace( + out_source, + message=( + "No package metadata matched component derivation " + "or output paths. Skipping nixpkgs metadata." + ), + ) + LOG.verbose("Matched nixpkgs metadata for %d component(s)", len(df)) + return df, out_source + + def _add_flake_input_package_meta( # noqa: PLR0913 + self, + components, + df, + source, + *, + flakeref, + buildtime, + impure, + ): + unmatched = _components_without_package_matches(components, df) + if unmatched is None or unmatched.empty: + return df + + lookup_keys = package_meta_lookup_keys_for_components(unmatched) + if not lookup_keys: + return df + if len(lookup_keys) > _PACKAGE_META_FLAKE_INPUT_LOOKUP_LIMIT: + LOG.debug( + "Skipping flake-input metadata scan for %d unmatched lookup(s)", + len(lookup_keys), + ) + return df + LOG.verbose( + "Looking up flake input metadata for %d unmatched package name(s)", + len(lookup_keys), + ) + + df_candidates = self._scan_package_source( + source, + lookup_keys, + flakeref=flakeref, + input_roots_only=True, + impure=impure, + ) + df_input = match_package_metadata_to_components( + unmatched, + df_candidates, + buildtime=buildtime, + ) + if df_input.empty: + return df + if df.empty: + return df_input + return pd.concat([df, df_input], ignore_index=True) def _resolve_source( self, @@ -91,104 +193,353 @@ def _resolve_source( ) if source is not None: return source - return self.source_resolver.resolve_flakeref_lock_source(flakeref) + return self.source_resolver.resolve_flakeref_lock_source( + flakeref, + impure=impure, + ) return self.source_resolver.path_target_without_source( target_path=target_path, original_ref=original_ref, ) - def _scan_source(self, source): - df, _source = self._scan_source_with_source(source) - return df - - def _scan_source_with_source(self, source): - if not source.path: - return None, source - if source.expression: - LOG.debug("Scanning meta-info using nix expression for: %s", source.path) - df = self._scan_expression( - source.expression, - cache_key=source.expression_cache_key, - impure=source.expression_impure, - ) - if df is not None and not df.empty: - return df, source - LOG.warning( - "Failed scanning evaluated package set: %s", - source.path, - ) - return None, replace( + def _scan_package_source( + self, + source, + lookup_keys, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + scan_impure = impure or source.expression_impure + with self.lock: + cache_scope = self._package_cache_scope( source, - message=( - "Evaluated package-set metadata scan failed. " - "Skipping nixpkgs metadata." - ), + flakeref=flakeref, + input_roots_only=input_roots_only, + impure=scan_impure, ) - LOG.debug("Scanning meta-info using nixpkgs path: %s", source.path) - return self._scan(source.path), source - - def _scan_expression(self, expression, *, cache_key=None, impure=False): - if cache_key is None: - with self.lock: - LOG.debug("cache disabled, scanning expression") - df = self._try_scan_expression(expression, impure=impure) - if df is None or df.empty: - LOG.warning("Failed scanning uncached nixmeta expression") - return None - return df - cache_key = f"expr:{cache_key}" - with self.lock: - df = self.cache.get(cache_key) - if df is not None and not df.empty: - LOG.debug("found from cache: %s", cache_key) - return df - LOG.debug("cache miss, scanning expression: %s", cache_key) - df = self._try_scan_expression(expression, impure=impure) - if df is None or df.empty: - LOG.warning("Failed scanning nixmeta expression: %s", cache_key) + exact_cache_key = None + df_cached = pd.DataFrame() + missing_lookup_keys = list(lookup_keys) + hit_count = 0 + if cache_scope is not None: + exact_cache_key = self._package_scan_cache_key( + cache_scope, + lookup_keys, + ) + df = self.cache.get(exact_cache_key) + if df is not None: + LOG.verbose( + "Package metadata cache hit for %d package name(s)", + len(lookup_keys), + ) + LOG.debug( + "found exact package metadata cache entry: %s", exact_cache_key + ) + return df + + ( + df_cached, + missing_lookup_keys, + hit_count, + ) = self._cached_package_metadata( + cache_scope, + lookup_keys, + ) + if not missing_lookup_keys: + LOG.verbose( + "Package metadata cache hit for %d package name(s)", + len(lookup_keys), + ) + LOG.debug( + "found package metadata cache references: %s", cache_scope + ) + return df_cached + if hit_count: + LOG.verbose( + "Package metadata cache partial hit for %d/%d package name(s); " + "scanning %d", + hit_count, + len(lookup_keys), + len(missing_lookup_keys), + ) + LOG.debug( + "partial package metadata cache hit: %s", + cache_scope, + ) + else: + LOG.verbose( + "Package metadata cache miss for %d package name(s); scanning", + len(lookup_keys), + ) + LOG.debug("cache miss, scanning package metadata: %s", cache_scope) + else: + LOG.verbose( + "Package metadata cache disabled for %d package name(s); scanning", + len(lookup_keys), + ) + LOG.debug( + "cache disabled, scanning package metadata for %d lookup(s)", + len(lookup_keys), + ) + df = try_scan_package_meta( + missing_lookup_keys, + flakeref=_package_scan_flakeref(source, flakeref), + nixpkgs_path=source.path, + pkgs_expression=source.expression, + input_roots_only=input_roots_only, + impure=scan_impure, + ) + if df is None: return None - self.cache.set(key=cache_key, value=df, ttl=_NIXMETA_NIXPKGS_TTL) - return df + result = _concat_package_metadata_frames([df_cached, df]) + if cache_scope is not None: + self._store_package_metadata( + cache_scope, + missing_lookup_keys, + df, + ) + if hit_count and exact_cache_key is not None: + self.cache.set( + key=exact_cache_key, + value=result, + ttl=_PACKAGE_META_TTL, + ) + LOG.debug( + "Stored combined package metadata cache entry: %s", + exact_cache_key, + ) + return result + + def _cached_package_metadata(self, cache_scope, lookup_keys): + lookup_cache_items = [ + (self._package_lookup_cache_id(lookup_key), lookup_key) + for lookup_key in lookup_keys + ] + df_index = self.cache.get(self._package_lookup_index_cache_key(cache_scope)) + ref_key_by_lookup_id = _package_lookup_index_refs(df_index) + missing_lookup_keys = [] + ref_keys = set() + hit_count = 0 + + for lookup_id, lookup_key in lookup_cache_items: + ref_key = ref_key_by_lookup_id.get(lookup_id) + if not ref_key: + missing_lookup_keys.append(lookup_key) + continue + ref_keys.add(ref_key) + + cached_ref_frames = _cache_get_many(self.cache, sorted(ref_keys)) + ref_frames = [] + seen_ref_keys = set() + for lookup_id, lookup_key in lookup_cache_items: + ref_key = ref_key_by_lookup_id.get(lookup_id) + if not ref_key: + continue + df_ref = cached_ref_frames.get(ref_key) + if df_ref is None: + missing_lookup_keys.append(lookup_key) + continue + hit_count += 1 + if ref_key in seen_ref_keys: + continue + seen_ref_keys.add(ref_key) + ref_frames.append(df_ref) + + return ( + _concat_package_metadata_frames(ref_frames), + missing_lookup_keys, + hit_count, + ) + + def _store_package_metadata(self, cache_scope, lookup_keys, df): + if not lookup_keys: + return + scan_cache_key = self._package_scan_cache_key(cache_scope, lookup_keys) + self.cache.set(key=scan_cache_key, value=df, ttl=_PACKAGE_META_TTL) + self._store_package_lookup_index(cache_scope, lookup_keys, scan_cache_key) + LOG.debug( + "Stored package metadata cache group for %d lookup(s): %s", + len(lookup_keys), + scan_cache_key, + ) + + def _store_package_lookup_index(self, cache_scope, lookup_keys, scan_cache_key): + index_cache_key = self._package_lookup_index_cache_key(cache_scope) + df_index = self.cache.get(index_cache_key) + df_new = pd.DataFrame( + [ + { + _PACKAGE_META_LOOKUP_ID_COLUMN: self._package_lookup_cache_id( + lookup_key + ), + _PACKAGE_META_CACHE_REF_COLUMN: scan_cache_key, + } + for lookup_key in lookup_keys + ] + ) + df_index = _merge_package_lookup_index(df_index, df_new) + self.cache.set(key=index_cache_key, value=df_index, ttl=_PACKAGE_META_TTL) @staticmethod - def _try_scan_expression(expression, *, impure=False): - try: - scanner = NixMetaScanner() - scanner.scan_expression(expression, impure=impure) - return scanner.to_df() - except SCAN_EXCEPTIONS: - LOG.debug("Failed scanning nixmeta expression", exc_info=True) + def _package_cache_scope( + source, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + if impure: return None + if source.expression: + source_key = source.expression_cache_key + else: + source_key = source.path or source.flakeref + if not source_key: + return None + flakeref_key = _package_cache_flake_key( + source, + flakeref, + require_stable=not source.expression, + ) + if flakeref_key is None: + return None + mode = "flake-inputs-only" if input_roots_only else "base" + return ( + f"package-meta:{source_key}:flake:{flakeref_key}:mode:{mode}:" + f"fingerprint:{package_meta_cache_fingerprint()}" + ) - def _scan(self, nixpkgs_path): - # In case sbomnix is run concurrently, we want to make sure there's - # only one instance of NixMetaScanner.scan_path() running at a time. - # The reason is, NixMetaScanner.scan_path() potentially invokes - # `nix-env -qa --meta --json -f /path/to/nixpkgs` which is very - # memory intensive. The locking needs to happen here (and not in - # NixMetaScanner) because this object caches the nixmeta info. - # First scan generates the cache, after which the consecutive scans - # will read the scan results from the cache, not having to run - # the nix-env command again, making the consecutive scans relatively - # fast and light-weight. - with self.lock: - df = self.cache.get(nixpkgs_path) - if df is not None and not df.empty: - LOG.debug("found from cache: %s", nixpkgs_path) - return df - LOG.debug("cache miss, scanning: %s", nixpkgs_path) - scanner = NixMetaScanner() - scanner.scan_path(nixpkgs_path) - df = scanner.to_df() - if df is None or df.empty: - LOG.warning("Failed scanning nixmeta: %s", nixpkgs_path) - return None - # Cache requires some TTL, so we set it to some value here. - # Although, we could as well store it indefinitely as it should - # not change given the same key (nixpkgs store path). - self.cache.set(key=nixpkgs_path, value=df, ttl=_NIXMETA_NIXPKGS_TTL) - return df + @classmethod + def _package_cache_key( + cls, + source, + lookup_keys, + *, + flakeref=None, + input_roots_only=False, + impure=False, + ): + cache_scope = cls._package_cache_scope( + source, + flakeref=flakeref, + input_roots_only=input_roots_only, + impure=impure, + ) + if cache_scope is None: + return None + return cls._package_scan_cache_key(cache_scope, lookup_keys) + @staticmethod + def _package_scan_cache_key(cache_scope, lookup_keys): + return f"{cache_scope}:lookups:{package_meta_candidate_key(lookup_keys)}" -############################################################################### + @staticmethod + def _package_lookup_index_cache_key(cache_scope): + return f"{cache_scope}:lookup-index" + + @staticmethod + def _package_lookup_cache_id(lookup_key): + return package_meta_candidate_key([lookup_key]) + + +def _stable_flakeref_for_package_cache(flakeref): + flake, _separator, _attr = str(flakeref or "").partition("#") + return ( + flake.startswith("/nix/store/") + or flake.startswith("path:/nix/store/") + or re.search(r"(?:[?&])(?:narHash|rev)=", flake) is not None + ) + + +def _components_without_package_matches(components, package_meta): + if components is None or components.empty: + return components + if package_meta is None or package_meta.empty: + return components + matched = set(package_meta[cols.STORE_PATH].astype(str)) + return components[~components[cols.STORE_PATH].astype(str).isin(matched)] + + +def _package_scan_flakeref(source, flakeref): + return source.flakeref_cache_key or flakeref or source.flakeref + + +def _package_cache_flakeref_key(source, flakeref, *, require_stable): + if source.flakeref_cache_key: + return source.flakeref_cache_key + flakeref_key = flakeref or source.flakeref or "" + if ( + not flakeref_key + or not require_stable + or _stable_flakeref_for_package_cache(flakeref_key) + ): + return flakeref_key + return None + + +def _package_cache_flake_key(source, flakeref, *, require_stable): + flakeref_key = _package_cache_flakeref_key( + source, + flakeref, + require_stable=require_stable, + ) + if flakeref_key is None: + return None + flake, _separator, _attr = str(flakeref_key).partition("#") + return flake + + +def _package_lookup_index_refs(df): + if ( + df is None + or df.empty + or _PACKAGE_META_LOOKUP_ID_COLUMN not in df.columns + or _PACKAGE_META_CACHE_REF_COLUMN not in df.columns + ): + return {} + refs = {} + df = df[[_PACKAGE_META_LOOKUP_ID_COLUMN, _PACKAGE_META_CACHE_REF_COLUMN]].dropna() + for row in df.itertuples(index=False): + lookup_id = str(row[0] or "") + ref_key = str(row[1] or "") + if lookup_id and ref_key: + refs[lookup_id] = ref_key + return refs + + +def _merge_package_lookup_index(df_index, df_new): + frames = [ + frame for frame in (df_index, df_new) if frame is not None and not frame.empty + ] + if not frames: + return pd.DataFrame( + { + _PACKAGE_META_LOOKUP_ID_COLUMN: pd.Series(dtype="object"), + _PACKAGE_META_CACHE_REF_COLUMN: pd.Series(dtype="object"), + } + ) + return ( + pd.concat(frames, ignore_index=True) + .dropna(subset=[_PACKAGE_META_LOOKUP_ID_COLUMN, _PACKAGE_META_CACHE_REF_COLUMN]) + .drop_duplicates(subset=[_PACKAGE_META_LOOKUP_ID_COLUMN], keep="last") + .reset_index(drop=True) + ) + + +def _cache_get_many(cache, keys): + if not keys: + return {} + get_many = getattr(cache, "get_many", None) + if get_many is not None: + return get_many(keys) + return {key: cache.get(key) for key in keys} + + +def _concat_package_metadata_frames(frames): + frames = [frame for frame in frames if frame is not None and not frame.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames, ignore_index=True).drop_duplicates(ignore_index=True) diff --git a/src/sbomnix/meta_source.py b/src/sbomnix/meta_source.py index 0d507a2..11cc688 100644 --- a/src/sbomnix/meta_source.py +++ b/src/sbomnix/meta_source.py @@ -20,7 +20,11 @@ ) from common.log import LOG from common.proc import exec_cmd, nix_cmd -from nixmeta.scanner import nixref_to_nixpkgs_path +from sbomnix.flake_metadata import ( + nixref_to_nixpkgs_path, + normalize_current_flake_shorthand, + normalize_local_flake_ref, +) META_NIXPKGS_NIX_PATH = "nix-path" @@ -37,6 +41,7 @@ class NixpkgsMetaSource: method: str path: str | None = None flakeref: str | None = None + flakeref_cache_key: str | None = None rev: str | None = None version: str | None = None message: str | None = None @@ -76,6 +81,16 @@ def nixpkgs_meta_source_with_path(source): return replace(source, version=read_nixpkgs_version(source.path)) +def log_nixpkgs_meta_source(source): + """Log a concise summary of a selected nixpkgs metadata source.""" + parts = [f"method '{source.method}'"] + if source.path: + parts.append(f"path '{source.path}'") + if source.flakeref: + parts.append(f"flakeref '{source.flakeref}'") + LOG.verbose("Selected nixpkgs metadata source: %s", ", ".join(parts)) + + class NixpkgsMetaSourceResolver: """Resolve a nixpkgs metadata source without scanning metadata.""" @@ -112,23 +127,31 @@ def resolve_meta_nixpkgs_option(self, meta_nixpkgs, *, target_path=None): def resolve_flakeref_target_source(self, flakeref, *, impure=False): """Resolve target-specific nixpkgs metadata for known flakeref outputs.""" + flakeref = normalize_current_flake_shorthand(flakeref) parsed = self._parse_nixos_toplevel_flakeref(flakeref) if not parsed: return None + LOG.info("Resolving nixpkgs path for '%s'", flakeref) flake, name = parsed name_attr = quote_nix_attr_segment(name) pkgs_path_ref = f"{flake}#nixosConfigurations.{name_attr}.pkgs.path" + LOG.info("Evaluating nixpkgs path for '%s'", pkgs_path_ref) pkgs_path = self._nix_eval_raw(pkgs_path_ref, impure=impure) if pkgs_path: expression_flake = self._flake_ref_for_expression( flake, impure=impure, ) - return nixpkgs_meta_source_with_path( + flakeref_cache_key = self._stable_flakeref_cache_key( + flakeref, + impure=impure, + ) + source = nixpkgs_meta_source_with_path( NixpkgsMetaSource( method="flakeref-target", path=pkgs_path, flakeref=pkgs_path_ref, + flakeref_cache_key=flakeref_cache_key, message="Scanning evaluated NixOS package set from flakeref", expression=self._nixos_pkgs_expression(expression_flake, name), expression_cache_key=self._nixos_pkgs_expression_cache_key( @@ -139,6 +162,8 @@ def resolve_flakeref_target_source(self, flakeref, *, impure=False): expression_impure=impure, ), ) + log_nixpkgs_meta_source(source) + return source return self._nixos_toplevel_without_source() @@ -174,23 +199,23 @@ def _nixos_pkgs_expression(flake, name): def _flake_ref_for_expression(self, flake, *, impure=False): if self._flake_ref_has_stable_lock(flake): return flake - if self._should_lock_flake_ref_for_expression(flake): + if self._should_lock_flake_ref(flake): locked_ref = self._locked_flake_ref_from_metadata(flake, impure=impure) if locked_ref: return locked_ref - return self._normalize_local_flake_ref_for_expression(flake) + return normalize_local_flake_ref(flake) @staticmethod def _flake_ref_has_stable_lock(flake): return re.search(r"(?:[?&])(?:narHash|rev)=", flake) is not None @classmethod - def _should_lock_flake_ref_for_expression(cls, flake): + def _should_lock_flake_ref(cls, flake): if cls._flake_ref_has_stable_lock(flake): return False if cls._is_existing_local_flake_ref(flake): return True - return re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or "") is not None + return bool(flake) @staticmethod def _is_existing_local_flake_ref(flake): @@ -223,6 +248,19 @@ def _locked_flake_ref_from_metadata(flake, *, impure=False): query["dir"] = locked_dir return f"path:{source_path}?{urlencode(query, safe='/')}" + @classmethod + def _stable_flakeref_cache_key(cls, flakeref, *, impure=False): + flakeref = normalize_current_flake_shorthand(flakeref) + flake, separator, attr = str(flakeref or "").partition("#") + if not flake: + return None + stable_flake = cls._stable_flake_ref_for_cache(flake) + if not stable_flake and cls._should_lock_flake_ref(flake): + stable_flake = cls._locked_flake_ref_from_metadata(flake, impure=impure) + if not stable_flake: + return None + return f"{stable_flake}{separator}{attr}" + @staticmethod def _nix_flake_metadata(flake, *, impure=False): LOG.debug("Reading flake metadata for nixpkgs metadata expression: %s", flake) @@ -241,33 +279,18 @@ def _nix_flake_metadata(flake, *, impure=False): LOG.debug("Failed parsing flake metadata for expression: %s", flake) return None - @staticmethod - def _normalize_local_flake_ref_for_expression(flake): - if flake.startswith("path:"): - path_text, separator, query = flake.removeprefix("path:").partition("?") - path = pathlib.Path(path_text).expanduser() - if not path.is_absolute(): - path_text = path.resolve().as_posix() - return f"path:{path_text}{separator}{query}" - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", flake or ""): - return flake - path = pathlib.Path(flake).expanduser() - if path.exists() or flake.startswith((".", "/", "~")): - return path.resolve().as_posix() - return flake - @classmethod def _nixos_pkgs_expression_cache_key(cls, flake, name, *, impure=False): if impure: return None - stable_ref = cls._stable_flake_ref_for_expression_cache(flake) + stable_ref = cls._stable_flake_ref_for_cache(flake) if not stable_ref: return None cache_parts = json.dumps([stable_ref, name], separators=(",", ":")) return f"nixos-pkgs:{cache_parts}" @staticmethod - def _stable_flake_ref_for_expression_cache(flake): + def _stable_flake_ref_for_cache(flake): if flake.startswith("path:/nix/store/"): return flake if flake.startswith("/nix/store/"): @@ -350,19 +373,26 @@ def _try_normalize_mutable_path(path): return nixpath return None - def resolve_flakeref_lock_source(self, nixref): + def resolve_flakeref_lock_source(self, nixref, *, impure=False): """Return the nixpkgs source selected by a flakeref lock graph.""" if nixref: - LOG.debug("Reading nixpkgs path from nixref: %s", nixref) + nixref = normalize_current_flake_shorthand(nixref) nixpath = nixref_to_nixpkgs_path(nixref) + flakeref_cache_key = self._stable_flakeref_cache_key( + nixref, + impure=impure, + ) if nixpath: - return nixpkgs_meta_source_with_path( + source = nixpkgs_meta_source_with_path( NixpkgsMetaSource( method="flakeref-lock", path=nixpath.as_posix(), flakeref=nixref, + flakeref_cache_key=flakeref_cache_key, ), ) + log_nixpkgs_meta_source(source) + return source return NixpkgsMetaSource(method="none") def resolve_default_source(self, nixref=None): diff --git a/src/sbomnix/package_meta.py b/src/sbomnix/package_meta.py index a68da76..4bbc9c3 100644 --- a/src/sbomnix/package_meta.py +++ b/src/sbomnix/package_meta.py @@ -250,7 +250,11 @@ def package_meta_lookup_keys_for_components( lookup = _lookup_key_for_component(component) if lookup is None: continue - if target_attr and _component_matches_target_path(component, target_path): + if ( + target_attr + and lookup["version"] + and _component_matches_target_path(component, target_path) + ): lookup["candidateAttrs"] = [target_attr] key = ( lookup["name"], @@ -313,6 +317,8 @@ def _flake_package_attr(flakeref): if not separator: return "" parts = attr.split(".") + if len(parts) == 1: + return parts[0] if ( len(parts) < 3 or parts[0] not in {"packages", "legacyPackages"} diff --git a/tests/test_builder_runtime.py b/tests/test_builder_runtime.py index 0cf9368..f939344 100644 --- a/tests/test_builder_runtime.py +++ b/tests/test_builder_runtime.py @@ -13,6 +13,7 @@ from sbomnix import builder as sbomnix_builder from sbomnix.builder import SbomBuilder from sbomnix.closure import dependency_rows_to_dataframe +from sbomnix.meta import NixpkgsMetaSource from sbomnix.runtime import RuntimeClosure TARGET_PATH = "/nix/store/11111111111111111111111111111111-target-1.0" @@ -257,3 +258,55 @@ def test_target_component_ref_rejects_missing_runtime_target_metadata(): with pytest.raises(MissingNixDerivationMetadataError, match=TARGET_PATH): builder._resolve_target_component_ref() + + +def test_join_meta_uses_package_component_identity(monkeypatch): + class FakeMeta: + def get_package_meta_with_source(self, **kwargs): + assert kwargs["components"].equals(builder.df_sbomdb) + return ( + pd.DataFrame( + [ + { + cols.STORE_PATH: TARGET_DERIVER, + cols.NAME: "target-from-meta", + cols.PNAME: "target-pname-from-meta", + cols.VERSION: "1.0-from-meta", + "meta_description": "package target metadata", + } + ] + ), + NixpkgsMetaSource(method="package-meta-nix"), + ) + + monkeypatch.setattr(sbomnix_builder, "Meta", FakeMeta) + builder = _builder_double() + builder.flakeref = "nixpkgs#target" + builder.original_ref = "nixpkgs#target" + builder.impure = False + builder.df_sbomdb = pd.DataFrame( + [ + { + cols.STORE_PATH: TARGET_DERIVER, + cols.NAME: "target", + cols.PNAME: "target", + cols.VERSION: "1.0", + cols.OUTPUTS: [TARGET_PATH], + }, + { + cols.STORE_PATH: "/nix/store/other.drv", + cols.NAME: "target", + cols.PNAME: "target", + cols.VERSION: "1.0", + cols.OUTPUTS: ["/nix/store/other"], + }, + ] + ) + + builder._join_meta() + + assert builder.df_sbomdb["meta_description"].isna().to_list() == [False, True] + assert builder.df_sbomdb["meta_description"].iloc[0] == "package target metadata" + assert builder.df_sbomdb["pname_meta"].iloc[0] == "target-pname-from-meta" + assert builder.df_sbomdb["version_meta"].iloc[0] == "1.0-from-meta" + assert "name_meta" not in builder.df_sbomdb.columns diff --git a/tests/test_nix_cli_argv.py b/tests/test_nix_cli_argv.py index 748421d..a89d6df 100644 --- a/tests/test_nix_cli_argv.py +++ b/tests/test_nix_cli_argv.py @@ -10,11 +10,9 @@ import pytest -from common.proc import exec_cmd, nix_cmd -from nixmeta import flake_metadata +from common.proc import exec_cmd from nixupdate import nix_outdated from sbomnix import derivers as sbomnix_derivers -from sbomnix.meta import Meta def test_exec_cmd_rejects_string_commands(): @@ -138,54 +136,6 @@ def fake_exec_cmd(cmd, **kwargs): ] -def test_get_flake_metadata_uses_argv_list(): - calls = [] - - def fake_exec_cmd(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - meta = flake_metadata.get_flake_metadata( - "/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - nix_cmd_fn=nix_cmd, - ) - - assert meta == {"path": "/nix/store/nixpkgs"} - assert calls == [ - ( - [ - "nix", - "flake", - "metadata", - "/tmp/my flake", - "--json", - "--extra-experimental-features", - "flakes", - "--extra-experimental-features", - "nix-command", - ], - {"raise_on_error": False, "return_error": True, "log_error": False}, - ) - ] - - -def test_get_flake_metadata_strips_nixpkgs_prefix_without_splitting_spaces(): - calls = [] - - def fake_exec_cmd(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - flake_metadata.get_flake_metadata( - "nixpkgs=/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - nix_cmd_fn=nix_cmd, - ) - - assert calls[0][0][3] == "/tmp/my flake" - - def test_run_nix_visualize_uses_argv_list(tmp_path, monkeypatch): calls = [] output_path = tmp_path / "graph output.csv" @@ -226,15 +176,3 @@ def fake_exec_cmd(cmd, **kwargs): {}, ) ] - - -def test_meta_reads_nix_path_entry_with_spaces(monkeypatch): - scanned = [] - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other:nixpkgs=/tmp/my flake") - monkeypatch.setattr(Meta, "_scan", lambda self, path: scanned.append(path) or path) - - resolved = Meta().get_nixpkgs_meta() - - assert resolved == "/tmp/my flake" - assert scanned == ["/tmp/my flake"] diff --git a/tests/test_package_meta.py b/tests/test_package_meta.py new file mode 100644 index 0000000..3d656b5 --- /dev/null +++ b/tests/test_package_meta.py @@ -0,0 +1,1013 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +"""Focused tests for package metadata lookup and identity matching.""" + +import json +import shutil +from dataclasses import replace + +import pandas as pd +import pytest + +from common import columns as cols +from sbomnix import flake_metadata, package_meta +from sbomnix import meta as meta_module +from sbomnix import meta_source as meta_source_module +from sbomnix.meta import Meta, NixpkgsMetaSource +from sbomnix.meta_source import NixpkgsMetaSourceResolver + + +def test_package_meta_lookup_keys_stay_compact(): + df_components = pd.DataFrame( + [ + { + cols.NAME: "python3.13-requests-2.32.5", + cols.PNAME: "python3.13-requests", + cols.VERSION: "2.32.5", + }, + { + cols.NAME: "libcap-ng-0.8.5", + cols.PNAME: "libcap-ng", + cols.VERSION: "0.8.5", + }, + ] + ) + + lookup_keys = package_meta.package_meta_lookup_keys_for_components(df_components) + + assert lookup_keys == [ + { + "name": "libcap-ng-0.8.5", + "pname": "libcap-ng", + "version": "0.8.5", + }, + { + "name": "python3.13-requests-2.32.5", + "pname": "python3.13-requests", + "version": "2.32.5", + }, + ] + + +def test_package_meta_lookup_keys_keep_dotted_package_names(): + df_components = pd.DataFrame( + [ + { + cols.NAME: "python3.13-zope.interface-7.2", + cols.PNAME: "zope.interface", + cols.VERSION: "7.2", + }, + { + cols.NAME: "source.json", + cols.PNAME: "source.json", + cols.VERSION: "", + }, + ] + ) + + lookup_keys = package_meta.package_meta_lookup_keys_for_components(df_components) + + assert lookup_keys == [ + { + "name": "python3.13-zope.interface-7.2", + "pname": "zope.interface", + "version": "7.2", + } + ] + + +def test_short_request_escapes_nix_antiquotation(monkeypatch): + captured = {} + + def fake_eval_request(request_arg, **kwargs): + captured["request_arg"] = request_arg + return pd.DataFrame() + + monkeypatch.setattr(package_meta, "_eval_request", fake_eval_request) + + package_meta._eval_package_meta_request( + {"flakeref": "flake#${bad}", "lookupKeys": []} + ) + + assert r"\${bad}" in captured["request_arg"] + + +def test_large_request_uses_wrapper_file_for_pure_eval(monkeypatch): + captured = {} + + def fake_eval_file(eval_file, **kwargs): + eval_path = package_meta.pathlib.Path(eval_file) + captured["request_exists"] = (eval_path.parent / "request.json").is_file() + captured["eval"] = eval_path.read_text(encoding="utf-8") + return pd.DataFrame() + + monkeypatch.setattr(package_meta, "_eval_file", fake_eval_file) + + package_meta._eval_package_meta_request({"lookupKeys": [{"name": "x" * 30_000}]}) + + assert captured["request_exists"] is True + assert "builtins.readFile ./request.json" in captured["eval"] + assert "requestFile" not in captured["eval"] + + +def test_try_scan_package_meta_normalizes_current_flake_shorthand( + monkeypatch, tmp_path +): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta( + [{"name": "hello-1.0", "pname": "hello", "version": "1.0"}], + flakeref="#hello", + ) + + assert captured["request"]["flakeref"] == f"{tmp_path.as_posix()}#hello" + assert "flakePackageAttrPath" not in captured["request"] + + +@pytest.mark.parametrize( + ("flakeref", "lookup_keys", "expected_system"), + [ + ( + "flake#packages.aarch64-linux.foo", + [{"name": "foo-1.0", "pname": "foo", "version": "1.0"}], + "aarch64-linux", + ), + ( + "flake#default", + [ + { + "name": "foo-1.0", + "pname": "foo", + "version": "1.0", + "system": "aarch64-linux", + } + ], + "aarch64-linux", + ), + ], +) +def test_try_scan_package_meta_uses_target_system( + monkeypatch, flakeref, lookup_keys, expected_system +): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta(lookup_keys, flakeref=flakeref) + + assert captured["request"]["system"] == expected_system + + +def test_try_scan_package_meta_splits_mixed_system_requests(monkeypatch): + captured = [] + + def fake_eval_package_meta_request(request, **kwargs): + captured.append(request) + return pd.DataFrame( + [ + { + cols.STORE_PATH: f"/nix/store/{request['system']}.drv", + package_meta.META_OUTPUT_PATH: "", + } + ] + ) + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + df = package_meta.try_scan_package_meta( + [ + { + "name": "pkg-x86-1.0", + "pname": "pkg-x86", + "version": "1.0", + "system": "x86_64-linux", + }, + { + "name": "source-patch", + "pname": "source-patch", + "version": "", + "system": "builtin", + }, + { + "name": "pkg-aarch-1.0", + "pname": "pkg-aarch", + "version": "1.0", + "system": "aarch64-linux", + }, + ], + flakeref="flake#default", + ) + + fallback_system = package_meta.nix_system() + expected_names_by_system = { + "aarch64-linux": ["pkg-aarch-1.0"], + "x86_64-linux": ["pkg-x86-1.0"], + } + expected_names_by_system.setdefault(fallback_system, []).append("source-patch") + names_by_system = { + request["system"]: [lookup["name"] for lookup in request["lookupKeys"]] + for request in captured + } + + assert names_by_system == expected_names_by_system + assert sorted(df[cols.STORE_PATH].to_list()) == [ + f"/nix/store/{system}.drv" for system in sorted(expected_names_by_system) + ] + + +def test_try_scan_package_meta_request_omits_direct_flake_attr_path(monkeypatch): + captured = {} + + def fake_eval_package_meta_request(request, **kwargs): + captured["request"] = request + return pd.DataFrame() + + monkeypatch.setattr( + package_meta, + "_eval_package_meta_request", + fake_eval_package_meta_request, + ) + + package_meta.try_scan_package_meta( + [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}], + flakeref="flake#expensive-target", + ) + + assert "flakePackageAttrPath" not in captured["request"] + + +def test_request_lookup_keys_expand_package_set_candidates(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "python3.13-requests-2.32.5", + "pname": "python3.13-requests", + "version": "2.32.5", + } + ] + )[0] + + assert lookup["pname"] == "requests" + assert lookup["candidateTiers"][0] == [ + { + "attr": "requests", + "packageSets": ["python313Packages", "python3Packages", ""], + } + ] + + +def test_request_lookup_keys_use_haskell_package_set_hint(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "vector-0.13.2.0", + "pname": "vector", + "version": "0.13.2.0", + } + ] + )[0] + + assert lookup["collectAllCanonical"] is True + assert lookup["candidateTiers"][-1] == [ + { + "attr": "vector", + "packageSets": ["haskellPackages"], + } + ] + + +def test_request_lookup_keys_probe_parsed_name_for_fallback_pname(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "foo-1.0", + "pname": "foo-1.0", + "version": "", + } + ] + )[0] + + assert lookup["pname"] == "foo" + assert lookup["candidateTiers"][0] == [ + { + "attr": "foo", + "packageSets": [""], + } + ] + + +def test_request_lookup_keys_keep_qt_hint_before_haskell_shape(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "qtbase-6.10.2", + "pname": "qtbase", + "version": "6.10.2", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "qtbase", + "packageSets": ["qt6Packages", "qt6", "kdePackages", ""], + } + ] + assert all( + "haskellPackages" not in candidate["packageSets"] + for tier in lookup["candidateTiers"] + for candidate in tier + ) + assert "collectAllCanonical" not in lookup + + +def test_request_lookup_keys_probe_kde_packages_for_k_prefixed_frameworks(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "kconfig-6.17.0", + "pname": "kconfig", + "version": "6.17.0", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "kconfig", + "packageSets": [""], + }, + { + "attr": "kconfig", + "packageSets": ["kdePackages"], + }, + ] + assert all( + "kdePackages" not in candidate["packageSets"] + for tier in lookup["candidateTiers"][1:] + for candidate in tier + ) + + +def test_request_lookup_keys_keep_haskell_hint_for_k_prefixed_packages(): + lookup = package_meta._request_lookup_keys( + [ + { + "name": "kan-extensions-5.2.7", + "pname": "kan-extensions", + "version": "5.2.7", + } + ] + )[0] + + assert lookup["candidateTiers"][0] == [ + { + "attr": "kan-extensions", + "packageSets": [""], + }, + { + "attr": "kan-extensions", + "packageSets": ["kdePackages"], + }, + ] + assert lookup["candidateTiers"][-1] == [ + { + "attr": "kan-extensions", + "packageSets": ["haskellPackages"], + } + ] + assert lookup["collectAllCanonical"] is True + + +@pytest.mark.parametrize( + ("buildtime", "expected_description"), + [ + (False, "runtime match"), + (True, "buildtime match"), + ], +) +def test_metadata_requires_exact_component_identity(buildtime, expected_description): + components = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/component-real.drv", + cols.OUTPUTS: ["/nix/store/component-real-out"], + } + ] + ) + metadata = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/wrong.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/component-real-out", + "meta_description": "runtime match", + }, + { + cols.STORE_PATH: "/nix/store/component-real.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/other-output", + "meta_description": "buildtime match", + }, + ] + ) + + matched = package_meta.match_package_metadata_to_components( + components, + metadata, + buildtime=buildtime, + ) + + assert matched.to_dict("records")[0]["meta_description"] == expected_description + assert package_meta.META_OUTPUT_PATH not in matched.columns + + +def test_buildtime_metadata_can_fall_back_to_exact_output_identity(): + components = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/component-real.drv", + cols.OUTPUTS: ["/nix/store/component-real-out"], + } + ] + ) + metadata = pd.DataFrame( + [ + { + cols.STORE_PATH: "/nix/store/metadata-drv.drv", + package_meta.META_OUTPUT_PATH: "/nix/store/component-real-out", + "meta_description": "output identity match", + } + ] + ) + + matched = package_meta.match_package_metadata_to_components( + components, + metadata, + buildtime=True, + ) + + assert matched.to_dict("records")[0]["meta_description"] == ( + "output identity match" + ) + assert matched.to_dict("records")[0][cols.STORE_PATH] == ( + "/nix/store/component-real.drv" + ) + + +def test_parse_package_metadata_records_identity_fields(): + df = package_meta.parse_package_metadata( + json.dumps( + { + "derivations": [ + { + "drvPath": "/nix/store/pkg.drv", + "path": "/nix/store/pkg-out", + "name": "pkg-1.0", + "pname": "pkg", + "version": "1.0", + "meta": { + "description": "Package", + "license": [{"spdxId": "MIT", "shortName": "MIT"}], + "maintainers": [{"email": "maintainer@example.invalid"}], + }, + } + ] + } + ) + ) + + record = df.to_dict("records")[0] + assert record[cols.STORE_PATH] == "/nix/store/pkg.drv" + assert record[package_meta.META_OUTPUT_PATH] == "/nix/store/pkg-out" + assert record["meta_description"] == "Package" + assert record["meta_license_spdxid"] == "MIT" + + +def test_package_meta_scans_flake_package_roots_and_preserves_output_metadata(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + dep_dir = tmp_path / "dep" + mk_shadow = """ +{ system, description }: +(derivation { + name = "shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; +}) // { + pname = "shadow"; + version = "1.0"; + meta.description = description; +} +""" + package_set = """{ system, config ? { } }: { + shadow = import ./mk-shadow.nix { inherit system; description = "imported"; }; +} +""" + flake = """ +{ + inputs.dep.url = "path:__DEP__"; + + outputs = { self, dep }: + let + system = "__SYSTEM__"; + in + { + packages.${system} = { + shadow = import ./mk-shadow.nix { + inherit system; + description = "flake-root"; + }; + }; + legacyPackages.${system}.default = throw "unrelated legacy default failed"; + }; +} +""".replace("__DEP__", dep_dir.as_posix()).replace("__SYSTEM__", system) + dep_flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system}."input-shadow" = (derivation { + name = "input-shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; + }) // { + pname = "input-shadow"; + version = "1.0"; + meta.description = "flake-input"; + }; + }; +} +""".replace("__SYSTEM__", system) + + package_set_path = tmp_path / "package-set.nix" + package_set_path.write_text(package_set, encoding="utf-8") + (tmp_path / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + dep_dir.mkdir() + (dep_dir / "flake.nix").write_text(dep_flake, encoding="utf-8") + (dep_dir / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + (flake_dir / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + + df = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow", "version": "1.0"}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["flake-root", "flake-root"] + assert df[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df[cols.VERSION].to_list() == ["1.0", "1.0"] + + df_fallback = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow-1.0", "version": ""}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df_fallback is not None + assert df_fallback["meta_description"].to_list() == ["flake-root", "flake-root"] + assert df_fallback[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df_fallback[cols.VERSION].to_list() == ["1.0", "1.0"] + + df_input = package_meta.try_scan_package_meta( + [ + { + "name": "input-shadow-1.0", + "pname": "input-shadow", + "version": "1.0", + } + ], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + input_roots_only=True, + impure=True, + ) + + assert df_input is not None + assert df_input["meta_description"].to_list() == ["flake-input", "flake-input"] + assert df_input[cols.PNAME].to_list() == ["input-shadow", "input-shadow"] + assert df_input[cols.VERSION].to_list() == ["1.0", "1.0"] + + +def test_package_meta_scans_target_flake_attr_when_name_differs(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system}.doc = (derivation { + name = "ghaf-docs-0.1.0"; + inherit system; + builder = "/bin/sh"; + args = [ "-c" "echo docs > $out" ]; + }) // { + pname = "ghaf-docs"; + version = "0.1.0"; + meta.description = "target flake package attr"; + }; + }; +} +""".replace("__SYSTEM__", system) + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + + components = pd.DataFrame( + [ + { + cols.NAME: "ghaf-docs-0.1.0", + cols.PNAME: "ghaf-docs", + cols.VERSION: "0.1.0", + "system": system, + cols.STORE_PATH: "/nix/store/placeholder.drv", + cols.OUTPUTS: ["/nix/store/placeholder-out"], + } + ] + ) + + for flakeref in ( + f"{flake_dir.as_posix()}#packages.{system}.doc", + f"{flake_dir.as_posix()}#doc", + ): + lookup_keys = package_meta.package_meta_lookup_keys_for_components( + components, + target_path="/nix/store/placeholder-out", + flakeref=flakeref, + ) + + assert lookup_keys[0]["candidateAttrs"] == ["doc"] + + df = package_meta.try_scan_package_meta( + lookup_keys, + flakeref=flakeref, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["target flake package attr"] + assert df[cols.NAME].to_list() == ["ghaf-docs-0.1.0"] + assert df[cols.PNAME].to_list() == ["ghaf-docs"] + + +def test_package_meta_skips_target_flake_attr_for_versionless_artifact(): + system = "x86_64-linux" + lookup_keys = package_meta.package_meta_lookup_keys_for_components( + pd.DataFrame( + [ + { + cols.NAME: "ghaf-host-disko-images", + cols.PNAME: "ghaf-host-disko-images", + cols.VERSION: "", + "system": system, + cols.STORE_PATH: "/nix/store/placeholder.drv", + cols.OUTPUTS: ["/nix/store/placeholder-out"], + } + ] + ), + target_path="/nix/store/placeholder-out", + flakeref=f"/flake#packages.{system}.lenovo-x1-carbon-gen11-debug", + ) + + assert lookup_keys == [ + { + "name": "ghaf-host-disko-images", + "pname": "ghaf-host-disko-images", + "version": "", + "system": system, + } + ] + + +def test_package_meta_ignores_broken_optional_flake_package_roots(tmp_path): + if shutil.which("nix") is None: + pytest.skip("nix is not available") + + system = package_meta.nix_system() + mk_shadow = """ +{ system, description }: +(derivation { + name = "shadow-1.0"; + inherit system; + builder = "/bin/sh"; + outputs = [ "out" "dev" ]; + args = [ "-c" "echo out > $out; echo dev > $dev" ]; +}) // { + pname = "shadow"; + version = "1.0"; + meta.description = description; +} +""" + package_set = """{ system, config ? { } }: { + shadow = import ./mk-shadow.nix { inherit system; description = "imported"; }; +} +""" + flake = """ +{ + outputs = { self }: + let + system = "__SYSTEM__"; + in + { + packages.${system} = throw "broken package root"; + legacyPackages.${system} = throw "broken legacy package root"; + }; +} +""".replace("__SYSTEM__", system) + + package_set_path = tmp_path / "package-set.nix" + package_set_path.write_text(package_set, encoding="utf-8") + (tmp_path / "mk-shadow.nix").write_text(mk_shadow, encoding="utf-8") + flake_dir = tmp_path / "flake" + flake_dir.mkdir() + (flake_dir / "flake.nix").write_text(flake, encoding="utf-8") + + df = package_meta.try_scan_package_meta( + [{"name": "shadow-1.0", "pname": "shadow", "version": "1.0"}], + flakeref=flake_dir.as_posix(), + nixpkgs_path=package_set_path, + impure=True, + ) + + assert df is not None + assert df["meta_description"].to_list() == ["imported", "imported"] + assert df[cols.PNAME].to_list() == ["shadow", "shadow"] + assert df[cols.VERSION].to_list() == ["1.0", "1.0"] + + +def test_expression_source_cache_requires_expression_identity(): + lookup_keys = [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}] + source = NixpkgsMetaSource( + method="flakeref-target", + path="/nix/store/source", + flakeref="flake#nixosConfigurations.host.pkgs.path", + expression='builtins.getFlake "flake"', + ) + + assert Meta._package_cache_key(source, lookup_keys) is None + + cache_key = Meta._package_cache_key( + replace( + source, + expression_cache_key='nixos-pkgs:["path:/nix/store/flake","host"]', + ), + lookup_keys, + ) + + assert cache_key is not None + assert "nixos-pkgs:" in cache_key + assert "/nix/store/source" not in cache_key + + +def test_mutable_flakeref_cache_uses_locked_identity(): + source = NixpkgsMetaSource(method="flakeref-lock", path="/nix/store/nixpkgs") + stable_source = replace( + source, + flakeref_cache_key="path:/nix/store/source?narHash=sha256-test#default", + ) + lookup_keys = [{"name": "pkg-1.0", "pname": "pkg", "version": "1.0"}] + + assert ( + Meta._package_cache_key( + source, + lookup_keys, + flakeref="./flake#default", + ) + is None + ) + + cache_key = Meta._package_cache_key( + stable_source, + lookup_keys, + flakeref="./flake#default", + ) + assert cache_key is not None + assert "path:/nix/store/source?narHash=sha256-test" in cache_key + assert "#default" not in cache_key + assert "./flake#default" not in cache_key + assert ( + meta_module._package_scan_flakeref( + stable_source, + "./flake#default", + ) + == "path:/nix/store/source?narHash=sha256-test#default" + ) + + +def test_package_metadata_cache_reuses_overlapping_lookup_sets(monkeypatch): + class NullLock: + lock_file = "test-lock" + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc, _traceback): + return False + + class FakeDfCache: + def __init__(self): + self.values = {} + self.set_calls = [] + + def get(self, key): + return self.values.get(key) + + def get_many(self, keys): + return {key: self.values.get(key) for key in keys} + + def set(self, key, value, ttl=None): + del ttl + self.set_calls.append(key) + self.values[key] = value.copy(deep=True) + + lookup_a = {"name": "alpha-1.0", "pname": "alpha", "version": "1.0"} + lookup_b = {"name": "beta-1.0", "pname": "beta", "version": "1.0"} + lookup_c = {"name": "gamma-1.0", "pname": "gamma", "version": "1.0"} + scan_calls = [] + + def fake_scan_package_meta(lookup_keys, **_kwargs): + scan_calls.append([lookup_key["name"] for lookup_key in lookup_keys]) + return pd.DataFrame( + [ + { + cols.NAME: lookup_key["name"], + cols.PNAME: lookup_key["pname"], + cols.VERSION: lookup_key["version"], + } + for lookup_key in lookup_keys + ] + ) + + monkeypatch.setattr( + meta_module, + "try_scan_package_meta", + fake_scan_package_meta, + ) + meta = Meta() + meta.lock = NullLock() + meta.cache = FakeDfCache() + source_alpha = NixpkgsMetaSource( + method="flakeref-lock", + path="/nix/store/nixpkgs", + flakeref_cache_key=( + "path:/nix/store/source?narHash=sha256-test#packages.x86_64-linux.alpha" + ), + ) + source_beta = NixpkgsMetaSource( + method="flakeref-lock", + path="/nix/store/nixpkgs", + flakeref_cache_key=( + "path:/nix/store/source?narHash=sha256-test#packages.x86_64-linux.beta" + ), + ) + + meta._scan_package_source( + source_alpha, + [lookup_a, lookup_b], + flakeref="./flake#packages.x86_64-linux.alpha", + ) + df_second = meta._scan_package_source( + source_beta, + [lookup_b, lookup_c], + flakeref="./flake#packages.x86_64-linux.beta", + ) + df_third = meta._scan_package_source( + source_beta, + [lookup_b, lookup_c], + flakeref="./flake#packages.x86_64-linux.beta", + ) + + assert scan_calls == [["alpha-1.0", "beta-1.0"], ["gamma-1.0"]] + assert sorted(df_second[cols.NAME].unique()) == [ + "alpha-1.0", + "beta-1.0", + "gamma-1.0", + ] + assert sorted(df_third[cols.NAME].unique()) == [ + "alpha-1.0", + "beta-1.0", + "gamma-1.0", + ] + assert len(meta.cache.set_calls) == 5 + assert not any(":lookup:" in key for key in meta.cache.set_calls) + assert sum(":lookup-index" in key for key in meta.cache.set_calls) == 2 + + +def test_registry_flakeref_cache_key_uses_locked_identity(monkeypatch): + def fake_locked_ref(flake, *, impure=False): + assert flake == "nixpkgs" + assert impure is False + return "path:/nix/store/registry-source?narHash=sha256-registry" + + monkeypatch.setattr( + NixpkgsMetaSourceResolver, + "_locked_flake_ref_from_metadata", + staticmethod(fake_locked_ref), + ) + + assert ( + NixpkgsMetaSourceResolver._stable_flakeref_cache_key("nixpkgs#firefox") + == "path:/nix/store/registry-source?narHash=sha256-registry#firefox" + ) + + +def test_current_flake_shorthand_source_resolution_uses_dot(monkeypatch, tmp_path): + nixpkgs_path = tmp_path / "nixpkgs" + (nixpkgs_path / "lib").mkdir(parents=True) + (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") + resolved_refs = [] + locked_refs = [] + + def fake_nixref_to_nixpkgs_path(nixref): + resolved_refs.append(nixref) + return nixpkgs_path + + def fake_locked_ref(flake, *, impure=False): + locked_refs.append((flake, impure)) + return "path:/nix/store/current-flake?narHash=sha256-current" + + monkeypatch.setattr( + meta_source_module, + "nixref_to_nixpkgs_path", + fake_nixref_to_nixpkgs_path, + ) + monkeypatch.setattr( + NixpkgsMetaSourceResolver, + "_locked_flake_ref_from_metadata", + staticmethod(fake_locked_ref), + ) + + source = NixpkgsMetaSourceResolver().resolve_flakeref_lock_source("#hello") + + assert resolved_refs == [".#hello"] + assert locked_refs == [(".", False)] + assert source.method == "flakeref-lock" + assert source.path == nixpkgs_path.as_posix() + assert source.flakeref == ".#hello" + assert source.flakeref_cache_key == ( + "path:/nix/store/current-flake?narHash=sha256-current#hello" + ) + assert source.version == "25.11" + + +def test_nixref_to_nixpkgs_path_normalizes_current_flake_shorthand(): + metadata_refs = [] + + def fake_get_flake_metadata(flakeref): + metadata_refs.append(flakeref) + return { + "path": "/nix/store/current-nixpkgs", + "description": "A collection of packages for the Nix package manager", + } + + nixpkgs_path = flake_metadata.nixref_to_nixpkgs_path( + "#hello", + get_flake_metadata_fn=fake_get_flake_metadata, + ) + + assert metadata_refs == ["."] + assert nixpkgs_path.as_posix() == "/nix/store/current-nixpkgs" From 5da3ce48f6522fa2c5b665eb2d0aa8a39994d332 Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Tue, 19 May 2026 13:53:40 +0300 Subject: [PATCH 5/7] sbomnix: remove explicit metadata source option Drop --meta-nixpkgs now that metadata enrichment is derived from the flakeref target and its package roots. Store-path targets still skip metadata because they do not identify the nixpkgs source that produced the path; users should use a flakeref target when metadata is required. Signed-off-by: Henri Rosten --- README.md | 17 +- doc/sbomnix_metadata.md | 53 ++ src/sbomnix/builder.py | 3 - src/sbomnix/main.py | 9 - src/sbomnix/meta.py | 8 - src/sbomnix/meta_source.py | 127 +---- tests/test_nixmeta_source.py | 788 ---------------------------- tests/test_nixmeta_source_export.py | 86 --- tests/test_sbom_vuln_enrichment.py | 37 -- 9 files changed, 65 insertions(+), 1063 deletions(-) create mode 100644 doc/sbomnix_metadata.md delete mode 100644 tests/test_nixmeta_source.py delete mode 100644 tests/test_nixmeta_source_export.py diff --git a/README.md b/README.md index d3c0d92..62350da 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ $ sbomnix github:NixOS/nixpkgs?ref=nixos-unstable#wget ``` #### Generate SBOM Based on Derivation File or Out-path -Flake references are the recommended target for `sbomnix`. When the target is a flake reference, `sbomnix` can resolve the nixpkgs version used to build the package and enrich the SBOM with metadata such as descriptions, licenses, maintainers, and homepage links. When the target is a store path, there is no information about which nixpkgs version produced it, so metadata enrichment is skipped by default; see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). +Flake references are the recommended target for `sbomnix`. When the target is a flake reference, `sbomnix` can resolve the nixpkgs version used to build the package and enrich the SBOM with metadata such as descriptions, licenses, maintainers, and homepage links. When the target is a store path, there is no information about which nixpkgs version produced it, so metadata enrichment is skipped; see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). By default `sbomnix` scans the given target and generates an SBOM including the runtime dependencies. Notice: determining the target runtime dependencies in Nix requires building the target. @@ -135,7 +135,7 @@ $ sbomnix github:NixOS/nixpkgs/nixos-unstable#wget --buildtime ```bash $ sbomnix /path/to/result ``` -Note: store paths carry no record of which nixpkgs version produced them, so nixpkgs metadata enrichment is skipped by default. Pass `--meta-nixpkgs` to supply a nixpkgs source explicitly, or see [Nixpkgs Metadata Source Selection](#nixpkgs-metadata-source-selection). +Note: store paths carry no record of which nixpkgs version produced them, so nixpkgs metadata enrichment is skipped. Use a flakeref target when nixpkgs metadata is required. #### Nixpkgs Metadata Source Selection `sbomnix` enriches packages with nixpkgs metadata, such as descriptions, @@ -147,19 +147,18 @@ toplevel flakerefs are handled through the selected NixOS package set, so overlays, package overrides, nixpkgs config, and system-specific package-set changes can be represented. -Store-path targets skip nixpkgs metadata by default; pass `--meta-nixpkgs` to -choose the source explicitly. - -`--meta-nixpkgs ` scans an explicit nixpkgs source. -`--meta-nixpkgs nix-path` scans the `nixpkgs=` entry from `NIX_PATH` as an -explicit opt-in source. `--exclude-meta` disables this enrichment and cannot be -combined with `--meta-nixpkgs`. +Store-path targets skip nixpkgs metadata because the store path does not +identify the nixpkgs source that produced it. `--exclude-meta` disables metadata +enrichment. CycloneDX and SPDX outputs record the selected metadata source in document metadata, including fields such as `nixpkgs:metadata_source_method`, `nixpkgs:path`, `nixpkgs:rev`, `nixpkgs:flakeref`, `nixpkgs:version`, and `nixpkgs:message`. +See [sbomnix metadata enrichment](./doc/sbomnix_metadata.md) for a short +overview of source selection, component matching, and metadata caching. + #### Visualize Package Dependencies `sbomnix` uses structured Nix JSON to find package dependencies where available. `nixgraph` can also be used as a stand-alone tool for visualizing diff --git a/doc/sbomnix_metadata.md b/doc/sbomnix_metadata.md new file mode 100644 index 0000000..2e460c3 --- /dev/null +++ b/doc/sbomnix_metadata.md @@ -0,0 +1,53 @@ + + +# sbomnix metadata enrichment + +`sbomnix` can enrich SBOM components with nixpkgs package metadata such as +descriptions, licenses, maintainers, and homepage links. Metadata enrichment is +best-effort: an SBOM can still be generated when metadata is unavailable or when +no metadata candidate matches a component exactly. + +## Metadata source selection + +For flakeref targets, `sbomnix` selects a nixpkgs metadata source from the +target context. For NixOS toplevel flakerefs, it uses the evaluated +`nixosConfigurations..pkgs` package set, so overlays, package overrides, +nixpkgs config, and system-specific settings can be represented. + +For other flakeref targets, `sbomnix` uses the target flake lock graph to find +the pinned nixpkgs source. Store-path targets skip nixpkgs metadata because a +store path does not identify which nixpkgs source produced it. + +## Component matching + +Component names, pnames, and versions are used as lookup hints to find possible +nixpkgs derivations. Those names are not trusted as the final match. The +metadata helper returns derivation and output identities, and `sbomnix` accepts +metadata only when the returned `drvPath` or `outPath` matches an SBOM component +exactly. + +This keeps metadata enrichment tied to the actual components in the SBOM while +still allowing targeted lookups in package sets. + +## Flake input packages + +Some targets include packages that come from flake inputs rather than the +primary nixpkgs package set. After scanning the selected package set, `sbomnix` +may scan package roots exported by the target flake inputs for components that +remain unmatched. + +## Caching and diagnostics + +Package metadata lookups are cached by metadata source identity, lookup set, +lookup mode, and helper implementation fingerprint. With `-v`, `sbomnix` logs +the selected metadata source, lookup counts, and whether package metadata came +from a cache hit or a fresh scan. + +CycloneDX and SPDX outputs record the selected metadata source in document +metadata, including fields such as `nixpkgs:metadata_source_method`, +`nixpkgs:path`, `nixpkgs:rev`, `nixpkgs:flakeref`, `nixpkgs:version`, and +`nixpkgs:message`. diff --git a/src/sbomnix/builder.py b/src/sbomnix/builder.py index bde9c65..f90e77e 100644 --- a/src/sbomnix/builder.py +++ b/src/sbomnix/builder.py @@ -88,7 +88,6 @@ def __init__( # noqa: PLR0913, PLR0917 depth=None, flakeref=None, original_ref=None, - meta_nixpkgs=None, impure=False, include_meta=True, include_vulns=False, @@ -113,7 +112,6 @@ def __init__( # noqa: PLR0913, PLR0917 self.dependency_index = None self.flakeref = flakeref self.original_ref = original_ref - self.meta_nixpkgs = meta_nixpkgs self.impure = impure self.meta = None # "disabled" records explicit opt-out; "none" means auto-selection @@ -330,7 +328,6 @@ def _join_meta(self): target_path=self.nix_path, flakeref=self.flakeref, original_ref=self.original_ref, - explicit_nixpkgs=self.meta_nixpkgs, impure=self.impure, ) self.nixpkgs_meta_source = source diff --git a/src/sbomnix/main.py b/src/sbomnix/main.py index 4d2b2af..b5da99f 100755 --- a/src/sbomnix/main.py +++ b/src/sbomnix/main.py @@ -50,12 +50,6 @@ def getargs(args=None): parser.add_argument( "--exclude-meta", help=helps, action="store_true", default=False ) - helps = ( - "Nixpkgs source used for metadata enrichment. Accepts a nixpkgs " - "flakeref, a nixpkgs source path, or nix-path. Overrides automatic " - "metadata-source detection." - ) - parser.add_argument("--meta-nixpkgs", help=helps, metavar="META_NIXPKGS") helps = "Exclude using heuristics-based CPE matches in the output" parser.add_argument( "--exclude-cpe-matching", help=helps, action="store_true", default=False @@ -89,8 +83,6 @@ def main(): def _run(args): - if args.exclude_meta and args.meta_nixpkgs: - raise SbomnixError("--exclude-meta cannot be used with --meta-nixpkgs") target = resolve_nix_target( args.NIXREF, buildtime=args.buildtime, impure=args.impure ) @@ -101,7 +93,6 @@ def _run(args): depth=args.depth, flakeref=target.flakeref, original_ref=target.original_ref, - meta_nixpkgs=args.meta_nixpkgs, impure=args.impure, include_meta=not args.exclude_meta, include_vulns=args.include_vulns, diff --git a/src/sbomnix/meta.py b/src/sbomnix/meta.py index 7a1acaf..5553903 100644 --- a/src/sbomnix/meta.py +++ b/src/sbomnix/meta.py @@ -57,7 +57,6 @@ def get_package_meta_with_source( # noqa: PLR0913 target_path=None, flakeref=None, original_ref=None, - explicit_nixpkgs=None, impure=False, ): """Return exact component metadata and selected metadata source.""" @@ -65,7 +64,6 @@ def get_package_meta_with_source( # noqa: PLR0913 target_path=target_path, flakeref=flakeref, original_ref=original_ref, - explicit_nixpkgs=explicit_nixpkgs, impure=impure, ) component_count = 0 if components is None else len(components) @@ -178,14 +176,8 @@ def _resolve_source( target_path=None, flakeref=None, original_ref=None, - explicit_nixpkgs=None, impure=False, ): - if explicit_nixpkgs: - return self.source_resolver.resolve_meta_nixpkgs_option( - explicit_nixpkgs, - target_path=target_path, - ) if flakeref: source = self.source_resolver.resolve_flakeref_target_source( flakeref, diff --git a/src/sbomnix/meta_source.py b/src/sbomnix/meta_source.py index 11cc688..897e030 100644 --- a/src/sbomnix/meta_source.py +++ b/src/sbomnix/meta_source.py @@ -2,17 +2,14 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Resolve nixpkgs metadata sources from target context and CLI options.""" +"""Resolve nixpkgs metadata sources from target context.""" import json -import os import pathlib import re from dataclasses import dataclass, replace -from subprocess import CalledProcessError from urllib.parse import urlencode -from common.errors import SbomnixError from common.flakeref import ( NIXOS_CONFIGURATION_TOPLEVEL_SUFFIX, parse_nixos_configuration_ref, @@ -26,13 +23,6 @@ normalize_local_flake_ref, ) -META_NIXPKGS_NIX_PATH = "nix-path" - -RESERVED_META_NIXPKGS_MODES = frozenset({META_NIXPKGS_NIX_PATH}) - -SCAN_EXCEPTIONS = (KeyError, OSError, CalledProcessError, TypeError, ValueError) -_NIXREF_RESOLUTION_EXCEPTIONS = (AttributeError, *SCAN_EXCEPTIONS) - @dataclass(frozen=True) class NixpkgsMetaSource: @@ -50,13 +40,6 @@ class NixpkgsMetaSource: expression_impure: bool = False -def classify_meta_nixpkgs(value): - """Classify a --meta-nixpkgs value as a reserved mode or explicit source.""" - if value in RESERVED_META_NIXPKGS_MODES: - return value - return "explicit" - - def read_nixpkgs_version(nixpkgs_path): """Read nixpkgs version from a source path if available.""" try: @@ -105,26 +88,11 @@ def path_target_without_source(target_path=None, original_ref=None): return NixpkgsMetaSource( method="none", message=( - "No nixpkgs metadata source was provided for store-path target. " - "Skipping nixpkgs metadata. Re-run with " - "--meta-nixpkgs to include metadata." + "No nixpkgs metadata source is available for store-path targets. " + "Skipping nixpkgs metadata. Use a flakeref target instead." ), ) - def resolve_meta_nixpkgs_option(self, meta_nixpkgs, *, target_path=None): - """Resolve an explicit --meta-nixpkgs source or reserved mode.""" - LOG.debug( - "Resolving explicit nixpkgs metadata source for target path=%s", - target_path, - ) - mode = classify_meta_nixpkgs(meta_nixpkgs) - if mode == META_NIXPKGS_NIX_PATH: - return self.resolve_nix_path_source( - message="NIX_PATH metadata source may not match the target", - required=True, - ) - return self.resolve_explicit_source(meta_nixpkgs) - def resolve_flakeref_target_source(self, flakeref, *, impure=False): """Resolve target-specific nixpkgs metadata for known flakeref outputs.""" flakeref = normalize_current_flake_shorthand(flakeref) @@ -173,8 +141,7 @@ def _nixos_toplevel_without_source(): method="none", message=( "Failed resolving target-specific nixpkgs metadata source from " - "NixOS configuration flakeref. Skipping nixpkgs metadata. Re-run " - "with --meta-nixpkgs to include metadata." + "NixOS configuration flakeref. Skipping nixpkgs metadata." ), ) @@ -315,64 +282,6 @@ def _nix_eval_raw(flakeref, *, impure=False): return None return ret.stdout.strip() or None - def resolve_explicit_source(self, meta_nixpkgs): - """Resolve an explicit --meta-nixpkgs path or flakeref.""" - path = pathlib.Path(meta_nixpkgs) - if path.exists(): - resolved_path = path.resolve() - if is_nix_store_path(resolved_path): - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=resolved_path.as_posix(), - ), - ) - nixpath = self._try_normalize_mutable_path(resolved_path) - if nixpath: - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=nixpath.as_posix(), - flakeref=resolved_path.as_posix(), - ), - ) - raise SbomnixError( - "Explicit --meta-nixpkgs path must resolve to an immutable " - f"/nix/store source before scanning: '{meta_nixpkgs}'" - ) - try: - nixpath = nixref_to_nixpkgs_path(meta_nixpkgs) - except _NIXREF_RESOLUTION_EXCEPTIONS as error: - raise SbomnixError( - f"Failed resolving --meta-nixpkgs source: '{meta_nixpkgs}'" - ) from error - if not nixpath: - raise SbomnixError( - f"Failed resolving --meta-nixpkgs source: '{meta_nixpkgs}'" - ) - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="explicit", - path=nixpath.as_posix(), - flakeref=meta_nixpkgs, - ), - ) - - @staticmethod - def _try_normalize_mutable_path(path): - try: - nixpath = nixref_to_nixpkgs_path(path.as_posix()) - except _NIXREF_RESOLUTION_EXCEPTIONS: - LOG.debug( - "Failed normalizing mutable nixpkgs path: %s", - path.as_posix(), - exc_info=True, - ) - return None - if nixpath and is_nix_store_path(nixpath): - return nixpath - return None - def resolve_flakeref_lock_source(self, nixref, *, impure=False): """Return the nixpkgs source selected by a flakeref lock graph.""" if nixref: @@ -394,31 +303,3 @@ def resolve_flakeref_lock_source(self, nixref, *, impure=False): log_nixpkgs_meta_source(source) return source return NixpkgsMetaSource(method="none") - - def resolve_default_source(self, nixref=None): - """Return the metadata source for the older direct Meta API.""" - if nixref: - return self.resolve_flakeref_lock_source(nixref) - if "NIX_PATH" in os.environ: - return self.resolve_nix_path_source() - return NixpkgsMetaSource(method="none") - - def resolve_nix_path_source(self, *, message=None, required=False): - """Return the nixpkgs source referenced by NIX_PATH.""" - LOG.debug("Reading nixpkgs path from NIX_PATH environment") - nix_path = os.environ.get("NIX_PATH", "") - m_nixpkgs = re.search(r"(?:^|:)nixpkgs=([^:]+)", nix_path) - if m_nixpkgs: - return nixpkgs_meta_source_with_path( - NixpkgsMetaSource( - method="nix-path", - path=m_nixpkgs.group(1), - message=message, - ), - ) - if required: - raise SbomnixError( - "NIX_PATH does not contain a nixpkgs= entry required by " - "--meta-nixpkgs nix-path" - ) - return NixpkgsMetaSource(method="none") diff --git a/tests/test_nixmeta_source.py b/tests/test_nixmeta_source.py deleted file mode 100644 index 388a0f4..0000000 --- a/tests/test_nixmeta_source.py +++ /dev/null @@ -1,788 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixpkgs metadata source selection.""" - -import json -import pathlib -from types import SimpleNamespace - -import pytest - -from common.errors import SbomnixError -from sbomnix import meta as sbomnix_meta -from sbomnix import meta_source as sbomnix_meta_source - - -def test_classify_meta_nixpkgs_reserved_modes_before_explicit_source(): - assert ( - sbomnix_meta.classify_meta_nixpkgs(sbomnix_meta.META_NIXPKGS_NIX_PATH) - == sbomnix_meta.META_NIXPKGS_NIX_PATH - ) - assert sbomnix_meta.classify_meta_nixpkgs("/nix/store/source") == "explicit" - - -def test_get_nixpkgs_meta_with_source_records_flakeref_lock(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - (nixpkgs_path / "lib").mkdir(parents=True) - (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: nixpkgs_path, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=".#target", - original_ref=".#target", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="flakeref-lock", - path=nixpkgs_path.as_posix(), - flakeref=".#target", - version="25.11", - ) - - -def test_get_nixpkgs_meta_with_source_records_opt_in_nix_path(monkeypatch): - scanned = [] - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other:nixpkgs=/tmp/my flake") - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=None, - original_ref="/nix/store/target", - explicit_nixpkgs=sbomnix_meta.META_NIXPKGS_NIX_PATH, - ) - - assert df_meta == "df" - assert scanned == ["/tmp/my flake"] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="nix-path", - path="/tmp/my flake", - message="NIX_PATH metadata source may not match the target", - ) - - -def test_explicit_nix_path_source_requires_nixpkgs_entry(monkeypatch): - def fail_if_scanned(self, path): - raise AssertionError(f"nix-path scan should not run: {path}") - - monkeypatch.setenv("NIX_PATH", "foo=/tmp/other") - monkeypatch.setattr(sbomnix_meta.Meta, "_scan", fail_if_scanned) - - with pytest.raises(SbomnixError, match="NIX_PATH.*nixpkgs="): - sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=None, - original_ref="/nix/store/target", - explicit_nixpkgs=sbomnix_meta.META_NIXPKGS_NIX_PATH, - ) - - -def test_path_target_without_source_skips_nix_path_metadata(monkeypatch): - def fail_if_scanned(self, path): - raise AssertionError(f"path-target scan should be skipped: {path}") - - monkeypatch.setenv("NIX_PATH", "nixpkgs=/tmp/nixpkgs") - monkeypatch.setattr(sbomnix_meta.Meta, "_scan", fail_if_scanned) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/resolved-target", - flakeref=None, - original_ref="./result", - ) - - assert df_meta is None - assert source.method == "none" - assert source.path is None - assert "store-path target" in source.message - assert "./result" not in source.message - assert "--meta-nixpkgs" in source.message - - -def test_explicit_store_path_source_records_explicit_method(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - (nixpkgs_path / "lib").mkdir(parents=True) - (nixpkgs_path / "lib" / ".version").write_text("25.11\n", encoding="utf-8") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - monkeypatch.setattr( - sbomnix_meta_source, - "is_nix_store_path", - lambda path: path.as_posix() == nixpkgs_path.as_posix(), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=nixpkgs_path.as_posix(), - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=nixpkgs_path.as_posix(), - version="25.11", - ) - - -def test_explicit_flakeref_source_resolves_nixpkgs_path(monkeypatch): - nixpkgs_path = "/nix/store/abc-source" - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: pathlib.Path(nixpkgs_path), - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs="github:NixOS/nixpkgs?rev=abc", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=nixpkgs_path, - flakeref="github:NixOS/nixpkgs?rev=abc", - ) - - -def test_mutable_explicit_path_is_normalized_before_scanning(monkeypatch, tmp_path): - mutable_path = tmp_path / "nixpkgs-checkout" - mutable_path.mkdir() - store_path = pathlib.Path("/nix/store/normalized-source") - scanned = [] - - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda nixref: store_path if nixref == mutable_path.as_posix() else None, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=mutable_path.as_posix(), - ) - - assert df_meta == "df" - assert scanned == [store_path.as_posix()] - assert source == sbomnix_meta.NixpkgsMetaSource( - method="explicit", - path=store_path.as_posix(), - flakeref=mutable_path.as_posix(), - ) - - -def test_mutable_explicit_path_is_rejected_if_not_cache_safe(monkeypatch, tmp_path): - mutable_path = tmp_path / "nixpkgs-checkout" - mutable_path.mkdir() - - monkeypatch.setattr( - sbomnix_meta_source, "nixref_to_nixpkgs_path", lambda _nixref: None - ) - - with pytest.raises(SbomnixError, match="immutable /nix/store source"): - sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - explicit_nixpkgs=mutable_path.as_posix(), - ) - - -def test_nixos_toplevel_flakeref_prefers_configuration_pkgs_path( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: (_ for _ in ()).throw( - AssertionError("lock-node lookup should not run") - ), - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is fake_df - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'] - ] - assert source.method == "flakeref-target" - assert source.path == nixpkgs_path.as_posix() - assert source.flakeref == '/flake#nixosConfigurations."host".pkgs.path' - assert source.message == "Scanning evaluated NixOS package set from flakeref" - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - None, - False, - ) - ] - - -def test_nixos_toplevel_expression_locks_relative_flake_refs( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - source_path = "/nix/store/root-source" - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '.#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - if cmd == ["nix", "flake", "metadata", ".", "--json"]: - return SimpleNamespace( - stdout=json.dumps( - { - "path": source_path, - "locked": {"narHash": "sha256-abc"}, - } - ), - returncode=0, - ) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=".#nixosConfigurations.host.config.system.build.toplevel", - original_ref=".#nixosConfigurations.host.config.system.build.toplevel", - ) - - locked_ref = f"path:{source_path}?narHash=sha256-abc" - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert source.flakeref == '.#nixosConfigurations."host".pkgs.path' - assert calls == [ - ["nix", "eval", "--raw", '.#nixosConfigurations."host".pkgs.path'], - ["nix", "flake", "metadata", ".", "--json"], - ] - cache_key = "nixos-pkgs:" + json.dumps( - [locked_ref, "host"], - separators=(",", ":"), - ) - assert expressions == [ - ( - f'let\n flake = builtins.getFlake "{locked_ref}";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_expression_preserves_locked_subflake_dir( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - source_path = "/nix/store/root-source" - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - 'path:.?dir=sub/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - if cmd == ["nix", "flake", "metadata", "path:.?dir=sub/flake", "--json"]: - return SimpleNamespace( - stdout=json.dumps( - { - "path": source_path, - "locked": { - "narHash": "sha256-abc", - "dir": "sub/flake", - }, - } - ), - returncode=0, - ) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - "path:.?dir=sub/flake#nixosConfigurations.host.config.system.build.toplevel" - ), - original_ref=( - "path:.?dir=sub/flake#nixosConfigurations.host.config.system.build.toplevel" - ), - ) - - locked_ref = f"path:{source_path}?narHash=sha256-abc&dir=sub/flake" - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert calls == [ - [ - "nix", - "eval", - "--raw", - 'path:.?dir=sub/flake#nixosConfigurations."host".pkgs.path', - ], - ["nix", "flake", "metadata", "path:.?dir=sub/flake", "--json"], - ] - cache_key = "nixos-pkgs:" + json.dumps( - [locked_ref, "host"], - separators=(",", ":"), - ) - assert expressions == [ - ( - f'let\n flake = builtins.getFlake "{locked_ref}";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_flakeref_handles_quoted_configuration_names( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host.example.com".pkgs.path', - ]: - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - '/flake#nixosConfigurations."host.example.com".config.system.build.toplevel' - ), - original_ref=( - '/flake#nixosConfigurations."host.example.com".config.system.build.toplevel' - ), - ) - - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert source.flakeref == ( - '/flake#nixosConfigurations."host.example.com".pkgs.path' - ) - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host.example.com".pkgs\n', - None, - False, - ) - ] - - -def test_nixos_toplevel_flakeref_metadata_eval_honors_impure(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - calls = [] - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - impure=True, - ) - - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert calls == [ - [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - "--impure", - ] - ] - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "/flake";\nin\n' - ' flake.nixosConfigurations."host".pkgs\n', - None, - True, - ) - ] - assert source.expression_cache_key is None - assert source.expression_impure is True - - -def test_nixos_toplevel_expression_cache_uses_only_stable_refs(monkeypatch, tmp_path): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - expressions = [] - fake_df = SimpleNamespace(empty=False) - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda self, expression, *, cache_key=None, impure=False: ( - expressions.append((expression, cache_key, impure)) or fake_df - ), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref=( - "github:example/flake?rev=abc" - '#nixosConfigurations."host:8080".config.system.build.toplevel' - ), - original_ref=( - "github:example/flake?rev=abc" - '#nixosConfigurations."host:8080".config.system.build.toplevel' - ), - ) - - cache_key = "nixos-pkgs:" + json.dumps( - ["github:example/flake?rev=abc", "host:8080"], - separators=(",", ":"), - ) - assert df_meta is fake_df - assert source.method == "flakeref-target" - assert expressions == [ - ( - 'let\n flake = builtins.getFlake "github:example/flake?rev=abc";\n' - "in\n" - ' flake.nixosConfigurations."host:8080".pkgs\n', - cache_key, - False, - ) - ] - - -def test_nixos_toplevel_expression_scan_failure_skips_metadata( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - scanned = [] - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout=f"{nixpkgs_path}\n", returncode=0) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan_expression", - lambda *_args, **_kwargs: None, - ) - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path), - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert scanned == [] - assert source.method == "flakeref-target" - assert source.expression is not None - assert source.path == nixpkgs_path.as_posix() - assert source.message == ( - "Evaluated package-set metadata scan failed. Skipping nixpkgs metadata." - ) - - -def test_nixos_toplevel_flakeref_without_pkgs_path_returns_message(monkeypatch): - calls = [] - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - if cmd == [ - "nix", - "eval", - "--raw", - '/flake#nixosConfigurations."host".pkgs.path', - ]: - return SimpleNamespace(stdout="", stderr="missing", returncode=1) - raise AssertionError(f"unexpected command: {cmd}") - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'], - ] - assert source.method == "none" - assert source.path is None - assert "NixOS configuration flakeref" in source.message - assert "--meta-nixpkgs" in source.message - - -def test_nixos_toplevel_flakeref_without_pkgs_returns_message( - monkeypatch, -): - calls = [] - - def fake_exec_cmd(cmd, **_kwargs): - calls.append(cmd) - return SimpleNamespace(stdout="", stderr="missing", returncode=1) - - monkeypatch.setattr( - sbomnix_meta_source, - "nix_cmd", - lambda *args, impure=False: ["nix", *args] + (["--impure"] if impure else []), - ) - monkeypatch.setattr(sbomnix_meta_source, "exec_cmd", fake_exec_cmd) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: None, - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host.config.system.build.toplevel", - original_ref="/flake#nixosConfigurations.host.config.system.build.toplevel", - ) - - assert df_meta is None - assert calls == [ - ["nix", "eval", "--raw", '/flake#nixosConfigurations."host".pkgs.path'], - ] - assert source.method == "none" - assert source.path is None - assert "NixOS configuration flakeref" in source.message - assert "--meta-nixpkgs" in source.message - - -def test_plain_nixos_configuration_attrset_is_not_target_inferred( - monkeypatch, - tmp_path, -): - nixpkgs_path = tmp_path / "lock-source" - nixpkgs_path.mkdir() - scanned = [] - - monkeypatch.setattr( - sbomnix_meta.Meta, - "_scan", - lambda self, path: scanned.append(path) or "df", - ) - monkeypatch.setattr( - sbomnix_meta_source, - "nixref_to_nixpkgs_path", - lambda _nixref: nixpkgs_path, - ) - - df_meta, source = sbomnix_meta.Meta().get_nixpkgs_meta_with_source( - target_path="/nix/store/target", - flakeref="/flake#nixosConfigurations.host", - original_ref="/flake#nixosConfigurations.host", - ) - - assert df_meta == "df" - assert scanned == [nixpkgs_path.as_posix()] - assert source.method == "flakeref-lock" - - -def test_meta_scan_uses_already_resolved_scanner_path(monkeypatch): - calls = [] - fake_df = SimpleNamespace(empty=False) - - class FakeScanner: - """Scanner stand-in that records normalized scan paths.""" - - def scan(self, path): - raise AssertionError(f"scan should not resolve path again: {path}") - - def scan_path(self, path): - calls.append(path) - - def to_df(self): - return fake_df - - meta = sbomnix_meta.Meta() - monkeypatch.setattr(meta.cache, "get", lambda _key: None) - monkeypatch.setattr(meta.cache, "set", lambda **_kwargs: None) - monkeypatch.setattr(sbomnix_meta, "NixMetaScanner", FakeScanner) - - assert meta._scan("/nix/store/source") is fake_df - assert calls == ["/nix/store/source"] diff --git a/tests/test_nixmeta_source_export.py b/tests/test_nixmeta_source_export.py deleted file mode 100644 index 7b2c315..0000000 --- a/tests/test_nixmeta_source_export.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for SBOM-level nixpkgs metadata source export.""" - -import uuid - -import pandas as pd - -from sbomnix.builder import SbomBuilder -from sbomnix.meta import NixpkgsMetaSource - - -def _make_minimal_sbom(): - sbomdb = object.__new__(SbomBuilder) - sbomdb.uid = "store_path" - sbomdb.nix_path = "/nix/store/target" - sbomdb.buildtime = False - sbomdb.target_deriver = "/nix/store/target.drv" - sbomdb.target_component_ref = "/nix/store/target.drv" - sbomdb.depth = None - sbomdb.uuid = uuid.uuid4() - sbomdb.sbom_type = "runtime_only" - sbomdb.nixpkgs_meta_source = NixpkgsMetaSource( - method="flakeref-target", - path="/nix/store/source", - rev="1234", - flakeref=".#target", - version="25.11", - message="base nixpkgs source metadata", - ) - sbomdb.df_sbomdb = pd.DataFrame( - [ - { - "store_path": "/nix/store/target.drv", - "pname": "target", - "name": "target", - "version": "1.0", - "outputs": ["/nix/store/target"], - "out": "/nix/store/target", - "purl": "", - "cpe": "", - "urls": "", - "patches": "", - } - ] - ) - return sbomdb - - -def test_cdx_document_records_nixpkgs_metadata_source(monkeypatch): - sbomdb = _make_minimal_sbom() - monkeypatch.setattr( - SbomBuilder, "lookup_dependencies", lambda *_args, **_kwargs: None - ) - - cdx = sbomdb.to_cdx_data() - - properties = {prop["name"]: prop["value"] for prop in cdx["metadata"]["properties"]} - assert properties["nixpkgs:metadata_source_method"] == "flakeref-target" - assert properties["nixpkgs:path"] == "/nix/store/source" - assert properties["nixpkgs:rev"] == "1234" - assert properties["nixpkgs:flakeref"] == ".#target" - assert properties["nixpkgs:version"] == "25.11" - assert properties["nixpkgs:message"] == "base nixpkgs source metadata" - - -def test_spdx_document_records_nixpkgs_metadata_source(monkeypatch): - sbomdb = _make_minimal_sbom() - monkeypatch.setattr( - SbomBuilder, "lookup_dependencies", lambda *_args, **_kwargs: None - ) - - spdx = sbomdb.to_spdx_data() - - assert "included dependencies: 'runtime_only'" in spdx["comment"] - assert ( - "nixpkgs metadata source: metadata_source_method=flakeref-target" - in spdx["comment"] - ) - assert "path=/nix/store/source" in spdx["comment"] - assert "rev=1234" in spdx["comment"] - assert "message=base nixpkgs source metadata" in spdx["comment"] - assert "warning=" not in spdx["comment"] diff --git a/tests/test_sbom_vuln_enrichment.py b/tests/test_sbom_vuln_enrichment.py index 344b6ce..afda342 100644 --- a/tests/test_sbom_vuln_enrichment.py +++ b/tests/test_sbom_vuln_enrichment.py @@ -12,7 +12,6 @@ import pandas as pd import pytest -from common.errors import SbomnixError from sbomnix import cli_utils as sbomnix_cli_utils from sbomnix import main as sbomnix_main from sbomnix import vuln_enrichment as sbomnix_vuln_enrichment @@ -30,38 +29,6 @@ def fatal(self, msg, *args): self.records.append(("fatal", msg, args)) -def test_sbomnix_getargs_accepts_meta_nixpkgs(): - args = sbomnix_main.getargs( - [ - "/nix/store/target", - "--meta-nixpkgs", - "nix-path", - ] - ) - - assert args.meta_nixpkgs == "nix-path" - - -def test_sbomnix_run_rejects_exclude_meta_with_meta_nixpkgs(): - args = SimpleNamespace( - NIXREF="/nix/store/target", - buildtime=False, - depth=None, - verbose=0, - include_vulns=False, - exclude_meta=True, - meta_nixpkgs="nix-path", - exclude_cpe_matching=False, - csv=None, - cdx=None, - spdx=None, - impure=True, - ) - - with pytest.raises(SbomnixError, match="--exclude-meta"): - sbomnix_main._run(args) - - def test_sbomnix_main_enriches_cdx_explicitly_when_include_vulns_is_set(monkeypatch): args = SimpleNamespace( NIXREF=".#target", @@ -70,7 +37,6 @@ def test_sbomnix_main_enriches_cdx_explicitly_when_include_vulns_is_set(monkeypa verbose=0, include_vulns=True, exclude_meta=False, - meta_nixpkgs=None, exclude_cpe_matching=False, csv=None, cdx="sbom.cdx.json", @@ -123,7 +89,6 @@ def to_csv(self, _path): "depth": None, "flakeref": ".#target", "original_ref": None, - "meta_nixpkgs": None, "impure": True, "include_meta": True, "include_vulns": True, @@ -149,7 +114,6 @@ def test_sbomnix_main_logs_generation_before_initializing_builder(monkeypatch): verbose=0, include_vulns=False, exclude_meta=False, - meta_nixpkgs=None, exclude_cpe_matching=False, csv=None, cdx=None, @@ -188,7 +152,6 @@ def __init__(self, **kwargs): "depth": None, "flakeref": ".#target", "original_ref": None, - "meta_nixpkgs": None, "impure": False, "include_meta": True, "include_vulns": False, From 5a74d6bc129a9ed422f27947b8058a9971171584 Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Tue, 19 May 2026 13:53:40 +0300 Subject: [PATCH 6/7] nixmeta: remove legacy metadata CLI Remove the standalone nixmeta command, its nix packaging entries, documentation, and tests. sbomnix now uses its own targeted package metadata resolver instead of the legacy nix-env metadata scanner, so the CLI is no longer part of the public tool set. Signed-off-by: Henri Rosten --- README.md | 6 +- doc/nixmeta.md | 37 ---- nix/apps.nix | 3 - nix/packages.nix | 1 - pyproject.toml | 2 - src/nixmeta/__init__.py | 3 - src/nixmeta/flake_metadata.py | 190 ---------------- src/nixmeta/main.py | 100 --------- src/nixmeta/metadata_json.py | 54 ----- src/nixmeta/scanner.py | 162 -------------- tests/integration/test_nixmeta_cli.py | 39 ---- tests/resources/nixmeta-package-set.nix | 49 ----- tests/test_cli_conventions.py | 4 - tests/test_cli_error_boundaries.py | 12 - tests/test_nixmeta_parsing.py | 55 ----- tests/test_nixmeta_progress.py | 279 ------------------------ tests/testpaths.py | 1 - 17 files changed, 4 insertions(+), 993 deletions(-) delete mode 100644 doc/nixmeta.md delete mode 100644 src/nixmeta/__init__.py delete mode 100644 src/nixmeta/flake_metadata.py delete mode 100755 src/nixmeta/main.py delete mode 100644 src/nixmeta/metadata_json.py delete mode 100755 src/nixmeta/scanner.py delete mode 100644 tests/integration/test_nixmeta_cli.py delete mode 100644 tests/resources/nixmeta-package-set.nix delete mode 100644 tests/test_nixmeta_parsing.py delete mode 100644 tests/test_nixmeta_progress.py diff --git a/README.md b/README.md index 62350da..5d2f22e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ SPDX-License-Identifier: CC-BY-SA-4.0 This repository is home to various command line tools and Python libraries that aim to help with software supply chain challenges: - [`sbomnix`](#generate-sbom) is a utility that generates SBOMs given a [Nix](https://nixos.org/) flake reference or store path. - [`nixgraph`](./doc/nixgraph.md) helps query and visualize dependency graphs for [Nix](https://nixos.org/) packages. -- [`nixmeta`](./doc/nixmeta.md) summarizes nixpkgs meta-attributes from the given nixpkgs version. - [`vulnxscan`](./doc/vulnxscan.md) is a vulnerability scanner demonstrating the usage of SBOMs in running vulnerability scans. - [`repology_cli`](./doc/repology_cli.md) and [`repology_cve`](./doc/repology_cli.md#repology-cve-search) are command line clients to [repology.org](https://repology.org/). - [`nix_outdated`](./doc/nix_outdated.md) is a utility that finds outdated nix dependencies for given out path, listing the outdated packages in priority order based on how many other packages depend on the given outdated package. @@ -71,7 +70,10 @@ $ cd sbomnix $ nix develop ``` -The devshell adds all CLI entry points (`sbomnix`, `nixgraph`, `nixmeta`, `vulnxscan`, `repology_cli`, `repology_cve`, `nix_outdated`, `provenance`) to `PATH`. They run against the local source tree, so any edits are picked up immediately without reinstalling. +The devshell adds all CLI entry points (`sbomnix`, `nixgraph`, +`vulnxscan`, `repology_cli`, `repology_cve`, `nix_outdated`, +`provenance`) to `PATH`. They run against the local source tree, so any +edits are picked up immediately without reinstalling. All tools support a consistent verbosity flag: no flag or `--verbose=0` shows INFO output, `-v` or `--verbose=1` enables VERBOSE progress diff --git a/doc/nixmeta.md b/doc/nixmeta.md deleted file mode 100644 index 5f4b38e..0000000 --- a/doc/nixmeta.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# Getting Started -To get started, follow the [Getting Started](../README.md#getting-started) section from the main [README](../README.md). - -As an example, to run the [`nixmeta`](../src/nixmeta/main.py) from the `tiiuae/sbomnix` repository: -```bash -# '--' signifies the end of argument list for `nix`. -# '--help' is the first argument to `nixmeta` -$ nix run github:tiiuae/sbomnix#nixmeta -- --help -``` - -# nixmeta -[`nixmeta`](../src/nixmeta/main.py) is a command line tool to summarize nixpkgs meta-attributes from the given nixpkgs version. The output is written to a csv file. Nixpkgs version is specified with [`flakeref`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake#flake-references). As an example, `--flakeref=github:NixOS/nixpkgs?ref=master` would make `nixmeta` output the meta-attributes from the nixpkgs version in the [master](https://github.com/NixOS/nixpkgs/tree/master) branch. Similarly, `--flakeref=github:NixOS/nixpkgs?ref=release-23.11` would output the meta-attributes from the nixpkgs version in the [release-23.11](https://github.com/NixOS/nixpkgs/tree/release-23.11) branch. Note that `--flakeref` does not necessarily have to reference `github:NixOS/nixpkgs` but any flakeref or even `NIX_ENV` environment variable can be used to specify the nixpkgs version. As an example, `--flakeref=github:tiiuae/sbomnix` would make `nixmeta` output the meta-attributes from the nixpkgs version [pinned by the sbomnix flake](https://github.com/tiiuae/sbomnix/blob/c243db5272fb01c4d97cbbb01a095ae514cd2dcb/flake.lock#L68) in its default branch. - -As an example, below command outputs nixpkgs meta-attributes from the nixpkgs version pinned by flake `github:NixOS/nixpkgs?ref=master`: - -```bash -$ nixmeta --flakeref=github:NixOS/nixpkgs?ref=master -INFO Finding meta-info for nixpkgs pinned in flake: github:NixOS/nixpkgs?ref=master -INFO Wrote: /home/foo/sbomnix-fork/nixmeta.csv -``` - -Output summarizes the meta-attributes of all the target nixpkgs packages enumerated by `nix-env --query --available`. -For each package, the output includes the following details: - -```bash -$ head -n2 nixmeta.csv | csvlook -| name | pname | version | meta_homepage | meta_unfree | meta_license_short | meta_license_spdxid | meta_maintainers_email | -| ---------- | ----- | ------- | -------------------- | ----------- | -------------------------------- | -------------------------------------- | ---------------------- | -| 0ad-0.0.26 | 0ad | 0.0.26 | https://play0ad.com/ | False | gpl2;lgpl21;mit;cc-by-sa-30;zlib | GPL-2.0;LGPL-2.1;MIT;CC-BY-SA-3.0;Zlib | nixpkgs@cvpetegem.be | - -``` diff --git a/nix/apps.nix b/nix/apps.nix index 0a3b44f..9e65844 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -29,9 +29,6 @@ # nix run .#nixgraph nixgraph = mkApp "${sbomnix}/bin/nixgraph" "Visualize nix package dependencies"; - # nix run .#nixmeta - nixmeta = mkApp "${sbomnix}/bin/nixmeta" "Summarize nixpkgs meta-attributes"; - # nix run .#vulnxscan vulnxscan = mkApp "${sbomnix}/bin/vulnxscan" "Scan nix artifacts or SBOMs for vulnerabilities"; diff --git a/nix/packages.nix b/nix/packages.nix index 220da82..2b9f2ee 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -103,7 +103,6 @@ ++ [ (mkDevEntry "sbomnix" "sbomnix.main") (mkDevEntry "nixgraph" "nixgraph.main") - (mkDevEntry "nixmeta" "nixmeta.main") (mkDevEntry "nix_outdated" "nixupdate.nix_outdated") (mkDevEntry "vulnxscan" "vulnxscan.vulnxscan_cli") (mkDevEntry "repology_cli" "repology.repology_cli") diff --git a/pyproject.toml b/pyproject.toml index cc7de47..5d98d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ Homepage = "https://github.com/tiiuae/sbomnix" [project.scripts] sbomnix = "sbomnix.main:main" nixgraph = "nixgraph.main:main" -nixmeta = "nixmeta.main:main" nix_outdated = "nixupdate.nix_outdated:main" vulnxscan = "vulnxscan.vulnxscan_cli:main" repology_cli = "repology.repology_cli:main" @@ -90,7 +89,6 @@ select = [ known-first-party = [ "common", "nixgraph", - "nixmeta", "nixupdate", "provenance", "repology", diff --git a/src/nixmeta/__init__.py b/src/nixmeta/__init__.py deleted file mode 100644 index 5cc74e1..0000000 --- a/src/nixmeta/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 diff --git a/src/nixmeta/flake_metadata.py b/src/nixmeta/flake_metadata.py deleted file mode 100644 index 30b7434..0000000 --- a/src/nixmeta/flake_metadata.py +++ /dev/null @@ -1,190 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Helpers for resolving nixpkgs metadata from flakerefs.""" - -import json -import pathlib -import re - -from common.log import LOG, LOG_SPAM -from common.proc import exec_cmd, nix_cmd - - -def get_flake_metadata(flakeref, *, exec_cmd_fn=exec_cmd, nix_cmd_fn=nix_cmd, log=LOG): - """Return ``nix flake metadata`` JSON for the given flakeref.""" - if flakeref.startswith("nixpkgs="): - flakeref = flakeref.removeprefix("nixpkgs=") - log.info("Reading flake metadata for '%s'", flakeref) - cmd = nix_cmd_fn("flake", "metadata", flakeref, "--json") - ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) - if ret is None or ret.returncode != 0: - log.warning("Failed reading flake metadata: %s", flakeref) - return None - meta_json = json.loads(ret.stdout) - log.log(LOG_SPAM, meta_json) - return meta_json - - -def is_nixpkgs_metadata(meta_json): - """Return true if the given metadata describes nixpkgs.""" - try: - if ( - "path" in meta_json - and "description" in meta_json - and meta_json["description"] - == "A collection of packages for the Nix package manager" - ): - return True - if ( - "path" in meta_json - and meta_json["locked"]["owner"] == "NixOS" - and meta_json["locked"]["repo"] == "nixpkgs" - ): - return True - except (KeyError, TypeError): - return False - return False - - -def _locked_obj_is_nixpkgs(node_name, locked_obj): - try: - if locked_obj.get("repo") == "nixpkgs": - return True - if node_name.startswith("nixpkgs") and locked_obj.get("type") == "path": - return True - except AttributeError: - return False - return False - - -def _input_node_names(value): - if isinstance(value, str): - return [value] - if isinstance(value, list) and value and isinstance(value[-1], str): - # Lock-file override chains store the resolved input node as the last item. - return [value[-1]] - return [] - - -def _get_flake_nixpkgs_obj(meta_json): - try: - nodes = meta_json["locks"]["nodes"] - root_name = meta_json["locks"]["root"] - root_inputs = nodes[root_name].get("inputs", {}) - except (KeyError, TypeError, AttributeError): - return None - - for node_name in _input_node_names(root_inputs.get("nixpkgs")): - try: - return nodes[node_name]["locked"] - except (KeyError, TypeError): - continue - - candidates = [] - for node_name, node in nodes.items(): - try: - locked_obj = node["locked"] - except (KeyError, TypeError): - continue - if _locked_obj_is_nixpkgs(node_name, locked_obj): - candidates.append(locked_obj) - if len(candidates) == 1: - return candidates[0] - return None - - -def _get_flake_nixpkgs_val(meta_json, key): - nixpkgs_obj = _get_flake_nixpkgs_obj(meta_json) - if nixpkgs_obj is None: - return None - try: - return nixpkgs_obj[key] - except (KeyError, TypeError): - return None - - -def _get_nixpkgs_flakeref_github(meta_json, *, log=LOG): - owner = _get_flake_nixpkgs_val(meta_json, "owner") - repo = _get_flake_nixpkgs_val(meta_json, "repo") - rev = _get_flake_nixpkgs_val(meta_json, "rev") - if None in [owner, repo, rev]: - log.debug( - "owner, repo, or rev not found: %s", - _get_flake_nixpkgs_obj(meta_json), - ) - return None - return f"github:{owner}/{repo}?rev={rev}" - - -def _get_nixpkgs_flakeref_git(meta_json, *, log=LOG): - url = _get_flake_nixpkgs_val(meta_json, "url") - rev = _get_flake_nixpkgs_val(meta_json, "rev") - ref = _get_flake_nixpkgs_val(meta_json, "ref") - if None in [url, rev, ref]: - log.debug("url, rev, or ref not found: %s", _get_flake_nixpkgs_obj(meta_json)) - return None - return f"git+{url}?ref={ref}&rev={rev}" - - -def _get_nixpkgs_flakeref_path(meta_json, *, log=LOG): - path = _get_flake_nixpkgs_val(meta_json, "path") - if path is None: - log.debug("path not found: %s", _get_flake_nixpkgs_obj(meta_json)) - return None - return f"path:{path}" - - -def _get_nixpkgs_flakeref_tarball(meta_json, *, log=LOG): - url = _get_flake_nixpkgs_val(meta_json, "url") - if url is None: - log.debug("url not found: %s", _get_flake_nixpkgs_obj(meta_json)) - return None - return f"{url}" - - -def get_nixpkgs_flakeref(meta_json, *, log=LOG): - """Given flake metadata, return the locked nixpkgs flakeref.""" - locked_type = _get_flake_nixpkgs_val(meta_json, "type") - if locked_type == "github": - return _get_nixpkgs_flakeref_github(meta_json, log=log) - if locked_type == "git": - return _get_nixpkgs_flakeref_git(meta_json, log=log) - if locked_type == "path": - return _get_nixpkgs_flakeref_path(meta_json, log=log) - if locked_type == "tarball": - return _get_nixpkgs_flakeref_tarball(meta_json, log=log) - log.debug("Unsupported nixpkgs locked type: %s", locked_type) - return None - - -def nixref_to_nixpkgs_path( - flakeref, - *, - get_flake_metadata_fn=get_flake_metadata, - log=LOG, - log_spam=LOG_SPAM, -): - """Return the nix store path of the nixpkgs pinned by ``flakeref``.""" - if not flakeref: - return None - log.info("Resolving nixpkgs path for '%s'", flakeref) - log.debug("Finding meta-info for nixpkgs pinned in nixref: %s", flakeref) - match = re.match(r"([^#]+)#", flakeref) - if match: - flakeref = match.group(1) - log.debug("Stripped target specifier: %s", flakeref) - meta_json = get_flake_metadata_fn(flakeref) - if not is_nixpkgs_metadata(meta_json): - log.debug("non-nixpkgs flakeref: %s", flakeref) - nixpkgs_flakeref = get_nixpkgs_flakeref(meta_json, log=log) - if not nixpkgs_flakeref: - log.warning("Failed parsing locked nixpkgs: %s", flakeref) - return None - log.log(log_spam, "using nixpkgs_flakeref: %s", nixpkgs_flakeref) - meta_json = get_flake_metadata_fn(nixpkgs_flakeref) - if not is_nixpkgs_metadata(meta_json): - log.warning("Failed reading nixpkgs metadata: %s", flakeref) - return None - return pathlib.Path(meta_json["path"]).absolute() diff --git a/src/nixmeta/main.py b/src/nixmeta/main.py deleted file mode 100755 index 1f07db4..0000000 --- a/src/nixmeta/main.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# SPDX-License-Identifier: Apache-2.0 - -"""Python script for summarizing nixpkgs meta-attributes""" - -import argparse -import pathlib - -from common.cli_args import add_verbose_argument, add_version_argument -from common.errors import SbomnixError -from common.log import LOG, set_log_verbosity -from common.proc import exit_unless_command_exists -from nixmeta.scanner import NixMetaScanner - -################################################################################ - - -def _getargs(args=None): - """Parse command line arguments""" - desc = ( - "Summarize nixpkgs meta-attributes from the given nixpkgs version " - "to a csv output file." - ) - epil = "Example: nixmeta --flakeref=github:NixOS/nixpkgs?ref=master" - parser = argparse.ArgumentParser(description=desc, epilog=epil) - helps = ( - "Flake reference specifying the location of the flake " - "from which the pinned nixpkgs target version is read. " - "The default value is the " - "current nixpkgs version in its 'nixos-unstable' branch. " - "For more details, see: " - "https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake" - "#flake-references and " - "https://nixos.wiki/wiki/Nix_channels " - "(default: --flakeref=github:NixOS/nixpkgs?ref=nixos-unstable)." - ) - parser.add_argument( - "-f", - "--flakeref", - help=helps, - type=str, - default="github:NixOS/nixpkgs?ref=nixos-unstable", - ) - helps = "Path to output file (default: --out=nixmeta.csv)." - parser.add_argument( - "-o", - "--out", - help=helps, - type=pathlib.Path, - default="nixmeta.csv", - ) - helps = ( - "Append to output file - removing duplicate entries - instead of " - "completely overwriting possible earlier output file." - ) - parser.add_argument( - "-a", - "--append", - help=helps, - action="store_true", - ) - add_version_argument(parser) - add_verbose_argument(parser) - return parser.parse_args(args) - - -############################################################################### - - -def main(): - """main entry point""" - args = _getargs() - set_log_verbosity(args.verbose) - try: - _run(args) - except SbomnixError as error: - LOG.fatal("%s", error) - raise SystemExit(1) from error - - -def _run(args): - # Fail early if the following commands are not in PATH - exit_unless_command_exists("nix") - exit_unless_command_exists("nix-env") - # Scan metadata from the flakeref pinned nixpkgs - LOG.info("Scanning nixpkgs metadata for '%s'", args.flakeref) - scanner = NixMetaScanner() - scanner.scan(args.flakeref) - # Output to csv file - scanner.to_csv(args.out, args.append) - - -################################################################################ - -if __name__ == "__main__": - main() - -################################################################################ diff --git a/src/nixmeta/metadata_json.py b/src/nixmeta/metadata_json.py deleted file mode 100644 index c5b22e4..0000000 --- a/src/nixmeta/metadata_json.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Helpers for flattening nix-env metadata JSON.""" - -import json - -import pandas as pd - -from common import columns as cols -from common.log import LOG - - -def parse_meta_entry(meta, key): - """Flatten nested metadata values for a single key into a string.""" - items = [] - if isinstance(meta, dict): - items.extend([parse_meta_entry(meta.get(key, ""), key)]) - elif isinstance(meta, list): - items.extend([parse_meta_entry(item, key) for item in meta]) - else: - return str(meta) - return ";".join(list(filter(None, items))) - - -def parse_json_metadata(json_filename, *, log=LOG): - """Parse package metadata from a ``nix-env --json`` output file.""" - with open(json_filename, "r", encoding="utf-8") as inf: - log.debug('Loading meta-info from "%s"', json_filename) - json_dict = json.loads(inf.read()) - dict_selected = {} - setcol = dict_selected.setdefault - for pkg in json_dict.values(): - setcol(cols.NAME, []).append(pkg.get("name", "")) - setcol("pname", []).append(pkg.get("pname", "")) - setcol(cols.VERSION, []).append(pkg.get("version", "")) - meta = pkg.get("meta", {}) - setcol("meta_homepage", []).append(parse_meta_entry(meta, key="homepage")) - setcol("meta_unfree", []).append(meta.get("unfree", "")) - setcol("meta_description", []).append(meta.get("description", "")) - setcol("meta_position", []).append(meta.get("position", "")) - meta_license = meta.get("license", {}) - setcol("meta_license_short", []).append( - parse_meta_entry(meta_license, key="shortName") - ) - setcol("meta_license_spdxid", []).append( - parse_meta_entry(meta_license, key="spdxId") - ) - meta_maintainers = meta.get("maintainers", {}) - setcol("meta_maintainers_email", []).append( - parse_meta_entry(meta_maintainers, key="email") - ) - return pd.DataFrame(dict_selected).astype(str) diff --git a/src/nixmeta/scanner.py b/src/nixmeta/scanner.py deleted file mode 100755 index d899016..0000000 --- a/src/nixmeta/scanner.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# SPDX-License-Identifier: Apache-2.0 - -"""Summarize nixpkgs meta-attributes""" - -import pathlib -import subprocess -from tempfile import NamedTemporaryFile - -import pandas as pd - -from common.df import df_from_csv_file, df_to_csv_file -from common.log import LOG, LOG_SPAM -from common.proc import exec_cmd, nix_cmd -from nixmeta.flake_metadata import get_flake_metadata, nixref_to_nixpkgs_path -from nixmeta.metadata_json import parse_json_metadata - -############################################################################### - - -def _run_nix_env_metadata(cmd, stdout): - """Run nix-env metadata scan while keeping successful eval warnings quiet.""" - ret = subprocess.run( - cmd, - encoding="utf-8", - check=True, - stdout=stdout, - stderr=subprocess.PIPE, - ) - if ret.stderr: - LOG.debug("nix-env metadata stderr:\n%s", ret.stderr.strip()) - - -class NixMetaScanner: - """Scan nixpkgs meta-info""" - - def __init__(self): - self.df_meta = None - - def scan(self, nixref): - """ - Scan nixpkgs meta-info using nixpkgs version pinned in nixref; - nixref can be a nix store path, flakeref or dynamical attribute set. - """ - nixpkgs_path = nixref_to_nixpkgs_path( - nixref, - get_flake_metadata_fn=lambda flakeref: get_flake_metadata( - flakeref, - exec_cmd_fn=exec_cmd, - nix_cmd_fn=nix_cmd, - log=LOG, - ), - log=LOG, - log_spam=LOG_SPAM, - ) - if not nixpkgs_path: - # try format which is understood by nix-env: - # https://ianthehenry.com/posts/how-to-learn-nix/chipping-away-at-flakes/ - # ownpkgs-nix-env.nix: - # { ... }: - # (builtins.getFlake "/tmp/ownpkgs-special-unstable"). - # outputs.packages.${builtins.currentSystem} - # and execute - # NIX_PATH="nixpkgs=/tmp/ownpkgs-special-unstable/ownpkgs-nix-env.nix" - # sbomnix /nix/store/outputpath-for-ownpkgs-special-unstable-flake-output - nixpkgs_path = pathlib.Path(nixref) - self.scan_path(nixpkgs_path) - - def scan_path(self, nixpkgs_path): - """Scan nixpkgs meta-info using an already resolved nixpkgs path.""" - nixpkgs_path = pathlib.Path(nixpkgs_path) - if not nixpkgs_path.exists(): - LOG.warning("Nixpkgs not in nix store: %s", nixpkgs_path.as_posix()) - return - LOG.debug("nixpkgs: %s", nixpkgs_path) - self._read_nixpkgs_meta(nixpkgs_path) - - def scan_expression(self, expression, *, impure=False): - """Scan nixpkgs meta-info using an expression returning a package set.""" - prefix = "nixmeta_expr_" - suffix = ".nix" - with NamedTemporaryFile( - mode="w", - delete=True, - encoding="utf-8", - prefix=prefix, - suffix=suffix, - ) as f: - f.write(expression) - f.flush() - self._read_nixpkgs_meta( - pathlib.Path(f.name), - enable_flakes=True, - impure=impure, - ) - - def to_csv(self, csv_path, append=False): - """Export meta-info to a csv file""" - csv_path = pathlib.Path(csv_path) - if append and csv_path.exists(): - df = df_from_csv_file(csv_path) - self.df_meta = pd.concat([self.df_meta, df], ignore_index=True) - self._drop_duplicates() - if self.df_meta is None or self.df_meta.empty: - LOG.info("Nothing to output") - return - csv_path.parent.mkdir(parents=True, exist_ok=True) - df_to_csv_file(self.df_meta, csv_path.absolute().as_posix()) - - def to_df(self): - """Return meta-info as dataframe""" - return self.df_meta - - def _read_nixpkgs_meta( - self, - nixpkgs_path, - *, - enable_flakes=False, - impure=False, - ): - prefix = "nixmeta_" - suffix = ".json" - with NamedTemporaryFile(delete=True, prefix=prefix, suffix=suffix) as f: - LOG.info("Reading nixpkgs metadata from '%s'", nixpkgs_path.as_posix()) - cmd = [ - "nix-env", - "-qa", - "--meta", - "--json", - "-f", - f"{nixpkgs_path.as_posix()}", - ] - if enable_flakes: - cmd.extend(["--option", "experimental-features", "nix-command flakes"]) - if impure: - cmd.append("--impure") - cmd.extend(["--arg", "config", "{allowAliases=false;}"]) - _run_nix_env_metadata(cmd, stdout=f) - LOG.debug("Generated meta.json: %s", f.name) - LOG.info("Parsing nixpkgs metadata") - self.df_meta = parse_json_metadata(f.name, log=LOG) - self._drop_duplicates() - - def _drop_duplicates(self): - if self.df_meta is None or self.df_meta.empty: - return - self.df_meta = self.df_meta.astype(str) - self.df_meta.fillna("", inplace=True) - uids = [ - "name", - "version", - "meta_license_short", - "meta_license_spdxid", - "meta_homepage", - ] - self.df_meta.sort_values(by=uids, inplace=True) - self.df_meta.drop_duplicates(subset=uids, keep="last", inplace=True) - - -############################################################################### diff --git a/tests/integration/test_nixmeta_cli.py b/tests/integration/test_nixmeta_cli.py deleted file mode 100644 index 18dd9fb..0000000 --- a/tests/integration/test_nixmeta_cli.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""CLI integration tests for nixmeta.""" - -import pytest - -from common.df import df_from_csv_file -from tests.testpaths import NIXMETA, RESOURCES_DIR - - -def test_nixmeta_help(_run_python_script): - """Test nixmeta command line argument: '-h'.""" - _run_python_script([NIXMETA, "-h"]) - - -@pytest.mark.slow -def test_nixmeta_sbomnix_flakeref(_run_python_script, test_work_dir): - """Test nixmeta with a small package-set path.""" - out_path = test_work_dir / "nixmeta.csv" - package_set = RESOURCES_DIR / "nixmeta-package-set.nix" - _run_python_script( - [ - NIXMETA, - "--out", - out_path.as_posix(), - "--flakeref", - package_set, - ] - ) - assert out_path.exists() - df_meta = df_from_csv_file(out_path) - assert df_meta is not None - assert set(df_meta["name"]) == { - "sbomnix-meta-first-1.0", - "sbomnix-meta-second-2.0", - } diff --git a/tests/resources/nixmeta-package-set.nix b/tests/resources/nixmeta-package-set.nix deleted file mode 100644 index dcbfe30..0000000 --- a/tests/resources/nixmeta-package-set.nix +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -{ ... }: - -let - mkPackage = - { - name, - pname, - version, - description, - homepage, - }: - builtins.derivation { - inherit name pname version; - system = builtins.currentSystem; - builder = "/bin/sh"; - args = [ - "-c" - "echo ${name} > $out" - ]; - meta = { - inherit description homepage; - license = { - shortName = "Apache-2.0"; - spdxId = "Apache-2.0"; - }; - }; - }; -in -{ - first = mkPackage { - name = "sbomnix-meta-first-1.0"; - pname = "sbomnix-meta-first"; - version = "1.0"; - description = "First sbomnix metadata fixture package"; - homepage = "https://example.test/sbomnix-meta-first"; - }; - - second = mkPackage { - name = "sbomnix-meta-second-2.0"; - pname = "sbomnix-meta-second"; - version = "2.0"; - description = "Second sbomnix metadata fixture package"; - homepage = "https://example.test/sbomnix-meta-second"; - }; -} diff --git a/tests/test_cli_conventions.py b/tests/test_cli_conventions.py index 31c6a16..cc7c353 100644 --- a/tests/test_cli_conventions.py +++ b/tests/test_cli_conventions.py @@ -13,7 +13,6 @@ from common.pkgmeta import _dev_version, get_py_pkg_version from nixgraph import main as nixgraph_main -from nixmeta import main as nixmeta_main from nixupdate import nix_outdated from provenance import main as provenance_main from repology import repology_cli, repology_cve @@ -31,7 +30,6 @@ def _stringify(value): CLI_ARG_CASES = [ (sbomnix_main.getargs, [".#pkg"]), (nixgraph_main.getargs, [".#pkg"]), - (nixmeta_main._getargs, []), (nix_outdated.getargs, [".#pkg"]), (vulnxscan_cli.getargs, [".#pkg"]), (osv_cli.getargs, ["sbom.json"]), @@ -49,7 +47,6 @@ def _stringify(value): [ sbomnix_main.getargs, nixgraph_main.getargs, - nixmeta_main._getargs, nix_outdated.getargs, vulnxscan_cli.getargs, osv_cli.getargs, @@ -112,7 +109,6 @@ def test_cli_verbose_level_two_forms_match(getargs, base_argv, verbose_argv): ("getargs", "argv", "expected_out"), [ (nixgraph_main.getargs, ["-o", "graph.dot", ".#pkg"], "graph.dot"), - (nixmeta_main._getargs, ["-o", "meta.csv"], "meta.csv"), (nix_outdated.getargs, ["-o", "nix_outdated.csv", ".#pkg"], "nix_outdated.csv"), (vulnxscan_cli.getargs, ["-o", "vulns.csv", ".#pkg"], "vulns.csv"), (osv_cli.getargs, ["-o", "osv.csv", "sbom.json"], "osv.csv"), diff --git a/tests/test_cli_error_boundaries.py b/tests/test_cli_error_boundaries.py index f071ed3..b7f2304 100644 --- a/tests/test_cli_error_boundaries.py +++ b/tests/test_cli_error_boundaries.py @@ -11,7 +11,6 @@ from common.errors import SbomnixError from nixgraph import main as nixgraph_main -from nixmeta import main as nixmeta_main from nixupdate import nix_outdated from provenance import main as provenance_main from vulnxscan import osv as osv_cli @@ -109,17 +108,6 @@ def test_osv_invalid_sbom_exits_nonzero(tmp_path, monkeypatch): ), "resolve_nix_target", ), - ( - nixmeta_main, - SimpleNamespace( - flakeref="github:NixOS/nixpkgs?ref=nixos-unstable", - out="nixmeta.csv", - append=False, - verbose=0, - ), - lambda monkeypatch: None, - "exit_unless_command_exists", - ), ( provenance_main, SimpleNamespace( diff --git a/tests/test_nixmeta_parsing.py b/tests/test_nixmeta_parsing.py deleted file mode 100644 index 1a569b9..0000000 --- a/tests/test_nixmeta_parsing.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixmeta parsing helpers.""" - -import json - -from nixmeta import metadata_json - - -def test_parse_json_metadata_flattens_nested_fields(tmp_path): - json_path = tmp_path / "meta.json" - json_path.write_text( - json.dumps( - { - "hello": { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta": { - "homepage": ["https://example.invalid/hello"], - "unfree": False, - "description": "GNU hello", - "position": "pkgs/tools/misc/hello/default.nix:1", - "license": [ - {"shortName": "GPLv3+", "spdxId": "GPL-3.0-or-later"} - ], - "maintainers": [ - {"email": "maintainer@example.invalid"}, - ], - }, - } - } - ), - encoding="utf-8", - ) - - df = metadata_json.parse_json_metadata(json_path) - - assert df.to_dict(orient="records") == [ - { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta_homepage": "https://example.invalid/hello", - "meta_unfree": "False", - "meta_description": "GNU hello", - "meta_position": "pkgs/tools/misc/hello/default.nix:1", - "meta_license_short": "GPLv3+", - "meta_license_spdxid": "GPL-3.0-or-later", - "meta_maintainers_email": "maintainer@example.invalid", - } - ] diff --git a/tests/test_nixmeta_progress.py b/tests/test_nixmeta_progress.py deleted file mode 100644 index 77590e8..0000000 --- a/tests/test_nixmeta_progress.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for nixmeta progress logging.""" - -import json -import subprocess -from types import SimpleNamespace - -from nixmeta import flake_metadata -from nixmeta import main as nixmeta_main -from nixmeta import scanner as nixmeta_scanner - - -class CapturingLogger: - def __init__(self): - self.records = [] - - def debug(self, msg, *args): - self.records.append(("debug", msg, args)) - - def info(self, msg, *args): - self.records.append(("info", msg, args)) - - def warning(self, msg, *args): - self.records.append(("warning", msg, args)) - - def fatal(self, msg, *args): - self.records.append(("fatal", msg, args)) - - def log(self, level, msg, *args): - self.records.append(("log", level, msg, args)) - - -def test_nixmeta_main_logs_scan_start(monkeypatch): - args = SimpleNamespace( - flakeref="github:NixOS/nixpkgs?ref=nixos-unstable", - out="nixmeta.csv", - append=False, - ) - logger = CapturingLogger() - events = [] - - class FakeScanner: - def scan(self, flakeref): - events.append(("scan", flakeref)) - - def to_csv(self, out, append): - events.append(("to_csv", out, append)) - - monkeypatch.setattr(nixmeta_main, "LOG", logger) - monkeypatch.setattr( - nixmeta_main, - "exit_unless_command_exists", - lambda command: events.append(("command", command)), - ) - monkeypatch.setattr(nixmeta_main, "NixMetaScanner", FakeScanner) - - nixmeta_main._run(args) - - assert ( - "info", - "Scanning nixpkgs metadata for '%s'", - ("github:NixOS/nixpkgs?ref=nixos-unstable",), - ) in logger.records - assert events == [ - ("command", "nix"), - ("command", "nix-env"), - ("scan", "github:NixOS/nixpkgs?ref=nixos-unstable"), - ("to_csv", "nixmeta.csv", False), - ] - - -def test_get_flake_metadata_logs_metadata_read(): - logger = CapturingLogger() - - def fake_exec_cmd(_cmd, **_kwargs): - return SimpleNamespace(stdout='{"path": "/nix/store/nixpkgs"}', returncode=0) - - meta = flake_metadata.get_flake_metadata( - "nixpkgs=/tmp/my flake", - exec_cmd_fn=fake_exec_cmd, - log=logger, - ) - - assert meta == {"path": "/nix/store/nixpkgs"} - assert ( - "info", - "Reading flake metadata for '%s'", - ("/tmp/my flake",), - ) in logger.records - - -def test_get_nixpkgs_flakeref_uses_root_nixpkgs_input_with_renamed_node(): - meta_json = { - "locks": { - "root": "root", - "nodes": { - "root": {"inputs": {"nixpkgs": "nixpkgs_3"}}, - "nixpkgs": { - "locked": { - "type": "github", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "wrong", - } - }, - "nixpkgs_3": { - "locked": { - "type": "github", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "right", - } - }, - }, - } - } - - assert ( - flake_metadata.get_nixpkgs_flakeref(meta_json) - == "github:NixOS/nixpkgs?rev=right" - ) - - -def test_nixmeta_scanner_logs_nix_env_progress(tmp_path, monkeypatch): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - logger = CapturingLogger() - commands = [] - - def fake_run_nix_env_metadata(_cmd, stdout): - commands.append(_cmd) - stdout.write( - json.dumps( - { - "hello": { - "name": "hello-2.12.1", - "pname": "hello", - "version": "2.12.1", - "meta": { - "homepage": "https://example.invalid/hello", - "unfree": False, - "description": "GNU hello", - "position": "pkgs/tools/misc/hello/default.nix:1", - "license": { - "shortName": "GPLv3+", - "spdxId": "GPL-3.0-or-later", - }, - "maintainers": { - "email": "maintainer@example.invalid", - }, - }, - } - } - ).encode("utf-8") - ) - stdout.flush() - - monkeypatch.setattr(nixmeta_scanner, "LOG", logger) - monkeypatch.setattr( - nixmeta_scanner, - "nixref_to_nixpkgs_path", - lambda *_args, **_kwargs: nixpkgs_path, - ) - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan("github:NixOS/nixpkgs?ref=nixos-unstable") - - assert ( - "info", - "Reading nixpkgs metadata from '%s'", - (nixpkgs_path.as_posix(),), - ) in logger.records - assert ("info", "Parsing nixpkgs metadata", ()) in logger.records - assert commands - assert scanner.to_df() is not None - - -def test_run_nix_env_metadata_captures_successful_stderr(monkeypatch, tmp_path): - calls = [] - logger = CapturingLogger() - - def fake_run(cmd, **kwargs): - calls.append((cmd, kwargs)) - return SimpleNamespace(stderr="warning: noisy eval\n") - - monkeypatch.setattr(nixmeta_scanner.subprocess, "run", fake_run) - monkeypatch.setattr(nixmeta_scanner, "LOG", logger) - out_path = tmp_path / "meta.json" - - with out_path.open("w", encoding="utf-8") as out: - nixmeta_scanner._run_nix_env_metadata(["nix-env"], stdout=out) - - assert calls - assert calls[0][1]["stderr"] is subprocess.PIPE - assert calls[0][1]["stdout"].name == out_path.as_posix() - assert ( - "debug", - "nix-env metadata stderr:\n%s", - ("warning: noisy eval",), - ) in logger.records - - -def test_nixmeta_scanner_tolerates_empty_metadata_json(tmp_path, monkeypatch): - nixpkgs_path = tmp_path / "nixpkgs" - nixpkgs_path.mkdir() - - def fake_run_nix_env_metadata(_cmd, stdout): - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_path(nixpkgs_path) - - assert scanner.to_df() is not None - assert scanner.to_df().empty - - -def test_nixmeta_expression_scan_enables_flakes(monkeypatch): - commands = [] - - def fake_run_nix_env_metadata(cmd, stdout): - commands.append(cmd) - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_expression('builtins.getFlake "github:NixOS/nixpkgs"') - - assert commands - assert [ - "--option", - "experimental-features", - "nix-command flakes", - ] in [commands[0][idx : idx + 3] for idx in range(len(commands[0]) - 2)] - - -def test_nixmeta_expression_scan_honors_impure(monkeypatch): - commands = [] - - def fake_run_nix_env_metadata(cmd, stdout): - commands.append(cmd) - stdout.write(b"{}") - stdout.flush() - - monkeypatch.setattr( - nixmeta_scanner, - "_run_nix_env_metadata", - fake_run_nix_env_metadata, - ) - - scanner = nixmeta_scanner.NixMetaScanner() - scanner.scan_expression( - 'builtins.getFlake "path:/tmp/local-flake"', - impure=True, - ) - - assert commands - assert "--impure" in commands[0] diff --git a/tests/testpaths.py b/tests/testpaths.py index e6e17e1..c2a3ded 100644 --- a/tests/testpaths.py +++ b/tests/testpaths.py @@ -18,7 +18,6 @@ SBOMNIX = SRCDIR / "sbomnix" / "main.py" NIXGRAPH = SRCDIR / "nixgraph" / "main.py" -NIXMETA = SRCDIR / "nixmeta" / "main.py" PROVENANCE = SRCDIR / "provenance" / "main.py" NIX_OUTDATED = SRCDIR / "nixupdate" / "nix_outdated.py" VULNXSCAN = SRCDIR / "vulnxscan" / "vulnxscan_cli.py" From d8311805bdad625f72aca9af528a39b3f1540f8e Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Wed, 20 May 2026 07:25:45 +0300 Subject: [PATCH 7/7] sbomnix: trace nix command and metadata timings Signed-off-by: Henri Rosten --- src/common/flakeref.py | 32 +++++++++++- src/common/proc.py | 32 ++++++++++++ src/sbomnix/builder.py | 78 +++++++++++++++++++++++++++--- src/sbomnix/main.py | 30 ++++++++++++ src/sbomnix/meta.py | 40 +++++++++++++-- src/sbomnix/package_meta.py | 35 ++++++++++++-- tests/test_sbom_vuln_enrichment.py | 3 ++ 7 files changed, 237 insertions(+), 13 deletions(-) diff --git a/src/common/flakeref.py b/src/common/flakeref.py index 9732599..4a9a1ec 100644 --- a/src/common/flakeref.py +++ b/src/common/flakeref.py @@ -7,6 +7,7 @@ import logging import pathlib import re +import time from common.errors import FlakeRefRealisationError, FlakeRefResolutionError from common.log import LOG, LOG_VERBOSE @@ -65,21 +66,29 @@ def try_resolve_flakeref( # noqa: PLR0913 else: log.log(LOG_VERBOSE, "Evaluating '%s'", flakeref) cmd = nix_cmd("eval", "--raw", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix eval failed for '%s' after %.3fs", flakeref, elapsed) if looks_like_flakeref: raise FlakeRefResolutionError(flakeref, ret.stderr if ret else "") log.debug("not a flakeref: '%s'", flakeref) return None nixpath = ret.stdout.strip() + log.log(LOG_VERBOSE, "Evaluated '%s' in %.3fs", flakeref, elapsed) log.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) if not force_realise: return nixpath log.info("Realising flakeref '%s'", flakeref) cmd = nix_cmd("build", "--no-link", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix build failed for '%s' after %.3fs", flakeref, elapsed) raise FlakeRefRealisationError(flakeref, ret.stderr if ret else "") + log.log(LOG_VERBOSE, "Realised flakeref '%s' in %.3fs", flakeref, elapsed) return nixpath @@ -92,8 +101,15 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): """Return the derivation path for a flakeref target.""" log.info("Evaluating flakeref '%s'", flakeref) cmd = nix_cmd("derivation", "show", flakeref, impure=impure) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug( + "nix derivation show failed for flakeref '%s' after %.3fs", + flakeref, + elapsed, + ) raise FlakeRefResolutionError(flakeref, ret.stderr if ret else "") drv_paths = parse_nix_derivation_show(ret.stdout) drv_path = next(iter(drv_paths), "") @@ -102,6 +118,12 @@ def _resolve_flakeref_derivation(flakeref, *, impure, exec_cmd_fn, log): flakeref, "nix derivation show returned no derivation path", ) + log.log( + LOG_VERBOSE, + "Resolved flakeref derivation in %.3fs to '%s'", + elapsed, + drv_path, + ) log.debug("flakeref='%s' maps to derivation='%s'", flakeref, drv_path) return drv_path @@ -127,8 +149,11 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): flakeref, impure=impure, ) + started = time.perf_counter() ret = exec_cmd_fn(cmd, raise_on_error=False, return_error=True, log_error=False) + elapsed = time.perf_counter() - started if ret is None or ret.returncode != 0: + log.debug("nix build failed for flakeref '%s' after %.3fs", flakeref, elapsed) raise FlakeRefRealisationError(flakeref, ret.stderr if ret else "") nixpath = _first_output_path(ret.stdout) if not nixpath: @@ -136,7 +161,12 @@ def _build_flakeref_path(flakeref, *, impure, exec_cmd_fn, log): flakeref, "nix build returned no output path", ) - log.log(LOG_VERBOSE, "Resolved flakeref to built path '%s'", nixpath) + log.log( + LOG_VERBOSE, + "Resolved flakeref to built path '%s' in %.3fs", + nixpath, + elapsed, + ) log.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) return nixpath diff --git a/src/common/proc.py b/src/common/proc.py index d676ac9..a63cbba 100644 --- a/src/common/proc.py +++ b/src/common/proc.py @@ -8,6 +8,7 @@ import os import shlex import subprocess +import time from collections.abc import Callable, Sequence from shutil import which from typing import IO, Literal, overload @@ -20,6 +21,18 @@ ExecCmdFn = Callable[..., ExecCmdResult] +def _stream_summary(value: str | None) -> str: + if value is None: + return "not captured" + return f"{len(value)} byte(s), {value.count(chr(10))} line(s)" + + +def _stream_preview(value: str, limit: int = 4000) -> str: + if len(value) <= limit: + return value.rstrip() + return f"{value[:limit].rstrip()}\n... truncated {len(value) - limit} byte(s)" + + @overload def exec_cmd( cmd: Sequence[CommandPart], @@ -62,6 +75,7 @@ def exec_cmd( raise TypeError("cmd must be an argv sequence, not a string-like value") argv = [os.fspath(part) for part in cmd] command_str = shlex.join(argv) + started = time.perf_counter() LOG.debug("Running: %s", command_str) try: if stdout: @@ -73,8 +87,26 @@ def exec_cmd( encoding="utf-8", check=True, ) + LOG.debug( + "Finished in %.3fs (rc=%s, stdout=%s, stderr=%s): %s", + time.perf_counter() - started, + ret.returncode, + _stream_summary(ret.stdout), + _stream_summary(ret.stderr), + command_str, + ) + if ret.stderr: + LOG.debug("Command stderr:\n%s", _stream_preview(ret.stderr)) return ret except subprocess.CalledProcessError as error: + LOG.debug( + "Failed in %.3fs (rc=%s, stdout=%s, stderr=%s): %s", + time.perf_counter() - started, + error.returncode, + _stream_summary(error.stdout), + _stream_summary(error.stderr), + command_str, + ) if log_error: LOG.error( "Error running shell command:\n cmd: '%s'\n stdout: %s\n stderr: %s", diff --git a/src/sbomnix/builder.py b/src/sbomnix/builder.py index f90e77e..fd36ea3 100644 --- a/src/sbomnix/builder.py +++ b/src/sbomnix/builder.py @@ -7,6 +7,7 @@ """SBOM builder orchestration.""" import logging +import time import uuid from dataclasses import dataclass from typing import Any @@ -96,6 +97,14 @@ def __init__( # noqa: PLR0913, PLR0917 # self.uid specifies the attribute that identifies SBOM components. # See the column names in # self.df_sbomdb (sbom.csv) for a list of all components' attributes. + started = time.perf_counter() + LOG.verbose( + "Initializing SBOM builder for '%s' (buildtime=%s, meta=%s, cpe=%s)", + nix_path, + buildtime, + include_meta, + include_cpe, + ) self.uid = cols.STORE_PATH self.nix_path = nix_path self.buildtime = buildtime @@ -135,12 +144,30 @@ def __init__( # noqa: PLR0913, PLR0917 self.sbom_type = "runtime_and_buildtime" if not self.buildtime: self.sbom_type = "runtime_only" + LOG.verbose( + "Initialized SBOM builder in %.3fs", + time.perf_counter() - started, + ) def _resolve_target_deriver(self, nix_path): + started = time.perf_counter() + LOG.verbose("Resolving target deriver for '%s'", nix_path) if self.buildtime: - return require_deriver(nix_path) + drv_path = require_deriver(nix_path) + LOG.verbose( + "Resolved build-time target deriver in %.3fs: %s", + time.perf_counter() - started, + drv_path, + ) + return drv_path try: - return find_deriver(nix_path) + drv_path = find_deriver(nix_path) + LOG.verbose( + "Resolved runtime target deriver in %.3fs: %s", + time.perf_counter() - started, + drv_path, + ) + return drv_path except SbomnixError: raise except RuntimeError: @@ -149,6 +176,10 @@ def _resolve_target_deriver(self, nix_path): nix_path, exc_info=True, ) + LOG.verbose( + "No loadable runtime target deriver found in %.3fs", + time.perf_counter() - started, + ) return None def _load_structured_closure(self, nix_path): @@ -171,8 +202,14 @@ def _load_recursive_buildtime_closure(self): """Load build-time dependencies from recursive derivation JSON.""" if self.target_deriver is None: raise MissingNixDeriverError(self.nix_path) + started = time.perf_counter() LOG.verbose("Loading build-time closure for '%s'", self.target_deriver) derivations, drv_infos = load_recursive(self.target_deriver) + LOG.verbose( + "Loaded %d recursive derivation record(s) in %.3fs", + len(drv_infos), + time.perf_counter() - started, + ) df_deps = derivation_dependencies_df(drv_infos) if self.depth: df_deps = self._filter_dependencies_to_depth( @@ -181,7 +218,9 @@ def _load_recursive_buildtime_closure(self): self.depth, ) LOG.verbose( - "Loaded build-time closure with %d dependency edge(s)", len(df_deps) + "Loaded build-time closure with %d dependency edge(s) in %.3fs", + len(df_deps), + time.perf_counter() - started, ) return StructuredClosure( df_deps=df_deps, @@ -190,8 +229,15 @@ def _load_recursive_buildtime_closure(self): def _load_runtime_path_info_closure(self, nix_path): """Load runtime dependencies from structured path-info JSON.""" + started = time.perf_counter() LOG.verbose("Loading runtime closure for '%s'", nix_path) runtime_closure = load_runtime_closure(nix_path) + LOG.verbose( + "Loaded raw runtime closure with %d dependency edge(s) and %d deriver(s) in %.3fs", + len(runtime_closure.df_deps), + len(runtime_closure.output_paths_by_drv), + time.perf_counter() - started, + ) output_paths_by_load_path = _runtime_output_paths_by_load_path( runtime_closure.output_paths_by_drv ) @@ -213,7 +259,11 @@ def _load_runtime_path_info_closure(self, nix_path): nix_path, self.depth, ) - LOG.verbose("Loaded runtime closure with %d dependency edge(s)", len(df_deps)) + LOG.verbose( + "Loaded runtime closure with %d dependency edge(s) in %.3fs", + len(df_deps), + time.perf_counter() - started, + ) return StructuredClosure( df_deps=df_deps, runtime_output_paths_by_load_path=output_paths_by_load_path, @@ -244,7 +294,9 @@ def _filter_dependencies_to_depth( def _init_components(self, include_meta): """Initialize the SBOM component dataframe.""" + started = time.perf_counter() paths = self._sbom_component_paths() + LOG.verbose("Initializing SBOM components from %d path(s)", len(paths)) # Populate store based on the dependencies if self._recursive_buildtime_derivations is not None: self.df_sbomdb = recursive_derivations_to_dataframe( @@ -257,7 +309,11 @@ def _init_components(self, include_meta): else: # _load_structured_closure always selects exactly one metadata source. raise AssertionError("Structured dependency metadata was not initialized") - LOG.verbose("Initialized %d SBOM component(s)", len(self.df_sbomdb)) + LOG.verbose( + "Initialized %d SBOM component(s) in %.3fs", + len(self.df_sbomdb), + time.perf_counter() - started, + ) # Join with meta information if include_meta: self._join_meta() @@ -268,8 +324,9 @@ def _init_components(self, include_meta): self.df_sbomdb_outputs_exploded = self.df_sbomdb.explode(cols.OUTPUTS) self._init_dependency_index() LOG.verbose( - "Prepared SBOM database with %d component(s)", + "Prepared SBOM database with %d component(s) in %.3fs", len(self.df_sbomdb), + time.perf_counter() - started, ) def _sbom_component_paths(self): @@ -321,6 +378,7 @@ def _join_meta(self): """Join component rows with nixpkgs metadata.""" if self.df_sbomdb is None: raise AssertionError("SBOM component metadata was not initialized") + started = time.perf_counter() self.meta = Meta() df_meta, source = self.meta.get_package_meta_with_source( components=self.df_sbomdb, @@ -331,6 +389,10 @@ def _join_meta(self): impure=self.impure, ) self.nixpkgs_meta_source = source + LOG.verbose( + "Resolved nixpkgs metadata candidates in %.3fs", + time.perf_counter() - started, + ) if df_meta is None or df_meta.empty: if source.message: LOG.info("%s", source.message) @@ -362,6 +424,10 @@ def _join_meta(self): suffixes=("", "_meta"), ) LOG.verbose("Joined nixpkgs metadata for %d component(s)", len(df_meta)) + LOG.verbose( + "Joined nixpkgs metadata in %.3fs", + time.perf_counter() - started, + ) def lookup_dependencies(self, drv, uid=cols.STORE_PATH): """Return indexed dependency values for one SBOM component.""" diff --git a/src/sbomnix/main.py b/src/sbomnix/main.py index b5da99f..4398a91 100755 --- a/src/sbomnix/main.py +++ b/src/sbomnix/main.py @@ -7,6 +7,7 @@ """Python script that generates SBOMs from nix packages""" import argparse +import time from common.cli_args import add_verbose_argument, add_version_argument, check_positive from common.errors import SbomnixError @@ -83,10 +84,19 @@ def main(): def _run(args): + started = time.perf_counter() + resolve_started = time.perf_counter() target = resolve_nix_target( args.NIXREF, buildtime=args.buildtime, impure=args.impure ) + LOG.verbose( + "Resolved CLI target in %.3fs: path='%s', flakeref='%s'", + time.perf_counter() - resolve_started, + target.path, + target.flakeref, + ) LOG.info("Generating SBOM for target '%s'", target.path) + build_started = time.perf_counter() sbom = SbomBuilder( nix_path=target.path, buildtime=args.buildtime, @@ -98,15 +108,35 @@ def _run(args): include_vulns=args.include_vulns, include_cpe=not args.exclude_cpe_matching, ) + LOG.verbose( + "Constructed SBOM model in %.3fs", + time.perf_counter() - build_started, + ) if args.cdx: + export_started = time.perf_counter() cdx = sbom.to_cdx_data() if args.include_vulns: sbom.enrich_cdx_with_vulnerabilities(cdx) sbom.write_json(args.cdx, cdx, printinfo=True) + LOG.verbose( + "Wrote CycloneDX output in %.3fs", + time.perf_counter() - export_started, + ) if args.spdx: + export_started = time.perf_counter() sbom.to_spdx(args.spdx) + LOG.verbose( + "Wrote SPDX output in %.3fs", + time.perf_counter() - export_started, + ) if args.csv: + export_started = time.perf_counter() sbom.to_csv(args.csv) + LOG.verbose( + "Wrote CSV output in %.3fs", + time.perf_counter() - export_started, + ) + LOG.verbose("Completed sbomnix run in %.3fs", time.perf_counter() - started) ################################################################################ diff --git a/src/sbomnix/meta.py b/src/sbomnix/meta.py index 5553903..393eddb 100644 --- a/src/sbomnix/meta.py +++ b/src/sbomnix/meta.py @@ -5,6 +5,7 @@ """Cache and scan nixpkgs meta information.""" import re +import time from dataclasses import replace import pandas as pd @@ -60,12 +61,18 @@ def get_package_meta_with_source( # noqa: PLR0913 impure=False, ): """Return exact component metadata and selected metadata source.""" + started = time.perf_counter() source = self._resolve_source( target_path=target_path, flakeref=flakeref, original_ref=original_ref, impure=impure, ) + LOG.verbose( + "Resolved nixpkgs metadata source method '%s' in %.3fs", + source.method, + time.perf_counter() - started, + ) component_count = 0 if components is None else len(components) LOG.verbose("Resolving package metadata for %d component(s)", component_count) out_source = replace( @@ -121,7 +128,11 @@ def get_package_meta_with_source( # noqa: PLR0913 "or output paths. Skipping nixpkgs metadata." ), ) - LOG.verbose("Matched nixpkgs metadata for %d component(s)", len(df)) + LOG.verbose( + "Matched nixpkgs metadata for %d component(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) return df, out_source def _add_flake_input_package_meta( # noqa: PLR0913 @@ -152,6 +163,7 @@ def _add_flake_input_package_meta( # noqa: PLR0913 len(lookup_keys), ) + started = time.perf_counter() df_candidates = self._scan_package_source( source, lookup_keys, @@ -164,6 +176,11 @@ def _add_flake_input_package_meta( # noqa: PLR0913 df_candidates, buildtime=buildtime, ) + LOG.verbose( + "Matched flake input metadata for %d component(s) in %.3fs", + len(df_input), + time.perf_counter() - started, + ) if df_input.empty: return df if df.empty: @@ -205,7 +222,13 @@ def _scan_package_source( impure=False, ): scan_impure = impure or source.expression_impure + lock_started = time.perf_counter() + LOG.debug("Acquiring package metadata cache lock: %s", self.lock.lock_file) with self.lock: + LOG.verbose( + "Acquired package metadata cache lock in %.3fs", + time.perf_counter() - lock_started, + ) cache_scope = self._package_cache_scope( source, flakeref=flakeref, @@ -276,6 +299,7 @@ def _scan_package_source( "cache disabled, scanning package metadata for %d lookup(s)", len(lookup_keys), ) + scan_started = time.perf_counter() df = try_scan_package_meta( missing_lookup_keys, flakeref=_package_scan_flakeref(source, flakeref), @@ -284,6 +308,12 @@ def _scan_package_source( input_roots_only=input_roots_only, impure=scan_impure, ) + row_count = "failed" if df is None else len(df) + LOG.verbose( + "Package metadata scan returned %s candidate row(s) in %.3fs", + row_count, + time.perf_counter() - scan_started, + ) if df is None: return None result = _concat_package_metadata_frames([df_cached, df]) @@ -294,13 +324,15 @@ def _scan_package_source( df, ) if hit_count and exact_cache_key is not None: + cache_started = time.perf_counter() self.cache.set( key=exact_cache_key, value=result, ttl=_PACKAGE_META_TTL, ) LOG.debug( - "Stored combined package metadata cache entry: %s", + "Stored combined package metadata cache entry in %.3fs: %s", + time.perf_counter() - cache_started, exact_cache_key, ) return result @@ -349,12 +381,14 @@ def _cached_package_metadata(self, cache_scope, lookup_keys): def _store_package_metadata(self, cache_scope, lookup_keys, df): if not lookup_keys: return + cache_started = time.perf_counter() scan_cache_key = self._package_scan_cache_key(cache_scope, lookup_keys) self.cache.set(key=scan_cache_key, value=df, ttl=_PACKAGE_META_TTL) self._store_package_lookup_index(cache_scope, lookup_keys, scan_cache_key) LOG.debug( - "Stored package metadata cache group for %d lookup(s): %s", + "Stored package metadata cache group for %d lookup(s) in %.3fs: %s", len(lookup_keys), + time.perf_counter() - cache_started, scan_cache_key, ) diff --git a/src/sbomnix/package_meta.py b/src/sbomnix/package_meta.py index 4bbc9c3..0695f84 100644 --- a/src/sbomnix/package_meta.py +++ b/src/sbomnix/package_meta.py @@ -18,6 +18,7 @@ import re import shutil import subprocess +import time from tempfile import TemporaryDirectory import pandas as pd @@ -85,6 +86,13 @@ def nix_system(): def _eval_package_meta_request(request, *, pkgs_expression=None, impure=False): request_json = json.dumps(request, sort_keys=True) + LOG.debug( + "Evaluating package metadata request for system=%s lookup_count=%d input_roots_only=%s json_bytes=%d", + request.get("system"), + len(request.get("lookupKeys") or []), + request.get("inputRootsOnly"), + len(request_json), + ) if len(request_json) <= 30_000: return _eval_request( f"request = {_nix_string_literal(request_json)};", @@ -126,6 +134,7 @@ def _nix_string_literal(value): def _eval_request(request_arg, *, pkgs_expression=None, impure=False): + started = time.perf_counter() ret = exec_cmd( _eval_command( _apply_expression(request_arg, pkgs_expression), @@ -133,15 +142,28 @@ def _eval_request(request_arg, *, pkgs_expression=None, impure=False): ), log_error=False, ) - return parse_package_metadata(ret.stdout) + df = parse_package_metadata(ret.stdout) + LOG.verbose( + "Evaluated inline package metadata request with %d row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df def _eval_file(eval_file, *, impure=False): + started = time.perf_counter() ret = exec_cmd( _eval_file_command(eval_file, impure), log_error=False, ) - return parse_package_metadata(ret.stdout) + df = parse_package_metadata(ret.stdout) + LOG.verbose( + "Evaluated file-backed package metadata request with %d row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df def _eval_command(apply_expression, impure): @@ -879,6 +901,7 @@ def try_scan_package_meta( # noqa: PLR0913 ): """Return package metadata dataframe, or None on failure.""" try: + started = time.perf_counter() lookup_keys = _normalized_lookup_keys(lookup_keys) if not lookup_keys: return pd.DataFrame() @@ -907,7 +930,13 @@ def try_scan_package_meta( # noqa: PLR0913 impure=impure, ) ) - return _concat_metadata_frames(frames) + df = _concat_metadata_frames(frames) + LOG.verbose( + "Read package metadata with %d candidate row(s) in %.3fs", + len(df), + time.perf_counter() - started, + ) + return df except ( FileNotFoundError, KeyError, diff --git a/tests/test_sbom_vuln_enrichment.py b/tests/test_sbom_vuln_enrichment.py index afda342..543b496 100644 --- a/tests/test_sbom_vuln_enrichment.py +++ b/tests/test_sbom_vuln_enrichment.py @@ -25,6 +25,9 @@ def __init__(self): def info(self, msg, *args): self.records.append(("info", msg, args)) + def verbose(self, _msg, *_args): + pass + def fatal(self, msg, *args): self.records.append(("fatal", msg, args))