From 89df90d48c9e5571523aec0f1fe15074d207e8a3 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 18 Mar 2025 16:51:31 +1000 Subject: [PATCH 01/16] feat: add GitHub attestation discovery Signed-off-by: Ben Selwyn-Smith --- src/macaron/artifact/local_artifact.py | 57 ++++++++- src/macaron/slsa_analyzer/analyzer.py | 117 ++++++++++++++++-- .../slsa_analyzer/git_service/api_client.py | 21 +++- .../maven_central_registry.py | 72 +++++++++++ .../slsa_analyzer/provenance/loader.py | 4 + .../cases/github_maven_attestation/policy.dl | 10 ++ .../cases/github_maven_attestation/test.yaml | 22 ++++ .../github_maven_attestation_local/policy.dl | 10 ++ .../github_maven_attestation_local/test.yaml | 28 +++++ 9 files changed, 326 insertions(+), 15 deletions(-) create mode 100644 tests/integration/cases/github_maven_attestation/policy.dl create mode 100644 tests/integration/cases/github_maven_attestation/test.yaml create mode 100644 tests/integration/cases/github_maven_attestation_local/policy.dl create mode 100644 tests/integration/cases/github_maven_attestation_local/test.yaml diff --git a/src/macaron/artifact/local_artifact.py b/src/macaron/artifact/local_artifact.py index ed37c335a..582799824 100644 --- a/src/macaron/artifact/local_artifact.py +++ b/src/macaron/artifact/local_artifact.py @@ -1,16 +1,21 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module declares types and utilities for handling local artifacts.""" import fnmatch import glob +import hashlib +import logging import os from packageurl import PackageURL from macaron.artifact.maven import construct_maven_repository_path from macaron.errors import LocalArtifactFinderError +from macaron.slsa_analyzer.package_registry import MavenCentralRegistry + +logger: logging.Logger = logging.getLogger(__name__) def construct_local_artifact_dirs_glob_pattern_maven_purl(maven_purl: PackageURL) -> list[str] | None: @@ -247,3 +252,53 @@ def get_local_artifact_dirs( ) raise LocalArtifactFinderError(f"Unsupported PURL type {purl_type}") + + +def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str], hash_algorithm_name: str) -> str | None: + """Compute the hash of the local artifact. + + Parameters + ---------- + purl: PackageURL + The PURL of the artifact being sought. + artifact_dirs: list[str] + The possible locations of the artifact. + hash_algorithm_name: str + The hash algorithm to use. + + Returns + ------- + str | None + The hash, or None if not found. + """ + if not artifact_dirs: + logger.debug("No artifact directories provided.") + return None + + if not purl.version: + logger.debug("PURL is missing version.") + return None + + artifact_target = None + if purl.type == "maven": + artifact_target = MavenCentralRegistry.get_artifact_file_name(purl) + + if not artifact_target: + logger.debug("PURL type not supported: %s", purl.type) + return None + + for artifact_dir in artifact_dirs: + full_path = os.path.join(artifact_dir, artifact_target) + if not os.path.exists(full_path): + continue + + with open(full_path, "rb") as file: + try: + hash_result = hashlib.file_digest(file, hash_algorithm_name) + except ValueError as error: + logger.debug("Error while hashing file: %s", error) + continue + + return hash_result.hexdigest() + + return None diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 8fd2e5f83..c5cfbd70b 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -4,11 +4,14 @@ """This module handles the cloning and analyzing a Git repo.""" import glob +import hashlib +import json import logging import os import re import sys import tempfile +import urllib.parse from collections.abc import Mapping from datetime import datetime, timezone from pathlib import Path @@ -20,7 +23,10 @@ from sqlalchemy.orm import Session from macaron import __version__ -from macaron.artifact.local_artifact import get_local_artifact_dirs +from macaron.artifact.local_artifact import ( + get_local_artifact_dirs, + get_local_artifact_hash, +) from macaron.config.global_config import global_config from macaron.config.target_config import Configuration from macaron.database.database_manager import DatabaseManager, get_db_manager, get_db_session @@ -41,6 +47,7 @@ ProvenanceError, PURLNotFoundError, ) +from macaron.json_tools import json_extract from macaron.output_reporter.reporter import FileReporter from macaron.output_reporter.results import Record, Report, SCMStatus from macaron.provenance import provenance_verifier @@ -66,12 +73,14 @@ from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 from macaron.slsa_analyzer.ci_service import CI_SERVICES from macaron.slsa_analyzer.database_store import store_analyze_context_to_db -from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService +from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService, GitHub from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR -from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES +from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, MavenCentralRegistry from macaron.slsa_analyzer.provenance.expectations.expectation_registry import ExpectationRegistry from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV01Payload +from macaron.slsa_analyzer.provenance.intoto.errors import LoadIntotoAttestationError +from macaron.slsa_analyzer.provenance.loader import load_provenance_payload from macaron.slsa_analyzer.provenance.slsa import SLSAProvenanceData from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.specs.ci_spec import CIInfo @@ -414,6 +423,17 @@ def run_single( status=SCMStatus.ANALYSIS_FAILED, ) + local_artifact_dirs = None + if parsed_purl and parsed_purl.type in self.local_artifact_repo_mapper: + local_artifact_repo_path = self.local_artifact_repo_mapper[parsed_purl.type] + try: + local_artifact_dirs = get_local_artifact_dirs( + purl=parsed_purl, + local_artifact_repo_path=local_artifact_repo_path, + ) + except LocalArtifactFinderError as error: + logger.debug(error) + # Prepare the repo. git_obj = None commit_finder_outcome = CommitFinderInfo.NOT_USED @@ -491,6 +511,37 @@ def run_single( git_service = self._determine_git_service(analyze_ctx) self._determine_ci_services(analyze_ctx, git_service) self._determine_build_tools(analyze_ctx, git_service) + + # Try to find an attestation from GitHub, if applicable. + if parsed_purl and not provenance_payload and analysis_target.repo_path and isinstance(git_service, GitHub): + # Try to discover GitHub attestation for the target software component. + url = None + try: + url = urllib.parse.urlparse(analysis_target.repo_path) + except TypeError as error: + logger.debug("Failed to parse repository path as URL: %s", error) + if url and url.hostname == "github.com": + artifact_hash = self.get_artifact_hash(parsed_purl, local_artifact_dirs, hashlib.sha256()) + if artifact_hash: + git_attestation_dict = git_service.api_client.get_attestation( + analyze_ctx.component.repository.full_name, artifact_hash + ) + if git_attestation_dict: + git_attestation_list = json_extract(git_attestation_dict, ["attestations"], list) + if git_attestation_list: + git_attestation = git_attestation_list[0] + + with tempfile.TemporaryDirectory() as temp_dir: + attestation_file = os.path.join(temp_dir, "attestation") + with open(attestation_file, "w", encoding="UTF-8") as file: + json.dump(git_attestation, file) + + try: + payload = load_provenance_payload(attestation_file) + provenance_payload = payload + except LoadIntotoAttestationError as error: + logger.debug("Failed to load provenance payload: %s", error) + if parsed_purl is not None: self._verify_repository_link(parsed_purl, analyze_ctx) self._determine_package_registries(analyze_ctx, package_registries_info) @@ -556,16 +607,8 @@ def run_single( analyze_ctx.dynamic_data["analyze_source"] = analyze_source analyze_ctx.dynamic_data["force_analyze_source"] = force_analyze_source - if parsed_purl and parsed_purl.type in self.local_artifact_repo_mapper: - local_artifact_repo_path = self.local_artifact_repo_mapper[parsed_purl.type] - try: - local_artifact_dirs = get_local_artifact_dirs( - purl=parsed_purl, - local_artifact_repo_path=local_artifact_repo_path, - ) - analyze_ctx.dynamic_data["local_artifact_paths"].extend(local_artifact_dirs) - except LocalArtifactFinderError as error: - logger.debug(error) + if local_artifact_dirs: + analyze_ctx.dynamic_data["local_artifact_paths"].extend(local_artifact_dirs) analyze_ctx.check_results = registry.scan(analyze_ctx) @@ -955,6 +998,54 @@ def create_analyze_ctx(self, component: Component) -> AnalyzeContext: return analyze_ctx + def get_artifact_hash( + self, purl: PackageURL, cached_artifacts: list[str] | None, hash_algorithm: Any + ) -> str | None: + """Get the hash of the artifact found from the passed PURL using local or remote files. + + Parameters + ---------- + purl: PackageURL + The PURL of the artifact. + cached_artifacts: list[str] | None + The list of local files that match the PURL. + hash_algorithm: Any + The hash algorithm to use. + + Returns + ------- + str | None + The hash of the artifact, or None if not found. + """ + if cached_artifacts: + # Try to get the hash from a local file. + artifact_hash = get_local_artifact_hash(purl, cached_artifacts, hash_algorithm.name) + + if artifact_hash: + return artifact_hash + + # Download the artifact. + if purl.type == "maven": + maven_registry = next( + ( + package_registry + for package_registry in PACKAGE_REGISTRIES + if isinstance(package_registry, MavenCentralRegistry) + ), + None, + ) + if not maven_registry: + return None + + return maven_registry.get_artifact_hash(purl, hash_algorithm) + + if purl.type == "pypi": + # TODO implement + return None + + logger.debug("Purl type '%s' not yet supported for GitHub attestation discovery.", purl.type) + return None + def _determine_git_service(self, analyze_ctx: AnalyzeContext) -> BaseGitService: """Determine the Git service used by the software component.""" remote_path = analyze_ctx.component.repository.remote_path if analyze_ctx.component.repository else None diff --git a/src/macaron/slsa_analyzer/git_service/api_client.py b/src/macaron/slsa_analyzer/git_service/api_client.py index 8e987e6ca..681a1f4e0 100644 --- a/src/macaron/slsa_analyzer/git_service/api_client.py +++ b/src/macaron/slsa_analyzer/git_service/api_client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The module provides API clients for VCS services, such as GitHub.""" @@ -659,6 +659,25 @@ def download_asset(self, url: str, download_path: str) -> bool: return True + def get_attestation(self, full_name: str, artifact_hash: str) -> dict: + """Download and return the attestation associated with the passed artifact hash, if any. + + Parameters + ---------- + full_name : str + The full name of the repo. + artifact_hash: str + The SHA256 hash of an artifact. + + Returns + ------- + dict + The attestation data, or an empty dict if not found. + """ + url = f"{GhAPIClient._REPO_END_POINT}/{full_name}/attestations/sha256:{artifact_hash}" + response_data = send_get_http(url, self.headers) + return response_data or {} + def get_default_gh_client(access_token: str) -> GhAPIClient: """Return a GhAPIClient instance with default values. diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 131051b66..593c15b88 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -6,10 +6,13 @@ import logging import urllib.parse from datetime import datetime, timezone +from typing import Any import requests from packageurl import PackageURL +from requests import RequestException +from macaron.artifact.maven import construct_maven_repository_path from macaron.config.defaults import defaults from macaron.errors import ConfigurationError, InvalidHTTPResponseError from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry @@ -236,3 +239,72 @@ def find_publish_timestamp(self, purl: str) -> datetime: raise InvalidHTTPResponseError(f"The timestamp returned by {url} is invalid") from error raise InvalidHTTPResponseError(f"Invalid response from Maven central for {url}.") + + @staticmethod + def get_artifact_file_name(purl: PackageURL) -> str | None: + """Return the artifact file name of the passed PURL based on the Maven registry standard. + + Parameters + ---------- + purl: PackageURL + The PURL of the artifact. + + Returns + ------- + str | None + The artifact file name, or None if invalid. + """ + if not purl.version: + return None + + return purl.name + "-" + purl.version + ".jar" + + def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None: + """Return the hash of the artifact found by the passed purl relevant to the registry's URL. + + Parameters + ---------- + purl: PackageURL + The purl of the artifact. + hash_algorithm: Any + The hash algorithm to use. + + Returns + ------- + str | None + The hash of the artifact, or None if not found. + """ + if not (purl.namespace and purl.version): + return None + + artifact_path = construct_maven_repository_path(purl.namespace, purl.name, purl.version) + file_name = MavenCentralRegistry.get_artifact_file_name(purl) + if not file_name: + return None + + artifact_url = self.registry_url + "/" + artifact_path + "/" + file_name + logger.debug("Search for artifact using URL: %s", artifact_url) + + try: + response = requests.get(artifact_url, stream=True, timeout=40) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + logger.debug("HTTP error occurred: %s", http_err) + return None + + if response.status_code != 200: + return None + + # Download file and compute hash as chunks are received. + try: + for chunk in response.iter_content(): + hash_algorithm.update(chunk) + except RequestException as error: + # Something went wrong with the request, abort. + logger.debug("Error while streaming target file: %s", error) + response.close() + return None + + artifact_hash: str = hash_algorithm.hexdigest() + logger.debug("Computed hash of artifact: %s", artifact_hash) + return artifact_hash diff --git a/src/macaron/slsa_analyzer/provenance/loader.py b/src/macaron/slsa_analyzer/provenance/loader.py index 19c256315..a46d8d8ef 100644 --- a/src/macaron/slsa_analyzer/provenance/loader.py +++ b/src/macaron/slsa_analyzer/provenance/loader.py @@ -101,6 +101,10 @@ def _load_provenance_file_content( if not provenance_payload: # PyPI Attestation. provenance_payload = json_extract(provenance, ["envelope", "statement"], str) + if not provenance_payload: + # GitHub Attestation. + # TODO Check if old method (above) actually works. + provenance_payload = json_extract(provenance, ["bundle", "dsseEnvelope", "payload"], str) if not provenance_payload: raise LoadIntotoAttestationError( 'Cannot find the "payload" field in the decoded provenance.', diff --git a/tests/integration/cases/github_maven_attestation/policy.dl b/tests/integration/cases/github_maven_attestation/policy.dl new file mode 100644 index 000000000..9df46219b --- /dev/null +++ b/tests/integration/cases/github_maven_attestation/policy.dl @@ -0,0 +1,10 @@ +/* Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_provenance_available_1"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:maven/io.liftwizard/liftwizard-checkstyle@2.1.22"). diff --git a/tests/integration/cases/github_maven_attestation/test.yaml b/tests/integration/cases/github_maven_attestation/test.yaml new file mode 100644 index 000000000..9913d930e --- /dev/null +++ b/tests/integration/cases/github_maven_attestation/test.yaml @@ -0,0 +1,22 @@ +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Discovering attestation of a Maven artifact on GitHub + +tags: +- macaron-python-package + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/io.liftwizard/liftwizard-checkstyle@2.1.22 + - -rp + - https://github.com/liftwizard/liftwizard +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl diff --git a/tests/integration/cases/github_maven_attestation_local/policy.dl b/tests/integration/cases/github_maven_attestation_local/policy.dl new file mode 100644 index 000000000..ff31abf90 --- /dev/null +++ b/tests/integration/cases/github_maven_attestation_local/policy.dl @@ -0,0 +1,10 @@ +/* Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_failed(component_id, "mcn_provenance_available_1"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:maven/io.liftwizard/liftwizard-checkstyle@2.1.22"). diff --git a/tests/integration/cases/github_maven_attestation_local/test.yaml b/tests/integration/cases/github_maven_attestation_local/test.yaml new file mode 100644 index 000000000..d66a089b2 --- /dev/null +++ b/tests/integration/cases/github_maven_attestation_local/test.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Discovering GitHub attestation of a local Maven artifact but failing because the artifact is wrong + +tags: +- macaron-python-package + +steps: +- name: Download artifact POM instead of the JAR + kind: shell + options: + cmd: curl --create-dirs -o ./output/.m2/repository/io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.jar https://repo1.maven.org/maven2/io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.pom +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/io.liftwizard/liftwizard-checkstyle@2.1.22 + - -rp + - https://github.com/liftwizard/liftwizard + - --local-maven-repo + - ./output/.m2 +- name: Run macaron verify-policy to verify no provenance was found + kind: verify + options: + policy: policy.dl From d960f66fad1fddc8691ff48211330e90e49fe8ef Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 19 Mar 2025 20:11:07 +1000 Subject: [PATCH 02/16] chore: add support for PyPI PURLs Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 54 ++++++++++-- .../checks/detect_malicious_metadata_check.py | 34 +++----- .../package_registry/pypi_registry.py | 87 ++++++++++++++++++- 3 files changed, 146 insertions(+), 29 deletions(-) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index c5cfbd70b..10f2cc379 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -76,7 +76,8 @@ from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService, GitHub from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR -from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, MavenCentralRegistry +from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, MavenCentralRegistry, PyPIRegistry +from macaron.slsa_analyzer.package_registry.pypi_registry import find_or_create_pypi_asset from macaron.slsa_analyzer.provenance.expectations.expectation_registry import ExpectationRegistry from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV01Payload from macaron.slsa_analyzer.provenance.intoto.errors import LoadIntotoAttestationError @@ -521,7 +522,9 @@ def run_single( except TypeError as error: logger.debug("Failed to parse repository path as URL: %s", error) if url and url.hostname == "github.com": - artifact_hash = self.get_artifact_hash(parsed_purl, local_artifact_dirs, hashlib.sha256()) + artifact_hash = self.get_artifact_hash( + parsed_purl, local_artifact_dirs, hashlib.sha256(), all_package_registries + ) if artifact_hash: git_attestation_dict = git_service.api_client.get_attestation( analyze_ctx.component.repository.full_name, artifact_hash @@ -999,7 +1002,11 @@ def create_analyze_ctx(self, component: Component) -> AnalyzeContext: return analyze_ctx def get_artifact_hash( - self, purl: PackageURL, cached_artifacts: list[str] | None, hash_algorithm: Any + self, + purl: PackageURL, + cached_artifacts: list[str] | None, + hash_algorithm: Any, + all_package_registries: list[PackageRegistryInfo], ) -> str | None: """Get the hash of the artifact found from the passed PURL using local or remote files. @@ -1011,6 +1018,8 @@ def get_artifact_hash( The list of local files that match the PURL. hash_algorithm: Any The hash algorithm to use. + all_package_registries: list[PackageRegistryInfo] + The list of package registry information. Returns ------- @@ -1040,8 +1049,43 @@ def get_artifact_hash( return maven_registry.get_artifact_hash(purl, hash_algorithm) if purl.type == "pypi": - # TODO implement - return None + pypi_registry = next( + ( + package_registry + for package_registry in PACKAGE_REGISTRIES + if isinstance(package_registry, PyPIRegistry) + ), + None, + ) + if not pypi_registry: + logger.debug("Missing registry for PyPI") + return None + + registry_info = next( + ( + info + for info in all_package_registries + if info.package_registry == pypi_registry and info.build_tool_name in {"pip", "poetry"} + ), + None, + ) + if not registry_info: + logger.debug("Missing registry information for PyPI") + return None + + pypi_asset = find_or_create_pypi_asset(purl.name, purl.version, registry_info) + if not pypi_asset: + return None + + pypi_asset.has_repository = True + if not pypi_asset.download(""): + return None + + source_url = pypi_asset.get_sourcecode_url("bdist_wheel") + if not source_url: + return None + + return pypi_registry.get_artifact_hash(source_url, hash_algorithm) logger.debug("Purl type '%s' not yet supported for GitHub attestation discovery.", purl.type) return None diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index 4358a076b..450cc44d5 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -34,6 +34,11 @@ from macaron.slsa_analyzer.package_registry.deps_dev import APIAccessError, DepsDevService from macaron.slsa_analyzer.package_registry.osv_dev import OSVDevService from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIPackageJsonAsset, PyPIRegistry +from macaron.slsa_analyzer.package_registry.pypi_registry import ( + PyPIPackageJsonAsset, + PyPIRegistry, + find_or_create_pypi_asset, +) from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo @@ -279,29 +284,16 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: case PackageRegistryInfo( build_tool_name="pip" | "poetry", build_tool_purl_type="pypi", - package_registry=PyPIRegistry() as pypi_registry, + package_registry=PyPIRegistry(), ) as pypi_registry_info: - # Retrieve the pre-existing AssetLocator object for the PyPI package JSON object, if it exists. - pypi_package_json = next( - ( - asset - for asset in pypi_registry_info.metadata - if isinstance(asset, PyPIPackageJsonAsset) - and asset.component_name == ctx.component.name - and asset.component_version == ctx.component.version - ), - None, + # Retrieve the pre-existing asset, or create a new one. + pypi_package_json = find_or_create_pypi_asset( + ctx.component.name, ctx.component.version, pypi_registry_info ) - if not pypi_package_json: - # Create an AssetLocator object for the PyPI package JSON object. - pypi_package_json = PyPIPackageJsonAsset( - component_name=ctx.component.name, - component_version=ctx.component.version, - has_repository=ctx.component.repository is not None, - pypi_registry=pypi_registry, - package_json={}, - package_sourcecode_path="", - ) + if pypi_package_json is None: + return CheckResultData(result_tables=[], result_type=CheckResultType.UNKNOWN) + + pypi_package_json.has_repository = ctx.component.repository is not None pypi_registry_info.metadata.append(pypi_package_json) diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 2c6af515c..0a2e2cdfd 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -14,6 +14,7 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime +from typing import Any import requests from bs4 import BeautifulSoup, Tag @@ -24,6 +25,7 @@ from macaron.json_tools import json_extract from macaron.malware_analyzer.datetime_parser import parse_datetime from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry +from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo from macaron.util import send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -263,6 +265,45 @@ def download_package_sourcecode(self, url: str) -> str: logger.debug("Temporary download and unzip of %s stored in %s", file_name, temp_dir) return temp_dir + def get_artifact_hash(self, artifact_url: str, hash_algorithm: Any) -> str | None: + """Return the hash of the artifact found at the passed URL. + + Parameters + ---------- + artifact_url + The URL of the artifact. + hash_algorithm: Any + The hash algorithm to use. + + Returns + ------- + str | None + The hash of the artifact, or None if not found. + """ + try: + response = requests.get(artifact_url, stream=True, timeout=40) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + logger.debug("HTTP error occurred: %s", http_err) + return None + + if response.status_code != 200: + logger.debug("Invalid response: %s", response.status_code) + return None + + try: + for chunk in response.iter_content(): + hash_algorithm.update(chunk) + except RequestException as error: + # Something went wrong with the request, abort. + logger.debug("Error while streaming source file: %s", error) + response.close() + return None + + artifact_hash: str = hash_algorithm.hexdigest() + logger.debug("Computed artifact hash: %s", artifact_hash) + return artifact_hash + def get_package_page(self, package_name: str) -> str | None: """Implement custom API to get package main page. @@ -499,15 +540,19 @@ def get_latest_version(self) -> str | None: """ return json_extract(self.package_json, ["info", "version"], str) - def get_sourcecode_url(self) -> str | None: + def get_sourcecode_url(self, package_type: str = "sdist") -> str | None: """Get the url of the source distribution. + Parameters + ---------- + package_type: str + The package type to retrieve the URL of. + Returns ------- str | None The URL of the source distribution. """ - urls: list | None = None if self.component_version: urls = json_extract(self.package_json, ["releases", self.component_version], list) else: @@ -516,7 +561,7 @@ def get_sourcecode_url(self) -> str | None: if not urls: return None for distribution in urls: - if distribution.get("packagetype") != "sdist": + if distribution.get("packagetype") != package_type: continue # We intentionally check if the url is None and use empty string if that's the case. source_url: str = distribution.get("url") or "" @@ -670,3 +715,39 @@ def iter_sourcecode(self) -> Iterator[tuple[str, bytes]]: contents = handle.read() yield filepath, contents + + +def find_or_create_pypi_asset( + asset_name: str, asset_version: str | None, pypi_registry_info: PackageRegistryInfo +) -> PyPIPackageJsonAsset | None: + """Find the asset in the provided package registry information, or create it. + + Parameters + ---------- + asset_name: str + The name of the asset. + asset_version: str | None + The version of the asset. + pypi_registry_info: + The package registry information. + + Returns + ------- + PyPIPackageJsonAsset | None + The asset, or None if not found. + """ + pypi_package_json = next( + (asset for asset in pypi_registry_info.metadata if isinstance(asset, PyPIPackageJsonAsset)), + None, + ) + if pypi_package_json: + return pypi_package_json + + package_registry = pypi_registry_info.package_registry + if not isinstance(package_registry, PyPIRegistry): + logger.debug("Failed to create PyPIPackageJson asset.") + return None + + asset = PyPIPackageJsonAsset(asset_name, asset_version, False, package_registry, {}, "") + pypi_registry_info.metadata.append(asset) + return asset From 79ec896423395e228573af2cf58dcf19ccaad258 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 31 Mar 2025 10:31:46 +1000 Subject: [PATCH 03/16] chore: add support for sha256 hashes in maven Signed-off-by: Ben Selwyn-Smith --- .../package_registry/maven_central_registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 593c15b88..2ad0cbf1e 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -282,8 +282,15 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None if not file_name: return None + # Maven supports but does not require a sha256 hash of uploaded artifacts. Check that first. artifact_url = self.registry_url + "/" + artifact_path + "/" + file_name - logger.debug("Search for artifact using URL: %s", artifact_url) + sha256_url = artifact_url + ".sha256" + logger.debug("Search for artifact hash using URL: %s", [sha256_url, artifact_url]) + + response = send_get_http_raw(sha256_url, {}) + if response and response.text: + logger.debug("Found hash of artifact: %s", response.text) + return response.text try: response = requests.get(artifact_url, stream=True, timeout=40) From ebfaf89a2808468066bdc3d0f08d88c355f9aa93 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 31 Mar 2025 11:09:50 +1000 Subject: [PATCH 04/16] chore: add pypi sha256 support Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 4 ++++ .../package_registry/pypi_registry.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 10f2cc379..09b9d3e26 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -1081,6 +1081,10 @@ def get_artifact_hash( if not pypi_asset.download(""): return None + artifact_hash = pypi_asset.get_sha256() + if artifact_hash: + return artifact_hash + source_url = pypi_asset.get_sourcecode_url("bdist_wheel") if not source_url: return None diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 0a2e2cdfd..9a0294ff0 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -716,6 +716,26 @@ def iter_sourcecode(self) -> Iterator[tuple[str, bytes]]: yield filepath, contents + def get_sha256(self) -> str | None: + """Get the sha256 hash of the artifact from its payload. + + Returns + ------- + str | None + The sha256 hash of the artifact, or None if not found. + """ + if not self.package_json and not self.download(""): + return None + + if not self.component_version: + artifact_hash = json_extract(self.package_json, ["urls", 0, "digests", "sha256"], str) + else: + artifact_hash = json_extract( + self.package_json, ["releases", self.component_version, "digests", "sha256"], str + ) + logger.debug("Found sha256 hash: %s", artifact_hash) + return artifact_hash + def find_or_create_pypi_asset( asset_name: str, asset_version: str | None, pypi_registry_info: PackageRegistryInfo From 3408034648865a1348a2f495c32c8b748daab11e Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 28 Apr 2025 10:17:48 +1000 Subject: [PATCH 05/16] chore: use maven hashes as reference only; add pypi integration test Signed-off-by: Ben Selwyn-Smith --- .../maven_central_registry.py | 13 ++++++++---- .../cases/github_pypi_attestation/policy.dl | 10 ++++++++++ .../cases/github_pypi_attestation/test.yaml | 20 +++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/integration/cases/github_pypi_attestation/policy.dl create mode 100644 tests/integration/cases/github_pypi_attestation/test.yaml diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 2ad0cbf1e..958946d35 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -282,15 +282,16 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None if not file_name: return None - # Maven supports but does not require a sha256 hash of uploaded artifacts. Check that first. + # Maven supports but does not require a sha256 hash of uploaded artifacts. artifact_url = self.registry_url + "/" + artifact_path + "/" + file_name sha256_url = artifact_url + ".sha256" logger.debug("Search for artifact hash using URL: %s", [sha256_url, artifact_url]) response = send_get_http_raw(sha256_url, {}) - if response and response.text: - logger.debug("Found hash of artifact: %s", response.text) - return response.text + sha256_hash = None + if response and (sha256_hash := response.text): + # As Maven hashes are user provided and not verified they serve as a reference only. + logger.debug("Found hash of artifact: %s", sha256_hash) try: response = requests.get(artifact_url, stream=True, timeout=40) @@ -313,5 +314,9 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None return None artifact_hash: str = hash_algorithm.hexdigest() + if sha256_hash and artifact_hash != sha256_hash: + logger.debug("Artifact hash and discovered hash do not match: %s != %s", artifact_hash, sha256_hash) + return None + logger.debug("Computed hash of artifact: %s", artifact_hash) return artifact_hash diff --git a/tests/integration/cases/github_pypi_attestation/policy.dl b/tests/integration/cases/github_pypi_attestation/policy.dl new file mode 100644 index 000000000..26cef7913 --- /dev/null +++ b/tests/integration/cases/github_pypi_attestation/policy.dl @@ -0,0 +1,10 @@ +/* Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_provenance_available_1"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:pypi/toga@0.5.0"). diff --git a/tests/integration/cases/github_pypi_attestation/test.yaml b/tests/integration/cases/github_pypi_attestation/test.yaml new file mode 100644 index 000000000..173361662 --- /dev/null +++ b/tests/integration/cases/github_pypi_attestation/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Discovering attestation of a PyPI artifact on GitHub + +tags: +- macaron-python-package + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:pypi/toga@0.5.0 +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 0c402b4ce16af49f62d61203dd6c29d1af342555 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 28 Apr 2025 12:46:42 +1000 Subject: [PATCH 06/16] chore: minor fixes Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 8 +- .../checks/detect_malicious_metadata_check.py | 1 - .../maven_central_registry.py | 6 +- .../test_maven_central_registry.py | 128 +++++++++++++----- 4 files changed, 99 insertions(+), 44 deletions(-) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 09b9d3e26..eb40828bb 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -523,7 +523,7 @@ def run_single( logger.debug("Failed to parse repository path as URL: %s", error) if url and url.hostname == "github.com": artifact_hash = self.get_artifact_hash( - parsed_purl, local_artifact_dirs, hashlib.sha256(), all_package_registries + parsed_purl, local_artifact_dirs, hashlib.sha256(), package_registries_info ) if artifact_hash: git_attestation_dict = git_service.api_client.get_attestation( @@ -1006,7 +1006,7 @@ def get_artifact_hash( purl: PackageURL, cached_artifacts: list[str] | None, hash_algorithm: Any, - all_package_registries: list[PackageRegistryInfo], + package_registries_info: list[PackageRegistryInfo], ) -> str | None: """Get the hash of the artifact found from the passed PURL using local or remote files. @@ -1018,7 +1018,7 @@ def get_artifact_hash( The list of local files that match the PURL. hash_algorithm: Any The hash algorithm to use. - all_package_registries: list[PackageRegistryInfo] + package_registries_info: list[PackageRegistryInfo] The list of package registry information. Returns @@ -1064,7 +1064,7 @@ def get_artifact_hash( registry_info = next( ( info - for info in all_package_registries + for info in package_registries_info if info.package_registry == pypi_registry and info.build_tool_name in {"pip", "poetry"} ), None, diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index 450cc44d5..2157fbe81 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -33,7 +33,6 @@ from macaron.slsa_analyzer.checks.check_result import CheckResultData, CheckResultType, Confidence, JustificationType from macaron.slsa_analyzer.package_registry.deps_dev import APIAccessError, DepsDevService from macaron.slsa_analyzer.package_registry.osv_dev import OSVDevService -from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIPackageJsonAsset, PyPIRegistry from macaron.slsa_analyzer.package_registry.pypi_registry import ( PyPIPackageJsonAsset, PyPIRegistry, diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 958946d35..bc419f921 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -274,15 +274,15 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None str | None The hash of the artifact, or None if not found. """ - if not (purl.namespace and purl.version): + if not purl.namespace: return None - artifact_path = construct_maven_repository_path(purl.namespace, purl.name, purl.version) file_name = MavenCentralRegistry.get_artifact_file_name(purl) - if not file_name: + if not (purl.version and file_name): return None # Maven supports but does not require a sha256 hash of uploaded artifacts. + artifact_path = construct_maven_repository_path(purl.namespace, purl.name, purl.version) artifact_url = self.registry_url + "/" + artifact_path + "/" + file_name sha256_url = artifact_url + ".sha256" logger.debug("Search for artifact hash using URL: %s", [sha256_url, artifact_url]) diff --git a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py index 62b9fdca0..2feffae96 100644 --- a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py +++ b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py @@ -7,11 +7,14 @@ import os import urllib.parse from datetime import datetime +from hashlib import sha256 from pathlib import Path import pytest +from packageurl import PackageURL from pytest_httpserver import HTTPServer +from macaron.artifact.maven import construct_maven_repository_path from macaron.config.defaults import load_defaults from macaron.errors import ConfigurationError, InvalidHTTPResponseError from macaron.slsa_analyzer.package_registry.maven_central_registry import MavenCentralRegistry @@ -35,6 +38,28 @@ def maven_central_instance() -> MavenCentralRegistry: ) +@pytest.fixture(name="maven_service") +def maven_service_(httpserver: HTTPServer, tmp_path: Path) -> None: + """Set up the Maven httpserver.""" + base_url_parsed = urllib.parse.urlparse(httpserver.url_for("")) + + user_config_input = f""" + [package_registry.maven_central] + request_timeout = 20 + search_netloc = {base_url_parsed.netloc} + search_scheme = {base_url_parsed.scheme} + registry_url_netloc = {base_url_parsed.netloc} + registry_url_scheme = {base_url_parsed.scheme} + """ + user_config_path = os.path.join(tmp_path, "config.ini") + with open(user_config_path, "w", encoding="utf-8") as user_config_file: + user_config_file.write(user_config_input) + # We don't have to worry about modifying the ``defaults`` object causing test + # pollution here, since we reload the ``defaults`` object before every test with the + # ``setup_test`` fixture. + load_defaults(user_config_path) + + def test_load_defaults(tmp_path: Path) -> None: """Test the ``load_defaults`` method.""" user_config_path = os.path.join(tmp_path, "config.ini") @@ -150,31 +175,14 @@ def test_is_detected( def test_find_publish_timestamp( resources_path: Path, httpserver: HTTPServer, - tmp_path: Path, + maven_service: dict, # pylint: disable=unused-argument purl: str, mc_json_path: str, query_string: str, expected_timestamp: str, ) -> None: """Test that the function finds the timestamp correctly.""" - base_url_parsed = urllib.parse.urlparse(httpserver.url_for("")) - maven_central = MavenCentralRegistry() - - # Set up responses of solrsearch endpoints using the httpserver plugin. - user_config_input = f""" - [package_registry.maven_central] - request_timeout = 20 - search_netloc = {base_url_parsed.netloc} - search_scheme = {base_url_parsed.scheme} - """ - user_config_path = os.path.join(tmp_path, "config.ini") - with open(user_config_path, "w", encoding="utf-8") as user_config_file: - user_config_file.write(user_config_input) - # We don't have to worry about modifying the ``defaults`` object causing test - # pollution here, since we reload the ``defaults`` object before every test with the - # ``setup_test`` fixture. - load_defaults(user_config_path) maven_central.load_defaults() with open(os.path.join(resources_path, "maven_central_files", mc_json_path), encoding="utf8") as page: @@ -208,35 +216,19 @@ def test_find_publish_timestamp( def test_find_publish_timestamp_errors( resources_path: Path, httpserver: HTTPServer, - tmp_path: Path, + maven_service: dict, # pylint: disable=unused-argument purl: str, mc_json_path: str, expected_msg: str, ) -> None: """Test that the function handles errors correctly.""" - base_url_parsed = urllib.parse.urlparse(httpserver.url_for("")) - maven_central = MavenCentralRegistry() - - # Set up responses of solrsearch endpoints using the httpserver plugin. - user_config_input = f""" - [package_registry.maven_central] - request_timeout = 20 - search_netloc = {base_url_parsed.netloc} - search_scheme = {base_url_parsed.scheme} - """ - user_config_path = os.path.join(tmp_path, "config.ini") - with open(user_config_path, "w", encoding="utf-8") as user_config_file: - user_config_file.write(user_config_input) - # We don't have to worry about modifying the ``defaults`` object causing test - # pollution here, since we reload the ``defaults`` object before every test with the - # ``setup_test`` fixture. - load_defaults(user_config_path) maven_central.load_defaults() with open(os.path.join(resources_path, "maven_central_files", mc_json_path), encoding="utf8") as page: mc_json_response = json.load(page) + # Set up responses of solrsearch endpoints using the httpserver plugin. httpserver.expect_request( "/solrsearch/select", query_string="q=g:org.apache.logging.log4j+AND+a:log4j-core+AND+v:3.0.0-beta2&core=gav&rows=1&wt=json", @@ -245,3 +237,67 @@ def test_find_publish_timestamp_errors( pat = f"^{expected_msg}" with pytest.raises(InvalidHTTPResponseError, match=pat): maven_central.find_publish_timestamp(purl=purl) + + +def test_get_artifact_file_name() -> None: + """Test the artifact file name function.""" + assert not MavenCentralRegistry().get_artifact_file_name(PackageURL.from_string("pkg:maven/test/example")) + + assert ( + MavenCentralRegistry().get_artifact_file_name(PackageURL.from_string("pkg:maven/text/example@1")) + == "example-1.jar" + ) + + +@pytest.mark.parametrize("purl_string", ["pkg:maven/example", "pkg:maven/example/test", "pkg:maven/example/test@1"]) +def test_get_artifact_hash_failures( + httpserver: HTTPServer, maven_service: dict, purl_string: str # pylint: disable=unused-argument +) -> None: + """Test failures of get artifact hash.""" + purl = PackageURL.from_string(purl_string) + + maven_registry = MavenCentralRegistry() + maven_registry.load_defaults() + + if ( + purl.namespace + and purl.version + and (file_name := MavenCentralRegistry().get_artifact_file_name(purl)) + and file_name + ): + artifact_path = "/" + construct_maven_repository_path(purl.namespace, purl.name, purl.version) + "/" + file_name + hash_algorithm = sha256() + hash_algorithm.update(b"example_data") + expected_hash = hash_algorithm.hexdigest() + httpserver.expect_request(artifact_path + ".sha256").respond_with_data(expected_hash) + httpserver.expect_request(artifact_path).respond_with_data(b"example_data_2") + + result = maven_registry.get_artifact_hash(purl, sha256()) + + assert not result + + +def test_get_artifact_hash_success( + httpserver: HTTPServer, maven_service: dict # pylint: disable=unused-argument +) -> None: + """Test success of get artifact hash.""" + purl = PackageURL.from_string("pkg:maven/example/test@1") + assert purl.namespace + assert purl.version + + maven_registry = MavenCentralRegistry() + maven_registry.load_defaults() + + file_name = MavenCentralRegistry().get_artifact_file_name(purl) + assert file_name + + artifact_path = "/" + construct_maven_repository_path(purl.namespace, purl.name, purl.version) + "/" + file_name + hash_algorithm = sha256() + hash_algorithm.update(b"example_data") + expected_hash = hash_algorithm.hexdigest() + httpserver.expect_request(artifact_path + ".sha256").respond_with_data(expected_hash) + httpserver.expect_request(artifact_path).respond_with_data(b"example_data") + + result = maven_registry.get_artifact_hash(purl, sha256()) + + assert result From 8222e303fa8035c085642b4f35cafbc31eb5b509 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 19 May 2025 15:29:37 +1000 Subject: [PATCH 07/16] chore: use correct path to sha256 hash for versioned pypi asset Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/package_registry/pypi_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 9a0294ff0..6dd418729 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -731,7 +731,7 @@ def get_sha256(self) -> str | None: artifact_hash = json_extract(self.package_json, ["urls", 0, "digests", "sha256"], str) else: artifact_hash = json_extract( - self.package_json, ["releases", self.component_version, "digests", "sha256"], str + self.package_json, ["releases", self.component_version, 0, "digests", "sha256"], str ) logger.debug("Found sha256 hash: %s", artifact_hash) return artifact_hash From 73b8587c9c1bcb200b1d9a9360da70ad66144970 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 30 May 2025 16:03:32 +1000 Subject: [PATCH 08/16] chore: address PR feedback Signed-off-by: Ben Selwyn-Smith --- src/macaron/artifact/local_artifact.py | 15 ++-- src/macaron/artifact/maven.py | 21 +++++- .../sourcecode/suspicious_setup.py | 2 +- src/macaron/repo_finder/repo_finder_pypi.py | 32 +++------ src/macaron/slsa_analyzer/analyzer.py | 66 ++++++++---------- .../maven_central_registry.py | 69 +++++++++---------- .../package_registry/pypi_registry.py | 29 ++++---- tests/artifact/test_maven.py | 15 +++- .../github_maven_attestation_local/test.yaml | 4 +- .../cases/github_pypi_attestation/test.yaml | 2 +- .../test_maven_central_registry.py | 25 ++----- 11 files changed, 137 insertions(+), 143 deletions(-) diff --git a/src/macaron/artifact/local_artifact.py b/src/macaron/artifact/local_artifact.py index 582799824..46772e970 100644 --- a/src/macaron/artifact/local_artifact.py +++ b/src/macaron/artifact/local_artifact.py @@ -11,9 +11,8 @@ from packageurl import PackageURL -from macaron.artifact.maven import construct_maven_repository_path +from macaron.artifact.maven import construct_maven_repository_path, construct_primary_jar_file_name from macaron.errors import LocalArtifactFinderError -from macaron.slsa_analyzer.package_registry import MavenCentralRegistry logger: logging.Logger = logging.getLogger(__name__) @@ -254,7 +253,7 @@ def get_local_artifact_dirs( raise LocalArtifactFinderError(f"Unsupported PURL type {purl_type}") -def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str], hash_algorithm_name: str) -> str | None: +def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str]) -> str | None: """Compute the hash of the local artifact. Parameters @@ -262,9 +261,7 @@ def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str], hash_alg purl: PackageURL The PURL of the artifact being sought. artifact_dirs: list[str] - The possible locations of the artifact. - hash_algorithm_name: str - The hash algorithm to use. + The list of directories that may contain the artifact file. Returns ------- @@ -281,7 +278,9 @@ def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str], hash_alg artifact_target = None if purl.type == "maven": - artifact_target = MavenCentralRegistry.get_artifact_file_name(purl) + artifact_target = construct_primary_jar_file_name(purl) + + # TODO add support for other PURL types here. if not artifact_target: logger.debug("PURL type not supported: %s", purl.type) @@ -294,7 +293,7 @@ def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str], hash_alg with open(full_path, "rb") as file: try: - hash_result = hashlib.file_digest(file, hash_algorithm_name) + hash_result = hashlib.file_digest(file, "sha256") except ValueError as error: logger.debug("Error while hashing file: %s", error) continue diff --git a/src/macaron/artifact/maven.py b/src/macaron/artifact/maven.py index dd97431f7..8b9b0721c 100644 --- a/src/macaron/artifact/maven.py +++ b/src/macaron/artifact/maven.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module declares types and utilities for Maven artifacts.""" @@ -196,3 +196,22 @@ def construct_maven_repository_path( if asset_name: path = "/".join([path, asset_name]) return path + + +def construct_primary_jar_file_name(purl: PackageURL) -> str | None: + """Return the name of the primary JAR for the passed PURL based on the Maven registry standard. + + Parameters + ---------- + purl: PackageURL + The PURL of the artifact. + + Returns + ------- + str | None + The artifact file name, or None if invalid. + """ + if not purl.version: + return None + + return purl.name + "-" + purl.version + ".jar" diff --git a/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py b/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py index 89d1909a3..ebde2a21f 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py @@ -59,7 +59,7 @@ def _get_setup_source_code(self, pypi_package_json: PyPIPackageJsonAsset) -> str response = requests.get(sourcecode_url, stream=True, timeout=40) response.raise_for_status() except requests.exceptions.HTTPError as http_err: - logger.debug("HTTP error occurred: %s", http_err) + logger.debug("HTTP error occurred when trying to download source: %s", http_err) return None if response.status_code != 200: diff --git a/src/macaron/repo_finder/repo_finder_pypi.py b/src/macaron/repo_finder/repo_finder_pypi.py index b11f9cbe6..db59becee 100644 --- a/src/macaron/repo_finder/repo_finder_pypi.py +++ b/src/macaron/repo_finder/repo_finder_pypi.py @@ -8,8 +8,8 @@ from macaron.repo_finder.repo_finder_enums import RepoFinderInfo from macaron.repo_finder.repo_validator import find_valid_repository_url -from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, PyPIRegistry -from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIPackageJsonAsset +from macaron.slsa_analyzer.package_registry import PyPIRegistry +from macaron.slsa_analyzer.package_registry.pypi_registry import find_or_create_pypi_asset from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo logger: logging.Logger = logging.getLogger(__name__) @@ -45,37 +45,21 @@ def find_repo( None, ) - if not pypi_info or not isinstance(pypi_info.package_registry, PyPIRegistry): - pypi_registry = next((registry for registry in PACKAGE_REGISTRIES if isinstance(registry, PyPIRegistry)), None) - else: - pypi_registry = pypi_info.package_registry - - if not pypi_registry: - logger.debug("PyPI package registry not available.") + if not pypi_info: return "", RepoFinderInfo.PYPI_NO_REGISTRY - pypi_asset = None - from_metadata = False - if pypi_info: - for existing_asset in pypi_info.metadata: - if not isinstance(existing_asset, PyPIPackageJsonAsset): - continue + if not purl.version: + return "", RepoFinderInfo.NO_VERSION_PROVIDED - if existing_asset.component_name == purl.name: - pypi_asset = existing_asset - from_metadata = True - break + pypi_asset = find_or_create_pypi_asset(purl.name, purl.version, pypi_info) if not pypi_asset: - pypi_asset = PyPIPackageJsonAsset(purl.name, purl.version, False, pypi_registry, {}, "") + # This should be unreachable, as the pypi_registry has already been confirmed to be of type PyPIRegistry. + return "", RepoFinderInfo.PYPI_NO_REGISTRY if not pypi_asset.package_json and not pypi_asset.download(dest=""): return "", RepoFinderInfo.PYPI_HTTP_ERROR - if not from_metadata and pypi_info: - # Save the asset for later use. - pypi_info.metadata.append(pypi_asset) - url_dict = pypi_asset.get_project_links() if not url_dict: return "", RepoFinderInfo.PYPI_JSON_ERROR diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index eb40828bb..655346010 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -4,14 +4,12 @@ """This module handles the cloning and analyzing a Git repo.""" import glob -import hashlib import json import logging import os import re import sys import tempfile -import urllib.parse from collections.abc import Mapping from datetime import datetime, timezone from pathlib import Path @@ -516,34 +514,26 @@ def run_single( # Try to find an attestation from GitHub, if applicable. if parsed_purl and not provenance_payload and analysis_target.repo_path and isinstance(git_service, GitHub): # Try to discover GitHub attestation for the target software component. - url = None - try: - url = urllib.parse.urlparse(analysis_target.repo_path) - except TypeError as error: - logger.debug("Failed to parse repository path as URL: %s", error) - if url and url.hostname == "github.com": - artifact_hash = self.get_artifact_hash( - parsed_purl, local_artifact_dirs, hashlib.sha256(), package_registries_info + artifact_hash = self.get_artifact_hash(parsed_purl, local_artifact_dirs, package_registries_info) + if artifact_hash: + git_attestation_dict = git_service.api_client.get_attestation( + analyze_ctx.component.repository.full_name, artifact_hash ) - if artifact_hash: - git_attestation_dict = git_service.api_client.get_attestation( - analyze_ctx.component.repository.full_name, artifact_hash - ) - if git_attestation_dict: - git_attestation_list = json_extract(git_attestation_dict, ["attestations"], list) - if git_attestation_list: - git_attestation = git_attestation_list[0] - - with tempfile.TemporaryDirectory() as temp_dir: - attestation_file = os.path.join(temp_dir, "attestation") - with open(attestation_file, "w", encoding="UTF-8") as file: - json.dump(git_attestation, file) - - try: - payload = load_provenance_payload(attestation_file) - provenance_payload = payload - except LoadIntotoAttestationError as error: - logger.debug("Failed to load provenance payload: %s", error) + if git_attestation_dict: + git_attestation_list = json_extract(git_attestation_dict, ["attestations"], list) + if git_attestation_list: + git_attestation = git_attestation_list[0] + + with tempfile.TemporaryDirectory() as temp_dir: + attestation_file = os.path.join(temp_dir, "attestation") + with open(attestation_file, "w", encoding="UTF-8") as file: + json.dump(git_attestation, file) + + try: + payload = load_provenance_payload(attestation_file) + provenance_payload = payload + except LoadIntotoAttestationError as error: + logger.debug("Failed to load provenance payload: %s", error) if parsed_purl is not None: self._verify_repository_link(parsed_purl, analyze_ctx) @@ -1005,30 +995,31 @@ def get_artifact_hash( self, purl: PackageURL, cached_artifacts: list[str] | None, - hash_algorithm: Any, package_registries_info: list[PackageRegistryInfo], ) -> str | None: """Get the hash of the artifact found from the passed PURL using local or remote files. + Provided local caches will be searched first. Artifacts will be downloaded if nothing is found within local + caches, or if no appropriate cache is provided for the target language. + Downloaded artifacts will be added to the passed package registry to prevent downloading them again. + Parameters ---------- purl: PackageURL The PURL of the artifact. cached_artifacts: list[str] | None The list of local files that match the PURL. - hash_algorithm: Any - The hash algorithm to use. package_registries_info: list[PackageRegistryInfo] The list of package registry information. Returns ------- str | None - The hash of the artifact, or None if not found. + The hash of the artifact, or None if no artifact can be found locally or remotely. """ if cached_artifacts: # Try to get the hash from a local file. - artifact_hash = get_local_artifact_hash(purl, cached_artifacts, hash_algorithm.name) + artifact_hash = get_local_artifact_hash(purl, cached_artifacts) if artifact_hash: return artifact_hash @@ -1046,7 +1037,7 @@ def get_artifact_hash( if not maven_registry: return None - return maven_registry.get_artifact_hash(purl, hash_algorithm) + return maven_registry.get_artifact_hash(purl) if purl.type == "pypi": pypi_registry = next( @@ -1073,6 +1064,9 @@ def get_artifact_hash( logger.debug("Missing registry information for PyPI") return None + if not purl.version: + return None + pypi_asset = find_or_create_pypi_asset(purl.name, purl.version, registry_info) if not pypi_asset: return None @@ -1089,7 +1083,7 @@ def get_artifact_hash( if not source_url: return None - return pypi_registry.get_artifact_hash(source_url, hash_algorithm) + return pypi_registry.get_artifact_hash(source_url) logger.debug("Purl type '%s' not yet supported for GitHub attestation discovery.", purl.type) return None diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index bc419f921..2fe3c5cea 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -2,17 +2,16 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The module provides abstractions for the Maven Central package registry.""" - +import hashlib import logging import urllib.parse from datetime import datetime, timezone -from typing import Any import requests from packageurl import PackageURL from requests import RequestException -from macaron.artifact.maven import construct_maven_repository_path +from macaron.artifact.maven import construct_maven_repository_path, construct_primary_jar_file_name from macaron.config.defaults import defaults from macaron.errors import ConfigurationError, InvalidHTTPResponseError from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry @@ -240,34 +239,27 @@ def find_publish_timestamp(self, purl: str) -> datetime: raise InvalidHTTPResponseError(f"Invalid response from Maven central for {url}.") - @staticmethod - def get_artifact_file_name(purl: PackageURL) -> str | None: - """Return the artifact file name of the passed PURL based on the Maven registry standard. + def get_artifact_hash(self, purl: PackageURL) -> str | None: + """Return the hash of the artifact found by the passed purl relevant to the registry's URL. - Parameters - ---------- - purl: PackageURL - The PURL of the artifact. + An artifact's URL will be as follows: + {registry_url}/{artifact_path}/{file_name} + Where: + - {registry_url} is determined by the setup/config of the registry. + - {artifact_path} is determined by the Maven repository layout. + (See: https://maven.apache.org/repository/layout.html and + https://maven.apache.org/guides/mini/guide-naming-conventions.html) + - {file_name} is {purl.name}-{purl.version}.jar (For a JAR artefact) - Returns + Example ------- - str | None - The artifact file name, or None if invalid. - """ - if not purl.version: - return None - - return purl.name + "-" + purl.version + ".jar" - - def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None: - """Return the hash of the artifact found by the passed purl relevant to the registry's URL. + PURL: pkg:maven/com.experlog/xapool@1.5.0 + URL: https://repo1.maven.org/maven2/com/experlog/xapool/1.5.0/xapool-1.5.0.jar Parameters ---------- purl: PackageURL The purl of the artifact. - hash_algorithm: Any - The hash algorithm to use. Returns ------- @@ -277,33 +269,34 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None if not purl.namespace: return None - file_name = MavenCentralRegistry.get_artifact_file_name(purl) + file_name = construct_primary_jar_file_name(purl) if not (purl.version and file_name): return None # Maven supports but does not require a sha256 hash of uploaded artifacts. artifact_path = construct_maven_repository_path(purl.namespace, purl.name, purl.version) artifact_url = self.registry_url + "/" + artifact_path + "/" + file_name - sha256_url = artifact_url + ".sha256" - logger.debug("Search for artifact hash using URL: %s", [sha256_url, artifact_url]) + artifact_sha256_url = artifact_url + ".sha256" + logger.debug("Search for artifact hash using URL: %s", [artifact_sha256_url, artifact_url]) - response = send_get_http_raw(sha256_url, {}) - sha256_hash = None - if response and (sha256_hash := response.text): + response = send_get_http_raw(artifact_sha256_url, {}) + retrieved_artifact_hash = None + if response and (retrieved_artifact_hash := response.text): # As Maven hashes are user provided and not verified they serve as a reference only. - logger.debug("Found hash of artifact: %s", sha256_hash) + logger.debug("Found hash of artifact: %s", retrieved_artifact_hash) try: response = requests.get(artifact_url, stream=True, timeout=40) response.raise_for_status() except requests.exceptions.HTTPError as http_err: - logger.debug("HTTP error occurred: %s", http_err) + logger.debug("HTTP error occurred when trying to download artifact: %s", http_err) return None if response.status_code != 200: return None # Download file and compute hash as chunks are received. + hash_algorithm = hashlib.sha256() try: for chunk in response.iter_content(): hash_algorithm.update(chunk) @@ -313,10 +306,14 @@ def get_artifact_hash(self, purl: PackageURL, hash_algorithm: Any) -> str | None response.close() return None - artifact_hash: str = hash_algorithm.hexdigest() - if sha256_hash and artifact_hash != sha256_hash: - logger.debug("Artifact hash and discovered hash do not match: %s != %s", artifact_hash, sha256_hash) + computed_artifact_hash: str = hash_algorithm.hexdigest() + if retrieved_artifact_hash and computed_artifact_hash != retrieved_artifact_hash: + logger.debug( + "Artifact hash and discovered hash do not match: %s != %s", + computed_artifact_hash, + retrieved_artifact_hash, + ) return None - logger.debug("Computed hash of artifact: %s", artifact_hash) - return artifact_hash + logger.debug("Computed hash of artifact: %s", computed_artifact_hash) + return computed_artifact_hash diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 6dd418729..113a2f2a0 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The module provides abstractions for the pypi package registry.""" - +import hashlib import logging import os import re @@ -14,7 +14,6 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime -from typing import Any import requests from bs4 import BeautifulSoup, Tag @@ -265,15 +264,13 @@ def download_package_sourcecode(self, url: str) -> str: logger.debug("Temporary download and unzip of %s stored in %s", file_name, temp_dir) return temp_dir - def get_artifact_hash(self, artifact_url: str, hash_algorithm: Any) -> str | None: + def get_artifact_hash(self, artifact_url: str) -> str | None: """Return the hash of the artifact found at the passed URL. Parameters ---------- artifact_url The URL of the artifact. - hash_algorithm: Any - The hash algorithm to use. Returns ------- @@ -284,13 +281,14 @@ def get_artifact_hash(self, artifact_url: str, hash_algorithm: Any) -> str | Non response = requests.get(artifact_url, stream=True, timeout=40) response.raise_for_status() except requests.exceptions.HTTPError as http_err: - logger.debug("HTTP error occurred: %s", http_err) + logger.debug("HTTP error occurred when trying to download artifact: %s", http_err) return None if response.status_code != 200: logger.debug("Invalid response: %s", response.status_code) return None + hash_algorithm = hashlib.sha256() try: for chunk in response.iter_content(): hash_algorithm.update(chunk) @@ -738,15 +736,15 @@ def get_sha256(self) -> str | None: def find_or_create_pypi_asset( - asset_name: str, asset_version: str | None, pypi_registry_info: PackageRegistryInfo + asset_name: str, asset_version: str, pypi_registry_info: PackageRegistryInfo ) -> PyPIPackageJsonAsset | None: - """Find the asset in the provided package registry information, or create it. + """Find the matching asset in the provided package registry information, or if not found, create and add it. Parameters ---------- asset_name: str The name of the asset. - asset_version: str | None + asset_version: str The version of the asset. pypi_registry_info: The package registry information. @@ -756,12 +754,17 @@ def find_or_create_pypi_asset( PyPIPackageJsonAsset | None The asset, or None if not found. """ - pypi_package_json = next( - (asset for asset in pypi_registry_info.metadata if isinstance(asset, PyPIPackageJsonAsset)), + asset = next( + ( + asset + for asset in pypi_registry_info.metadata + if isinstance(asset, PyPIPackageJsonAsset) and asset.component_name == asset_name + ), None, ) - if pypi_package_json: - return pypi_package_json + + if asset: + return asset package_registry = pypi_registry_info.package_registry if not isinstance(package_registry, PyPIRegistry): diff --git a/tests/artifact/test_maven.py b/tests/artifact/test_maven.py index 6014c20ad..b39856e4e 100644 --- a/tests/artifact/test_maven.py +++ b/tests/artifact/test_maven.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for types and utilities for Maven artifacts.""" @@ -6,7 +6,11 @@ import pytest from packageurl import PackageURL -from macaron.artifact.maven import MavenSubjectPURLMatcher, construct_maven_repository_path +from macaron.artifact.maven import ( + MavenSubjectPURLMatcher, + construct_maven_repository_path, + construct_primary_jar_file_name, +) from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, validate_intoto_payload @@ -161,3 +165,10 @@ def test_to_group_folder_path( ) -> None: """Test the ``to_gorup_folder_path`` method.""" assert construct_maven_repository_path(group_id) == expected_group_path + + +def test_construct_primary_jar_file_name() -> None: + """Test the artifact file name function.""" + assert not construct_primary_jar_file_name(PackageURL.from_string("pkg:maven/test/example")) + + assert construct_primary_jar_file_name(PackageURL.from_string("pkg:maven/text/example@1")) == "example-1.jar" diff --git a/tests/integration/cases/github_maven_attestation_local/test.yaml b/tests/integration/cases/github_maven_attestation_local/test.yaml index d66a089b2..d442b546c 100644 --- a/tests/integration/cases/github_maven_attestation_local/test.yaml +++ b/tests/integration/cases/github_maven_attestation_local/test.yaml @@ -2,10 +2,12 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. description: | - Discovering GitHub attestation of a local Maven artifact but failing because the artifact is wrong + Discovering GitHub attestation of a local Maven artifact but failing because the artifact is wrong. In this case + we download the artifact's POM file and save it as a JAR file. tags: - macaron-python-package +- macaron-docker-image steps: - name: Download artifact POM instead of the JAR diff --git a/tests/integration/cases/github_pypi_attestation/test.yaml b/tests/integration/cases/github_pypi_attestation/test.yaml index 173361662..7cf096192 100644 --- a/tests/integration/cases/github_pypi_attestation/test.yaml +++ b/tests/integration/cases/github_pypi_attestation/test.yaml @@ -13,7 +13,7 @@ steps: options: command_args: - -purl - - pkg:pypi/toga@0.5.0 + - pkg:pypi/toga@0.4.8 - name: Run macaron verify-policy to verify passed/failed checks kind: verify options: diff --git a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py index 2feffae96..40b51c9ae 100644 --- a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py +++ b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py @@ -14,7 +14,7 @@ from packageurl import PackageURL from pytest_httpserver import HTTPServer -from macaron.artifact.maven import construct_maven_repository_path +from macaron.artifact.maven import construct_maven_repository_path, construct_primary_jar_file_name from macaron.config.defaults import load_defaults from macaron.errors import ConfigurationError, InvalidHTTPResponseError from macaron.slsa_analyzer.package_registry.maven_central_registry import MavenCentralRegistry @@ -239,16 +239,6 @@ def test_find_publish_timestamp_errors( maven_central.find_publish_timestamp(purl=purl) -def test_get_artifact_file_name() -> None: - """Test the artifact file name function.""" - assert not MavenCentralRegistry().get_artifact_file_name(PackageURL.from_string("pkg:maven/test/example")) - - assert ( - MavenCentralRegistry().get_artifact_file_name(PackageURL.from_string("pkg:maven/text/example@1")) - == "example-1.jar" - ) - - @pytest.mark.parametrize("purl_string", ["pkg:maven/example", "pkg:maven/example/test", "pkg:maven/example/test@1"]) def test_get_artifact_hash_failures( httpserver: HTTPServer, maven_service: dict, purl_string: str # pylint: disable=unused-argument @@ -259,12 +249,7 @@ def test_get_artifact_hash_failures( maven_registry = MavenCentralRegistry() maven_registry.load_defaults() - if ( - purl.namespace - and purl.version - and (file_name := MavenCentralRegistry().get_artifact_file_name(purl)) - and file_name - ): + if purl.namespace and purl.version and (file_name := construct_primary_jar_file_name(purl)) and file_name: artifact_path = "/" + construct_maven_repository_path(purl.namespace, purl.name, purl.version) + "/" + file_name hash_algorithm = sha256() hash_algorithm.update(b"example_data") @@ -272,7 +257,7 @@ def test_get_artifact_hash_failures( httpserver.expect_request(artifact_path + ".sha256").respond_with_data(expected_hash) httpserver.expect_request(artifact_path).respond_with_data(b"example_data_2") - result = maven_registry.get_artifact_hash(purl, sha256()) + result = maven_registry.get_artifact_hash(purl) assert not result @@ -288,7 +273,7 @@ def test_get_artifact_hash_success( maven_registry = MavenCentralRegistry() maven_registry.load_defaults() - file_name = MavenCentralRegistry().get_artifact_file_name(purl) + file_name = construct_primary_jar_file_name(purl) assert file_name artifact_path = "/" + construct_maven_repository_path(purl.namespace, purl.name, purl.version) + "/" + file_name @@ -298,6 +283,6 @@ def test_get_artifact_hash_success( httpserver.expect_request(artifact_path + ".sha256").respond_with_data(expected_hash) httpserver.expect_request(artifact_path).respond_with_data(b"example_data") - result = maven_registry.get_artifact_hash(purl, sha256()) + result = maven_registry.get_artifact_hash(purl) assert result From c27b97fb4988dfb571c349dd281f96a22bf40abd Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 2 Jun 2025 12:41:42 +1000 Subject: [PATCH 09/16] chore: minor fix Signed-off-by: Ben Selwyn-Smith --- .../checks/detect_malicious_metadata_check.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index 2157fbe81..a8fc8bc0b 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -286,9 +286,11 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: package_registry=PyPIRegistry(), ) as pypi_registry_info: # Retrieve the pre-existing asset, or create a new one. - pypi_package_json = find_or_create_pypi_asset( - ctx.component.name, ctx.component.version, pypi_registry_info - ) + pypi_package_json = None + if ctx.component.version: + pypi_package_json = find_or_create_pypi_asset( + ctx.component.name, ctx.component.version, pypi_registry_info + ) if pypi_package_json is None: return CheckResultData(result_tables=[], result_type=CheckResultType.UNKNOWN) From 6b6ff7bbe8549d5c2ffa6797a143d441335a8f1a Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 2 Jun 2025 13:42:24 +1000 Subject: [PATCH 10/16] chore: add GitHub attestation schema link; add local artifact hash unit test Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 68 +++++++++++++++++---------- tests/artifact/test_local_artifact.py | 20 ++++++++ 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 655346010..1b09212b8 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -4,7 +4,6 @@ """This module handles the cloning and analyzing a Git repo.""" import glob -import json import logging import os import re @@ -78,7 +77,6 @@ from macaron.slsa_analyzer.package_registry.pypi_registry import find_or_create_pypi_asset from macaron.slsa_analyzer.provenance.expectations.expectation_registry import ExpectationRegistry from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV01Payload -from macaron.slsa_analyzer.provenance.intoto.errors import LoadIntotoAttestationError from macaron.slsa_analyzer.provenance.loader import load_provenance_payload from macaron.slsa_analyzer.provenance.slsa import SLSAProvenanceData from macaron.slsa_analyzer.registry import registry @@ -516,24 +514,7 @@ def run_single( # Try to discover GitHub attestation for the target software component. artifact_hash = self.get_artifact_hash(parsed_purl, local_artifact_dirs, package_registries_info) if artifact_hash: - git_attestation_dict = git_service.api_client.get_attestation( - analyze_ctx.component.repository.full_name, artifact_hash - ) - if git_attestation_dict: - git_attestation_list = json_extract(git_attestation_dict, ["attestations"], list) - if git_attestation_list: - git_attestation = git_attestation_list[0] - - with tempfile.TemporaryDirectory() as temp_dir: - attestation_file = os.path.join(temp_dir, "attestation") - with open(attestation_file, "w", encoding="UTF-8") as file: - json.dump(git_attestation, file) - - try: - payload = load_provenance_payload(attestation_file) - provenance_payload = payload - except LoadIntotoAttestationError as error: - logger.debug("Failed to load provenance payload: %s", error) + provenance_payload = self.get_github_attestation_payload(analyze_ctx, git_service, artifact_hash) if parsed_purl is not None: self._verify_repository_link(parsed_purl, analyze_ctx) @@ -994,7 +975,7 @@ def create_analyze_ctx(self, component: Component) -> AnalyzeContext: def get_artifact_hash( self, purl: PackageURL, - cached_artifacts: list[str] | None, + local_artifact_dirs: list[str] | None, package_registries_info: list[PackageRegistryInfo], ) -> str | None: """Get the hash of the artifact found from the passed PURL using local or remote files. @@ -1007,8 +988,8 @@ def get_artifact_hash( ---------- purl: PackageURL The PURL of the artifact. - cached_artifacts: list[str] | None - The list of local files that match the PURL. + local_artifact_dirs: list[str] | None + The list of directories that may contain the artifact file. package_registries_info: list[PackageRegistryInfo] The list of package registry information. @@ -1017,9 +998,9 @@ def get_artifact_hash( str | None The hash of the artifact, or None if no artifact can be found locally or remotely. """ - if cached_artifacts: + if local_artifact_dirs: # Try to get the hash from a local file. - artifact_hash = get_local_artifact_hash(purl, cached_artifacts) + artifact_hash = get_local_artifact_hash(purl, local_artifact_dirs) if artifact_hash: return artifact_hash @@ -1088,6 +1069,43 @@ def get_artifact_hash( logger.debug("Purl type '%s' not yet supported for GitHub attestation discovery.", purl.type) return None + def get_github_attestation_payload( + self, analyze_ctx: AnalyzeContext, git_service: GitHub, artifact_hash: str + ) -> InTotoPayload | None: + """Get the GitHub attestation associated with the given PURL, or None if it cannot be found. + + The schema of GitHub attestation can be found on the API page: + https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-attestations + + Parameters + ---------- + analyze_ctx: AnalyzeContext + The analysis context. + git_service: GitHub + The Git service to retrieve the attestation from. + artifact_hash: str + The hash of the related artifact. + + Returns + ------- + InTotoPayload | None + The attestation payload, if found. + """ + git_attestation_dict = git_service.api_client.get_attestation( + analyze_ctx.component.repository.full_name, artifact_hash + ) + + if not git_attestation_dict: + return None + + git_attestation_list = json_extract(git_attestation_dict, ["attestations"], list) + if not git_attestation_list: + return None + + git_attestation: str = git_attestation_list[0] + + return load_provenance_payload(git_attestation) + def _determine_git_service(self, analyze_ctx: AnalyzeContext) -> BaseGitService: """Determine the Git service used by the software component.""" remote_path = analyze_ctx.component.repository.remote_path if analyze_ctx.component.repository else None diff --git a/tests/artifact/test_local_artifact.py b/tests/artifact/test_local_artifact.py index 1468bef42..3124afc3c 100644 --- a/tests/artifact/test_local_artifact.py +++ b/tests/artifact/test_local_artifact.py @@ -15,7 +15,9 @@ construct_local_artifact_dirs_glob_pattern_pypi_purl, find_artifact_dirs_from_python_venv, get_local_artifact_dirs, + get_local_artifact_hash, ) +from macaron.artifact.maven import construct_primary_jar_file_name from macaron.errors import LocalArtifactFinderError @@ -249,3 +251,21 @@ def test_get_local_artifact_paths_succeeded_pypi(tmp_path: Path) -> None: ) assert sorted(result) == sorted(pypi_artifact_paths) + + +def test_get_local_artifact_hash() -> None: + """Test the get local artifact hash function.""" + artifact_purl = PackageURL.from_string("pkg:maven/test/test@1") + artifact_jar_name = construct_primary_jar_file_name(artifact_purl) + + assert artifact_jar_name + + with tempfile.TemporaryDirectory() as temp_dir: + artifact_path = os.path.join(temp_dir, artifact_jar_name) + with open(artifact_path, "w", encoding="utf8") as file: + file.write("1") + + # A file containing: "1". + target_hash = "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" + + assert target_hash == get_local_artifact_hash(artifact_purl, [temp_dir]) From aa640dd449b9a8965cfad6965ea9e224dcb9f436 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 3 Jun 2025 14:44:21 +1000 Subject: [PATCH 11/16] chore: update test Signed-off-by: Ben Selwyn-Smith --- .../checks/detect_malicious_metadata_check.py | 10 ++++------ .../slsa_analyzer/package_registry/pypi_registry.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index a8fc8bc0b..c9c44ae7c 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -286,12 +286,10 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: package_registry=PyPIRegistry(), ) as pypi_registry_info: # Retrieve the pre-existing asset, or create a new one. - pypi_package_json = None - if ctx.component.version: - pypi_package_json = find_or_create_pypi_asset( - ctx.component.name, ctx.component.version, pypi_registry_info - ) - if pypi_package_json is None: + pypi_package_json = find_or_create_pypi_asset( + ctx.component.name, ctx.component.version, pypi_registry_info + ) + if not pypi_package_json: return CheckResultData(result_tables=[], result_type=CheckResultType.UNKNOWN) pypi_package_json.has_repository = ctx.component.repository is not None diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 113a2f2a0..b57a12ca3 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -736,7 +736,7 @@ def get_sha256(self) -> str | None: def find_or_create_pypi_asset( - asset_name: str, asset_version: str, pypi_registry_info: PackageRegistryInfo + asset_name: str, asset_version: str | None, pypi_registry_info: PackageRegistryInfo ) -> PyPIPackageJsonAsset | None: """Find the matching asset in the provided package registry information, or if not found, create and add it. @@ -744,7 +744,7 @@ def find_or_create_pypi_asset( ---------- asset_name: str The name of the asset. - asset_version: str + asset_version: str | None The version of the asset. pypi_registry_info: The package registry information. From cdee7fcae4d8a64620fa289d324ff8f513f14ca6 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 3 Jun 2025 15:09:07 +1000 Subject: [PATCH 12/16] chore: add doc comment for modification of package registry info Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/package_registry/pypi_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index b57a12ca3..6a41f6bbe 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -747,7 +747,7 @@ def find_or_create_pypi_asset( asset_version: str | None The version of the asset. pypi_registry_info: - The package registry information. + The package registry information. If a new asset is created, it will be added to the metadata of this registry. Returns ------- From e09ed8b12a54cd34d14b71b1d0da2229d6c65a2f Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 3 Jun 2025 15:40:39 +1000 Subject: [PATCH 13/16] chore: use correct import location for error class Signed-off-by: Ben Selwyn-Smith --- .../slsa_analyzer/package_registry/package_registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/macaron/slsa_analyzer/package_registry/package_registry.py b/src/macaron/slsa_analyzer/package_registry/package_registry.py index 9e71fc595..ca0adfa62 100644 --- a/src/macaron/slsa_analyzer/package_registry/package_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/package_registry.py @@ -7,9 +7,9 @@ from abc import ABC, abstractmethod from datetime import datetime -from macaron.errors import InvalidHTTPResponseError +from macaron.errors import APIAccessError, InvalidHTTPResponseError from macaron.json_tools import json_extract -from macaron.slsa_analyzer.package_registry.deps_dev import APIAccessError, DepsDevService +from macaron.slsa_analyzer.package_registry.deps_dev import DepsDevService logger: logging.Logger = logging.getLogger(__name__) From 76ec56707628afeddc00553d46774122526ca572 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 4 Jun 2025 09:26:14 +1000 Subject: [PATCH 14/16] chore: fix cyclic import issue; rearrange unsubscriptable issue bypassers Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/checks/build_script_check.py | 4 ---- .../slsa_analyzer/checks/provenance_commit_check.py | 4 ---- src/macaron/slsa_analyzer/checks/provenance_repo_check.py | 4 ++++ .../slsa_analyzer/package_registry/pypi_registry.py | 7 ++++++- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/macaron/slsa_analyzer/checks/build_script_check.py b/src/macaron/slsa_analyzer/checks/build_script_check.py index 44f675ce4..ccd61cca1 100644 --- a/src/macaron/slsa_analyzer/checks/build_script_check.py +++ b/src/macaron/slsa_analyzer/checks/build_script_check.py @@ -27,10 +27,6 @@ class BuildScriptFacts(CheckFacts): __tablename__ = "_build_script_check" - # This check is disabled here due to a bug in pylint. The Mapped class triggers a false positive. - # It may arbitrarily become true that this is no longer needed in this check, or will be needed in another check. - # pylint: disable=unsubscriptable-object - #: The primary key. id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 diff --git a/src/macaron/slsa_analyzer/checks/provenance_commit_check.py b/src/macaron/slsa_analyzer/checks/provenance_commit_check.py index 61fb6b8a3..b2b5d7297 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_commit_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_commit_check.py @@ -22,10 +22,6 @@ class ProvenanceDerivedCommitFacts(CheckFacts): __tablename__ = "_provenance_derived_commit_check" - # This check is disabled here due to a bug in pylint. The Mapped class triggers a false positive. - # It may arbitrarily become true that this is no longer needed in this check, or will be needed in another check. - # pylint: disable=unsubscriptable-object - #: The primary key. id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 diff --git a/src/macaron/slsa_analyzer/checks/provenance_repo_check.py b/src/macaron/slsa_analyzer/checks/provenance_repo_check.py index 063e68c2b..1f35fef39 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_repo_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_repo_check.py @@ -22,6 +22,10 @@ class ProvenanceDerivedRepoFacts(CheckFacts): __tablename__ = "_provenance_derived_repo_check" + # This check is disabled here due to a bug in pylint. The Mapped class triggers a false positive. + # It may arbitrarily become true that this is no longer needed in this check, or will be needed in another check. + # pylint: disable=unsubscriptable-object + #: The primary key. id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 6a41f6bbe..13156f7f7 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -2,6 +2,8 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The module provides abstractions for the pypi package registry.""" +from __future__ import annotations + import hashlib import logging import os @@ -14,6 +16,7 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime +from typing import TYPE_CHECKING import requests from bs4 import BeautifulSoup, Tag @@ -24,9 +27,11 @@ from macaron.json_tools import json_extract from macaron.malware_analyzer.datetime_parser import parse_datetime from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry -from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo from macaron.util import send_get_http_raw +if TYPE_CHECKING: + from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo + logger: logging.Logger = logging.getLogger(__name__) From 2fcc15d660aa0a6fcc2265cda5182661657dce15 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 4 Jun 2025 11:30:04 +1000 Subject: [PATCH 15/16] chore: minor fixes Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 9 +++++++- src/macaron/repo_finder/repo_finder_pypi.py | 19 +++++++++++----- src/macaron/slsa_analyzer/analyzer.py | 17 ++++++++++---- .../slsa_analyzer/provenance/loader.py | 22 +++++++++++++++++-- .../cases/github_pypi_attestation/policy.dl | 2 +- 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index e6c026554..ad28452e0 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -111,6 +111,8 @@ def find_repo( logger.debug("Analyzing %s with Repo Finder: %s", purl, type(repo_finder)) found_repo, outcome = repo_finder.find_repo(purl) + print(package_registries_info) + if not found_repo: found_repo, outcome = find_repo_alternative(purl, outcome, package_registries_info) @@ -166,7 +168,12 @@ def find_repo_alternative( found_repo, outcome = repo_finder_pypi.find_repo(purl, package_registries_info) if not found_repo: - logger.debug("Could not find repository using type specific (%s) methods for PURL: %s", purl.type, purl) + logger.debug( + "Could not find repository using type specific (%s) methods for PURL %s. Outcome: %s", + purl.type, + purl, + outcome, + ) return found_repo, outcome diff --git a/src/macaron/repo_finder/repo_finder_pypi.py b/src/macaron/repo_finder/repo_finder_pypi.py index db59becee..978c483b0 100644 --- a/src/macaron/repo_finder/repo_finder_pypi.py +++ b/src/macaron/repo_finder/repo_finder_pypi.py @@ -8,8 +8,8 @@ from macaron.repo_finder.repo_finder_enums import RepoFinderInfo from macaron.repo_finder.repo_validator import find_valid_repository_url -from macaron.slsa_analyzer.package_registry import PyPIRegistry -from macaron.slsa_analyzer.package_registry.pypi_registry import find_or_create_pypi_asset +from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, PyPIRegistry +from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIPackageJsonAsset, find_or_create_pypi_asset from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo logger: logging.Logger = logging.getLogger(__name__) @@ -44,14 +44,21 @@ def find_repo( ), None, ) - - if not pypi_info: - return "", RepoFinderInfo.PYPI_NO_REGISTRY + if not pypi_info: + return "", RepoFinderInfo.PYPI_NO_REGISTRY if not purl.version: return "", RepoFinderInfo.NO_VERSION_PROVIDED - pypi_asset = find_or_create_pypi_asset(purl.name, purl.version, pypi_info) + # Create the asset. + if pypi_info: + pypi_asset = find_or_create_pypi_asset(purl.name, purl.version, pypi_info) + else: + # If this function has been reached via find-source, we do not store the asset. + pypi_registry = next((registry for registry in PACKAGE_REGISTRIES if isinstance(registry, PyPIRegistry)), None) + if not pypi_registry: + return "", RepoFinderInfo.PYPI_NO_REGISTRY + pypi_asset = PyPIPackageJsonAsset(purl.name, purl.version, False, pypi_registry, {}) if not pypi_asset: # This should be unreachable, as the pypi_registry has already been confirmed to be of type PyPIRegistry. diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 1b09212b8..23a252c97 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -76,8 +76,13 @@ from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, MavenCentralRegistry, PyPIRegistry from macaron.slsa_analyzer.package_registry.pypi_registry import find_or_create_pypi_asset from macaron.slsa_analyzer.provenance.expectations.expectation_registry import ExpectationRegistry -from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV01Payload -from macaron.slsa_analyzer.provenance.loader import load_provenance_payload +from macaron.slsa_analyzer.provenance.intoto import ( + InTotoPayload, + InTotoV01Payload, + ValidateInTotoPayloadError, + validate_intoto_payload, +) +from macaron.slsa_analyzer.provenance.loader import decode_provenance from macaron.slsa_analyzer.provenance.slsa import SLSAProvenanceData from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.specs.ci_spec import CIInfo @@ -1102,9 +1107,13 @@ def get_github_attestation_payload( if not git_attestation_list: return None - git_attestation: str = git_attestation_list[0] + payload = decode_provenance(git_attestation_list[0]) - return load_provenance_payload(git_attestation) + try: + return validate_intoto_payload(payload) + except ValidateInTotoPayloadError as error: + logger.debug("Invalid attestation payload: %s", error) + return None def _determine_git_service(self, analyze_ctx: AnalyzeContext) -> BaseGitService: """Determine the Git service used by the software component.""" diff --git a/src/macaron/slsa_analyzer/provenance/loader.py b/src/macaron/slsa_analyzer/provenance/loader.py index a46d8d8ef..106cc03b5 100644 --- a/src/macaron/slsa_analyzer/provenance/loader.py +++ b/src/macaron/slsa_analyzer/provenance/loader.py @@ -80,15 +80,33 @@ def _load_provenance_file_content( try: decompressed_file_content = gzip.decompress(file_content) decoded_file_content = decompressed_file_content.decode() - provenance = json.loads(decoded_file_content) + return decode_provenance(json.loads(decoded_file_content)) except (gzip.BadGzipFile, EOFError, zlib.error): decoded_file_content = file_content.decode() - provenance = json.loads(decoded_file_content) + return decode_provenance(json.loads(decoded_file_content)) except (json.JSONDecodeError, TypeError, UnicodeDecodeError) as error: raise LoadIntotoAttestationError( "Cannot deserialize the file content as JSON.", ) from error + +def decode_provenance(provenance: dict) -> dict[str, JsonType]: + """Find and decode the provenance payload. + + Parameters + ---------- + provenance: dict + The contents of the provenance from which the payload will be decoded. + + Returns + ------- + The decoded payload. + + Raises + ------ + LoadIntotoAttestationError + If the payload could not be decoded. + """ # The GitHub Attestation stores the DSSE envelope in `dsseEnvelope` property. dsse_envelope = provenance.get("dsseEnvelope", None) if dsse_envelope: diff --git a/tests/integration/cases/github_pypi_attestation/policy.dl b/tests/integration/cases/github_pypi_attestation/policy.dl index 26cef7913..960a55e88 100644 --- a/tests/integration/cases/github_pypi_attestation/policy.dl +++ b/tests/integration/cases/github_pypi_attestation/policy.dl @@ -7,4 +7,4 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_provenance_available_1"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:pypi/toga@0.5.0"). + is_component(component_id, "pkg:pypi/toga@0.4.8"). From a9370446b6325cd09dc23b0f4af181601daf22db Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 4 Jun 2025 14:28:31 +1000 Subject: [PATCH 16/16] chore: minor fixes Signed-off-by: Ben Selwyn-Smith --- src/macaron/artifact/local_artifact.py | 2 ++ src/macaron/repo_finder/repo_finder.py | 2 -- src/macaron/repo_finder/repo_finder_pypi.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/macaron/artifact/local_artifact.py b/src/macaron/artifact/local_artifact.py index 46772e970..0df7e6248 100644 --- a/src/macaron/artifact/local_artifact.py +++ b/src/macaron/artifact/local_artifact.py @@ -281,6 +281,8 @@ def get_local_artifact_hash(purl: PackageURL, artifact_dirs: list[str]) -> str | artifact_target = construct_primary_jar_file_name(purl) # TODO add support for other PURL types here. + # Other purl types can be easily supported if user provided artifacts are accepted from the command line. + # See https://github.com/oracle/macaron/issues/498. if not artifact_target: logger.debug("PURL type not supported: %s", purl.type) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index ad28452e0..9017a4ae0 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -111,8 +111,6 @@ def find_repo( logger.debug("Analyzing %s with Repo Finder: %s", purl, type(repo_finder)) found_repo, outcome = repo_finder.find_repo(purl) - print(package_registries_info) - if not found_repo: found_repo, outcome = find_repo_alternative(purl, outcome, package_registries_info) diff --git a/src/macaron/repo_finder/repo_finder_pypi.py b/src/macaron/repo_finder/repo_finder_pypi.py index 978c483b0..c0c273154 100644 --- a/src/macaron/repo_finder/repo_finder_pypi.py +++ b/src/macaron/repo_finder/repo_finder_pypi.py @@ -58,7 +58,7 @@ def find_repo( pypi_registry = next((registry for registry in PACKAGE_REGISTRIES if isinstance(registry, PyPIRegistry)), None) if not pypi_registry: return "", RepoFinderInfo.PYPI_NO_REGISTRY - pypi_asset = PyPIPackageJsonAsset(purl.name, purl.version, False, pypi_registry, {}) + pypi_asset = PyPIPackageJsonAsset(purl.name, purl.version, False, pypi_registry, {}, "") if not pypi_asset: # This should be unreachable, as the pypi_registry has already been confirmed to be of type PyPIRegistry.