diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 87a109717..fcf45dec9 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -9,7 +9,7 @@ metadata: pipelinesascode.tekton.dev/on-comment: "/test-heavy" pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]" - pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/max-keep-runs: "100" spec: timeouts: pipeline: 10h30m0s # Timeout for the entire PipelineRun diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 6f1672020..39a70dc18 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -15,7 +15,7 @@ metadata: pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]" # How many runs we want to keep. - pipelinesascode.tekton.dev/max-keep-runs: "5" + pipelinesascode.tekton.dev/max-keep-runs: "100" spec: timeouts: pipeline: 2h30m0s # Timeout for the entire PipelineRun diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 01dbbfcd8..34bf19687 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -144,6 +144,10 @@ spec: fieldPath: metadata.namespace - name: GOMODCACHE value: /exploit-iq-package-cache/go/pkg/mod + - name: GOCACHE + value: /exploit-iq-package-cache/go/cache + - name: MAVEN_OPTS + value: "-Dmaven.repo.local=/exploit-iq-package-cache/maven" - name: ENABLE_MLOPS value: "true" - name: CREDENTIAL_ENCRYPTION_KEY diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 379b94bb0..734e0d795 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -15,6 +15,7 @@ import ast import configparser +import concurrent.futures import glob as glob_module import hashlib import json @@ -29,7 +30,7 @@ import tomllib import urllib.error import urllib.request -import zipfile +import math from abc import ABC from abc import abstractmethod from collections import defaultdict @@ -53,6 +54,70 @@ from collections import deque from exploit_iq_commons.logging.loggers_factory import LoggingFactory + + +def _available_cpus() -> int: + """Return the number of CPUs available to this process. + + Respects container CPU limits from cgroup v2/v1 before falling back to + process affinity and Python CPU APIs. + """ + + # cgroup v2 + try: + with open("/sys/fs/cgroup/cpu.max", encoding="utf-8") as f: + quota_s, period_s = f.read().strip().split() + + if quota_s != "max": + quota = int(quota_s) + period = int(period_s) + + if quota > 0 and period > 0: + return max(1, math.ceil(quota / period)) + except (OSError, ValueError): + pass + + # cgroup v1 + for base in ( + "/sys/fs/cgroup/cpu", + "/sys/fs/cgroup/cpu,cpuacct", + ): + try: + with open(f"{base}/cpu.cfs_quota_us", encoding="utf-8") as f: + quota = int(f.read().strip()) + + with open(f"{base}/cpu.cfs_period_us", encoding="utf-8") as f: + period = int(f.read().strip()) + + # cgroup v1 uses quota == -1 to mean "no CPU quota". + if quota > 0 and period > 0: + return max(1, math.ceil(quota / period)) + except (OSError, ValueError): + pass + + # CPU affinity fallback. + try: + return max(1, len(os.sched_getaffinity(0))) + except (AttributeError, OSError): + pass + + # Python 3.13+ fallback. + # Keep this after cgroup checks: in OpenShift/Kubernetes it may expose + # the full node-visible CPU set rather than the pod CPU limit. + if hasattr(os, "process_cpu_count"): + count = os.process_cpu_count() + if count: + return max(1, count) + + return os.cpu_count() or 4 + +def _extract_source_jar(jar: Path, dest: Path) -> None: + """Extract a single source JAR into dest directory.""" + dest.mkdir(exist_ok=True) + result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) + if result.returncode != 0: + LoggingFactory.get_agent_logger(__name__).warning( + "Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) from exploit_iq_commons.utils.java_utils import add_missing_jar_string from exploit_iq_commons.utils.java_utils import is_maven_gav from exploit_iq_commons.utils.java_utils import parse_depgraph_line @@ -76,7 +141,6 @@ def _get_go_repo_lock(manifest_path) -> threading.Lock: ROOT_LEVEL_SENTINEL = 'root-top-level-exploit-iq' - TRANSITIVE_ENV_NAME = 'transitive_env' INSTALLED_PACKAGES_FILE = 'installed_packages.txt' @@ -169,7 +233,7 @@ def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None: ] if any(p.is_file() for p in c_candidates): for root, dirs, files in os.walk(git_repo_path): - dirs[:] = [d for d in dirs if not d.startswith('.')] + dirs[:] = [d for d in dirs if d not in _WALK_EXCLUDE_DIRS and not d.startswith('.')] if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files): return MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] return None @@ -1034,18 +1098,18 @@ def install_dependencies(self, manifest_path: Path): source_path = self.DEP_SOURCE_DIR process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", - "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + "-DincludeScope=runtime", "-Dmaven.artifact.threads=10", + f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) if process_object.returncode > 0: - # Remove stale node_modules owned by a different UID to prevent - # EPERM when frontend-maven-plugin runs pnpm install during build. for child in manifest_path.rglob("node_modules"): if child.is_dir() and child.name == "node_modules": shutil.rmtree(child, ignore_errors=True) logger.debug("Removed stale %s", child) process_object = subprocess.run([mvn_command, "clean", "install", - "-DskipTests", "-s", settings_path], cwd=manifest_path) + "-DskipTests", "-Dmaven.artifact.threads=10", + "-s", settings_path], cwd=manifest_path) if process_object.returncode > 0: formatted_error_msg = ( f"Failed to build project" @@ -1056,7 +1120,8 @@ def install_dependencies(self, manifest_path: Path): process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", - "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + "-DincludeScope=runtime", "-Dmaven.artifact.threads=10", + f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) if process_object.returncode > 0: formatted_error_msg = ( @@ -1067,18 +1132,21 @@ def install_dependencies(self, manifest_path: Path): raise Exception(formatted_error_msg) full_source_path = manifest_path / source_path + jars_to_extract = [] for jar in full_source_path.glob("*-sources.jar"): if jar.stat().st_size > 0: - dest = full_source_path / jar.stem # folder named after jar - + dest = full_source_path / jar.stem if not dest.exists(): - dest.mkdir(exist_ok=True) - result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) - if result.returncode != 0: - logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) + jars_to_extract.append((jar, dest)) else: logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) + if jars_to_extract: + max_workers = min(_available_cpus() * 2, 8) + logger.info("Extracting %d source JARs with %d workers", len(jars_to_extract), max_workers) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool: + pool.map(lambda args: _extract_source_jar(*args), jars_to_extract) + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: mvn_command = resolve_mvn_command(manifest_path) settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml") @@ -1091,6 +1159,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: mvn_command, "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", "-s", settings_path, + "-Dmaven.artifact.threads=10", "-DgraphFormat=text", "-DshowGroupIds", "-DshowVersions", @@ -1109,6 +1178,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: mvn_command, "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", "-s", settings_path, + "-Dmaven.artifact.threads=10", "-DgraphFormat=text", "-DshowGroupIds", "-DshowVersions", @@ -1640,21 +1710,6 @@ def _try_file(path: Path, extractor) -> str | None: return None - def _ensure_venv(self, manifest_path: Path) -> str: - """Ensure transitive_env exists with a working python binary.""" - venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' - if Path(venv_python).exists(): - return venv_python - logger.warning("Venv python not found at %s — creating venv", venv_python) - python_version = self.determine_python_version(str(manifest_path)) - if not python_version: - import sys - python_version = f"{sys.version_info.major}.{sys.version_info.minor}" - logger.info("Python version undetermined; using current interpreter %s", python_version) - logger.info("Creating transitive_env with Python %s using uv", python_version) - run_command(["uv", "venv" ,TRANSITIVE_ENV_NAME, "--python", python_version] ,cwd=manifest_path) - return venv_python - def install_dependencies(self, manifest_path: Path): """Install Python dependencies for the given repository into a virtual environment. @@ -1698,7 +1753,7 @@ def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, for lock_file in (UV_LOCK, POETRY_LOCK): if (manifest_path / lock_file).exists(): working_dir = manifest_path - create_requirements_command = ["uv","export","--format","requirements-txt","--no-dev"] + create_requirements_command = ["uv","export","--format","requirements-txt","no-dev","2>/dev/null"] requirements_txt_result = run_command(args=create_requirements_command, cwd=working_dir) if requirements_txt_result is not None: @@ -1735,8 +1790,8 @@ def _install_from_requirements_txt(self, req_txt: Path, manifest_path: Path, def _write_installed_packages(self, manifest_path: Path) -> None: """Write a freeze-format snapshot of the venv to installed_packages.txt.""" - venv_python = f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python" - pip_freeze = run_command(["uv", "pip", "freeze", "--python", venv_python]) + pip_full_path= f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip" + pip_freeze = run_command([pip_full_path,"list","--format=freeze"]) if pip_freeze: (manifest_path / INSTALLED_PACKAGES_FILE).write_text(pip_freeze) logger.info("Wrote installed packages snapshot to %s/%s", manifest_path, INSTALLED_PACKAGES_FILE) @@ -1778,7 +1833,7 @@ def _find_module_dirs(self, package_name: str, site_packages: Path) -> list[str] if package_name.startswith('types-'): base = package_name[6:] - candidates = [f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()] + candidates = list(dict.fromkeys([f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()])) elif package_name.startswith('mypy-boto3-'): base = package_name[11:] candidates = [f'mypy_boto3_{base}'] diff --git a/src/vuln_analysis/functions/cve_verify_vuln_package.py b/src/vuln_analysis/functions/cve_verify_vuln_package.py index e58cdcade..edcc97be3 100644 --- a/src/vuln_analysis/functions/cve_verify_vuln_package.py +++ b/src/vuln_analysis/functions/cve_verify_vuln_package.py @@ -348,7 +348,7 @@ class CveProcessingConfig: 4. RPM NEVRA/module strings may embed upstream version separately from distro tags. 5. If description clearly shows the installed version is at or past the fix, answer not vulnerable. -Respond with whether the installed version is vulnerable and explain your reasoning.""" +Respond with whether the installed version is vulnerable. Keep the reason to one sentence.""" class CVEVerifyVulnPackageConfig(FunctionBaseConfig, name="cve_verify_vuln_package"):