Parallel JAR extraction, Maven threads, caching, agent prompt

.tekton/on-cm-runner.yaml

@@ -9,7 +9,7 @@ metadata:
     pipelinesascode.tekton.dev/on-comment: "/test-heavy"
 
     pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]"
-    pipelinesascode.tekton.dev/max-keep-runs: "3"
+    pipelinesascode.tekton.dev/max-keep-runs: "100"
 spec:
   timeouts:
       pipeline: 10h30m0s # Timeout for the entire PipelineRun

.tekton/on-pull-request.yaml

@@ -15,7 +15,7 @@ metadata:
     pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]"
 
     # How many runs we want to keep.
-    pipelinesascode.tekton.dev/max-keep-runs: "5"
+    pipelinesascode.tekton.dev/max-keep-runs: "100"
 spec:
   timeouts:
     pipeline: 2h30m0s # Timeout for the entire PipelineRun

kustomize/base/exploit_iq_service.yaml

@@ -144,6 +144,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: GOMODCACHE
               value: /exploit-iq-package-cache/go/pkg/mod
+            - name: GOCACHE
+              value: /exploit-iq-package-cache/go/cache
+            - name: MAVEN_OPTS
+              value: "-Dmaven.repo.local=/exploit-iq-package-cache/maven"
             - name: ENABLE_MLOPS
               value: "true"
             - name: CREDENTIAL_ENCRYPTION_KEY

src/exploit_iq_commons/utils/dep_tree.py

@@ -15,6 +15,7 @@
 
 import ast
 import configparser
+import concurrent.futures
 import glob as glob_module
 import hashlib
 import json
@@ -29,7 +30,7 @@
 import tomllib
 import urllib.error
 import urllib.request
-import zipfile
+import math
 from abc import ABC
 from abc import abstractmethod
 from collections import defaultdict
@@ -53,6 +54,70 @@
 from collections import deque
 
 from exploit_iq_commons.logging.loggers_factory import LoggingFactory
+
+
+def _available_cpus() -> int:
+    """Return the number of CPUs available to this process.
+
+    Respects container CPU limits from cgroup v2/v1 before falling back to
+    process affinity and Python CPU APIs.
+    """
+
+    # cgroup v2
+    try:
+        with open("/sys/fs/cgroup/cpu.max", encoding="utf-8") as f:
+            quota_s, period_s = f.read().strip().split()
+
+        if quota_s != "max":
+            quota = int(quota_s)
+            period = int(period_s)
+
+            if quota > 0 and period > 0:
+                return max(1, math.ceil(quota / period))
+    except (OSError, ValueError):
+        pass
+
+    # cgroup v1
+    for base in (
+            "/sys/fs/cgroup/cpu",
+            "/sys/fs/cgroup/cpu,cpuacct",
+    ):
+        try:
+            with open(f"{base}/cpu.cfs_quota_us", encoding="utf-8") as f:
+                quota = int(f.read().strip())
+
+            with open(f"{base}/cpu.cfs_period_us", encoding="utf-8") as f:
+                period = int(f.read().strip())
+
+            # cgroup v1 uses quota == -1 to mean "no CPU quota".
+            if quota > 0 and period > 0:
+                return max(1, math.ceil(quota / period))
+        except (OSError, ValueError):
+            pass
+
+    # CPU affinity fallback.
+    try:
+        return max(1, len(os.sched_getaffinity(0)))
+    except (AttributeError, OSError):
+        pass
+
+    # Python 3.13+ fallback.
+    # Keep this after cgroup checks: in OpenShift/Kubernetes it may expose
+    # the full node-visible CPU set rather than the pod CPU limit.
+    if hasattr(os, "process_cpu_count"):
+        count = os.process_cpu_count()
+        if count:
+            return max(1, count)
+
+    return os.cpu_count() or 4
+
+def _extract_source_jar(jar: Path, dest: Path) -> None:
+    """Extract a single source JAR into dest directory."""
+    dest.mkdir(exist_ok=True)
+    result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest)
+    if result.returncode != 0:
+        LoggingFactory.get_agent_logger(__name__).warning(
+            "Failed to extract sources jar: %s (exit code %d)", jar, result.returncode)
 from exploit_iq_commons.utils.java_utils import add_missing_jar_string
 from exploit_iq_commons.utils.java_utils import is_maven_gav
 from exploit_iq_commons.utils.java_utils import parse_depgraph_line
@@ -76,7 +141,6 @@ def _get_go_repo_lock(manifest_path) -> threading.Lock:
 
 
 ROOT_LEVEL_SENTINEL = 'root-top-level-exploit-iq'
-
 TRANSITIVE_ENV_NAME = 'transitive_env'
 INSTALLED_PACKAGES_FILE = 'installed_packages.txt'
 
@@ -169,7 +233,7 @@ def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None:
     ]
     if any(p.is_file() for p in c_candidates):
         for root, dirs, files in os.walk(git_repo_path):
-            dirs[:] = [d for d in dirs if not d.startswith('.')]
+            dirs[:] = [d for d in dirs if d not in _WALK_EXCLUDE_DIRS and not d.startswith('.')]
             if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files):
                 return MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1]
     return None
@@ -1034,18 +1098,18 @@ def install_dependencies(self, manifest_path: Path):
         source_path = self.DEP_SOURCE_DIR
 
         process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources",
-                                         "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path)
+                                         "-DincludeScope=runtime", "-Dmaven.artifact.threads=10",
+                                         f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path)
 
         if process_object.returncode > 0:
-            # Remove stale node_modules owned by a different UID to prevent
-            # EPERM when frontend-maven-plugin runs pnpm install during build.
             for child in manifest_path.rglob("node_modules"):
                 if child.is_dir() and child.name == "node_modules":
                     shutil.rmtree(child, ignore_errors=True)
                     logger.debug("Removed stale %s", child)
 
             process_object = subprocess.run([mvn_command, "clean", "install",
-                                             "-DskipTests", "-s", settings_path], cwd=manifest_path)
+                                             "-DskipTests", "-Dmaven.artifact.threads=10",
+                                             "-s", settings_path], cwd=manifest_path)
             if process_object.returncode > 0:
                 formatted_error_msg = (
                     f"Failed to build project"
@@ -1056,7 +1120,8 @@ def install_dependencies(self, manifest_path: Path):
 
             process_object = subprocess.run([mvn_command, "-s", settings_path,
                                              "dependency:copy-dependencies", "-Dclassifier=sources",
-                                             "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path)
+                                             "-DincludeScope=runtime", "-Dmaven.artifact.threads=10",
+                                             f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path)
 
             if process_object.returncode > 0:
                 formatted_error_msg = (
@@ -1067,18 +1132,21 @@ def install_dependencies(self, manifest_path: Path):
                 raise Exception(formatted_error_msg)
 
         full_source_path = manifest_path / source_path
+        jars_to_extract = []
         for jar in full_source_path.glob("*-sources.jar"):
             if jar.stat().st_size > 0:
-                dest = full_source_path / jar.stem  # folder named after jar
-
+                dest = full_source_path / jar.stem
                 if not dest.exists():
-                    dest.mkdir(exist_ok=True)
-                    result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest)
-                    if result.returncode != 0:
-                        logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode)
+                    jars_to_extract.append((jar, dest))
             else:
                 logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar)
 
+        if jars_to_extract:
+            max_workers = min(_available_cpus() * 2, 8)
+            logger.info("Extracting %d source JARs with %d workers", len(jars_to_extract), max_workers)
+            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+                pool.map(lambda args: _extract_source_jar(*args), jars_to_extract)
+
     def build_tree(self, manifest_path: Path) -> dict[str, list[str]]:
         mvn_command = resolve_mvn_command(manifest_path)
         settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml")
@@ -1091,6 +1159,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]:
                     mvn_command,
                     "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate",
                     "-s", settings_path,
+                    "-Dmaven.artifact.threads=10",
                     "-DgraphFormat=text",
                     "-DshowGroupIds",
                     "-DshowVersions",
@@ -1109,6 +1178,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]:
                     mvn_command,
                     "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate",
                     "-s", settings_path,
+                    "-Dmaven.artifact.threads=10",
                     "-DgraphFormat=text",
                     "-DshowGroupIds",
                     "-DshowVersions",
@@ -1640,21 +1710,6 @@ def _try_file(path: Path, extractor) -> str | None:
 
         return None
 
-    def _ensure_venv(self, manifest_path: Path) -> str:
-        """Ensure transitive_env exists with a working python binary."""
-        venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python'
-        if Path(venv_python).exists():
-            return venv_python
-        logger.warning("Venv python not found at %s — creating venv", venv_python)
-        python_version = self.determine_python_version(str(manifest_path))
-        if not python_version:
-            import sys
-            python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
-            logger.info("Python version undetermined; using current interpreter %s", python_version)
-        logger.info("Creating transitive_env with Python %s using uv", python_version)
-        run_command(["uv", "venv" ,TRANSITIVE_ENV_NAME, "--python", python_version] ,cwd=manifest_path)
-        return venv_python
-
     def install_dependencies(self, manifest_path: Path):
         """Install Python dependencies for the given repository into a virtual environment.
 
@@ -1698,7 +1753,7 @@ def _install_from_best_manifest(self, manifest_path: Path, venv_python: str,
         for lock_file in (UV_LOCK, POETRY_LOCK):
             if (manifest_path / lock_file).exists():
                 working_dir = manifest_path
-                create_requirements_command = ["uv","export","--format","requirements-txt","--no-dev"]
+                create_requirements_command = ["uv","export","--format","requirements-txt","no-dev","2>/dev/null"]
                 requirements_txt_result = run_command(args=create_requirements_command, cwd=working_dir)
 
                 if requirements_txt_result is not None:
@@ -1735,8 +1790,8 @@ def _install_from_requirements_txt(self, req_txt: Path, manifest_path: Path,
 
     def _write_installed_packages(self, manifest_path: Path) -> None:
         """Write a freeze-format snapshot of the venv to installed_packages.txt."""
-        venv_python = f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python"
-        pip_freeze = run_command(["uv", "pip", "freeze", "--python", venv_python])
+        pip_full_path= f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip"
+        pip_freeze = run_command([pip_full_path,"list","--format=freeze"])
         if pip_freeze:
             (manifest_path / INSTALLED_PACKAGES_FILE).write_text(pip_freeze)
             logger.info("Wrote installed packages snapshot to %s/%s", manifest_path, INSTALLED_PACKAGES_FILE)
@@ -1778,7 +1833,7 @@ def _find_module_dirs(self, package_name: str, site_packages: Path) -> list[str]
 
         if package_name.startswith('types-'):
             base = package_name[6:]
-            candidates = [f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()]
+            candidates = list(dict.fromkeys([f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()]))
         elif package_name.startswith('mypy-boto3-'):
             base = package_name[11:]
             candidates = [f'mypy_boto3_{base}']

src/vuln_analysis/functions/cve_verify_vuln_package.py

@@ -348,7 +348,7 @@ class CveProcessingConfig:
 4. RPM NEVRA/module strings may embed upstream version separately from distro tags.
 5. If description clearly shows the installed version is at or past the fix, answer not vulnerable.
 
-Respond with whether the installed version is vulnerable and explain your reasoning."""
+Respond with whether the installed version is vulnerable. Keep the reason to one sentence."""
 
 
 class CVEVerifyVulnPackageConfig(FunctionBaseConfig, name="cve_verify_vuln_package"):