From 40b7ff8aef20958ac382ded1f6a15e797ef538ec Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Thu, 24 Jul 2025 18:29:20 +0000 Subject: [PATCH 001/286] Added new vdc skip function. --- src/vuln_analysis/data_models/dependencies.py | 1 + .../functions/cve_check_vuln_deps.py | 1 + .../functions/cve_check_vuln_deps_skip.py | 83 +++++++++++++++++++ src/vuln_analysis/register.py | 1 + src/vuln_analysis/utils/llm_engine_utils.py | 30 +++---- 5 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 src/vuln_analysis/functions/cve_check_vuln_deps_skip.py diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/vuln_analysis/data_models/dependencies.py index 1af1325f0..57d68c179 100644 --- a/src/vuln_analysis/data_models/dependencies.py +++ b/src/vuln_analysis/data_models/dependencies.py @@ -55,5 +55,6 @@ class VulnerableDependencies(BaseModel): vulnerable for a given vuln_id. """ vuln_id: str + run_agent: bool vuln_package_intel_sources: list[str] vulnerable_sbom_packages: list[VulnerableSBOMPackage] diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index e769266df..2f7d453da 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -114,6 +114,7 @@ async def _calc_dep(cve_intel: CveIntel): # Add the vulnerable dependencies for this CVE to the overall list return VulnerableDependencies(vuln_id=vuln_id, + run_agent=len(vulnerable_sbom_packages) > 0 or len(vuln_package_intel_sources) == 0, vuln_package_intel_sources=vuln_package_intel_sources, vulnerable_sbom_packages=vulnerable_sbom_packages) diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py new file mode 100644 index 000000000..882434b22 --- /dev/null +++ b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging + +import aiohttp +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +logger = logging.getLogger(__name__) + + +class CVEVulnerableDepsChecksSkipConfig(FunctionBaseConfig, name="cve_check_vuln_deps_skip"): + """ + Skips running the Vulnerable Dependency Checker + """ + pass + +@register_function(config_type=CVEVulnerableDepsChecksSkipConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_check_vuln_deps_skip(config: CVEVulnerableDepsChecksSkipConfig, builder: Builder): # pylint: disable=unused-argument + + from vuln_analysis.data_models.cve_intel import CveIntel + from vuln_analysis.data_models.dependencies import VulnerableDependencies + from vuln_analysis.data_models.dependencies import VulnerableSBOMPackage + from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.vulnerable_dependency_checker import VulnerableDependencyChecker + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + + sbom = message.info.sbom.packages + image = f"{message.input.image.name}:{message.input.image.tag}" + + if not sbom: + logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) + message.info.vulnerable_dependencies = [ + VulnerableDependencies(vuln_id=vuln.vuln_id, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) + for vuln in message.input.scan.vulns + ] + return message + + async def _inner(): + + async with aiohttp.ClientSession() as session: + + async def _create_empty_vuln_deps(cve_intel: CveIntel): + vuln_id = cve_intel.vuln_id + + # Add the empty vulnerable dependencies for this CVE to the overall list + return VulnerableDependencies(vuln_id=vuln_id, + run_agent=True, + vuln_package_intel_sources=[], + vulnerable_sbom_packages=[]) + + # Check vulnerable dependencies for each CVE + vulnerable_dependencies: list[VulnerableDependencies] = await asyncio.gather( + *[_create_empty_vuln_deps(cve_intel) for cve_intel in message.info.intel]) + + return vulnerable_dependencies + + # Update the message info with the vulnerable dependencies list + message.info.vulnerable_dependencies = asyncio.run(_inner()) + return message + + yield FunctionInfo.from_fn(_arun, + input_schema=AgentMorpheusEngineInput, + description=("Returns a list of empty vulnerable dependencies for each CVE")) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index a6d938609..fd21befec 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -31,6 +31,7 @@ # pylint: disable=unused-import from vuln_analysis.functions import cve_agent from vuln_analysis.functions import cve_check_vuln_deps +from vuln_analysis.functions import cve_check_vuln_deps_skip from vuln_analysis.functions import cve_checklist from vuln_analysis.functions import cve_fetch_intel from vuln_analysis.functions import cve_file_output diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 786596944..1fba26762 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -47,31 +47,31 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE vulnerable_sbom_packages = v.vulnerable_sbom_packages vulnerable_dependencies.append([p.name for p in vulnerable_sbom_packages]) - has_vuln_package_info_flags = [len(v.vuln_package_intel_sources) > 0 for v in message.info.vulnerable_dependencies] + vdc_run_agent = [v.run_agent for v in message.info.vulnerable_dependencies] - has_sufficient_intel_flags = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] + sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] + + run_agent = [all(tup) for tup in zip(vdc_run_agent, sufficient_intel_run_agent)] vuln_ids = [vuln.vuln_id for vuln in am_input.scan.vulns] - filtered_vuln_ids = [] - for vuln_id, vuln_deps, has_vuln_package_info, has_sufficient_intel in zip( - vuln_ids, vulnerable_dependencies, has_vuln_package_info_flags, - has_sufficient_intel_flags): - if len(vuln_deps) > 0 or (not has_vuln_package_info and has_sufficient_intel): - filtered_vuln_ids.append(vuln_id) + cves_for_agent = [] + for vuln_id, agent_check in zip(vuln_ids, run_agent): + if agent_check: + cves_for_agent.append(vuln_id) # Drop duplicate vuln_ids - unique_vulns = list(OrderedSet(filtered_vuln_ids)) - if len(filtered_vuln_ids) > len(unique_vulns): + unique_vulns = list(OrderedSet(cves_for_agent)) + if len(cves_for_agent) > len(unique_vulns): logger.warning( "Input contains duplicate vuln_ids. Passing only the first instance of each vuln_id to the LLM Engine.") - filtered_vuln_ids = unique_vulns + cves_for_agent = unique_vulns filtered_intel = [] for i in message.info.intel: - if i.vuln_id in filtered_vuln_ids: + if i.vuln_id in cves_for_agent: filtered_intel.append(i) - logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(filtered_vuln_ids)) + logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(cves_for_agent)) return AgentMorpheusEngineState(code_vdb_path=message.info.vdb.code_vdb_path, doc_vdb_path=message.info.vdb.doc_vdb_path, @@ -162,7 +162,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, vulnerable_dependencies: list[VulnerableDependencies] = message.info.vulnerable_dependencies - no_vulns = [vuln_dep.vuln_id for vuln_dep in vulnerable_dependencies if len(vuln_dep.vulnerable_sbom_packages) == 0] + vdc_skip = [vuln_dep.vuln_id for vuln_dep in vulnerable_dependencies if not vuln_dep.run_agent] deficient_intel = [i.vuln_id for i in message.info.intel if not i.has_sufficient_intel_for_agent] @@ -185,7 +185,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, justification=result.justifications[vuln_id])) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) - elif vuln_id in no_vulns: + elif vuln_id in vdc_skip: output.append(build_no_vuln_packages_output(vuln_id)) else: assert False, "CVE has vulnerable dependencies but there is no workflow output." From a106a8015b025cd6e26065d29cf7a567aecac168 Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Fri, 25 Jul 2025 17:14:21 +0000 Subject: [PATCH 002/286] Clean up and documentation. --- README.md | 3 ++- src/vuln_analysis/functions/cve_check_vuln_deps_skip.py | 4 +++- src/vuln_analysis/utils/llm_engine_utils.py | 8 +++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index aaf842b1d..2f9ee2402 100644 --- a/README.md +++ b/README.md @@ -535,7 +535,8 @@ The configuration defines how the workflow operates, including functions, LLMs, - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`. - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. - `cve_process_sbom`: Prepares and validates input SBOM. - - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. + - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. + - `cve_check_vuln_deps_skip`: Skip the cross-reference checks of the SBOM for known vulnerabilities and run every CVE through the agent. - Core LLM engine functions: - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis. - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database. diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py index 882434b22..c362710ce 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py @@ -62,7 +62,9 @@ async def _inner(): async def _create_empty_vuln_deps(cve_intel: CveIntel): vuln_id = cve_intel.vuln_id - # Add the empty vulnerable dependencies for this CVE to the overall list + # Add the empty vulnerable dependencies for this CVE to the overall list. + # Setting run_agent to True guarantees that the CVE will run through the agent, + # granted that there is sufficient intel. return VulnerableDependencies(vuln_id=vuln_id, run_agent=True, vuln_package_intel_sources=[], diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 1fba26762..e8492fe5e 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -42,15 +42,13 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE am_input: AgentMorpheusInput = message.input - vulnerable_dependencies = [] - for v in message.info.vulnerable_dependencies: - vulnerable_sbom_packages = v.vulnerable_sbom_packages - vulnerable_dependencies.append([p.name for p in vulnerable_sbom_packages]) - + # Scan through the VDC output for CVE's to run through the agent vdc_run_agent = [v.run_agent for v in message.info.vulnerable_dependencies] + # Look through the intel for CVE's with sufficient intel for the agent sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] + # Create the filter list by combining the VDC and intel criteria run_agent = [all(tup) for tup in zip(vdc_run_agent, sufficient_intel_run_agent)] vuln_ids = [vuln.vuln_id for vuln in am_input.scan.vulns] From de9ff132433e9b43bd30050a3f1a19d2e9d27394 Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Fri, 25 Jul 2025 13:10:59 -0700 Subject: [PATCH 003/286] Added missing 'run_agent' to vdc skip function. --- src/vuln_analysis/functions/cve_check_vuln_deps_skip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py index c362710ce..596e928a5 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py @@ -31,7 +31,7 @@ class CVEVulnerableDepsChecksSkipConfig(FunctionBaseConfig, name="cve_check_vuln """ Skips running the Vulnerable Dependency Checker """ - pass + missing_sbom_default: bool Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") @register_function(config_type=CVEVulnerableDepsChecksSkipConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_check_vuln_deps_skip(config: CVEVulnerableDepsChecksSkipConfig, builder: Builder): # pylint: disable=unused-argument @@ -50,7 +50,7 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: if not sbom: logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) message.info.vulnerable_dependencies = [ - VulnerableDependencies(vuln_id=vuln.vuln_id, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) + VulnerableDependencies(vuln_id=vuln.vuln_id, run_agent=config.missing_sbom_default, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) for vuln in message.input.scan.vulns ] return message From 3fe5694448ffcf32e29c4e54c33d9d9f0d6edae6 Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Fri, 25 Jul 2025 13:12:40 -0700 Subject: [PATCH 004/286] Missing constructor call. --- src/vuln_analysis/data_models/dependencies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/vuln_analysis/data_models/dependencies.py index 57d68c179..cf67a3bac 100644 --- a/src/vuln_analysis/data_models/dependencies.py +++ b/src/vuln_analysis/data_models/dependencies.py @@ -49,6 +49,8 @@ class VulnerableDependencies(BaseModel): Information about the vulnerable SBOM packages associated with the vuln_id. - vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list. + - run_agent: whether or not this vuln_id should be run through the agent based on the vulnerable + dependency findings. - vuln_package_intel_sources: list of sources (e.g. "ghsa", "nvd", "ubuntu", "rhsa") that provided the vulnerable package/version intel for the vuln_id. - vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are From 927c4ac66af505739624af2ef910dc73552b8bfc Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Fri, 25 Jul 2025 13:44:02 -0700 Subject: [PATCH 005/286] Missing '=' added --- src/vuln_analysis/functions/cve_check_vuln_deps_skip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py index 596e928a5..c03b98743 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py @@ -31,7 +31,7 @@ class CVEVulnerableDepsChecksSkipConfig(FunctionBaseConfig, name="cve_check_vuln """ Skips running the Vulnerable Dependency Checker """ - missing_sbom_default: bool Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") + missing_sbom_default: bool = Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") @register_function(config_type=CVEVulnerableDepsChecksSkipConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_check_vuln_deps_skip(config: CVEVulnerableDepsChecksSkipConfig, builder: Builder): # pylint: disable=unused-argument From 8260df804587510720e7e5d9c883c00f2cdbc638 Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Mon, 28 Jul 2025 07:45:17 -0700 Subject: [PATCH 006/286] Switched skipping to be a parameter of the current VDC. --- src/vuln_analysis/configs/config.yml | 1 + .../morpheus:23.11-runtime.json | 3 + .../functions/cve_check_vuln_deps.py | 13 ++- .../functions/cve_check_vuln_deps_skip.py | 85 ------------------- src/vuln_analysis/register.py | 1 - 5 files changed, 16 insertions(+), 87 deletions(-) delete mode 100644 src/vuln_analysis/functions/cve_check_vuln_deps_skip.py diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 08503cdec..76f6af494 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -30,6 +30,7 @@ functions: _type: cve_process_sbom cve_check_vuln_deps : _type: cve_check_vuln_deps + skip: False cve_checklist: _type: cve_checklist llm_name: checklist_llm diff --git a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json index 2805c6047..21ebe640d 100644 --- a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json +++ b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json @@ -58,6 +58,9 @@ }, { "vuln_id": "GHSA-3ww4-gg4f-jr7f" + }, + { + "vuln_id": "CVE-2023-51767" } ] } diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 2f7d453da..272d835b1 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -31,7 +31,9 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep """ Defines a function that cross-references every entry in the SBOM for known vulnerabilities. """ + missing_sbom_default: bool = Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") + skip: bool = Field(default=False, description="Whether or not the vulnerability dependency checker should be skipped.") @register_function(config_type=CVEVulnerableDepsChecksConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -51,7 +53,7 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: if not sbom: logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) message.info.vulnerable_dependencies = [ - VulnerableDependencies(vuln_id=vuln.vuln_id, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) + VulnerableDependencies(vuln_id=vuln.vuln_id, run_agent=config.missing_sbom_default, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) for vuln in message.input.scan.vulns ] return message @@ -70,6 +72,15 @@ async def _inner(): async def _calc_dep(cve_intel: CveIntel): vuln_id = cve_intel.vuln_id + if config.skip: + # Add the empty vulnerable dependencies for this CVE to the overall list. + # Setting run_agent to True guarantees that the CVE will run through the agent, + # granted that there is sufficient intel. + return VulnerableDependencies(vuln_id=vuln_id, + run_agent=True, + vuln_package_intel_sources=[], + vulnerable_sbom_packages=[]) + vuln_deps = [] vuln_package_intel_sources = [] diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py b/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py deleted file mode 100644 index c03b98743..000000000 --- a/src/vuln_analysis/functions/cve_check_vuln_deps_skip.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import logging - -import aiohttp -from aiq.builder.builder import Builder -from aiq.builder.framework_enum import LLMFrameworkEnum -from aiq.builder.function_info import FunctionInfo -from aiq.cli.register_workflow import register_function -from aiq.data_models.function import FunctionBaseConfig -from pydantic import Field - -logger = logging.getLogger(__name__) - - -class CVEVulnerableDepsChecksSkipConfig(FunctionBaseConfig, name="cve_check_vuln_deps_skip"): - """ - Skips running the Vulnerable Dependency Checker - """ - missing_sbom_default: bool = Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") - -@register_function(config_type=CVEVulnerableDepsChecksSkipConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) -async def cve_check_vuln_deps_skip(config: CVEVulnerableDepsChecksSkipConfig, builder: Builder): # pylint: disable=unused-argument - - from vuln_analysis.data_models.cve_intel import CveIntel - from vuln_analysis.data_models.dependencies import VulnerableDependencies - from vuln_analysis.data_models.dependencies import VulnerableSBOMPackage - from vuln_analysis.data_models.input import AgentMorpheusEngineInput - from vuln_analysis.utils.vulnerable_dependency_checker import VulnerableDependencyChecker - - async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - - sbom = message.info.sbom.packages - image = f"{message.input.image.name}:{message.input.image.tag}" - - if not sbom: - logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) - message.info.vulnerable_dependencies = [ - VulnerableDependencies(vuln_id=vuln.vuln_id, run_agent=config.missing_sbom_default, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) - for vuln in message.input.scan.vulns - ] - return message - - async def _inner(): - - async with aiohttp.ClientSession() as session: - - async def _create_empty_vuln_deps(cve_intel: CveIntel): - vuln_id = cve_intel.vuln_id - - # Add the empty vulnerable dependencies for this CVE to the overall list. - # Setting run_agent to True guarantees that the CVE will run through the agent, - # granted that there is sufficient intel. - return VulnerableDependencies(vuln_id=vuln_id, - run_agent=True, - vuln_package_intel_sources=[], - vulnerable_sbom_packages=[]) - - # Check vulnerable dependencies for each CVE - vulnerable_dependencies: list[VulnerableDependencies] = await asyncio.gather( - *[_create_empty_vuln_deps(cve_intel) for cve_intel in message.info.intel]) - - return vulnerable_dependencies - - # Update the message info with the vulnerable dependencies list - message.info.vulnerable_dependencies = asyncio.run(_inner()) - return message - - yield FunctionInfo.from_fn(_arun, - input_schema=AgentMorpheusEngineInput, - description=("Returns a list of empty vulnerable dependencies for each CVE")) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index fd21befec..a6d938609 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -31,7 +31,6 @@ # pylint: disable=unused-import from vuln_analysis.functions import cve_agent from vuln_analysis.functions import cve_check_vuln_deps -from vuln_analysis.functions import cve_check_vuln_deps_skip from vuln_analysis.functions import cve_checklist from vuln_analysis.functions import cve_fetch_intel from vuln_analysis.functions import cve_file_output From 378086abb00a8c169e5777ea2e7e9268bef72adc Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:24:03 -0700 Subject: [PATCH 007/286] Added more explicit VDC skipping. Cleaned up README and config --- README.md | 1 - src/vuln_analysis/configs/config.yml | 2 +- src/vuln_analysis/data_models/dependencies.py | 2 -- .../functions/cve_check_vuln_deps.py | 19 ++++++------------- src/vuln_analysis/utils/llm_engine_utils.py | 13 ++++++++++--- src/vuln_analysis/utils/output_formatter.py | 3 +++ 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 2f9ee2402..94dfa6f27 100644 --- a/README.md +++ b/README.md @@ -536,7 +536,6 @@ The configuration defines how the workflow operates, including functions, LLMs, - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. - `cve_process_sbom`: Prepares and validates input SBOM. - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. - - `cve_check_vuln_deps_skip`: Skip the cross-reference checks of the SBOM for known vulnerabilities and run every CVE through the agent. - Core LLM engine functions: - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis. - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database. diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 76f6af494..cc1e66371 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -30,7 +30,7 @@ functions: _type: cve_process_sbom cve_check_vuln_deps : _type: cve_check_vuln_deps - skip: False + skip: false cve_checklist: _type: cve_checklist llm_name: checklist_llm diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/vuln_analysis/data_models/dependencies.py index cf67a3bac..f8ff1e085 100644 --- a/src/vuln_analysis/data_models/dependencies.py +++ b/src/vuln_analysis/data_models/dependencies.py @@ -49,7 +49,6 @@ class VulnerableDependencies(BaseModel): Information about the vulnerable SBOM packages associated with the vuln_id. - vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list. - - run_agent: whether or not this vuln_id should be run through the agent based on the vulnerable dependency findings. - vuln_package_intel_sources: list of sources (e.g. "ghsa", "nvd", "ubuntu", "rhsa") that provided the vulnerable package/version intel for the vuln_id. @@ -57,6 +56,5 @@ class VulnerableDependencies(BaseModel): vulnerable for a given vuln_id. """ vuln_id: str - run_agent: bool vuln_package_intel_sources: list[str] vulnerable_sbom_packages: list[VulnerableSBOMPackage] diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 272d835b1..708f75cf3 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -31,7 +31,6 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep """ Defines a function that cross-references every entry in the SBOM for known vulnerabilities. """ - missing_sbom_default: bool = Field(default=True, description="If SBOM package info is missing, this determines if the CVE is run by the agent or not.") retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") skip: bool = Field(default=False, description="Whether or not the vulnerability dependency checker should be skipped.") @@ -50,10 +49,15 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: sbom = message.info.sbom.packages image = f"{message.input.image.name}:{message.input.image.tag}" + if config.skip: + logger.info("`config.skip` is set to True. Skipping vulnerable dependency check for image %s.", image) + message.info.vulnerable_dependencies = None + return message + if not sbom: logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) message.info.vulnerable_dependencies = [ - VulnerableDependencies(vuln_id=vuln.vuln_id, run_agent=config.missing_sbom_default, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) + VulnerableDependencies(vuln_id=vuln.vuln_id, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) for vuln in message.input.scan.vulns ] return message @@ -71,16 +75,6 @@ async def _inner(): async def _calc_dep(cve_intel: CveIntel): vuln_id = cve_intel.vuln_id - - if config.skip: - # Add the empty vulnerable dependencies for this CVE to the overall list. - # Setting run_agent to True guarantees that the CVE will run through the agent, - # granted that there is sufficient intel. - return VulnerableDependencies(vuln_id=vuln_id, - run_agent=True, - vuln_package_intel_sources=[], - vulnerable_sbom_packages=[]) - vuln_deps = [] vuln_package_intel_sources = [] @@ -125,7 +119,6 @@ async def _calc_dep(cve_intel: CveIntel): # Add the vulnerable dependencies for this CVE to the overall list return VulnerableDependencies(vuln_id=vuln_id, - run_agent=len(vulnerable_sbom_packages) > 0 or len(vuln_package_intel_sources) == 0, vuln_package_intel_sources=vuln_package_intel_sources, vulnerable_sbom_packages=vulnerable_sbom_packages) diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index e8492fe5e..959b7fc81 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -43,7 +43,10 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE am_input: AgentMorpheusInput = message.input # Scan through the VDC output for CVE's to run through the agent - vdc_run_agent = [v.run_agent for v in message.info.vulnerable_dependencies] + if message.info.vulnerable_dependencies is None: + vdc_run_agent = [True for _ in am_input.scan.vulns] + else: + vdc_run_agent = [len(v.vulnerable_sbom_packages) > 0 or len(v.vuln_package_intel_sources) == 0 for v in message.info.vulnerable_dependencies] # Look through the intel for CVE's with sufficient intel for the agent sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] @@ -158,9 +161,13 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: def postprocess_engine_output(message: AgentMorpheusEngineInput, result: AgentMorpheusEngineState) -> AgentMorpheusOutput: - vulnerable_dependencies: list[VulnerableDependencies] = message.info.vulnerable_dependencies + vulnerable_dependencies: list[VulnerableDependencies] | None = message.info.vulnerable_dependencies - vdc_skip = [vuln_dep.vuln_id for vuln_dep in vulnerable_dependencies if not vuln_dep.run_agent] + if vulnerable_dependencies is None: + vdc_skip = [] + else: + vdc_skip = [v.vuln_id for v in vulnerable_dependencies + if len(v.vulnerable_sbom_packages) == 0 and len(v.vuln_package_intel_sources) > 0] deficient_intel = [i.vuln_id for i in message.info.intel if not i.has_sufficient_intel_for_agent] diff --git a/src/vuln_analysis/utils/output_formatter.py b/src/vuln_analysis/utils/output_formatter.py index 1773dde02..6604236ee 100644 --- a/src/vuln_analysis/utils/output_formatter.py +++ b/src/vuln_analysis/utils/output_formatter.py @@ -454,6 +454,9 @@ def _add_vulnerable_sboms(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ + if model_dict.info.vulnerable_dependencies is None: + return + vulnerable_sboms_per_cve_id = { i.vuln_id: i.vulnerable_sbom_packages for i in model_dict.info.vulnerable_dependencies From aa70bea2a8bf0c9516b9766677e854814ffe266f Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:39:37 -0700 Subject: [PATCH 008/286] Added vdc skip param to config-tracing --- src/vuln_analysis/configs/config-tracing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 85be3db1f..d6503f1d9 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -40,6 +40,7 @@ functions: _type: cve_process_sbom cve_check_vuln_deps : _type: cve_check_vuln_deps + skip: false cve_checklist: _type: cve_checklist llm_name: checklist_llm From 03e426d201b91e3af859f13ab9429db09e2a2e77 Mon Sep 17 00:00:00 2001 From: shawn-davis <12801620+shawn-davis@users.noreply.github.com> Date: Wed, 30 Jul 2025 15:49:17 -0400 Subject: [PATCH 009/286] Update src/vuln_analysis/data_models/dependencies.py Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com> --- src/vuln_analysis/data_models/dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/vuln_analysis/data_models/dependencies.py index f8ff1e085..1af1325f0 100644 --- a/src/vuln_analysis/data_models/dependencies.py +++ b/src/vuln_analysis/data_models/dependencies.py @@ -49,7 +49,6 @@ class VulnerableDependencies(BaseModel): Information about the vulnerable SBOM packages associated with the vuln_id. - vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list. - dependency findings. - vuln_package_intel_sources: list of sources (e.g. "ghsa", "nvd", "ubuntu", "rhsa") that provided the vulnerable package/version intel for the vuln_id. - vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are From 180e49a1b5184ecc6cb7dfa426056f5d741b96d1 Mon Sep 17 00:00:00 2001 From: shawn-davis <12801620+shawn-davis@users.noreply.github.com> Date: Wed, 30 Jul 2025 15:49:59 -0400 Subject: [PATCH 010/286] Update README.md Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com> --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94dfa6f27..26c9a045b 100644 --- a/README.md +++ b/README.md @@ -536,6 +536,7 @@ The configuration defines how the workflow operates, including functions, LLMs, - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. - `cve_process_sbom`: Prepares and validates input SBOM. - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. + - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false. - Core LLM engine functions: - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis. - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database. From c47551818c722729b2a4e07ef949138a2bb4fe56 Mon Sep 17 00:00:00 2001 From: shdavis <12801620+shawn-davis@users.noreply.github.com> Date: Wed, 30 Jul 2025 20:43:21 +0000 Subject: [PATCH 011/286] Formatting and clean-up --- .../functions/cve_check_vuln_deps.py | 3 +- src/vuln_analysis/utils/llm_engine_utils.py | 33 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 708f75cf3..9a650ea27 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -32,7 +32,8 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep Defines a function that cross-references every entry in the SBOM for known vulnerabilities. """ retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") - skip: bool = Field(default=False, description="Whether or not the vulnerability dependency checker should be skipped.") + skip: bool = Field(default=False, + description="Whether or not the vulnerability dependency checker should be skipped.") @register_function(config_type=CVEVulnerableDepsChecksConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 959b7fc81..25e1abe7a 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -46,7 +46,10 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE if message.info.vulnerable_dependencies is None: vdc_run_agent = [True for _ in am_input.scan.vulns] else: - vdc_run_agent = [len(v.vulnerable_sbom_packages) > 0 or len(v.vuln_package_intel_sources) == 0 for v in message.info.vulnerable_dependencies] + vdc_run_agent = [ + len(v.vulnerable_sbom_packages) > 0 or len(v.vuln_package_intel_sources) == 0 + for v in message.info.vulnerable_dependencies + ] # Look through the intel for CVE's with sufficient intel for the agent sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] @@ -55,24 +58,24 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE run_agent = [all(tup) for tup in zip(vdc_run_agent, sufficient_intel_run_agent)] vuln_ids = [vuln.vuln_id for vuln in am_input.scan.vulns] - cves_for_agent = [] - for vuln_id, agent_check in zip(vuln_ids, run_agent): - if agent_check: - cves_for_agent.append(vuln_id) + vulns_for_agent = [] + for vuln_id, should_run_agent in zip(vuln_ids, run_agent): + if should_run_agent: + vulns_for_agent.append(vuln_id) # Drop duplicate vuln_ids - unique_vulns = list(OrderedSet(cves_for_agent)) - if len(cves_for_agent) > len(unique_vulns): + unique_vulns = list(OrderedSet(vulns_for_agent)) + if len(vulns_for_agent) > len(unique_vulns): logger.warning( "Input contains duplicate vuln_ids. Passing only the first instance of each vuln_id to the LLM Engine.") - cves_for_agent = unique_vulns + vulns_for_agent = unique_vulns filtered_intel = [] for i in message.info.intel: - if i.vuln_id in cves_for_agent: + if i.vuln_id in vulns_for_agent: filtered_intel.append(i) - logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(cves_for_agent)) + logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(vulns_for_agent)) return AgentMorpheusEngineState(code_vdb_path=message.info.vdb.code_vdb_path, doc_vdb_path=message.info.vdb.doc_vdb_path, @@ -164,10 +167,12 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, vulnerable_dependencies: list[VulnerableDependencies] | None = message.info.vulnerable_dependencies if vulnerable_dependencies is None: - vdc_skip = [] + vdc_skipped_vulns = [] else: - vdc_skip = [v.vuln_id for v in vulnerable_dependencies - if len(v.vulnerable_sbom_packages) == 0 and len(v.vuln_package_intel_sources) > 0] + vdc_skipped_vulns = [ + v.vuln_id for v in vulnerable_dependencies + if len(v.vulnerable_sbom_packages) == 0 and len(v.vuln_package_intel_sources) > 0 + ] deficient_intel = [i.vuln_id for i in message.info.intel if not i.has_sufficient_intel_for_agent] @@ -190,7 +195,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, justification=result.justifications[vuln_id])) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) - elif vuln_id in vdc_skip: + elif vuln_id in vdc_skipped_vulns: output.append(build_no_vuln_packages_output(vuln_id)) else: assert False, "CVE has vulnerable dependencies but there is no workflow output." From 63579c97d22e723ba51a129bb1f9ef2480340d73 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 3 Aug 2025 14:43:14 +0300 Subject: [PATCH 012/286] initial changes verifying core functions Signed-off-by: Zvi Grinberg --- .../configs/config-http-openai.yml | 171 ++++++++++++++++++ src/vuln_analysis/utils/document_embedding.py | 2 +- 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 src/vuln_analysis/configs/config-http-openai.yml diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml new file mode 100644 index 000000000..8629b101b --- /dev/null +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + +functions: + cve_generate_vdbs: + _type: cve_generate_vdbs + agent_name: cve_agent_executor # Used to determine which tools are enabled + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + cve_process_sbom: + _type: cve_process_sbom + cve_check_vuln_deps : + _type: cve_check_vuln_deps + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Container Image Code QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Container Image Developer Guide QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Lexical Search Container Image Code QA System: + _type: lexical_code_search + top_k: 5 + Internet Search: + _type: serp_wrapper + max_retries: 5 + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Internet Search + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /reports + +llms: + checklist_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + summarize_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_generate_vdbs_name: cve_generate_vdbs + cve_fetch_intel_name: cve_fetch_intel + cve_process_sbom_name: cve_process_sbom + cve_check_vuln_deps_name: cve_check_vuln_deps + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index d06d16f1e..0470e9002 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -462,7 +462,7 @@ def build_vdbs(self, # Create embeddings for each source type for source_type in ["code", "doc"]: - if ignore_code_embedding and source_type == "code": + if ignore_code_embedding: continue # Filter the source documents From 4e99213150cdd0296ba0c9a9ac5553b68b56ff96 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 3 Aug 2025 17:36:41 +0300 Subject: [PATCH 013/286] chore: skip check_vuln_deps stage by config fix: allow http output endpoint to return http status >= 200 <= 202 Signed-off-by: Zvi Grinberg --- src/vuln_analysis/configs/config-http-openai.yml | 1 + src/vuln_analysis/functions/cve_http_output.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 8629b101b..7038869a8 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -31,6 +31,7 @@ functions: _type: cve_process_sbom cve_check_vuln_deps : _type: cve_check_vuln_deps + skip: true cve_checklist: _type: cve_checklist llm_name: checklist_llm diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index f80ac7729..60683fb2d 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -14,6 +14,7 @@ # limitations under the License. import logging +from http import HTTPStatus from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo @@ -46,7 +47,7 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: try: http_utils.request_with_retry(request_kwargs={ "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers - }) + }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) except Exception as e: logger.error('Unable to send output response to %s. Error: %s', url, e) else: From fa4c4e1c2836086c4de817532626468dc51d19b9 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 7 Aug 2025 17:55:14 +0300 Subject: [PATCH 014/286] feat: migrate transitive code search tool to NAT Signed-off-by: Zvi Grinberg --- .../configs/config-http-openai.yml | 16 + src/vuln_analysis/data_models/state.py | 2 + src/vuln_analysis/functions/cve_agent.py | 5 +- .../functions/cve_generate_vdbs.py | 6 +- src/vuln_analysis/register.py | 2 + .../tools/transitive_code_search.py | 133 ++++ .../utils/chain_of_calls_retriever.py | 385 ++++++++++++ .../utils/clients/ubuntu_client.py | 2 +- src/vuln_analysis/utils/dep_tree.py | 164 +++++ src/vuln_analysis/utils/document_embedding.py | 78 ++- .../utils/function_name_extractor.py | 117 ++++ .../utils/functions_parsers/__init__.py | 14 + .../golang_functions_parsers.py | 581 ++++++++++++++++++ .../lang_functions_parsers.py | 108 ++++ .../lang_functions_parsers_factory.py | 18 + .../utils/go_segmenters_with_methods.py | 77 +++ src/vuln_analysis/utils/llm_engine_utils.py | 3 +- .../utils/transitive_code_searcher_tool.py | 123 ++++ 18 files changed, 1806 insertions(+), 28 deletions(-) create mode 100644 src/vuln_analysis/tools/transitive_code_search.py create mode 100644 src/vuln_analysis/utils/chain_of_calls_retriever.py create mode 100644 src/vuln_analysis/utils/dep_tree.py create mode 100644 src/vuln_analysis/utils/function_name_extractor.py create mode 100644 src/vuln_analysis/utils/functions_parsers/__init__.py create mode 100644 src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py create mode 100644 src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py create mode 100644 src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py create mode 100644 src/vuln_analysis/utils/go_segmenters_with_methods.py create mode 100644 src/vuln_analysis/utils/transitive_code_searcher_tool.py diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 7038869a8..4002b1408 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -15,6 +15,12 @@ general: use_uvloop: true + telemetry: + tracing: + langsmith: + _type: langsmith + project: default + api_key: ${LANGSMITH_API_KEY} functions: cve_generate_vdbs: @@ -24,6 +30,7 @@ functions: base_git_dir: .cache/am_cache/git base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -35,6 +42,12 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm + Transitive code search tool: + _type: transitive_code_search + enable_transitive_search: true + Calling Function Name Extractor: + _type: calling_function_name_extractor + enable_functions_usage_search: true Container Image Code QA System: _type: local_vdb_retriever embedder_name: nim_embedder @@ -53,6 +66,7 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 + cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -61,6 +75,8 @@ functions: - Container Image Developer Guide QA System - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - Internet Search + - Transitive code search tool + - Calling Function Name Extractor max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index ac61bc7d4..e7ee0bc1b 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -18,6 +18,7 @@ from pydantic import BaseModel from vuln_analysis.data_models.cve_intel import CveIntel +from vuln_analysis.data_models.input import AgentMorpheusEngineInput class AgentMorpheusEngineState(BaseModel): @@ -25,6 +26,7 @@ class AgentMorpheusEngineState(BaseModel): doc_vdb_path: str | None = None code_index_path: str | None = None cve_intel: list[CveIntel] + original_input: AgentMorpheusEngineInput | None = None checklist_plans: dict[str, list[str]] = {} checklist_results: dict[str, list[dict[str, typing.Any]]] = {} final_summaries: dict[str, str] = {} diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 29dd006fd..f214b95dc 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -75,7 +75,10 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, tool for tool in tools if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or - (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None)) + (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None) + # (tool.name == "Transitive code search tool" and state.code_index_path is None) or + # (tool.name == "Calling Function Name Extractor" and state.code_index_path is None) + ) ] agent = create_react_agent(llm=llm, diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index c3134a09d..d2bb55ce6 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -45,6 +45,9 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): description="The directory used for storing vector database files.") base_code_index_dir: str = Field(default=".cache/am_cache/code_index", description="The directory used for storing code index files.") + base_pickle_dir: str = Field(default=".cache/am_cache/pickle", + description="The directory used for storing code index files.") + ignore_errors: bool = Field(default=False, description="Whether to ignore errors during generation.") ignore_code_embedding: bool = Field(default=False, description="Whether to ignore building code vector database.") ignore_code_index: bool = Field(default=False, description="Whether to ignore building code index.") @@ -79,7 +82,8 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): embedder = DocumentEmbedding(embedding=embedding, vdb_directory=config.base_vdb_dir, - git_directory=config.base_git_dir) + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir) def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: DocumentEmbedding, output_path: Path): logger.info("Collecting documents from git repos. Source Infos: %s", diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index a6d938609..a7709d09f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -40,6 +40,8 @@ from vuln_analysis.functions import cve_process_sbom from vuln_analysis.functions import cve_summarize from vuln_analysis.tools import lexical_full_search +# This is actually registers the tool in the type registry of NAT! +from vuln_analysis.tools import transitive_code_search from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp # pylint: enable=unused-import diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py new file mode 100644 index 000000000..a1d1a02aa --- /dev/null +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextvars +import logging +from pathlib import Path +from vuln_analysis.functions.cve_agent import ctx_state +from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from langchain.docstore.document import Document + +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from ..utils.chain_of_calls_retriever import ChainOfCallsRetriever +from vuln_analysis.utils.dep_tree import Ecosystem +from vuln_analysis.utils.document_embedding import DocumentEmbedding +from ..utils.function_name_extractor import FunctionNameExtractor + +logger = logging.getLogger(__name__) + +cached_coc_retriever = contextvars.ContextVar("cached_coc_retriever", default=None) + + +class TransitiveCodeSearchToolConfig(FunctionBaseConfig, name="transitive_code_search"): + """ + Transitive code search tool used to search source code. + """ + enable_transitive_search: bool = Field(default=True, description="Whether to enable transitive code search or not.") + + +class CallingFunctionNameExtractorToolConfig(FunctionBaseConfig, name="calling_function_name_extractor"): + """ + Calling function name extractor tool used to search source code function specific usages. + """ + enable_functions_usage_search: bool = Field(default=True, description="Whether to enable function usage code search" + " or not.") + + +def get_call_of_chains_retriever(documents_embedder, si): + documents: list[Document] + third_party_deps_installed: bool + if cached_coc_retriever.get() is None: + for source_info in si: + if source_info.type == "code": + git_repo = documents_embedder.get_repo_path(source_info) + third_party_deps_installed \ + = TransitiveCodeSearcher.download_dependencies(git_repo_path=git_repo) + documents = documents_embedder.collect_documents(source_info) + coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=Ecosystem.GO, manifest_path=git_repo) + cached_coc_retriever.set(coc_retriever) + return coc_retriever + else: + return cached_coc_retriever.get() + + +@register_function(config_type=TransitiveCodeSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def transitive_search(config: TransitiveCodeSearchToolConfig, + builder: Builder): # pylint: disable=unused-argument + + async def _arun(query: str) -> tuple: + if config.enable_transitive_search: + workflow_state: AgentMorpheusEngineState = ctx_state.get() + si = workflow_state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + result = transitive_code_searcher.search(query) + return result + + yield FunctionInfo.from_fn( + _arun, + description=("This tool is useful when you need to search the container image's code for " + "a given function or method in a given package or library, if it's being used in" + " application or called from application code base. " + "This requires both package name and function/method name, separated by " + "',' as input. Input example - 'package_name,function_name'." + " It returns a tuple of first boolean element indicating whether the function" + " is being called or reachable from an application code base or not, and " + "the second element is a list of documents comprising the call of " + "hierarchy path, showing the calls hierarchy from application " + "to a function in the package,in case the first element value in the tuple is True." + "Do not use this list of documents in subsequent or later checks")) + + +@register_function(config_type=CallingFunctionNameExtractorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, + builder: Builder): # pylint: disable=unused-argument + + async def _arun(query: str) -> list: + if config.enable_functions_usage_search: + workflow_state: AgentMorpheusEngineState = ctx_state.get() + si = workflow_state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + function_name_extractor = FunctionNameExtractor(coc_retriever) + result = function_name_extractor.fetch_list(query) + return result + else: + return [] + + yield FunctionInfo.from_fn( + _arun, + description=("This tool is useful when you need to search the container image's code " + "for a specific usage or occurrences of standard library function or method within" + " a specific package, in order " + "to get a list of calling functions within that package." + " It gets one parameter, containing the package name to search in and " + "the function usage string, for example - " + "input_package,sysPackage.functionName(arg, 'textLiteral') " + "It returns list of all functions containing usages in the given package," + " for example - ['input_package,function1','input_package,function2']. " + "If there are no usages returns empty list." + "Each one of the items in the returned list can be checked with another search " + "tool, in order to determine eventually whether usage " + "of a standard library function is called or not from an application code base." + " Even if the package name in the results is not the same as the expected one," + " use it anyway.")) diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py new file mode 100644 index 000000000..01ec88b36 --- /dev/null +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -0,0 +1,385 @@ +import re +from pathlib import Path +from typing import Any, List, Optional + +from langchain_core.documents import Document + +import logging + +from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from .functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from .functions_parsers.lang_functions_parsers_factory import ( + get_language_function_parser, +) + +PARENTS_INDEX = 0 + +EXCLUSIONS_INDEX = 1 + +logger = logging.getLogger(f"morpheus.{__name__}") + + +def calculate_hashable_string_for_function(function_file_name: str, function_name_to_search: str) -> str: + return f"{function_file_name};{function_name_to_search}" + + +def get_extension_of_file(file_path: str): + extension_start = file_path.rfind(".") + return file_path[extension_start:] + + +def get_functions_for_package(package_name: str, documents: list[Document], language_parser: LanguageFunctionsParser, + callee_function_file_name="", function_to_search="", sources_location_packages=True): + # Retrieve documents of packages only ( functions + code) + if sources_location_packages: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (document.metadata.get('source').startswith(language_parser.dir_name_for_3rd_party_packages()) and + document.metadata.get('content_type') == 'functions_classes' and + language_parser.is_function(document) and + is_function_callable(document, language_parser, callee_function_file_name) and + language_parser.is_supported_file_extensions(doc_extension) and + document_belongs_to_package(language_parser, document, package_name) and + function_called_from_caller_body(document, function_to_search, language_parser)): + yield document + # Retrieve documents of application only ( functions + code) + else: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (language_parser.is_root_package(document) + and document.metadata.get('content_type') == 'functions_classes' + and language_parser.is_function(document) + and language_parser.is_supported_file_extensions(doc_extension) + and function_called_from_caller_body(document, function_to_search, language_parser)): + yield document + + +def is_function_callable(document: Document, language_parser, callee_function_file_name: str) -> bool: + return (language_parser.is_exported_function(document) or + document.metadata['source'].lower() == callee_function_file_name.lower()) + + +def function_called_from_caller_body(document, function_to_search, language_parser: LanguageFunctionsParser) -> bool: + if function_to_search.strip() == "": + return True + function_word = language_parser.get_function_reserved_word() + func_header_template = rf"${function_word} (\(.*\))?\s?${function_to_search}" + return (re.search(pattern=function_to_search, string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE) + # verify caller function or method is not the function + and not re.search(pattern=func_header_template, + string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE)) + + +def document_belongs_to_package(language_parser: LanguageFunctionsParser, document: Document, + package_name: str) -> bool: + return (not language_parser.is_root_package(document) and + language_parser.get_package_name(function=document, package_name=package_name)) + + +class ChainOfCallsRetriever: + """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being + called from the application code base or not. + """ + last_visited_parent_package_indexes: Optional[dict] + documents: Optional[List[Document]] + documents_of_full_sources: Optional[dict] + documents_of_types: Optional[list] + language_parser: Optional[LanguageFunctionsParser] + dependency_tree: Optional[DependencyTree] + tree_dict: Optional[dict] + ecosystem: Optional[Ecosystem] + manifest_path: Optional[Path] + package_name: Optional[str] + found_path: Optional[bool] + types_classes_fields_mapping: Optional[dict[tuple, list[tuple]]] + functions_local_variables_index: Optional[dict[str, dict]] + + def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_path: Path): + """ + + :param documents: + List of documents containing the functions/methods and classes/types of the application code + + application dependencies' code + :param ecosystem: Ecosystem + The programming language of the list of documents + :param manifest_path: str + The manifest of the application, defining all the direct and transitive dependencies of the application, being + used here to build the dependency tree for a more efficient lookup and search. + """ + + logger.debug("Creating Chain of Calls Retriever") + logger.debug("Starting building Chain of Calls Retriever") + self.ecosystem = ecosystem + logger.debug("Chain of Calls Retriever - creating dependency tree") + # Build dependency tree based on the parameter programming language/package manager. + self.dependency_tree = DependencyTree(ecosystem=ecosystem) + logger.debug("Chain of Calls Retriever - get language parser") + self.language_parser = get_language_function_parser(ecosystem) + self.manifest_path = manifest_path + # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. + if self.dependency_tree.builder is None: + raise RuntimeError("Couldn't continue as dependency tree weren't generated") + + self.tree_dict = dict() + + # Build a dependency tree using the dependency tree builder logic. + for package, parents in self.dependency_tree.builder.build_tree(manifest_path=manifest_path).items(): + parents.extend([package]) + self.tree_dict[package] = list() + self.tree_dict[package].append(parents) + self.tree_dict[package].append([]) + logger.debug("Chain of Calls Retriever - populating functions documents") + allowed_files_extensions = self.language_parser.supported_files_extensions() + # filter out unsupported files extensions. + filtered_documents = [doc for doc in documents + if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) + .endswith(ext)])] + # filter out types and full code documents, retaining only functions/methods documents in this attribute. + self.documents = [doc for doc in filtered_documents + if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] + # filter out full code documents and functions/methods docs, retaining only types/classes docs + self.documents_of_types = [doc for doc in filtered_documents + if doc.page_content.startswith(self.language_parser.get_type_reserved_word())] + # boolean attribute that indicates whether a path was found or not, initially set to False. + self.found_path = False + # Filter out all documents but full source docs. + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in filtered_documents + if doc.metadata.get('content_type') == 'simplified_code'} + logger.debug("Chain of Calls Retriever - after documents_of_full_sources") + self.last_visited_parent_package_indexes = dict() + # Constructing a map of types and classes to their attributes/members/fields + self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields( + self.documents_of_types) + # Create a data structure containing dict of key=(function_name@source_file),value = dict of + # local variables names mapped to (types, (values or expressions)) + self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents) + logger.debug("Chain of Calls Retriever - after functions_local_variables_index") + + def __find_caller_function(self, document_function: Document, function_package: str) -> Document: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :return: a document of a function that is calling document_function + """ + package_names = self.language_parser.get_package_names(document_function) + direct_parents = list() + # gets list of all direct parents of function + for package_name in package_names: + list_of_packages = self.tree_dict.get(package_name) + if list_of_packages is not None: + direct_parents.extend(list_of_packages[PARENTS_INDEX]) + # Add same package itself to search path. + # direct_parents.extend([function_package]) + # gets list of documents to search in only from parents of function' package. + function_name_to_search = self.language_parser.get_function_name(document_function) + function_file_name = document_function.metadata.get('source') + relevant_docs_to_search_in = list() + last_visited_package_index = (self.last_visited_parent_package_indexes + .get(calculate_hashable_string_for_function(function_file_name, + function_name_to_search), 0)) + package_exclusions = self.tree_dict.get(function_package)[EXCLUSIONS_INDEX] + # Search for caller functions only at parents according to dependency tree. + for package_index, package in enumerate(direct_parents[last_visited_package_index:]): + sources_location_packages = True + if self.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: + sources_location_packages = False + + # Collect all potential caller functions + for doc in get_functions_for_package(package_name=package, + documents=self.get_possible_docs(function_name_to_search, package, + package_exclusions, + sources_location_packages), + language_parser=self.language_parser, + sources_location_packages=sources_location_packages, + function_to_search=function_name_to_search, + callee_function_file_name=function_file_name): + relevant_docs_to_search_in.append(doc) + # Perform the search only on the subset of potential caller functions + for doc in relevant_docs_to_search_in: + function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, + callee_function= + function_name_to_search, + callee_function_package= + function_package, + code_documents= + self.documents_of_full_sources, + type_documents= + self.documents_of_types, + callee_function_file_name= + function_file_name, + fields_of_types= + self.types_classes_fields_mapping + , + functions_local_variables_index= + self.functions_local_variables_index) + + # If current document is found to be calling document_function in function_package, then return it as a + # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. + if function_is_being_called: + package_exclusions.append(doc) + # update index of last scanned package for backtracking + # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) + # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index + return doc + + # If didn't find a matching caller function document, returns None. + return None + + + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all + # excluded functions, and all function that their body doesn't contain the target function name to search for. + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool) \ + -> (list[ + Document], bool): + flatten_docs_functions_names = [self.language_parser.get_function_name(doc) for doc in exclusions] + if sources_location_packages: + filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') + and self.language_parser.is_function(doc) and + not self.language_parser.get_function_name(doc) in flatten_docs_functions_names] + else: + filter_1 = [doc for doc in self.documents if self.language_parser.is_function(doc) and + not self.language_parser.get_function_name(doc) in flatten_docs_functions_names] + + return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + """Sync implementations for retriever.""" + self.found_path = False + (package_name, function) = tuple(query.split(",")) + found_package = False + matching_documents = [] + # Check if input package is in dependency tree + for package in self.tree_dict: + if package_name.lower() in package.lower(): + package_name = package + found_package = True + break + # If it's , then create a document for it. + if found_package: + target_function_doc = self.__find_initial_function(function, package_name=package_name, + documents=self.documents, + + language_parser=self.language_parser) + # If not, there is a chance that the package is some standard library in the ecosystem. + else: + # Try to create dummy package for ecosystem standard library function, in such case, build a document for + # vulnerable function in standard lib package of language. + target_function_doc = Document(page_content=f"func {function + '()' + '{}'}" + , metadata={"source": package_name, + "ecosystem": self.ecosystem}) + importing_docs = [value for (file, value) in self.documents_of_full_sources.items() + if re.search( + rf"(import {package_name}|import\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + , value.page_content, flags=re.MULTILINE)] + root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] + prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() + # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. + parents = set([self.language_parser.get_package_names(doc)[1] for doc in importing_docs if + doc.metadata['source'].startswith( + prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[1] + in self.tree_dict.keys()]) + for doc in importing_docs: + if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): + parents.add(root_package[0]) + break + self.tree_dict[package_name] = [parents, []] + end_loop = False + current_package_name = package_name + # If an initial document (that represents the vulnerable input function in the input package) was created + # then add it to the path and start constructing path that comprise a chain of calls targeting this function. + if target_function_doc is not None: + matching_documents.append(target_function_doc) + # Otherwise, don't even start the process as the target function is not in dependencies tree or not a standard + # library + else: + end_loop = True + logger.error(f"Cannot find initial function=${function}, in package=${package_name}") + # main loop. + while True: + if end_loop: + break + # Find a caller function and containing package in the dependency tree according to hierarchy + found_document = self.__find_caller_function(document_function=target_function_doc, + function_package=current_package_name) + # If found, then add it to path + if found_document is not None: + matching_documents.append(found_document) + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(found_document): + end_loop = True + self.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + target_function_doc = found_document + # extract package name from function document + current_package_name = self.__determine_doc_package_name(target_function_doc) + else: + # end loop because didn't find a caller for initial function + if len(matching_documents) == 1: + end_loop = True + # Backtrack - means that we're going back because current function has no callers anywhere, so we + # need to remove it, and continue with other possible potential called functions from its caller + else: + dead_end_node = matching_documents.pop() + # Excludes dead end function node from future searches, as it led to nowhere. + self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + target_function_doc = matching_documents[-1] + current_package_name = self.__determine_doc_package_name(target_function_doc) + + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, self.found_path + + def __determine_doc_package_name(self, target_function_doc): + return [package_name for package_name in + self.language_parser.get_package_names(target_function_doc) + if self.tree_dict.get(package_name, None) is not None][0] + + def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], + language_parser: LanguageFunctionsParser) -> Document: + relevant_docs = [doc for doc in documents if doc.metadata.get('source').__contains__(package_name) and + doc.page_content.__contains__(function_name)] + package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] + for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): + + # document_function_calls_input_function = True + if function_name.lower() == language_parser.get_function_name(document).lower(): + # if language_parser.search_for_called_function(document, callee_function=function_name): + package_exclusions.append(document) + return document + else: + return None + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) ->list[str]: + results = [] + for i, package_function in enumerate(reversed(call_hierarchy_list)): + packages_names = self.language_parser.get_package_names(package_function) + if len(packages_names) > 1: + maximum_length_package = max(len(packages_names[0]), len(packages_names[1])) + if maximum_length_package == len(packages_names[0]): + package_name = packages_names[0] + else: + package_name = packages_names[1] + elif len(packages_names) > 0: + package_name = packages_names[0] + else: + package_name = package_function.metadata['source'] + function_name = self.language_parser.get_function_name(package_function) + current_level = f"(package={package_name},function={function_name},depth={i})" + results.append(current_level) + + return results + + diff --git a/src/vuln_analysis/utils/clients/ubuntu_client.py b/src/vuln_analysis/utils/clients/ubuntu_client.py index a90848332..d0964a63a 100644 --- a/src/vuln_analysis/utils/clients/ubuntu_client.py +++ b/src/vuln_analysis/utils/clients/ubuntu_client.py @@ -32,7 +32,7 @@ def __init__(self, *, base_url: str | None = None, session: aiohttp.ClientSession | None = None, - retry_count: int = 10, + retry_count: int = 1, sleep_time: float = 0.1, respect_retry_after_header: bool = True, retry_on_client_errors: bool = True): diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py new file mode 100644 index 000000000..8d8b03e4d --- /dev/null +++ b/src/vuln_analysis/utils/dep_tree.py @@ -0,0 +1,164 @@ +import subprocess +from abc import ABC, abstractmethod +from enum import Enum +from pathlib import Path +import logging + +logger = logging.getLogger(__name__) + + +ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' + + +class Ecosystem(Enum): + GO = 1 + PYTHON = 2 + JAVASCRIPT = 3 + JAVA = 4 + + +GOLANG_MANIFEST = "go.mod" +JAVA_MANIFEST = "pom.xml" +JS_MANIFEST = "package.json" +PYTHON_MANIFEST = "requirements.txt" + +MANIFESTS_TO_ECOSYSTEMS = { + "go.mod": Ecosystem.GO, + "requirements.txt": Ecosystem.PYTHON, + "package.json": Ecosystem.JAVASCRIPT, + "pom.xml": Ecosystem.JAVA +} + + +class DependencyTreeBuilder(ABC): + """ + An abstract base class that represents an object that knows how to build a dependency tree for a given ecosystem. + Should be subclassed for each supported ecosystem. + """ + + @abstractmethod + # Build a sort of "upside down" tree - a dict containing mapping of each package to a list of all consuming packages + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + pass + + @abstractmethod + # Return only package name, without version + def extract_package_name(self, package_name: str) -> str: + pass + + @abstractmethod + # A Method that knows how to install the app dependencies based on the manifest path and ecosystem. + def install_dependencies(self, manifest_path: Path): + pass + + +class GoDependencyTreeBuilder(DependencyTreeBuilder): + + def install_dependencies(self, manifest_path: Path): + self.download_go_mod_vendor(manifest_path) + + @staticmethod + def _parent_son_separator(line: str) -> list[str]: + return line.split(" ") + + def _get_parent(self, line: str): + parts = self._parent_son_separator(line) + return parts[0] + + def _get_son(self, line: str): + parts = self._parent_son_separator(line) + return parts[1] + + def __init__(self): + pass + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + # Get go version from manifest + # TODO - download go binary of this version and run the go mod graph using it for optimal performance. + go_version = self.determine_go_version(manifest_path) + + go_mod_tree = self.get_go_mod_graph_tree(manifest_path) + lines = go_mod_tree.splitlines() + root_package_name = self._get_parent(lines[0]) + tree = dict() + for line in lines: + line.split(" ") + parent = self.extract_package_name(self._get_parent(line)) + son = self.extract_package_name(self._get_son(line)) + if tree.get(son, None) is None: + tree[son] = [parent] + else: + tree.get(son).append(parent) + # Mark the top level for + tree[root_package_name] = [ROOT_LEVEL_SENTINEL] + return tree + + @staticmethod + def determine_go_version(manifest_path): + go_version = "" + with open(str(manifest_path) + "/go.mod") as go_mod_file: + for line in go_mod_file: + if line.startswith("go"): + go_version = line.split(" ")[1] + break + return go_version + + @staticmethod + def get_go_mod_graph_tree(manifest_path) -> str: + """ + run the go mod graph command to retirive all dependencies from go.mod, and their hirearchy + :param manifest_path: path to the go.mod manifest. + :return: a string representation of the go.mod graph, including dependencies and their hierarchy. + """ + return subprocess.run( + ["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], + capture_output=True, text=True).stdout + + def download_go_mod_vendor(self, manifest_path): + """ + Run the 'go mod vendor' command, in order to download go' app dependencies, in case vendor directory doesn't + already exist in git repo, it downloads it to git repo root dir where the go.mod file located. + :param manifest_path: path to go.mod manifest in repo directory. + """ + subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; go mod vendor"]) + + def extract_package_name(self, package_name: str) -> str: + if package_name.__contains__("@"): + version_start = package_name.index("@") + return package_name[: version_start] + else: + return package_name + + +def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTreeBuilder: + """ + A Factory method function that gets an programming language as argument, and returns a DependencyTreeBuilder' + instance related to the ecosystem. + :param programming_language: ecosystem to get the dependency tree builder for. + :return: DependencyTreeBuilder instance for the ecosystem, If not implemented, raise a NotImplementedError. + """ + if programming_language == Ecosystem.GO.value: + return GoDependencyTreeBuilder() + else: + raise NotImplementedError(f'Unsupported Programming Language for transitive search -> {programming_language}') + + +class DependencyTree: + """ + A class that represents a dependency tree to access an appropriate dependency tree builder, based on ecosystem. + """ + builder: DependencyTreeBuilder + ecosystem: Ecosystem + + def __init__(self, ecosystem: Ecosystem): + """ + + :param ecosystem: the ecosystem of the dependency tree. + """ + self.ecosystem = ecosystem + try: + # gets a dependency tree builder object based on ecosystem. + self.builder = get_dependency_tree_builder(ecosystem.value) + except ValueError as err: + logger.warning(f"Unable to build dependency tree - ${str(err)}") + self.builder = None diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index 0470e9002..dafca919e 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -17,6 +17,7 @@ import json import logging import os +import pickle import sys import time import typing @@ -36,6 +37,7 @@ from langchain_core.document_loaders.blob_loaders import Blob from vuln_analysis.data_models.input import SourceDocumentsInfo +from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader @@ -47,15 +49,42 @@ logger = logging.getLogger(__name__) +def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref) -> tuple: + if not os.path.exists(base_pickle_dir): + os.makedirs(base_pickle_dir) + + cached_documents_path = \ + (f"{base_pickle_dir}/" + f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" + f"{repo_ref}") + if os.path.isfile(cached_documents_path): + with open(cached_documents_path, 'rb') as doc_file: + documents = pickle.load(doc_file) # deserialize using load() + logger.debug(f"Cache Hit on Documents, Loaded existing docs at {cached_documents_path}") + return documents, True + + return [], False + + +def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: list[Document]): + cached_documents_path = \ + (f"{base_pickle_dir}/" + f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" + f"{repo_ref}") + logger.debug(f"Saved loaded Documents into Cache => {cached_documents_path}") + with open(cached_documents_path, 'wb') as doc_file: # open a text file + pickle.dump(documents, doc_file) + + class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter): """ A version of langchain's RecursiveCharacterTextSplitter that supports multiple languages. """ def __init__( - self, - keep_separator: bool = True, - **kwargs, + self, + keep_separator: bool = True, + **kwargs, ) -> None: """Create a new RecursiveCharacterTextSplitter.""" super().__init__(is_separator_regex=True, keep_separator=keep_separator, **kwargs) @@ -107,7 +136,7 @@ class ExtendedLanguageParser(LanguageParser): "javascript": ExtendedJavaScriptSegmenter, "js": ExtendedJavaScriptSegmenter, } - + additional_segmenters["go"] = GoSegmenterWithMethods LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, **additional_segmenters, @@ -208,13 +237,9 @@ class DocumentEmbedding: repositories and chunked into smaller pieces for embedding. """ - def __init__(self, - *, - embedding: "Embeddings", - vdb_directory: PathLike = "./.cache/am_cache/vdb", - git_directory: PathLike = "./.cache/am_cache/git", - chunk_size: int = 800, - chunk_overlap: int = 160): + def __init__(self, *, embedding: "Embeddings", vdb_directory: PathLike = "./.cache/am_cache/vdb", + git_directory: PathLike = "./.cache/am_cache/git", chunk_size: int = 800, chunk_overlap: int = 160, + pickle_cache_directory: PathLike = "./.cache/am_cache/pickle"): """ Create a new DocumentEmbedding instance. @@ -231,6 +256,7 @@ def __init__(self, Maximum size of a single chunk, by default 1000 chunk_overlap : int, optional Overlap between chunks, by default 200 + :param pickle_cache_directory: """ self._embedding = embedding @@ -238,6 +264,7 @@ def __init__(self, self._git_directory = Path(git_directory) self._chunk_size = chunk_size self._chunk_overlap = chunk_overlap + self._pickle_cache_directory = Path(pickle_cache_directory) @property def embedding(self): @@ -337,22 +364,26 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: """ repo_path = self.get_repo_path(source_info) + documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, + source_info.git_repo, source_info.ref) + if documents_were_in_cache or len(documents) > 0: + return documents + else: + blob_loader = SourceCodeGitLoader(repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude) - blob_loader = SourceCodeGitLoader(repo_path=repo_path, - clone_url=source_info.git_repo, - ref=source_info.ref, - include=source_info.include, - exclude=source_info.exclude) - - blob_parser = ExtendedLanguageParser() - - loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + blob_parser = ExtendedLanguageParser() - documents = loader.load() + loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) - logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + documents = loader.load() - return documents + logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + save_to_cache(self._pickle_cache_directory, source_info.git_repo, source_info.ref, documents) + return documents def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathLike): """ @@ -379,7 +410,6 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL # Warn if the output path already exists and we will overwrite it if (output_path.exists()): - logger.warning("Vector Database already exists and will be overwritten: %s", output_path) documents = [] diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py new file mode 100644 index 000000000..ca7799a1f --- /dev/null +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -0,0 +1,117 @@ +import os +import re + +from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever + + +def traverse_all_parameters(function_ending_index_end, function_prefix_index_end, function_string): + current_idx = function_prefix_index_end + function_builder = function_string[:function_prefix_index_end] + if current_idx == function_ending_index_end: + function_builder += ")" + while current_idx < function_ending_index_end: + end_of_arg_ind = function_string[current_idx:].find(",") + if end_of_arg_ind > -1: + value = handle_argument(function_string[current_idx: current_idx + end_of_arg_ind].strip()) + function_builder += f"\\s?{value}," + current_idx = current_idx + end_of_arg_ind + 1 + else: + # last argument + value = handle_argument(function_string[current_idx:function_ending_index_end].strip()) + function_builder += f"\\s?{value}\\s?)" + current_idx = function_ending_index_end + + return function_builder + + +def infer_if_short_package_name_match(package: str, package_names: list[str], final_package: set[str]) -> bool: + package_with_version_suffix = r"[./][vV][1-9]+$" + match = re.search(package_with_version_suffix, package) + if match: + parts = package.split(match.group()) + short_package_name = parts[-2].split("/")[-1] + else: + parts = package.split("/") + short_package_name = parts[-1].split("/")[-1] + + for current_package in package_names: + if short_package_name in current_package: + final_package.add(current_package) + return True + + else: + return False + + +def handle_argument(param: str) -> str: + """ + This function get argument string.if it's a literal, returns it as is,otherwise, returns wildcard to match anything. + as literals are intended to be searched as specified, and variables/identifiers names can be any other name. + :param param: str + :return: returns the right matcher. + """ + if (param.startswith("'") and param.endswith("'")) or (param.startswith('"') and param.endswith('"')): + return param + elif param.__contains__("(") and param.__contains__(")"): + function_prefix_index_end = param.find("(") + 1 + function_ending_index_end = param.rfind(")") + return traverse_all_parameters(function_ending_index_end, function_prefix_index_end, param) + else: + return ".*" + + +class FunctionNameExtractor: + + def __init__(self, coc_retriever: ChainOfCallsRetriever): + self.lang_parser = coc_retriever.language_parser + self.docs = coc_retriever.documents + + def fetch_list(self, query: str) -> list[str]: + """ +use this tool to search for real specific usages of standard library functions/methods in a given package. +It searches for exact string matching, except variables placed in arguments, which can be anyname in essence, +and not necessarily the ones that are in the search string. + +returns list of calling functions in the package, e.g - ['packageName,functionName','packageName,functionName2'] + """ + parts_input = query.split(",", 1) + package = parts_input[0] + function = parts_input[1] + final_package = set() + package_functions = [doc for doc in self.docs if + self.lang_parser.get_package_name(doc, package) + or + infer_if_short_package_name_match(package, self.lang_parser.get_package_names(doc), + + final_package)] + the_final_package = list(final_package) + # If short package name was inferred , use it also in searching all the way + if len(the_final_package) > 0 and the_final_package[0] != package: + package = the_final_package[0] + + if function.__contains__(".") and function.__contains__("(") and function.__contains__(")"): + function_string = str(function) + function_prefix_index_end = function_string.find("(") + 1 + function_ending_index_end = function_string.rfind(")") + function_builder = traverse_all_parameters(function_ending_index_end, function_prefix_index_end + , function_string) + else: + return [] + + containing_functions = list() + comment_line_character = self.lang_parser.get_comment_line_notation() + for func in package_functions: + + if match := re.search(rf"{function_builder}", func.page_content, flags=re.MULTILINE): + current_offset = match.start() - 1 + while func.page_content[current_offset] != os.linesep: + current_offset -= 1 + + # Make sure to filter out matches that are code comments + if func.page_content[current_offset: match.start()].strip().startswith(comment_line_character): + pass + else: + new_function_name = self.lang_parser.get_function_name(func) + containing_functions.append(f"{package},{new_function_name}") + + return containing_functions diff --git a/src/vuln_analysis/utils/functions_parsers/__init__.py b/src/vuln_analysis/utils/functions_parsers/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py new file mode 100644 index 000000000..94258de38 --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -0,0 +1,581 @@ +import os +import re + +from langchain_core.documents import Document + +from .lang_functions_parsers import LanguageFunctionsParser + +EMBEDDED_TYPE = "embedded_type" + +PARAMETER = "parameter" + +RETURN_TYPES = "return_types" + +LOCAL_VAR_USAGE = "local_var_usage" +LOCAL_IMPLICIT = "local_implicit" +LOCAL_INDIRECT_TYPES_INDICATIONS = [LOCAL_VAR_USAGE, LOCAL_IMPLICIT] +PRIMITIVE_TYPES = ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', + 'complex64', 'complex128', 'byte', 'rune', 'uint', 'int', 'uintptr', 'string', 'ptr', 'bool'] +MAP_TYPE_REGEX = r"map\[[a-zA-Z0-9]+\][a-zA-Z0-9]+ " +SLICE_TYPES_REGEX = r"\[\][a-zA-Z0-9]+ " + + +# def parse_type_struct(fieldName: str, ) -> : dict[str,str] + +def get_package_name_file(function: Document): + content = function.page_content + index_of_package = content.find("package") + index_of_end_of_line = index_of_package + content[index_of_package:].find(os.linesep) + if index_of_package > -1: + return content[index_of_package + 8:index_of_end_of_line].strip() + + +def check_types_from_callee_package(params: list[tuple], type_documents: list[Document], callee_package: str, + code_documents: dict[str, Document], parameter: str, + callee_function_file_name: str) -> bool: + callee_function_doc = code_documents.get(callee_function_file_name) + callee_function_file_package_name = get_package_name_file(callee_function_doc) + for param in params: + param_name, param_type = param + if param_name == parameter: + param_type_stripped = str(param_type).replace("*", "").replace("&", "") + parts = param_type_stripped.split(".") + # Only type without package + if len(parts) == 1: + for the_type in type_documents: + if the_type.page_content.startwith(f"type {parts[0]}"): + code_with_type_file = code_documents.get(the_type.metadata['source']) + type_file_package_name = get_package_name_file(code_with_type_file) + if type_file_package_name == callee_function_file_package_name: + return True + # type with package qualifier + else: + for the_type in type_documents: + if the_type.page_content.startwith(f"type {parts[1]}"): + code_with_type_file = code_documents.get(the_type.metadata['source']) + package_match = handle_imports(code_with_type_file, parts[0], callee_package) + return package_match + else: + return False + + +def handle_imports(code_content: str, identifier: str, callee_package: str) -> bool: + start_search_for_import = max(code_content.rfind("//import"), code_content.rfind("// import")) + if start_search_for_import > -1: + after_last_import_comment_index = start_search_for_import + 4 + else: + after_last_import_comment_index = 0 + after_last_import_comment = code_content[after_last_import_comment_index:] + first_import = after_last_import_comment.find("import") + last_import = after_last_import_comment.rfind("import") + # return after_last_import_comment, first_import, last_import + if first_import == last_import: + block_of_import = after_last_import_comment[first_import + 6:] + after_left_bracket = block_of_import.find("(") + before_right_bracket = block_of_import.find(")") + inside_block = block_of_import[after_left_bracket + 1: before_right_bracket] + position_of_identifier_in_import_block = inside_block.find(identifier) + if position_of_identifier_in_import_block != -1: + previous_end_of_line = inside_block[:position_of_identifier_in_import_block].rfind( + os.linesep) + alias_package_extended = inside_block[previous_end_of_line:].strip() + index_of_end_of_line = alias_package_extended.find(os.linesep) + row_of_identifier = alias_package_extended[:index_of_end_of_line] + if len(row_of_identifier.split(" ")) > 1: + package_name_of_alias = row_of_identifier.split(" ")[1].strip('"') + else: + package_name_of_alias = row_of_identifier.split(" ")[0].strip('"') + + if (package_name_of_alias.strip().lower() == callee_package.strip().lower() + or callee_package.strip().lower() in package_name_of_alias.strip().lower()): + return True + + # Checks if there is a dedicated import with alias and imported package name + else: + identifier_import_position = code_content.find(f"import {identifier}") + if identifier_import_position != -1: + start_of_package_name = code_content[identifier_import_position + len("import ") + + len(identifier):] + index_of_end_of_line = start_of_package_name.find(os.linesep) + package_name_to_check = start_of_package_name[:index_of_end_of_line] + if package_name_to_check.strip().lower() == callee_package.strip().lower(): + return True + # import without alias, in this case maybe package name contain alias + else: + # re.search(regex, caller_function_body, re.MULTILINE) + matching = re.search(rf"import [\'\"].*{identifier}[\'\"]", code_content) + if matching and matching.group(0): + import_line = code_content[matching.start():] + import_package_line = import_line[:import_line.find(os.linesep)].strip() + package_name = import_package_line.split(r"\s")[1] + if package_name.strip().lower() == callee_package.strip().lower(): + return True + return False + + +class GoLanguageFunctionsParser(LanguageFunctionsParser): + + def __trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], + callee_package: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str) -> bool: + variables_mappings = functions_local_variables_index[caller_function_index] + parts = expression.split(".") + result = False + + (resolved_type, struct_initializer_expression, + value, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-1) + + if (var_properties is not None + and (struct_initializer_expression or + resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or value == PARAMETER)): + result = self.__lookup_package(callee_package, resolved_type, struct_initializer_expression, + type_documents, value) + + # Property/member is not in function, check if it's member/property of a type + elif var_properties is None and len(parts) > 1: + (resolved_type, struct_initializer_expression, + value, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-2) + if (var_properties is not None + and (resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or value == PARAMETER)): + field_name = parts[-1] + possible_types = {key: value for (key, value) in fields_of_types.items() + if resolved_type in key + and any([field for field in value if field_name in field])} + for the_type, mappings in possible_types.items(): + for mapping in mappings: + if field_name in mapping: + returned_matched_types = self.__get_type_docs_matched_with_callee_package(callee_package, + mapping[1], + type_documents) + if len(returned_matched_types) > 0: + result = True + + elif var_properties is not None: + value = var_properties.get("value", None) + if value is not None and value.strip() != "": + return self.__trace_down_package(value, code_documents, type_documents, callee_package, + fields_of_types, functions_local_variables_index, + caller_function_index) + + return result + + def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): + var_properties = variables_mappings.get(parts[the_part], None) + if var_properties is not None: + resolved_type = var_properties.get("type") + value = var_properties.get("value") + struct_initializer_expression = re.search(r"(&|\\*)?\w+\s*{", value) + resolved_type = str(resolved_type).replace("&", "").replace("*", "") + return resolved_type, struct_initializer_expression, value, var_properties + else: + return None, None, None, None + + def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, + value) -> bool: + result = False + if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + docs = self.__get_type_docs_matched_with_callee_package(callee_package, resolved_type, type_documents) + + if len(docs) > 0: + result = True + elif value == PARAMETER: + result = False + + elif struct_initializer_expression: + struct_type = (struct_initializer_expression.group(0).replace("{", "") + .replace("&", "").replace("*", "")) + docs = self.__get_type_docs_matched_with_callee_package(callee_package, struct_type, type_documents) + if len(docs) > 0: + result = True + return result + + def __get_type_docs_matched_with_callee_package(self, callee_package, checked_type, type_documents) -> list[ + Document]: + return [a_type for a_type in type_documents if callee_package in a_type.metadata['source'] and + (self.__get_type_name(a_type) == checked_type or self.__get_type_name(a_type) in checked_type)] + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + for func_method in functions_methods_documents: + func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + all_vars = dict() + for row in func_method.page_content.splitlines(): + if not self.is_comment_line(row): + # Extract arguments and receiver argument of type as parameters + if row.startswith("func"): + match = re.finditer(r"(func|\w+)\s*\([a-zA-Z0-9\s*,.\[\]]+\)" + , func_method.page_content[:func_method.page_content.find("{")] + , flags=re.MULTILINE) + for current_match in match: + current_args = (current_match.group(0).replace("\n\t", "") + .replace("\t", "").replace("\n", "")) + current_params = re.search(r"\(.*\)", current_args) + params = (current_params.group(0).replace("(", "") + .replace(")", "").split(",")) + params_tuple = tuple(params) + param_type: str = "" + for param in reversed(params_tuple): + data = param.strip().split(" ") + param_name = data[0] + if len(data) > 1: + param_type = data[1] + the_value = PARAMETER + all_vars[param_name] = {"value": the_value, "type": param_type} + # Gets return types from function + index_of_start_func = func_method.page_content.find(row) + last_left_curly_bracket = index_of_start_func + func_method.page_content.find("{") + last_right_bracket = func_method.page_content[ + index_of_start_func:last_left_curly_bracket].rfind(")") + return_parameters = func_method.page_content[index_of_start_func + last_right_bracket + 1: + last_left_curly_bracket - 1].strip() + return_parameters = return_parameters.replace(")", "").replace("(", "") + if return_parameters.strip() != "": + all_vars[RETURN_TYPES] = return_parameters.split(",") + else: + all_vars[RETURN_TYPES] = [] + + elif row.strip().startswith("var ") and not re.search(r"var\s*\(", row.strip()): + row_without_var_prefix = row.strip()[3:].strip() + parts = row_without_var_prefix.split() + # variable name + left_side = parts[0] + if len(parts) == 2 and parts[1].__contains__("="): + assignment = row.strip().split("=") + all_vars[(left_side.strip())] = {"value": assignment[1], + "type": assignment[0].replace("*", "")} + else: + if len(parts) == 2: + all_vars[(left_side.strip())] = {"value": "", "type": parts[1].replace("*", "")} + elif row.strip().__contains__(":=") and row.strip().__contains__("if"): + index_of_start_if = func_method.page_content.find(row) + # Go until delimiter of assignment and start of boolean expression + end_of_assignment = func_method.page_content[index_of_start_if:].find(";") + parts = row.strip().split(":=") + left_side = parts[0].replace("if", "") + right_side = func_method.page_content[index_of_start_if + 2: + index_of_start_if + end_of_assignment - 1].strip() + + all_vars[(left_side.strip())] = {" value": right_side.strip().replace + ("\n\t", "").replace("\t", ""), "type": LOCAL_IMPLICIT + } + + elif (row.strip().__contains__(":=") and not row.strip().__contains__("if") + and not row.strip().__contains__("for ") and not row.strip().__contains__("range ") + and not row.strip().__contains__("range ") and not row.strip().__contains__("case ")): + parts = row.strip().split(":=") + left_side = parts[0] + right_side = parts[1] + all_vars[(left_side.strip())] = {"value": right_side.strip(), "type": LOCAL_IMPLICIT} + elif (row.strip().__contains__("=") and not row.strip().__contains__("==") + and not row.strip().__contains__("!=") + and not row.strip().__contains__("if") and not row.strip().__contains__("for ") + and not row.strip().__contains__("range ") and not row.strip().__contains__("case ")): + parts = row.strip().split("=") + left_side = parts[0] + right_side = parts[1] + all_vars[(left_side.strip())] = {"value": right_side.strip(), "type": ("%s" % LOCAL_VAR_USAGE)} + else: + pass + + mappings[func_key] = all_vars + + return mappings + + def __is_struct_or_interface_type(self, doc: Document): + return re.search(r"^type\s+[a-zA-Z0-9_]+\s+(struct|interface)", doc.page_content) + + def __get_type_kind(self, the_type: Document) -> str: + if self.__is_struct_or_interface_type(the_type): + parts_of_type_header = the_type.page_content.split() + return parts_of_type_header[2] if len(parts_of_type_header) > 2 else "" + # not composite type, either primitive or wrapper + else: + if match := re.search(r"^type\s+ \(", the_type.page_content): + right_bracket = match.group(0).find(")") + left_bracket = match.group(0).find("(") + parts = match.group(0)[left_bracket:right_bracket].strip().split(sep=" ", maxsplit=1) + return parts[1] + else: + parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content + return parts[2] + + def __get_type_name(self, the_type: Document) -> str: + if self.__is_struct_or_interface_type(the_type): + parts_of_type_header = the_type.page_content.split() + return parts_of_type_header[1] if len(parts_of_type_header) > 1 else "" + elif match := re.search(r"^type\s+ \(", the_type.page_content): + right_bracket = match.group(0).find(")") + left_bracket = match.group(0).find("(") + parts = match.group(0)[left_bracket:right_bracket].strip().split(sep=" ", maxsplit=1) + return parts[0] + else: + parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content + return parts[1] + + def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for the_type in types: + the_kind = self.__get_type_kind(the_type) + type_name = self.__get_type_name(the_type) + type_key = type_name + if the_kind.lower() == "interface": + type_key = f"interface;{type_name}" + + start_index = the_type.page_content.find("{") + 2 + end_index = the_type.page_content.rfind("}") + current_index = start_index + fields_list = list() + if end_index > - 1: + while current_index < end_index: + index_of_eol = the_type.page_content[current_index:].find(os.linesep) + end_of_row = end_index + if index_of_eol > -1: + end_of_row = index_of_eol + current_line = the_type.page_content[current_index:current_index + end_of_row] + if not self.is_comment_line(current_line) and not current_line.strip() == "": + right_bracket_completion = "" + if type_key.startswith("interface"): + parts = current_line.split(sep=")", maxsplit=1) + right_bracket_completion = ")" + else: + parts = current_line.split(sep=" ", maxsplit=1) + if len(parts) == 2: + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + parts[1].lstrip()[:parts[1].lstrip().find(" ")])) + elif len(parts) == 1: + fields_list.append((EMBEDDED_TYPE, parts[0])) + elif len(parts) > 2: + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + parts[1][:parts[1].find(" ")].strip())) + else: + pass + + current_index = current_index + end_of_row + 1 + if type_key.strip() != "" and len(fields_list) != 0: + types_mapping[(type_key, the_type.metadata['source'])] = fields_list + # Primitive type or wrapper of another type + else: + types_mapping[(type_key.strip(), the_type.metadata['source'])] = [(type_name.strip(), the_kind.strip())] + return types_mapping + + def get_function_reserved_word(self) -> str: + return "func" + + def get_type_reserved_word(self) -> str: + return "type" + + def is_searchable_file_name(self, function: Document) -> bool: + file_path = str(function.metadata['source']) + return "test" not in file_path[file_path.rfind("/") + 1:].split(".")[0].lower() + + def is_function(self, function: Document) -> bool: + return function.page_content.startswith("func") + + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + def supported_files_extensions(self) -> list[str]: + return [".go"] + + def get_comment_line_notation(self) -> str: + return "//" + + def dir_name_for_3rd_party_packages(self) -> str: + return "vendor" + + def is_exported_function(self, function: Document) -> bool: + function_name = self.get_function_name(function) + return re.search("[A-Z][a-z0-9-]*", function_name) + + def get_function_name(self, function: Document) -> str: + try: + index_of_function_opening = function.page_content.index("{") + except ValueError as e: + function_line = function.page_content.find(os.linesep) + # print(f"function {function.page_content[:function_line]} => contains no body ") + return function.page_content[:function_line] + + function_header = function.page_content[:index_of_function_opening] + # function is a method of a type + if function_header.startswith("func ("): + index_of_first_right_bracket = function_header.index(")") + skip_receiver_arg = function_header[index_of_first_right_bracket + 1:] + index_of_first_left_bracket = skip_receiver_arg.index("(") + return skip_receiver_arg[:index_of_first_left_bracket].strip() + # regular function not tied to a certain type + else: + try: + index_of_first_left_bracket = function_header.index("(") + # Go Generic function + except ValueError: + try: + index_of_first_left_bracket = function_header.index("[") + except ValueError: + raise ValueError(f"Invalid function header - {function_header}") + func_with_name = function_header[:index_of_first_left_bracket] + if len(func_with_name.split(" ")) > 1: + return func_with_name.split(" ")[1] + # TODO Try to extract anonymous function var + # else: + + def search_for_called_function(self, caller_function: Document, callee_function: str, callee_function_package: str, + code_documents: list[Document], type_documents: list[Document], + callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict]) -> bool: + index_of_function_opening = caller_function.page_content.index("{") + index_of_function_closing = caller_function.page_content.rfind("}") + caller_function_body = str( + caller_function.page_content[index_of_function_opening + 1: index_of_function_closing]) + re.search("", caller_function_body) + regex = fr'[a-zA-Z0-9_\[\]\(\).]*.?{callee_function}\(' + matching = re.search(regex, caller_function_body, re.MULTILINE) + if matching and matching.group(0): + return self.__check_identifier_resolved_to_callee_function_package(function=caller_function, + identifier_function=matching.group(0), + callee_package=callee_function_package, + code_documents=code_documents, + caller_function_body=caller_function_body, + type_documents=type_documents, + callee_function_file_name= + callee_function_file_name, + fields_of_types= + fields_of_types, + functions_local_variables_index= + functions_local_variables_index) + else: + return False + + def __check_identifier_resolved_to_callee_function_package(self, function: Document, identifier_function: str, + callee_package: str, code_documents: dict[str, Document], + caller_function_body: str, + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[ + str, dict]) -> bool: + caller_function_file = function.metadata.get('source') + caller_function_name = self.get_function_name(function) + caller_function_index = f"{caller_function_name}@{caller_function_file}" + parts = identifier_function.split(".") + # If there is no qualifier identifier , then check whether callee function and caller + # function are in the same package + if len(parts) == 1: + try: + callee_function = code_documents[callee_function_file_name] + callee_function_package = get_package_name_file(callee_function).strip() + except KeyError as e: + # Standard library function , there is no function code, thus the source name is the package name in + # this case + callee_function_package = callee_function_file_name + + caller_function = code_documents[function.metadata.get('source')] + caller_function_package = get_package_name_file(caller_function).strip() + if (callee_package in self.get_package_names(function) and + callee_function_package == caller_function_package): + return True + # There is a qualifier identifier. + else: + identifier = parts[-2] + if identifier.startswith("return"): + identifier = identifier.replace("return", " ").strip() + if "(" in identifier: + identifier = identifier[identifier.index("(") + 1:] + + # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , + # if it's the same as callee package name + # TODO - reduce list to only files in package + for doc in code_documents: + if function.metadata.get('source') == code_documents[doc].metadata.get('source'): + # maybe identifier is the package itself in the file + regex = f"package {identifier}" + code_content = code_documents[doc].page_content + matching = re.search(regex, code_content, re.MULTILINE) + if matching and matching.group(0): + return True + + identifier_is_imported = handle_imports(code_content, identifier, callee_package) + if identifier_is_imported: + return True + + ## otherwise, the identifier is in the caller function body or in signature/receiver function argument. + + else: + # TODO check if identifier is defined in the same + # function and dig into structures and identifiers defined by variables + function_header = function.page_content[:function.page_content.index("{")] + regex_arguments = r"\([a-zA-Z0-9\s*,.]+\)" + match_regex = re.finditer(rf"^(\s*|\n){identifier}\s*(:=|=)\s*[^=]+\n*$", + caller_function_body, flags=re.MULTILINE) + matches = [match.group(0) for match in match_regex] + + if len(matches) > 0: + # match_variable = matches[-1] + # split = match_variable.split(":=") + # if split[0] == match_variable: + # split = match_variable.split("=") + # if len(split) > 1: + return self.__trace_down_package(expression=identifier.strip(), code_documents=code_documents, + type_documents=type_documents, callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_function_index) + + + # Checks if match some argument in function or receiver parameter ( without parenthesis of return + # values) + elif re.search(regex_arguments, function_header): + return self.__trace_down_package(expression=identifier.strip(), code_documents=code_documents, + type_documents=type_documents, callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_function_index) + # parameters = [tuple(e.replace(")", "").replace("(", "").split(",")) for e + # in re.findall(regex_arguments, function_header)[:2]] + # return check_types_from_callee_package(parameter=identifier, params=parameters, + # type_documents=type_documents, + # callee_package=callee_package, + # code_documents=code_documents, + # callee_function_file_name=callee_function_file_name + # ) + + return False + + def get_package_names(self, function: Document) -> list[str]: + package_names = list() + full_doc_path = str(function.metadata['source']) + parts = full_doc_path.split("/") + version = "" + if len(parts) > 4: + match = re.search(r"[vV][0-9]{1,2}", parts[4]) + if match and match.group(0): + version = f"/{match.group(0)}" + + if parts[0].startswith(self.dir_name_for_3rd_party_packages()) and len(parts) > 3: + package_names.append(f"{parts[1]}/{parts[2]}{version}") + package_names.append(f"{parts[1]}/{parts[2]}/{parts[3]}{version}") + else: + try: + package_names.append(f"{parts[0]}/{parts[1]}{version}") + package_names.append(f"{parts[0]}/{parts[1]}/{parts[2]}{version}") + # Standard library package + except IndexError as index_excp: + if len(parts) > 1: + package_names.append(f"{parts[0]}/{parts[1]}{version}") + else: + package_names.append(f"{parts[0]}{version}") + + return package_names + + def get_package_name(self, function: Document, package_name: str) -> str: + package_names = self.get_package_names(function) + for package in package_names: + if package_name.lower() in package.lower() and package_name.lower() == package.lower(): + return package.lower() + return None + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + return line.strip().startswith("//") diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py new file mode 100644 index 000000000..1d28da980 --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py @@ -0,0 +1,108 @@ +from abc import ABC, abstractmethod + +from langchain_core.documents import Document + + +class LanguageFunctionsParser(ABC): + + # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing + # mappings of all local variables defined in these functions/methods, to their type/value/expression. + # Implementation should be consistent with search_for_called_function, as functions_methods_documents is being used + # in this method as functions_local_variables_index + @abstractmethod + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + pass + + # This method returns (for the given ecosystem) , a mapping of all types/classes in all packages to a dict + # containing mapping of all fields names in the type/class to their defining types. + # a list of + @abstractmethod + def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + pass + + @abstractmethod + def get_function_name(self, function: Document) -> str: + pass + + # This method get a caller function document, a callee function, and a callee package, and returns True + # if caller function + @abstractmethod + def search_for_called_function(self, caller_function: Document, callee_function: str, callee_function_package: str, + code_documents: list[Document], type_documents: list[Document], + callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict]) -> bool: + pass + + # This method gets a function document, and return a list with the containing package name. if + # more than one potential package name is possible, then it should return it also + @abstractmethod + def get_package_names(self, function: Document) -> list[str]: + pass + + # This method gets a function document as an argument, a package name as an argument , and return the containing + # package name, when you have multiple options. + @abstractmethod + def get_package_name(self, function: Document, package_name: str) -> str: + pass + + # This method get a function document, and returns true, only if it's a function in the root package ( application) + @abstractmethod + def is_root_package(self, function: Document) -> bool: + pass + + # This method gets a line from a source code, and returns True if it's a comment line for the ecosystem, + # otherwise it returns False. + @abstractmethod + def is_comment_line(self, line: str) -> bool: + pass + + # This method returns the ecosystem' comment character(s) + @abstractmethod + def get_comment_line_notation(self) -> str: + pass + + # This method gets a function document as an argument, and returns True if this contained function is exported to be + # Used in other files/packages ( returns True only if it can be called from outside the source file) + @abstractmethod + def is_exported_function(self, function: Document) -> bool: + pass + + # This method get a source document, and returns True only if the document type + # is a function ( not type/class/full_source) for the ecosystem. + @abstractmethod + def is_function(self, function: Document) -> bool: + pass + + # This method returns the name of the directory containing the 3rd party packages/libs source code files for the + # ecosystem + @abstractmethod + def dir_name_for_3rd_party_packages(self) -> str: + pass + + # This method returns the list of the supported source file extensions for the ecosystem. + @abstractmethod + def supported_files_extensions(self) -> list[str]: + pass + + # This method gets an extension as an argument, and returns True only if it's supported in the ecosystem. + @abstractmethod + def is_supported_file_extensions(self, extension: str) -> bool: + pass + + # This method gets a document function as an argument, and returns True only if this function should be taken into + # Account as candidate to be included in chains of calls paths ( for example , tests and dev resources files should + # be excluded from the search process as they're not indication for exploitability as these are not executed on + # runtime at target environment. + @abstractmethod + def is_searchable_file_name(self, function: Document) -> bool: + pass + + # This method returns the function/method reserved word in code for the ecosystem/programming language. + @abstractmethod + def get_function_reserved_word(self) -> str: + pass + + # This function returns the type reserved word in code for the ecosystem/programming language. + @abstractmethod + def get_type_reserved_word(self) -> str: + pass diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py new file mode 100644 index 000000000..460dd866e --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py @@ -0,0 +1,18 @@ +from ..dep_tree import Ecosystem +from .golang_functions_parsers import GoLanguageFunctionsParser +from .lang_functions_parsers import LanguageFunctionsParser + + +def get_language_function_parser(ecosystem: Ecosystem) -> LanguageFunctionsParser: + """ + Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated + with the ecosystem. + :param ecosystem: the desired programming language parser. + :return: + The right language functions parser associated to the ecosystem, if not exists, return ABC parent that + doesn't do anything + """ + if ecosystem == Ecosystem.GO: + return GoLanguageFunctionsParser() + else: + return LanguageFunctionsParser() diff --git a/src/vuln_analysis/utils/go_segmenters_with_methods.py b/src/vuln_analysis/utils/go_segmenters_with_methods.py new file mode 100644 index 000000000..4a8bae901 --- /dev/null +++ b/src/vuln_analysis/utils/go_segmenters_with_methods.py @@ -0,0 +1,77 @@ +import re +from typing import List + +from langchain_community.document_loaders.parsers.language.go import GoSegmenter + + +def parse_all_methods(code: str) -> list[str]: + """ + This function gets a source code content as multi-line string, and returns a list of all golang methods in the source. + :arg code : multi-line source string of a go source file + :return: List of all golang methods in the source, including signatures + bodies ( implementation) + """ + # regex = r"func\s*\([a-zA-Z0-9\s]*\) [a-zA-X]+\([a-zA-Z0-9\s]*\)([a-zA-Z0-9\s]*){" + regex = r"func\s*\([a-zA-Z0-9\s\*.]+\) [a-zA-Z]+\([,a-zA-Z0-9\s\[\].]*\)\s*(\(?[a-zA-Z0-9\s.,*]+\)?)?\s*{" + methods = get_all_functions(code, regex) + return methods + + +def parse_all_anonymous_functions(code: str) -> list[str]: + """ + This function gets a source code content as multi-line string, and returns a list of all golang anonymous functions + in that source, if exists. + :param code: multi-line source string of a go source file + :return: List of all anonymous functions in the source, if any. + """ + regex = r"(var)? [a-zA-Z0-9_]+\s*(:)?=\s*func\s*.*{" + functions = get_all_functions(code, regex) + return functions + + +def get_all_functions(code: str, regex: str): + """ + This function gets a source code content as multi-line string, and a regex that identify golang functions/methods + boundaries, and construct a list of all golang functions/methods ( complete implementation, signatures + body) + :param code: multi-line source string of a go source file + :param regex: the regex to identify the needed golang type of functions - anonymous functions/methods + :return: a list containing all golang functions/methods ( each one with a complete implementation, signature + body) + """ + matches = re.finditer(regex, code) + functions = list() + for matchNum, match in enumerate(matches, start=1): + curly_brackets_counter = 1 + internal_offset = 0 + current_method_body = code[match.end():] + while curly_brackets_counter > 0 or internal_offset == 0: + left_bracket_ind = current_method_body[internal_offset:].find("{") + right_bracket_ind = current_method_body[internal_offset:].find("}") + if left_bracket_ind < right_bracket_ind and left_bracket_ind != -1: + curly_brackets_counter += 1 + internal_offset = internal_offset + left_bracket_ind + 1 + else: + curly_brackets_counter -= 1 + internal_offset = internal_offset + right_bracket_ind + 1 + current_method = code[match.start(): match.end() + 1 + internal_offset] + functions.append(current_method) + return functions + + +class GoSegmenterWithMethods(GoSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + def extract_functions_classes(self) -> List[str]: + """ + This method extracts all the golang methods and anonymous functions from a source code, and appends them to + the list of all golang regular functions calculated in parent class' method. + : + :return: a list of all golang functions/methods/anonymous functions ( each one with a complete implementation, + signature + body) + """ + function_classes = super().extract_functions_classes() + methods_classes = parse_all_methods(self.code) + parse_anonymous_functions = parse_all_anonymous_functions(self.code) + function_classes.extend(methods_classes) + function_classes.extend(parse_anonymous_functions) + return function_classes diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 25e1abe7a..643f4f9e6 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -80,7 +80,8 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE return AgentMorpheusEngineState(code_vdb_path=message.info.vdb.code_vdb_path, doc_vdb_path=message.info.vdb.doc_vdb_path, code_index_path=message.info.vdb.code_index_path, - cve_intel=filtered_intel) + cve_intel=filtered_intel, + original_input=message) def parse_agent_morpheus_engine_output(vuln_id: str, diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py new file mode 100644 index 000000000..2f8b98f5d --- /dev/null +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from pathlib import Path + +from langchain.docstore.document import Document + +from .chain_of_calls_retriever import ChainOfCallsRetriever +from .dep_tree import ( + GOLANG_MANIFEST, + JAVA_MANIFEST, + JS_MANIFEST, + MANIFESTS_TO_ECOSYSTEMS, + PYTHON_MANIFEST, + Ecosystem, + get_dependency_tree_builder, +) + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class TransitiveCodeSearcher: + """ + Transitive code Searcher for code using a Chain Of Calls Retriever object + + Parameters + ---------- + chain_of_calls_retriever : ChainOfCallsRetriever + Chain Of Calls Retriever object that knows how to search in code + + """ + + def __init__(self, chain_of_calls_retriever: ChainOfCallsRetriever): + self.chain_of_calls_retriever = chain_of_calls_retriever + + @staticmethod + def download_dependencies(git_repo_path: Path) -> bool: + """ Download all dependencies according to manifest file in the Git repository + Parameters + ---------- + git_repo_path : Path + Git repository path to fetch the application manifests from + + Returns whether dependencies were downloaded or not. + """ + ecosystem: Ecosystem + # Check the root dir of the repo for existence of manifests, the precedence of which manifest file t o check is + # according to the order from top to bottom + if os.path.isfile(git_repo_path / GOLANG_MANIFEST): + ecosystem = MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] + elif os.path.isfile(git_repo_path / PYTHON_MANIFEST): + ecosystem = MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] + elif os.path.isfile(git_repo_path / JS_MANIFEST): + ecosystem = MANIFESTS_TO_ECOSYSTEMS[JS_MANIFEST] + elif os.path.isfile(git_repo_path / JAVA_MANIFEST): + ecosystem = MANIFESTS_TO_ECOSYSTEMS[JAVA_MANIFEST] + else: + logger.info("Didn't find manifest to install, skipping.. ") + return False + + try: + logger.info(f"Started installing packages for {ecosystem}") + tree_builder = get_dependency_tree_builder(ecosystem.value) + tree_builder.install_dependencies(git_repo_path) + logger.info(f"Finished installing packages for {ecosystem}") + return True + except NotImplementedError as err: + logger.info(f"Didn't install package using manifest, underlying error --> {str(err)} ") + return False + + def search(self, query: str) -> tuple[bool, list[Document]]: + """search for functions/methods in transitive packages if they're reachable from base application or not + + Parameters + ---------- + query : str + query to search, in the form of "package_name,function_name". + + Returns + ------- + bool + Whether the function or method in transitive package is reachable or not from base application + """ + logger.debug(f"Now using transitive code search tool with input =>{query}") + try: + call_hierarchy_list, found_path = self.chain_of_calls_retriever.get_relevant_documents(query) + except ValueError as ex: + found_path = False + logger.debug(f"error intercepted from COC Retriever =>{str(ex)}") + # logger.debug(f"error intercepted from COC Retriever =>{str(ex)}", extra=MULTI_LINE_MESSAGE_TRUE) + return found_path, [] + + path_details = self.chain_of_calls_retriever.print_call_hierarchy(call_hierarchy_list) + log_message = f"\n Retriever found path={found_path}\n path size={len(call_hierarchy_list)}\n" \ + f"==============================================\nPrints Hierarchy call functions " \ + f"path\n==============================================\n" + + for node in path_details: + log_message = log_message + node + "\n" + + logger.debug(log_message) + # logger.debug(log_message, extra=MULTI_LINE_MESSAGE_TRUE) + log_message_2 = "Path Contents Content: \n" + "==============================================\n" + content_of_files_in_path = f"{log_message_2}" + for i, function_method in enumerate(reversed(call_hierarchy_list)): + content_of_files_in_path = f"{content_of_files_in_path} File {i + 1}: {function_method.metadata['source']}\n" \ + f"-------------------------------------------\n{function_method.page_content}\n" + logger.debug(content_of_files_in_path) + # logger.debug(content_of_files_in_path, extra=MULTI_LINE_MESSAGE_TRUE) + return found_path, call_hierarchy_list From e72b142d5cd8df41bec809ddf1630e0773a26419 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 11 Aug 2025 18:47:43 +0300 Subject: [PATCH 015/286] chore: add OTEL tracing with provider arize phoenix Signed-off-by: Zvi Grinberg --- src/vuln_analysis/configs/config-http-openai.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 4002b1408..e06235c04 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -15,12 +15,18 @@ general: use_uvloop: true + telemetry: tracing: - langsmith: - _type: langsmith - project: default - api_key: ${LANGSMITH_API_KEY} + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent +# tracing: +# langsmith: +# _type: langsmith +# project: default +# api_key: ${LANGSMITH_API_KEY} functions: cve_generate_vdbs: From 03a3638b19d4a65383d52d188538f68ba34d5354 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 12 Aug 2025 09:07:36 +0300 Subject: [PATCH 016/286] fix: bug of 3rd party libs' sources are not being loaded Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/document_embedding.py | 3 +++ src/vuln_analysis/utils/source_code_git_loader.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index dafca919e..17f88049b 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -40,6 +40,7 @@ from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader +from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover @@ -369,6 +370,8 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: if documents_were_in_cache or len(documents) > 0: return documents else: + # Download 3rd party dependencies to be part of the git repo' dir. + TransitiveCodeSearcher.download_dependencies(self.repo_path) blob_loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=source_info.git_repo, ref=source_info.ref, diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index 61d27293d..2665ca1e7 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -128,7 +128,6 @@ def load_repo(self): repo.git.checkout("FETCH_HEAD") logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) - self._repo = repo return repo @@ -167,7 +166,7 @@ def yield_blobs(self) -> typing.Iterator[Blob]: exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) # Filter out files that are not in the repo - include_files = include_files.intersection(all_files_in_repo) + # include_files = include_files.intersection(all_files_in_repo) # Take the include files and remove the exclude files. final_files = include_files - exclude_files From e4c2dad448ef3aef33372f82b632cb2ecbce6e02 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 14 Aug 2025 14:43:24 +0300 Subject: [PATCH 017/286] fix: make transitive search code configurable the right way Signed-off-by: Zvi Grinberg --- .../configs/config-http-openai.yml | 1 + src/vuln_analysis/functions/cve_agent.py | 10 ++--- .../tools/transitive_code_search.py | 37 ++++++++----------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index e06235c04..56487c4df 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -89,6 +89,7 @@ functions: replace_exceptions: true replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false +# transitive_search_tool_enabled: false verbose: false cve_summarize: _type: cve_summarize diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index f214b95dc..952ad3772 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -23,6 +23,7 @@ from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig +from aiq.profiler.decorators.function_tracking import track_function from langchain.agents import AgentExecutor from langchain.agents import create_react_agent from langchain.agents.agent import RunnableAgent @@ -61,6 +62,7 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): default=False, description= "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") + transitive_search_tool_enabled: bool = Field(default=True, description="Whether to enable transitive code search or not.") verbose: bool = Field(default=False, description="Set to true for verbose output") @@ -75,9 +77,9 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, tool for tool in tools if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or - (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None) - # (tool.name == "Transitive code search tool" and state.code_index_path is None) or - # (tool.name == "Calling Function Name Extractor" and state.code_index_path is None) + (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None) or + (tool.name == "Transitive code search tool" and not config.transitive_search_tool_enabled) or + (tool.name == "Calling Function Name Extractor" and not config.transitive_search_tool_enabled) ) ] @@ -103,7 +105,6 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, async def _process_steps(agent, steps, semaphore): - async def _process_step(step): if semaphore: async with semaphore: @@ -168,7 +169,6 @@ def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, @register_function(config_type=CVEAgentExecutorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent(config: CVEAgentExecutorToolConfig, builder: Builder): - semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index a1d1a02aa..718bc9f8a 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -41,15 +41,12 @@ class TransitiveCodeSearchToolConfig(FunctionBaseConfig, name="transitive_code_s """ Transitive code search tool used to search source code. """ - enable_transitive_search: bool = Field(default=True, description="Whether to enable transitive code search or not.") class CallingFunctionNameExtractorToolConfig(FunctionBaseConfig, name="calling_function_name_extractor"): """ Calling function name extractor tool used to search source code function specific usages. """ - enable_functions_usage_search: bool = Field(default=True, description="Whether to enable function usage code search" - " or not.") def get_call_of_chains_retriever(documents_embedder, si): @@ -74,14 +71,14 @@ async def transitive_search(config: TransitiveCodeSearchToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> tuple: - if config.enable_transitive_search: - workflow_state: AgentMorpheusEngineState = ctx_state.get() - si = workflow_state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) - transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) - result = transitive_code_searcher.search(query) - return result + + state: AgentMorpheusEngineState = ctx_state.get() + si = state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + result = transitive_code_searcher.search(query) + return result yield FunctionInfo.from_fn( _arun, @@ -103,16 +100,14 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> list: - if config.enable_functions_usage_search: - workflow_state: AgentMorpheusEngineState = ctx_state.get() - si = workflow_state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) - function_name_extractor = FunctionNameExtractor(coc_retriever) - result = function_name_extractor.fetch_list(query) - return result - else: - return [] + + state: AgentMorpheusEngineState = ctx_state.get() + si = state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + function_name_extractor = FunctionNameExtractor(coc_retriever) + result = function_name_extractor.fetch_list(query) + return result yield FunctionInfo.from_fn( _arun, From e5bca4e0a0b0f18e6413d38c7ea0d820e293e887 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 12 Aug 2025 12:24:35 +0300 Subject: [PATCH 018/286] feat: migrate plugin based intel retrieval Signed-off-by: Zvi Grinberg --- .../configs/config-http-openai.yml | 5 ++ src/vuln_analysis/data_models/cve_intel.py | 6 ++ src/vuln_analysis/data_models/plugin.py | 78 ++++++++++++++++++ .../data_models/plugins/intel_plugin.py | 81 +++++++++++++++++++ .../functions/cve_fetch_intel.py | 11 ++- .../functions/cve_generate_vdbs.py | 3 +- src/vuln_analysis/utils/intel_retriever.py | 20 ++++- src/vuln_analysis/utils/prompting.py | 1 + 8 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 src/vuln_analysis/data_models/plugin.py create mode 100644 src/vuln_analysis/data_models/plugins/intel_plugin.py diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 56487c4df..752619a4c 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -40,6 +40,11 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : diff --git a/src/vuln_analysis/data_models/cve_intel.py b/src/vuln_analysis/data_models/cve_intel.py index 43b2ddfe5..863b405ca 100644 --- a/src/vuln_analysis/data_models/cve_intel.py +++ b/src/vuln_analysis/data_models/cve_intel.py @@ -216,6 +216,11 @@ def description_fields(self): return [] +class IntelPluginData(BaseModel): + label: str + description: str + + class CveIntel(BaseModel): """ Information about a CVE (Common Vulnerabilities and Exposures) entry. @@ -231,6 +236,7 @@ class CveIntel(BaseModel): rhsa: CveIntelRhsa | None = None ubuntu: CveIntelUbuntu | None = None epss: CveIntelEpss | None = None + plugin_data: list[IntelPluginData] = [] @computed_field() @property diff --git a/src/vuln_analysis/data_models/plugin.py b/src/vuln_analysis/data_models/plugin.py new file mode 100644 index 000000000..ff09c0754 --- /dev/null +++ b/src/vuln_analysis/data_models/plugin.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing +from abc import ABC, abstractmethod +from pydoc import locate + +import aiohttp + +from .common import TypedBaseModel +from .cve_intel import IntelPluginData + + +_T = typing.TypeVar('_T', bound='PluginSchema') + + +class PluginConfig(TypedBaseModel[typing.Literal["plugin"]]): + plugin_name: str + + plugin_config: dict[str, typing.Any] = {} + + +class PluginSchema(ABC): + + @classmethod + def locate(cls: type[_T], plugin_name: str, **kwargs) -> _T: + '''Locate plugin''' + pluginClass: type | None = locate(plugin_name) + + if not pluginClass: + raise ValueError(f"Plugin not found: {plugin_name}") + if not issubclass(pluginClass, cls): + raise ValueError(f"{pluginClass} object must be a subclass of {cls}") + + return pluginClass(**kwargs) + + +class InputPluginSchema(PluginSchema): + + @abstractmethod + def build_input(self): + # add the plugin specific input building logic here + pass + + +class OutputPluginSchema(PluginSchema): + + @abstractmethod + def build_output(self, config): + # add the plugin specific output building logic here + pass + + +class IntelPluginSchema(PluginSchema): + + def __init__(self, session: aiohttp.ClientSession, config: dict = {}): + self._session = session + self._config = config + + @abstractmethod + async def retrieve(self, vuln_id: str) -> IntelPluginData: + """ + Asynchronously retrieve the details of a given CVE Id + """ + pass diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py new file mode 100644 index 000000000..7889c02e8 --- /dev/null +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +import logging + +import requests +from pydantic import BaseModel + +from ..cve_intel import IntelPluginData +from ..plugin import IntelPluginSchema + +logger = logging.getLogger(__name__) + + +class SimpleHttpIntelPluginConfig(BaseModel): + source: str + endpoint: str + api_key: str | None = None + token_path: str | None = None + verify_path: str | None = None + + +class SimpleHttpIntelPlugin(IntelPluginSchema): + """ + This is a simple Http intel retriever that sends an http GET to the configured + endpoint and returns the text content as a result. + The configuration expects an endpoint with the `{vuln_id}` template parameter + that will be replaced by the current vuln_id. e.g. https://example.com/{vuln_id}/details + Additionally if the `api_key` is provided, it will be sent as an HTTP Authorization Header + with value: Bearer + """ + + def __init__(self, session, config): + self._session = session + self._config = SimpleHttpIntelPluginConfig(**config) + + async def retrieve(self, vuln_id: str) -> IntelPluginData: + endpoint = self._config.endpoint.format(vuln_id=vuln_id) + try: + headers = {"Accept": "text/plain"} + if self._config.api_key: + headers["Authorization"] = f"Bearer {self._config.api_key}" + elif self._config.token_path: + headers["Authorization"] = f"{await self.get_token_value()}" + response = requests.get( + endpoint, headers=headers, verify=await self.get_verify_path() + ) + response.raise_for_status() + if not response.ok: + logger.error( + f"Unable to request intel data from {endpoint}. Error: {response.status_code} - {response.reason}" + ) + else: + logger.debug(f"Successfully requested intel data to: {endpoint}") + return IntelPluginData(label=self._config.source, description=response.text) + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching intel data from {endpoint}: {e}") + return IntelPluginData( + label=self._config.source, + description="No data available for this intel source", + ) + + async def get_verify_path(self) -> str | None: + if self._config.verify_path: + return self._config.verify_path + else: + return None + + async def get_token_value(self): + try: + with open(self._config.token_path, "r") as file: + return f"Bearer {file.read().strip()}" + except Exception as e: + logger.warning(f"Unable to read OAuth token: {e}") + return None diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index 8141c4a39..a6e406645 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -15,6 +15,7 @@ import asyncio import logging +import typing import aiohttp from aiq.builder.builder import Builder @@ -24,6 +25,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field +from vuln_analysis.data_models.common import TypedBaseModel +from vuln_analysis.data_models.plugin import PluginConfig + logger = logging.getLogger(__name__) @@ -32,6 +36,7 @@ class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): Defines a function that fetches details about CVEs from NIST and CVE Details websites. """ retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") + intel_plugin_config: PluginConfig = Field(description="Intel Plugin configuration") @register_function(config_type=CVEFetchIntelConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -41,12 +46,10 @@ async def cve_fetch_intel(config: CVEFetchIntelConfig, builder: Builder): # pyl from vuln_analysis.utils.intel_retriever import IntelRetriever async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - async def _inner(): - async with aiohttp.ClientSession() as session: - - intel_retriever = IntelRetriever(session=session, retry_on_client_errors=config.retry_on_client_errors) + intel_retriever = IntelRetriever(session=session, retry_on_client_errors=config.retry_on_client_errors, + plugins_config=[config.intel_plugin_config]) intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index d2bb55ce6..ece747e3e 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -46,7 +46,7 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): base_code_index_dir: str = Field(default=".cache/am_cache/code_index", description="The directory used for storing code index files.") base_pickle_dir: str = Field(default=".cache/am_cache/pickle", - description="The directory used for storing code index files.") + description="The directory used for storing code index files.") ignore_errors: bool = Field(default=False, description="Whether to ignore errors during generation.") ignore_code_embedding: bool = Field(default=False, description="Whether to ignore building code vector database.") @@ -55,7 +55,6 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): @register_function(config_type=CVEGenerateVDBsToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): - from vuln_analysis.data_models.info import AgentMorpheusInfo from vuln_analysis.data_models.input import AgentMorpheusEngineInput from vuln_analysis.data_models.input import AgentMorpheusInput diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index 245452c0b..e8326d0f4 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -20,7 +20,7 @@ import aiohttp -from ..data_models.cve_intel import CveIntel +from ..data_models.cve_intel import CveIntel, IntelPluginData from ..data_models.cve_intel import CveIntelEpss from ..data_models.cve_intel import CveIntelNvd from ..data_models.cve_intel import CveIntelRhsa @@ -30,6 +30,7 @@ from .clients.nvd_client import NVDClient from .clients.rhsa_client import RHSAClient from .clients.ubuntu_client import UbuntuClient +from ..data_models.plugin import PluginConfig, IntelPluginSchema logger = logging.getLogger(__name__) @@ -45,6 +46,7 @@ def __init__(self, ghsa_api_key: str | None = None, lang_code: str = 'en', max_retries: int = 10, + plugins_config: list[PluginConfig] | None = None, retry_on_client_errors: bool = True): """ Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. @@ -71,6 +73,10 @@ def __init__(self, self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries, retry_on_client_errors=retry_on_client_errors) + self._intel_plugins = [] + if plugins_config is not None: + self._intel_plugins = [IntelPluginSchema.locate( + plugin_name=cfg.plugin_name, session=session, config=cfg.plugin_config) for cfg in plugins_config] @asynccontextmanager async def _get_session(self, session: aiohttp.ClientSession | None = None): @@ -161,6 +167,16 @@ async def _get_epss_score(self, intel: CveIntel) -> CveIntelEpss | None: return None + async def append_plugin_data(self, plugin: IntelPluginSchema, intel: CveIntel) -> IntelPluginData: + + try: + intel.plugin_data.append(await plugin.retrieve(intel.vuln_id)) + return intel.plugin_data + + except Exception as e: + logger.exception("Error fetching data from plugin %s : %s", intel.vuln_id, e, exc_info=True) + return None + async def retrieve(self, vuln_id: str) -> CveIntel: """ Asynchronously retrieve all relevant details for a given CVE ID. @@ -194,6 +210,8 @@ async def retrieve(self, vuln_id: str) -> CveIntel: self._get_epss_score(intel=intel) ] + coros.extend([self.append_plugin_data(intel=intel, plugin=plugin) for plugin in self._intel_plugins]) + await asyncio.gather(*coros) return intel diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index b746bd3c7..fac1b02e7 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -189,6 +189,7 @@ def build_prompt(self) -> str: # IfPromptBuilder('ubuntu_notices', 'Ubuntu Priority Reason: '), # Disabling for now since its very long IfPromptBuilder('ubuntu_ubuntu_description', 'Ubuntu Security Note: '), IfPromptBuilder('vulnerable_dependencies', 'Identified Vulnerable Dependencies: '), + IfPromptBuilder('plugin_data', 'Extra information: '), ] additional_intel_prompting = '\n'.join([pb.build_prompt() for pb in additional_intel_fields]) From f00aae3e0bd28739456faaa72e14463566849cd7 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 15 Aug 2025 01:34:18 +0300 Subject: [PATCH 019/286] fix: in case there are no docs , need to disable all tools relying on docs fix: call download dependencies hook only once, just after loading git repo snapshot Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_agent.py | 9 ++++++--- .../functions/cve_generate_vdbs.py | 17 +++++++++++------ .../tools/transitive_code_search.py | 2 -- src/vuln_analysis/utils/document_embedding.py | 3 --- .../utils/source_code_git_loader.py | 3 +++ 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 952ad3772..8b174a0d0 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -62,7 +62,8 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): default=False, description= "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") - transitive_search_tool_enabled: bool = Field(default=True, description="Whether to enable transitive code search or not.") + transitive_search_tool_enabled: bool = Field(default=True, + description="Whether to enable transitive code search or not.") verbose: bool = Field(default=False, description="Set to true for verbose output") @@ -78,8 +79,10 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None) or - (tool.name == "Transitive code search tool" and not config.transitive_search_tool_enabled) or - (tool.name == "Calling Function Name Extractor" and not config.transitive_search_tool_enabled) + (tool.name == "Transitive code search tool" and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) or + (tool.name == "Calling Function Name Extractor" and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) ) ] diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index ece747e3e..88695ed80 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -84,7 +84,8 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): git_directory=config.base_git_dir, pickle_cache_directory=config.base_pickle_dir) - def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: DocumentEmbedding, output_path: Path): + def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: DocumentEmbedding, + output_path: Path) -> bool : logger.info("Collecting documents from git repos. Source Infos: %s", json.dumps([x.model_dump(mode="json") for x in source_infos])) @@ -105,6 +106,7 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume # Warn and return early if no documents were collected if len(documents) == 0: logger.warning("No documents were collected, skipping code indexing for source infos: '%s'", source_infos) + return False # Apply chunking on the source documents chunked_documents = embedder._chunk_documents(documents) @@ -118,13 +120,14 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume indexing_start_time = time.time() full_text_search.add_documents_from_langchain_chunks(chunked_documents) logger.info("Completed code indexing in %.2f seconds for '%s'", time.time() - indexing_start_time, output_path) + return True def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: code_index_path: Path | None = None # Filter to only code sources source_infos = [si for si in source_infos if si.type == 'code'] - + documents_exists: bool = True if source_infos: # Use DocumentEmbedding class with embedding=None for its utility functions @@ -136,11 +139,13 @@ def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: base_path=config.base_code_index_dir, hash_value=embedder.hash_source_documents_info(source_infos)) if (not code_index_path.exists() or os.environ.get("MORPHEUS_ALWAYS_REBUILD_VDB", "0") == "1"): - _create_code_index(source_infos, embedder, code_index_path) + documents_exists = _create_code_index(source_infos, embedder, code_index_path) else: logger.info("Cache hit on code index. Loading existing code index: %s", code_index_path) - - return code_index_path + if documents_exists: + return code_index_path + else: + return None async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: """ @@ -185,7 +190,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: base_image) # Build code index if not ignored - if (not config.ignore_code_index): + if not config.ignore_code_index: code_index_path = _build_code_index(source_infos) if code_index_path is None: diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 718bc9f8a..b5f256947 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -56,8 +56,6 @@ def get_call_of_chains_retriever(documents_embedder, si): for source_info in si: if source_info.type == "code": git_repo = documents_embedder.get_repo_path(source_info) - third_party_deps_installed \ - = TransitiveCodeSearcher.download_dependencies(git_repo_path=git_repo) documents = documents_embedder.collect_documents(source_info) coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=Ecosystem.GO, manifest_path=git_repo) cached_coc_retriever.set(coc_retriever) diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index 17f88049b..55c3a933e 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -371,17 +371,14 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: return documents else: # Download 3rd party dependencies to be part of the git repo' dir. - TransitiveCodeSearcher.download_dependencies(self.repo_path) blob_loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=source_info.git_repo, ref=source_info.ref, include=source_info.include, exclude=source_info.exclude) - blob_parser = ExtendedLanguageParser() loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) - documents = loader.load() logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index 2665ca1e7..0618c9175 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -24,6 +24,8 @@ from langchain_core.document_loaders.blob_loaders import Blob from tqdm import tqdm +from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher + PathLike = typing.Union[str, os.PathLike] logger = logging.getLogger(__name__) @@ -128,6 +130,7 @@ def load_repo(self): repo.git.checkout("FETCH_HEAD") logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) + TransitiveCodeSearcher.download_dependencies(self.repo_path) self._repo = repo return repo From 0238b980a66fcaddd43807ed129ffd001106b51e Mon Sep 17 00:00:00 2001 From: Theodor Mihalache Date: Tue, 19 Aug 2025 10:08:16 +0300 Subject: [PATCH 020/286] APPENG-3354 - Merge Low Quality CVE score Signed-off-by: Theodor Mihalache --- .../configs/config-http-openai.yml | 15 +++ src/vuln_analysis/configs/config-tracing.yml | 14 +++ src/vuln_analysis/configs/config.yml | 14 +++ src/vuln_analysis/data_models/cve_intel.py | 30 +++++ src/vuln_analysis/data_models/output.py | 1 + src/vuln_analysis/data_models/state.py | 1 + .../functions/cve_calculate_intel_score.py | 55 +++++++++ src/vuln_analysis/functions/cve_checklist.py | 4 +- src/vuln_analysis/register.py | 21 +++- src/vuln_analysis/utils/data_utils.py | 26 ++++ src/vuln_analysis/utils/intel_source_score.py | 115 ++++++++++++++++++ src/vuln_analysis/utils/llm_engine_utils.py | 73 ++++++++++- 12 files changed, 358 insertions(+), 11 deletions(-) create mode 100644 src/vuln_analysis/functions/cve_calculate_intel_score.py create mode 100644 src/vuln_analysis/utils/intel_source_score.py diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 752619a4c..3c85be747 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -106,6 +106,12 @@ functions: _type: cve_http_output url: http://localhost:8080 endpoint: /reports + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false llms: checklist_llm: @@ -156,6 +162,14 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -169,6 +183,7 @@ workflow: _type: cve_agent cve_generate_vdbs_name: cve_generate_vdbs cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index d6503f1d9..fa465c43b 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -88,6 +88,12 @@ functions: file_path: .tmp/output.json markdown_dir: .tmp/vulnerability_markdown_reports overwrite: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false llms: checklist_llm: @@ -132,6 +138,13 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -145,6 +158,7 @@ workflow: _type: cve_agent cve_generate_vdbs_name: cve_generate_vdbs cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index cc1e66371..6fd39d527 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -78,6 +78,12 @@ functions: file_path: .tmp/output.json markdown_dir: .tmp/vulnerability_markdown_reports overwrite: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false llms: checklist_llm: @@ -122,6 +128,13 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -135,6 +148,7 @@ workflow: _type: cve_agent cve_generate_vdbs_name: cve_generate_vdbs cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist diff --git a/src/vuln_analysis/data_models/cve_intel.py b/src/vuln_analysis/data_models/cve_intel.py index 863b405ca..8050ffe26 100644 --- a/src/vuln_analysis/data_models/cve_intel.py +++ b/src/vuln_analysis/data_models/cve_intel.py @@ -15,6 +15,8 @@ import abc import functools +import json +import re import typing from pydantic import BaseModel @@ -237,6 +239,7 @@ class CveIntel(BaseModel): ubuntu: CveIntelUbuntu | None = None epss: CveIntelEpss | None = None plugin_data: list[IntelPluginData] = [] + intel_score: int = 0 @computed_field() @property @@ -338,3 +341,30 @@ def get_ghsa_id(self): return self.vuln_id return None + + @property + def plugins_intel_data(self) -> list[str]: + return [f"{pd.label}: {pd.description}" for pd in self.plugin_data] + + def get_intel_score(self) -> int: + return self.intel_score + + def parse_plugin_data(self, plugin_list: list[str]) -> dict: + merged = {} + for plugin_str in plugin_list: + if not isinstance(plugin_str, str): + continue # skip non-strings + + match = re.search(r'{.*}', plugin_str, re.DOTALL) + if match: + try: + plugin_dict = json.loads(match.group()) + for k, v in plugin_dict.items(): + if k in merged: + merged[k] += f" {v}" + else: + merged[k] = str(v) + except json.JSONDecodeError: + continue # skip malformed JSON + + return merged \ No newline at end of file diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index 7d8481329..ab0a3b01f 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -68,6 +68,7 @@ class AgentMorpheusEngineOutput(BaseModel): checklist: list[ChecklistItemOutput] summary: str justification: JustificationOutput + intel_score: int class AgentMorpheusOutput(AgentMorpheusEngineInput): diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index e7ee0bc1b..84253e209 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -31,3 +31,4 @@ class AgentMorpheusEngineState(BaseModel): checklist_results: dict[str, list[dict[str, typing.Any]]] = {} final_summaries: dict[str, str] = {} justifications: dict[str, dict[str, str]] = {} + poor_quality_intel_vul: dict[str, int] = {} diff --git a/src/vuln_analysis/functions/cve_calculate_intel_score.py b/src/vuln_analysis/functions/cve_calculate_intel_score.py new file mode 100644 index 000000000..458af643b --- /dev/null +++ b/src/vuln_analysis/functions/cve_calculate_intel_score.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging + +import aiohttp +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +logger = logging.getLogger(__name__) + + +class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel_score"): + """ + Defines a function that calculates the intel quality generating a score for it. + """ + llm_name: str = Field(description="The LLM model to use with the CVE agent.") + generate_intel_score: bool = Field(default=True, description="Whether to generate a CVE intel score for the agent morpheus analysis or not.") + intel_low_score: int = Field(default=51, description="The intel low score threshold to stop analysis.") + insist_analysis: bool = Field(default=False, description="Whether to continue the analysis even when the intel score is below 'intel_low_score' threshold.") + +@register_function(config_type=CVECalculateIntelScoreConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_calculate_intel_score(config: CVECalculateIntelScoreConfig, builder: Builder): # pylint: disable=unused-argument + + from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.intel_source_score import IntelScorer + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + if config.generate_intel_score: + intel_scorer = IntelScorer(config, builder) + + message.info.intel = [await intel_scorer.calculate_intel_score(intel) for intel in message.info.intel] + + return message + + yield FunctionInfo.from_fn(_arun, + input_schema=AgentMorpheusEngineInput, + description="Calculates the CVE source score.") diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 23453c9d0..1675aa0cc 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -24,6 +24,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field +from vuln_analysis.utils import data_utils + logger = logging.getLogger(__name__) @@ -60,7 +62,7 @@ async def generate_checklist_for_cve(cve_intel): return cve_intel["vuln_id"], checklist[0] async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: - intel_df = pd.json_normalize([x.model_dump(mode="json") for x in state.cve_intel], sep="_") + intel_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe(state.cve_intel) workflow_cve_intel = intel_df.to_dict(orient='records') results = await asyncio.gather(*(generate_checklist_for_cve(cve_intel) for cve_intel in workflow_cve_intel)) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index a7709d09f..9ad40c293 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -45,7 +45,7 @@ from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp # pylint: enable=unused-import -from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output +from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input logger = logging.getLogger(__name__) @@ -57,6 +57,7 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): """ cve_generate_vdbs_name: str = Field(description="Function name to generate vector databases") cve_fetch_intel_name: str = Field(description="Function name to fetch intel") + cve_calculate_intel_score_name: str = Field(description="Function name to calculate intel score") cve_process_sbom_name: str = Field(description="Function name to process SBOMs") cve_check_vuln_deps_name: str = Field(description="Function name to check vulnerable dependencies") cve_checklist_name: str = Field(description="Function name to generate checklist") @@ -81,6 +82,7 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): # Access functions that will be used in this workflow cve_generate_vdbs_fn = builder.get_function(name=config.cve_generate_vdbs_name) cve_fetch_intel_fn = builder.get_function(name=config.cve_fetch_intel_name) + cve_calculate_intel_score_fn = builder.get_function(name=config.cve_calculate_intel_score_name) cve_process_sbom_fn = builder.get_function(name=config.cve_process_sbom_name) cve_check_vuln_deps_fn = builder.get_function(name=config.cve_check_vuln_deps_name) cve_checklist_fn = builder.get_function(name=config.cve_checklist_name) @@ -105,6 +107,11 @@ async def fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngi return await cve_fetch_intel_fn.ainvoke(state.model_dump()) + async def calculate_intel_score_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Calculate score for intel source""" + + return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) + async def process_sbom_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Process SBOMs for CVE input""" @@ -163,8 +170,12 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): subgraph_input = preprocess_engine_input(message) - results = await subgraph.ainvoke(subgraph_input) - subgraph_output = AgentMorpheusEngineState(**results) + subgraph_input = finalize_preprocess_engine_input(message, subgraph_input, builder) + if len(subgraph_input.cve_intel) > 0: + results = await subgraph.ainvoke(subgraph_input) + subgraph_output = AgentMorpheusEngineState(**results) + else: + subgraph_output = subgraph_input output = postprocess_engine_output(message=message, result=subgraph_output) return output @@ -173,6 +184,7 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph_builder.add_node("add_start_time", add_start_time_node) graph_builder.add_node("generate_vdbs", generate_vdbs_node) graph_builder.add_node("fetch_intel", fetch_intel_node) + graph_builder.add_node("calculate_intel_score_node", calculate_intel_score_node) graph_builder.add_node("process_sbom", process_sbom_node) graph_builder.add_node("check_vuln_deps", check_vuln_deps_node) graph_builder.add_node("llm_engine", call_llm_engine_subgraph_node) @@ -182,7 +194,8 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph_builder.add_edge(START, "add_start_time") graph_builder.add_edge("add_start_time", "generate_vdbs") graph_builder.add_edge("generate_vdbs", "fetch_intel") - graph_builder.add_edge("fetch_intel", "process_sbom") + graph_builder.add_edge("fetch_intel", "calculate_intel_score_node") + graph_builder.add_edge("calculate_intel_score_node", "process_sbom") graph_builder.add_edge("process_sbom", "check_vuln_deps") graph_builder.add_edge("check_vuln_deps", "llm_engine") graph_builder.add_edge("llm_engine", "add_completed_time") diff --git a/src/vuln_analysis/utils/data_utils.py b/src/vuln_analysis/utils/data_utils.py index 060e77c7e..729420ce2 100644 --- a/src/vuln_analysis/utils/data_utils.py +++ b/src/vuln_analysis/utils/data_utils.py @@ -17,9 +17,14 @@ import logging import typing +import pandas as pd +from pandas import DataFrame + from pydantic import BaseModel as BaseModel from pydantic.v1 import BaseModel as BaseModelV1 +from vuln_analysis.data_models.cve_intel import CveIntel + logger = logging.getLogger(__name__) @@ -74,3 +79,24 @@ def safe_getattr(obj, attr_path, default=None): if obj == default: break return obj + +def merge_intel_and_plugin_data_convert_to_dataframe(intel_list: list[CveIntel]) -> DataFrame: + return pd.json_normalize([ + { + "vuln_id": x.vuln_id, + "ghsa_id": x.get_ghsa_id(), + "cve_id": x.get_cve_id(), + + # Flatten model_data, then merge plugin_data values if key exists + **{ + key: f"{str(model_flat.get(key, ''))} {str(plugin_data.get(key, ''))}".strip() + for key in model_flat + } + } + + for x in intel_list + for model_flat, plugin_data in [ + (pd.json_normalize(x.model_dump(mode="json"), sep="_").to_dict(orient="records")[0], + x.parse_plugin_data(x.plugins_intel_data)) + ] + ], sep="_") \ No newline at end of file diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py new file mode 100644 index 000000000..f415ac7d1 --- /dev/null +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -0,0 +1,115 @@ +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import logging + +from aiq.builder.builder import Builder +from langchain_core.language_models.base import BaseLanguageModel + +from ..data_models.cve_intel import CveIntel +from ..functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig +from ..utils import data_utils +from ..utils.prompting import additional_intel_prompting +from aiq.builder.framework_enum import LLMFrameworkEnum + +logger = logging.getLogger(__name__) + +class IntelScorer: + def __init__(self, + config: CVECalculateIntelScoreConfig, + builder: Builder): + self._config = config + self._builder = builder + + async def calculate_intel_score(self, intel: CveIntel) -> CveIntel: + llm = await self._builder.get_llm(llm_name=self._config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + assert isinstance(llm, BaseLanguageModel) + + response = await llm.ainvoke(self.__get_calculate_score_prompt(intel)) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nresponse: %s", str(response.content)) + + score = self.__extract_score(response.content) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nScore: %d", score) + + intel.intel_score = score + + return intel + + def __get_calculate_score_prompt(self, intel: CveIntel) -> str: + return """ + You are an expert in cybersecurity evaluating the quality of a CVE. Calculate a score for the given CVE on a scale from 0 to 100 based on: + + * Technical specificity (20 points) — How precise and in-depth are the technical details? + * Clarity and grammar (10 points) — Is the text clear, well-structured, and free of major grammatical issues? + * Mention of affected component and impact (15 points) — Does it clearly state what is affected and the consequence? + * Reproducibility (15 points) — Would an attacker understand how to exploit the issue from the description? + * Presence of vulnerable function/method (15 points) — Is a specific function, method, or code snippet identified? + * Existence of mitigation (10 points) — Are there any described mitigations, patches, or workarounds? + * Details on environment (10 points) — Is there context about the affected environment (e.g., OS, version)? + * Relevant configuration details (5 points) — Are configuration factors or misconfigurations involved? + + Ensure that you return only JSON response with no markdown symbols: + 1. The individual scores per criterion. + 2. A short justification for each sub-score. + 3. The total score out of 100. + + Ensure the final score does not exceed 100, the weights are respected and the score is returned in total_score tag. + + Use the following 3 examples to calculate the score: + + 1. High Score Example + CVE ID: CVE-2025-30204\n- CVE Description: golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function\'s argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-405: Asymmetric Resource Consumption (Amplification) (4.17)\n- GHSA Details: [{\'first_patched_version\': \'5.2.2\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt/v5\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'>= 5.0.0-rc.1, < 5.2.2\'}, {\'first_patched_version\': \'4.5.2\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt/v4\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 4.5.2\'}, {\'first_patched_version\': None, \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'>= 3.2.0, <= 3.2.2\'}]\n- CWE Description: The product does not properly control situations in which an adversary can cause the product to consume or produce excessive resources without requiring the adversary to invest equivalent work or otherwise prove authorization, i.e., the adversary\'s influence is "asymmetric."\n- This can lead to poor performance due to "amplification" of resource consumption, typically in a non-linear fashion. This situation is worsened if the product allows malicious users or attackers to consume more resources than their access level permits.\n- RHSA Description: golang-jwt/jwt: jwt-go allows excessive memory allocation during header parsing\n- RHSA Details: ["golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function\'s argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2.", \'A flaw was found in the golang-jwt implementation of JSON Web Tokens (JWT). In affected versions, a malicious request with specially crafted Authorization header data may trigger an excessive consumption of resources on the host system. This issue can cause significant performance degradation or an application crash, leading to a denial of service.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-agent-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-reporter-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:openshift_builds:1\', \'fix_state\': \'Affected\', \'package_name\': \'openshift-builds/openshift-builds-git-cloner-rhel9\', \'product_name\': \'Builds for Red Hat OpenShift\'}, {\'cpe\': \'cpe:/a:redhat:openshift_builds:1\', \'fix_state\': \'Affected\', \'package_name\': \'openshift-builds/openshift-builds-image-bundler-rhel9\', \'product_name\': \'Builds for Red Hat... + Score: 80 + + 2. Medium Score Example + CVE ID: CVE-2022-29810\n- CVE Description: The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter.\n- CVSS Vector: CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N\n- CWE Name: CWE-532: Insertion of Sensitive Information into Log File (4.17)\n- GHSA Details: [{\'first_patched_version\': \'1.5.11\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/hashicorp/go-getter\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 1.5.11\'}]\n- Known Affected Software: [{\'package\': \'go-getter\', \'system\': None, \'versionEndExcluding\': \'1.5.11\', \'versionEndIncluding\': None, \'versionStartExcluding\': None, \'versionStartIncluding\': None}]\n- CWE Description: The product writes sensitive information to a log file.\n- Notable Vulnerable Software Vendors: [\'Hashicorp\']\n- RHSA Description: go-getter: writes SSH credentials into logfile, exposing sensitive credentials to local uses\n- RHSA Details: [\'The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter.\', \'A flaw was found in go-getter, where the go-getter library can write SSH credentials into its log file. This flaw allows a local user with access to read log files to read sensitive credentials, which may lead to privilege escalation or account takeover.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/agent-service-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/clusterlifecycle-state-metrics-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhacm2/managedcluster-import-controller-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/multicloud-manager-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/multiclusterhub-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\':... + Score: 62 + + 3. Low Score Example + CVE ID: CVE-2022-2385\n- CVE Description: A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H\n- CWE Name: [{\'cwe_id\': \'CWE-20\', \'name\': \'Improper Input Validation\'}]\n- GHSA Details: [{\'first_patched_version\': \'0.5.9\', \'package\': {\'ecosystem\': \'go\', \'name\': \'sigs.k8s.io/aws-iam-authenticator\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 0.5.9\'}]\n- Known Affected Software: [{\'package\': \'aws-iam-authenticator\', \'system\': \'kubernetes\', \'versionEndExcluding\': \'0.5.9\', \'versionEndIncluding\': None, \'versionStartExcluding\': None, \'versionStartIncluding\': \'0.5.2\'}]\n- Notable Vulnerable Software Vendors: [\'Kubernetes\']\n- RHSA Description: aws-iam-authenticator: AccessKeyID validation bypass\n- RHSA Details: [\'A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges.\', \'A flaw was found in aws-iam-authenticator. This issue occurs when an allow-listed IAM identity may be able to modify their username and escalate privileges.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:openshift:4\', \'fix_state\': \'Affected\', \'package_name\': \'openshift4/ose-hypershift-rhel9\', \'product_name\': \'Red Hat OpenShift Container Platform 4\'}] + Score: 20 + + Given CVE Details:\n + """ + self.__render_template(additional_intel_prompting, intel) + + def __extract_score(self, text: str) -> int: + text = text.replace("```", "").replace("json", "").strip() + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\ntext: %s", str(text)) + data = json.loads(text) + + total_score = data['total_score'] + if isinstance(total_score, dict): + total_score = total_score['score'] + + return total_score + + def __render_template(self, template_str: str, intel: CveIntel) -> str: + from jinja2 import Template + + template_jinja = Template(template_str, trim_blocks=True, lstrip_blocks=True) + full_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe([intel]) + flat_dict = full_df.to_dict(orient="records")[0] + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nflat_dict: %s", str(flat_dict)) + + rendered_input_list = template_jinja.render(**flat_dict) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nrendered_input_list: %s", str(rendered_input_list)) + + return rendered_input_list \ No newline at end of file diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 643f4f9e6..cc40ea04c 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -26,6 +26,9 @@ from vuln_analysis.data_models.output import ChecklistItemOutput from vuln_analysis.data_models.output import JustificationOutput from vuln_analysis.data_models.state import AgentMorpheusEngineState +from aiq.builder.builder import Builder + +from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig logger = logging.getLogger(__name__) @@ -87,7 +90,8 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE def parse_agent_morpheus_engine_output(vuln_id: str, checklist_results: list[dict[str, typing.Any]], summary: str, - justification: dict[str, str]) -> AgentMorpheusEngineOutput: + justification: dict[str, str], + intel_score: int) -> AgentMorpheusEngineOutput: """ Parse the output fields for a single vulnerability into an AgentMorpheusEngineOutput object. """ @@ -105,7 +109,8 @@ def parse_agent_morpheus_engine_output(vuln_id: str, return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=checklist_output, summary=summary, - justification=justification_output) + justification=justification_output, + intel_score=intel_score) def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -123,7 +128,8 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: intermediate_steps=None) ], summary=summary, - justification=justification) + justification=justification, + intel_score=0) def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -141,7 +147,8 @@ def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: intermediate_steps=None) ], summary=summary, - justification=justification) + justification=justification, + intel_score=0) def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -159,7 +166,23 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: intermediate_steps=None) ], summary=summary, - justification=justification) + justification=justification, + intel_score=0) + + +def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheusEngineOutput: + summary = ("There is poor quality intel available to determine vulnerability. There is not enough gathered intel" + " for the agent to make an informed decision.") + justification = JustificationOutput(label="poor_quality_intel", + reason="Poor quality intel for CVE", + status="UNKNOWN") + return AgentMorpheusEngineOutput( + vuln_id=vuln_id, + checklist=[], + summary=summary, + justification=justification, + intel_score=intel_score + ) def postprocess_engine_output(message: AgentMorpheusEngineInput, @@ -187,15 +210,22 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, else: output: list[AgentMorpheusEngineOutput] = [] output_vuln_ids = list(result.final_summaries.keys()) + + poor_quality_intel_vul = result.poor_quality_intel_vul + intel_map = {cve.vuln_id: cve for cve in message.info.intel} + for vuln_id in input_vuln_ids: if vuln_id in output_vuln_ids: output.append( parse_agent_morpheus_engine_output(vuln_id=vuln_id, checklist_results=result.checklist_results[vuln_id], summary=result.final_summaries[vuln_id], - justification=result.justifications[vuln_id])) + justification=result.justifications[vuln_id], + intel_score=intel_map[vuln_id].intel_score)) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) + elif vuln_id in poor_quality_intel_vul: + output.append(build_low_intel_score_output(vuln_id, poor_quality_intel_vul[vuln_id])) elif vuln_id in vdc_skipped_vulns: output.append(build_no_vuln_packages_output(vuln_id)) else: @@ -208,3 +238,34 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, out.justification.label) return AgentMorpheusOutput(input=message.input, info=message.info, output=output) + +def finalize_preprocess_engine_input(message: AgentMorpheusEngineInput, engine_state: AgentMorpheusEngineState, builder: Builder) -> AgentMorpheusEngineState: + config = builder.get_function_config("cve_calculate_intel_score") + assert isinstance(config, CVECalculateIntelScoreConfig) + + if config.generate_intel_score: + cve_list = [] + poor_quality_intel_vul: dict[str, int] = {} + + # Create a map of the full cve's for convenience + intel_map = {cve.vuln_id: cve for cve in message.info.intel} + + for filtered_cve in engine_state.cve_intel: + full_cve = intel_map.get(filtered_cve.vuln_id) + + if full_cve: + # Check the intel score is below the threshold + if full_cve.get_intel_score() < config.intel_low_score: + poor_quality_intel_vul[filtered_cve.vuln_id] = full_cve.get_intel_score() + + # Check if to force analysis + if config.insist_analysis: + cve_list.append(filtered_cve) + else: + cve_list.append(filtered_cve) + + engine_state.cve_intel = cve_list + engine_state.poor_quality_intel_vul = poor_quality_intel_vul + + return engine_state + From bd9ee312257e5aceb5176ab3d04dd75b194af06b Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 19 Aug 2025 12:42:13 +0300 Subject: [PATCH 021/286] fix: generate go dep tree error Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/dep_tree.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 8d8b03e4d..5df89a0aa 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -110,9 +110,13 @@ def get_go_mod_graph_tree(manifest_path) -> str: :param manifest_path: path to the go.mod manifest. :return: a string representation of the go.mod graph, including dependencies and their hierarchy. """ - return subprocess.run( - ["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], - capture_output=True, text=True).stdout + process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], + capture_output=True, text=True) + if process_object.returncode > 0: + process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" + f"go mod graph"], + capture_output=True, text=True) + return process_object.stdout def download_go_mod_vendor(self, manifest_path): """ From ec26beb9d27e51e07feb71490a0ccb761cbd7cbc Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 20 Aug 2025 00:41:19 +0300 Subject: [PATCH 022/286] fix: make init phase of transitive tool out of tool run fix: add error handling to getting go dependency tree process Signed-off-by: Zvi Grinberg --- src/vuln_analysis/data_models/state.py | 1 + src/vuln_analysis/functions/cve_agent.py | 1 - .../tools/transitive_code_search.py | 45 ++++++++++--------- src/vuln_analysis/utils/dep_tree.py | 25 ++++++++--- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index 84253e209..59cbb399e 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -26,6 +26,7 @@ class AgentMorpheusEngineState(BaseModel): doc_vdb_path: str | None = None code_index_path: str | None = None cve_intel: list[CveIntel] + transitive_code_searcher: typing.Any | None = None original_input: AgentMorpheusEngineInput | None = None checklist_plans: dict[str, list[str]] = {} checklist_results: dict[str, list[dict[str, typing.Any]]] = {} diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 8b174a0d0..e00acaa77 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -17,7 +17,6 @@ import contextvars import logging import typing - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index b5f256947..37decd834 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import contextvars + import logging -from pathlib import Path from vuln_analysis.functions.cve_agent import ctx_state from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from aiq.builder.builder import Builder @@ -22,7 +21,6 @@ from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -from pydantic import Field from langchain.docstore.document import Document @@ -34,7 +32,7 @@ logger = logging.getLogger(__name__) -cached_coc_retriever = contextvars.ContextVar("cached_coc_retriever", default=None) + class TransitiveCodeSearchToolConfig(FunctionBaseConfig, name="transitive_code_search"): @@ -51,17 +49,12 @@ class CallingFunctionNameExtractorToolConfig(FunctionBaseConfig, name="calling_f def get_call_of_chains_retriever(documents_embedder, si): documents: list[Document] - third_party_deps_installed: bool - if cached_coc_retriever.get() is None: - for source_info in si: - if source_info.type == "code": - git_repo = documents_embedder.get_repo_path(source_info) - documents = documents_embedder.collect_documents(source_info) - coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=Ecosystem.GO, manifest_path=git_repo) - cached_coc_retriever.set(coc_retriever) - return coc_retriever - else: - return cached_coc_retriever.get() + for source_info in si: + if source_info.type == "code": + git_repo = documents_embedder.get_repo_path(source_info) + documents = documents_embedder.collect_documents(source_info) + coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=Ecosystem.GO, manifest_path=git_repo) + return coc_retriever @register_function(config_type=TransitiveCodeSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -69,12 +62,16 @@ async def transitive_search(config: TransitiveCodeSearchToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> tuple: - + transitive_code_searcher: TransitiveCodeSearcher state: AgentMorpheusEngineState = ctx_state.get() - si = state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) - transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + if state.transitive_code_searcher is None: + si = state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + state.transitive_code_searcher = transitive_code_searcher + else: + transitive_code_searcher = state.transitive_code_searcher result = transitive_code_searcher.search(query) return result @@ -98,11 +95,15 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> list: - state: AgentMorpheusEngineState = ctx_state.get() si = state.original_input.input.image.source_info documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + coc_retriever: ChainOfCallsRetriever + if state.transitive_code_searcher is None: + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + else: + transitive_search_temp = state.transitive_code_searcher + coc_retriever = transitive_search_temp.chain_of_calls_retriever function_name_extractor = FunctionNameExtractor(coc_retriever) result = function_name_extractor.fetch_list(query) return result diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 5df89a0aa..7f75e2693 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -110,12 +110,25 @@ def get_go_mod_graph_tree(manifest_path) -> str: :param manifest_path: path to the go.mod manifest. :return: a string representation of the go.mod graph, including dependencies and their hierarchy. """ - process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], - capture_output=True, text=True) - if process_object.returncode > 0: - process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" - f"go mod graph"], - capture_output=True, text=True) + process_object: subprocess.CompletedProcess + formatted_error_msg: str + try: + process_object = subprocess.run( + ["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], + capture_output=True, text=True) + if process_object.returncode > 0: + process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" + f"go mod graph"], + capture_output=True, text=True) + if process_object.returncode > 0: + formatted_error_msg = f"Failed to generate dependencies tree from go.mod manifest at {manifest_path}," \ + f" error details => {process_object.stderr}" + raise Exception(formatted_error_msg) + except Exception as e: + error_message_exception = f"Failed to generate dependencies tree from because go.mod manifest wasn't found at {manifest_path}," \ + f" error details => {repr(e)}" + logging.error(error_message_exception) + raise e return process_object.stdout def download_go_mod_vendor(self, manifest_path): From 5e61ecafb4b79e9b2362e8fb55221e04162386a3 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 15 Aug 2025 02:04:51 +0300 Subject: [PATCH 023/286] feat: migrate adding extended logging and traceId to logs feat: migrate centralized error handling and catching decorator Signed-off-by: Zvi Grinberg --- .../data_models/plugins/intel_plugin.py | 3 +- src/vuln_analysis/functions/cve_agent.py | 7 +- .../functions/cve_check_vuln_deps.py | 5 +- .../functions/cve_fetch_intel.py | 6 +- .../functions/cve_file_output.py | 3 +- .../functions/cve_generate_vdbs.py | 7 +- .../functions/cve_http_output.py | 6 +- src/vuln_analysis/functions/cve_justify.py | 6 +- .../functions/cve_process_sbom.py | 3 +- src/vuln_analysis/functions/cve_summarize.py | 6 +- src/vuln_analysis/logging/__init__.py | 0 src/vuln_analysis/logging/loggers_factory.py | 66 +++++++++++++++++++ src/vuln_analysis/register.py | 2 +- .../tools/lexical_full_search.py | 3 +- src/vuln_analysis/tools/local_vdb.py | 3 +- src/vuln_analysis/tools/serp.py | 3 +- .../tools/transitive_code_search.py | 3 +- src/vuln_analysis/utils/async_http_utils.py | 3 +- .../utils/chain_of_calls_retriever.py | 3 +- .../utils/checklist_prompt_generator.py | 3 +- .../utils/clients/first_client.py | 3 +- .../utils/clients/ghsa_client.py | 3 +- src/vuln_analysis/utils/clients/nvd_client.py | 3 +- .../utils/clients/rhsa_client.py | 3 +- src/vuln_analysis/utils/data_utils.py | 2 +- src/vuln_analysis/utils/dep_tree.py | 3 +- src/vuln_analysis/utils/document_embedding.py | 5 +- .../utils/error_handling_decorator.py | 26 ++++++++ src/vuln_analysis/utils/full_text_search.py | 3 +- src/vuln_analysis/utils/http_utils.py | 3 +- src/vuln_analysis/utils/intel_retriever.py | 3 +- src/vuln_analysis/utils/js_extended_parser.py | 3 +- .../utils/justification_parser.py | 3 +- src/vuln_analysis/utils/llm_engine_utils.py | 6 +- .../utils/source_code_git_loader.py | 4 +- .../utils/transitive_code_searcher_tool.py | 10 +-- .../utils/vulnerable_dependency_checker.py | 3 +- 37 files changed, 181 insertions(+), 46 deletions(-) create mode 100644 src/vuln_analysis/logging/__init__.py create mode 100644 src/vuln_analysis/logging/loggers_factory.py create mode 100644 src/vuln_analysis/utils/error_handling_decorator.py diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py index 7889c02e8..b928c470d 100644 --- a/src/vuln_analysis/data_models/plugins/intel_plugin.py +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -15,7 +15,8 @@ from ..cve_intel import IntelPluginData from ..plugin import IntelPluginSchema -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class SimpleHttpIntelPluginConfig(BaseModel): diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index e00acaa77..d3be34f7d 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -22,18 +22,18 @@ from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -from aiq.profiler.decorators.function_tracking import track_function + from langchain.agents import AgentExecutor from langchain.agents import create_react_agent from langchain.agents.agent import RunnableAgent from langchain_core.exceptions import OutputParserException from langchain_core.prompts import PromptTemplate from pydantic import Field - from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.prompting import get_agent_prompt +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id -logger = logging.getLogger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) ctx_state = contextvars.ContextVar("ctx_state", default="default_value") @@ -174,6 +174,7 @@ async def cve_agent(config: CVEAgentExecutorToolConfig, builder: Builder): semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) ctx_state.set(state) agent = await _create_agent(config, builder, state) results = await asyncio.gather(*(_process_steps(agent, steps, semaphore) diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 9a650ea27..56964db8c 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -24,7 +24,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_deps"): @@ -47,6 +49,7 @@ async def cve_check_vuln_deps(config: CVEVulnerableDepsChecksConfig, builder: Bu async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + trace_id.set(message.input.scan.id) sbom = message.info.sbom.packages image = f"{message.input.image.name}:{message.input.image.tag}" diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index a6e406645..bf42f657d 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -25,10 +25,11 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.data_models.common import TypedBaseModel from vuln_analysis.data_models.plugin import PluginConfig -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): @@ -48,6 +49,7 @@ async def cve_fetch_intel(config: CVEFetchIntelConfig, builder: Builder): # pyl async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: async def _inner(): async with aiohttp.ClientSession() as session: + trace_id.set(message.input.scan.id) intel_retriever = IntelRetriever(session=session, retry_on_client_errors=config.retry_on_client_errors, plugins_config=[config.intel_plugin_config]) diff --git a/src/vuln_analysis/functions/cve_file_output.py b/src/vuln_analysis/functions/cve_file_output.py index 2fe53e69f..c3ea6788e 100644 --- a/src/vuln_analysis/functions/cve_file_output.py +++ b/src/vuln_analysis/functions/cve_file_output.py @@ -21,7 +21,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class CVEFileOutputConfig(FunctionBaseConfig, name="cve_file_output"): diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 88695ed80..06c246498 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -14,7 +14,6 @@ # limitations under the License. import json -import logging import os import time from pathlib import Path @@ -27,7 +26,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): @@ -173,7 +174,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: source_infos = message.image.source_info try: - + trace_id.set(message.scan.id) # Build VDBs vdb_code_path, vdb_doc_path = embedder.build_vdbs(source_infos, config.ignore_code_embedding) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index 60683fb2d..e7d071678 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -22,7 +22,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): @@ -40,7 +42,7 @@ async def output_to_http(config: CVEHttpOutputConfig, builder: Builder): # pyli from vuln_analysis.utils import http_utils async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: - + trace_id.set(message.input.scan.id) model_json = message.model_dump_json(by_alias=True) url = config.url + config.endpoint headers = {'Content-type': 'application/json'} diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py index 50a49abaf..e7f50828a 100644 --- a/src/vuln_analysis/functions/cve_justify.py +++ b/src/vuln_analysis/functions/cve_justify.py @@ -23,7 +23,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEJustifyToolConfig(FunctionBaseConfig, name="cve_justify"): @@ -53,8 +55,8 @@ async def justify_cve(summary): return justification_text.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) results = await asyncio.gather(*(justify_cve(summary) for summary in state.final_summaries.values())) - parsed_justification = await asyncio.gather(jp._parse_justification(results)) # format justification output diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index 736673add..ee70572e3 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -23,7 +23,8 @@ from pydantic import Field from pydantic import NonNegativeInt -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class CVEProcessSBOMConfig(FunctionBaseConfig, name="cve_process_sbom"): diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index e5dc6f077..af5161a3f 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -25,7 +25,9 @@ from vuln_analysis.utils.string_utils import get_checklist_item_string -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"): @@ -55,7 +57,7 @@ async def summarize_cve(results): return final_summary.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: - + trace_id.set(state.original_input.input.scan.id) results = await asyncio.gather(*(summarize_cve(results) for results in state.checklist_results.items())) state.final_summaries = dict(zip(state.checklist_results.keys(), results)) return state diff --git a/src/vuln_analysis/logging/__init__.py b/src/vuln_analysis/logging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/logging/loggers_factory.py b/src/vuln_analysis/logging/loggers_factory.py new file mode 100644 index 000000000..de0482bc3 --- /dev/null +++ b/src/vuln_analysis/logging/loggers_factory.py @@ -0,0 +1,66 @@ +import logging +import sys +import textwrap +from contextvars import ContextVar + +MULTI_LINE_MESSAGE = "MultiLineMessage" +MULTI_LINE_MESSAGE_TRUE = {MULTI_LINE_MESSAGE: True} +old_factory = logging.getLogRecordFactory() +trace_id: ContextVar[str] = ContextVar('trace_id', default=' ') + + +class AgentStreamHandler(logging.StreamHandler): + + def __init__(self, stream=None): + super().__init__(stream) + + def emit(self, record): + # escape new lines to remove from multi lines message logs end of lines, preserving only the \n character itself + # This one is in order to support multiline string messages to be logged in centralized logging in the same log + # record, withut being truncated after first new line. + # if record.__dict__.get("%s" % MULTI_LINE_MESSAGE, None): + # record.msg = record.msg.replace("\n", "\\n") + try: + msg = self.format(record) + if record.__dict__.get("%s" % MULTI_LINE_MESSAGE, None): + msg = msg.replace("\n", "\\n") + stream = self.stream + stream.write(msg + self.terminator) + self.flush() + except RecursionError: + raise + except Exception: + self.handleError(record) + + +def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.traceId = trace_id.get() + if kwargs.get("%s" % MULTI_LINE_MESSAGE, None): + record.message = textwrap.dedent(record.message) + return record + + +class LoggingFactory: + instances = 0 + + def __init__(self): + logging.setLogRecordFactory(record_factory) + LoggingFactory.instances += 1 + + @classmethod + def get_agent_logger(cls, logger_name: str): + if cls.instances == 0: + cls.__init__(cls) + + logger = logging.getLogger(logger_name) + logger.propagate = False + handler = AgentStreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s - threadName=%(threadName)s- traceId=%(traceId)s - %(name)s - %(' + 'levelname)s -' + ' %(message)s') + handler.setFormatter(formatter) + if len(logger.handlers) == 0: + logger.addHandler(handler) + + return logger diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 9ad40c293..416db7d29 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -45,7 +45,7 @@ from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp # pylint: enable=unused-import -from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input +from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input logger = logging.getLogger(__name__) diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 2dca0253d..f570bc002 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -22,7 +22,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class LexicalSearchToolConfig(FunctionBaseConfig, name="lexical_code_search"): diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 6bff4f448..76b889b0c 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -24,7 +24,8 @@ from vuln_analysis.data_models.vdb_type import VdbType -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class LocalVDBRetrieverToolConfig(FunctionBaseConfig, name="local_vdb_retriever"): diff --git a/src/vuln_analysis/tools/serp.py b/src/vuln_analysis/tools/serp.py index 6b1f2d7d3..cca482c11 100644 --- a/src/vuln_analysis/tools/serp.py +++ b/src/vuln_analysis/tools/serp.py @@ -20,7 +20,8 @@ from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class SerpWrapperToolConfig(FunctionBaseConfig, name="serp_wrapper"): diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 37decd834..f2c54bf0f 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -30,7 +30,8 @@ from vuln_analysis.utils.document_embedding import DocumentEmbedding from ..utils.function_name_extractor import FunctionNameExtractor -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/async_http_utils.py b/src/vuln_analysis/utils/async_http_utils.py index 7dbbd7e25..288819b9c 100644 --- a/src/vuln_analysis/utils/async_http_utils.py +++ b/src/vuln_analysis/utils/async_http_utils.py @@ -21,7 +21,8 @@ import aiohttp -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) @asynccontextmanager diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 01ec88b36..44774f413 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -16,7 +16,8 @@ EXCLUSIONS_INDEX = 1 -logger = logging.getLogger(f"morpheus.{__name__}") +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") def calculate_hashable_string_for_function(function_file_name: str, function_name_to_search: str) -> str: diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 6eb7abfd5..0078b1e26 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -24,7 +24,8 @@ from vuln_analysis.utils.prompting import get_mod_examples from vuln_analysis.utils.string_utils import attempt_fix_list_string -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) DEFAULT_CHECKLIST_PROMPT = MOD_FEW_SHOT.format(examples=get_mod_examples()) diff --git a/src/vuln_analysis/utils/clients/first_client.py b/src/vuln_analysis/utils/clients/first_client.py index c4bfe4a49..1d7f1c446 100644 --- a/src/vuln_analysis/utils/clients/first_client.py +++ b/src/vuln_analysis/utils/clients/first_client.py @@ -22,7 +22,8 @@ from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class FirstClient(IntelClient): diff --git a/src/vuln_analysis/utils/clients/ghsa_client.py b/src/vuln_analysis/utils/clients/ghsa_client.py index f7af369c5..a17cf7f9e 100644 --- a/src/vuln_analysis/utils/clients/ghsa_client.py +++ b/src/vuln_analysis/utils/clients/ghsa_client.py @@ -22,7 +22,8 @@ from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class GHSAClient(IntelClient): diff --git a/src/vuln_analysis/utils/clients/nvd_client.py b/src/vuln_analysis/utils/clients/nvd_client.py index a9fde72c7..96ee6ed5a 100644 --- a/src/vuln_analysis/utils/clients/nvd_client.py +++ b/src/vuln_analysis/utils/clients/nvd_client.py @@ -28,7 +28,8 @@ from vuln_analysis.utils.url_utils import url_join from vuln_analysis.utils.clients.intel_client import IntelClient -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class NVDClient(IntelClient): diff --git a/src/vuln_analysis/utils/clients/rhsa_client.py b/src/vuln_analysis/utils/clients/rhsa_client.py index 606b48dba..956d92d04 100644 --- a/src/vuln_analysis/utils/clients/rhsa_client.py +++ b/src/vuln_analysis/utils/clients/rhsa_client.py @@ -22,7 +22,8 @@ from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class RHSAClient(IntelClient): diff --git a/src/vuln_analysis/utils/data_utils.py b/src/vuln_analysis/utils/data_utils.py index 729420ce2..7fca6c937 100644 --- a/src/vuln_analysis/utils/data_utils.py +++ b/src/vuln_analysis/utils/data_utils.py @@ -99,4 +99,4 @@ def merge_intel_and_plugin_data_convert_to_dataframe(intel_list: list[CveIntel]) (pd.json_normalize(x.model_dump(mode="json"), sep="_").to_dict(orient="records")[0], x.parse_plugin_data(x.plugins_intel_data)) ] - ], sep="_") \ No newline at end of file + ], sep="_") diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 7f75e2693..f8709819a 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -4,7 +4,8 @@ from pathlib import Path import logging -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index 55c3a933e..8bbeaa276 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -41,13 +41,16 @@ from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from vuln_analysis.logging.loggers_factory import LoggingFactory if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover PathLike = typing.Union[str, os.PathLike] -logger = logging.getLogger(__name__) + + +logger = LoggingFactory.get_agent_logger(__name__) def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref) -> tuple: diff --git a/src/vuln_analysis/utils/error_handling_decorator.py b/src/vuln_analysis/utils/error_handling_decorator.py new file mode 100644 index 000000000..f92625790 --- /dev/null +++ b/src/vuln_analysis/utils/error_handling_decorator.py @@ -0,0 +1,26 @@ +import functools + +from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE + +logger = LoggingFactory.get_agent_logger(__name__) + + +def catch_pipeline_errors_async(func): + @functools.wraps(func) + async def catch_errors(self, *args, **kwargs): + try: + logger.debug(f"Before running function {func.__name__} in file {func.__module__}") + if args == () and not kwargs == {}: + return await func(self, **kwargs) + elif kwargs == {} and not args == (): + return await func(self, *args) + else: + return await func(self, *args, **kwargs) + + except Exception as exc: + logger.exception(f"Intercepted a pipeline exception in function %s on file %s --> \n, Aborting " + f"pipeline.... %s", func.__name__, + func.__module__, exc, extra=MULTI_LINE_MESSAGE_TRUE) + raise + + return catch_errors diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 58b6fcb21..bb5c96582 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -28,7 +28,8 @@ from tantivy import SchemaBuilder from tqdm import tqdm -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) variable_pattern = re.compile(r"([A-Z][a-z]+|[a-z]+|[A-Z]+(?=[A-Z]|$))") diff --git a/src/vuln_analysis/utils/http_utils.py b/src/vuln_analysis/utils/http_utils.py index 8a0f4a088..cae6ece36 100644 --- a/src/vuln_analysis/utils/http_utils.py +++ b/src/vuln_analysis/utils/http_utils.py @@ -22,7 +22,8 @@ import requests import urllib3 -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class MimeTypes(Enum): diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index e8326d0f4..d782c3a0c 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -32,7 +32,8 @@ from .clients.ubuntu_client import UbuntuClient from ..data_models.plugin import PluginConfig, IntelPluginSchema -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class IntelRetriever: diff --git a/src/vuln_analysis/utils/js_extended_parser.py b/src/vuln_analysis/utils/js_extended_parser.py index ba597eb87..cf7713125 100644 --- a/src/vuln_analysis/utils/js_extended_parser.py +++ b/src/vuln_analysis/utils/js_extended_parser.py @@ -21,7 +21,8 @@ import esprima from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): diff --git a/src/vuln_analysis/utils/justification_parser.py b/src/vuln_analysis/utils/justification_parser.py index 8186b4369..0711ac9f1 100644 --- a/src/vuln_analysis/utils/justification_parser.py +++ b/src/vuln_analysis/utils/justification_parser.py @@ -16,7 +16,8 @@ import logging from textwrap import dedent -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class JustificationParser: diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index cc40ea04c..d20a2164c 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -30,7 +30,9 @@ from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineState: @@ -188,8 +190,8 @@ def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheu def postprocess_engine_output(message: AgentMorpheusEngineInput, result: AgentMorpheusEngineState) -> AgentMorpheusOutput: + trace_id.set(message.input.scan.id) vulnerable_dependencies: list[VulnerableDependencies] | None = message.info.vulnerable_dependencies - if vulnerable_dependencies is None: vdc_skipped_vulns = [] else: diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index 0618c9175..766fda168 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -25,10 +25,12 @@ from tqdm import tqdm from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from vuln_analysis.logging.loggers_factory import LoggingFactory PathLike = typing.Union[str, os.PathLike] -logger = logging.getLogger(__name__) + +logger = LoggingFactory.get_agent_logger(__name__) class SourceCodeGitLoader(BlobLoader): diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py index 2f8b98f5d..f5b35b41e 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -29,7 +29,9 @@ get_dependency_tree_builder, ) -logger = logging.getLogger(f"morpheus.{__name__}") +from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") class TransitiveCodeSearcher: @@ -100,7 +102,7 @@ def search(self, query: str) -> tuple[bool, list[Document]]: except ValueError as ex: found_path = False logger.debug(f"error intercepted from COC Retriever =>{str(ex)}") - # logger.debug(f"error intercepted from COC Retriever =>{str(ex)}", extra=MULTI_LINE_MESSAGE_TRUE) + logger.debug(f"error intercepted from COC Retriever =>{str(ex)}", extra=MULTI_LINE_MESSAGE_TRUE) return found_path, [] path_details = self.chain_of_calls_retriever.print_call_hierarchy(call_hierarchy_list) @@ -112,12 +114,12 @@ def search(self, query: str) -> tuple[bool, list[Document]]: log_message = log_message + node + "\n" logger.debug(log_message) - # logger.debug(log_message, extra=MULTI_LINE_MESSAGE_TRUE) + logger.debug(log_message, extra=MULTI_LINE_MESSAGE_TRUE) log_message_2 = "Path Contents Content: \n" + "==============================================\n" content_of_files_in_path = f"{log_message_2}" for i, function_method in enumerate(reversed(call_hierarchy_list)): content_of_files_in_path = f"{content_of_files_in_path} File {i + 1}: {function_method.metadata['source']}\n" \ f"-------------------------------------------\n{function_method.page_content}\n" logger.debug(content_of_files_in_path) - # logger.debug(content_of_files_in_path, extra=MULTI_LINE_MESSAGE_TRUE) + logger.debug(content_of_files_in_path, extra=MULTI_LINE_MESSAGE_TRUE) return found_path, call_hierarchy_list diff --git a/src/vuln_analysis/utils/vulnerable_dependency_checker.py b/src/vuln_analysis/utils/vulnerable_dependency_checker.py index 9e3ac3c4a..ac477f8e8 100644 --- a/src/vuln_analysis/utils/vulnerable_dependency_checker.py +++ b/src/vuln_analysis/utils/vulnerable_dependency_checker.py @@ -32,7 +32,8 @@ from vuln_analysis.utils.string_utils import package_names_match from vuln_analysis.utils.url_utils import url_join -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) REQUESTS_TIMEOUT = 2 From 538a78ebd70a097c13783e5d2451871e1cbf97b2 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 18 Aug 2025 20:19:43 +0300 Subject: [PATCH 024/286] chore: Change deployments variants to accomodate config and execution changes required by ExploitIQ powered by the NAT Framework https://issues.redhat.com/browse/APPENG-3356 Signed-off-by: Zvi Grinberg --- kustomize/README.md | 290 ++++++++++++++++++ kustomize/base/anyuid-scc-permission.yaml | 11 + kustomize/base/excludes.json | 62 ++++ kustomize/base/exploit-iq-config.yml | 199 ++++++++++++ kustomize/base/exploit_iq_client.yaml | 142 +++++++++ kustomize/base/exploit_iq_client_db.yaml | 66 ++++ kustomize/base/exploit_iq_service.yaml | 155 ++++++++++ .../base/image-pull-secret-permission.yaml | 0 kustomize/base/includes.json | 70 +++++ kustomize/base/ips-patch-client.json | 6 + kustomize/base/ips-patch.json | 6 + kustomize/base/kustomization.yaml | 69 +++++ kustomize/base/nginx.yaml | 148 +++++++++ kustomize/base/nginx/nginx_cache.conf | 123 ++++++++ kustomize/base/nginx/nginx_ssl.conf | 24 ++ .../templates/routes/intel.conf.template | 110 +++++++ .../nginx/templates/routes/nemo.conf.template | 22 ++ .../nginx/templates/routes/nim.conf.template | 39 +++ .../templates/routes/nvidia.conf.template | 45 +++ .../templates/routes/openai.conf.template | 24 ++ .../template-variables.conf.template | 55 ++++ kustomize/base/service-account.yaml | 9 + kustomize/config-http-openai-local.yml | 201 ++++++++++++ kustomize/img.png | Bin 0 -> 44368 bytes kustomize/network-policy.yaml | 17 + .../batch-processing/kustomization.yaml | 86 ++++++ .../templates/routes/intel.conf.template | 110 +++++++ .../nginx/templates/routes/nemo.conf.template | 19 ++ .../nginx/templates/routes/nim.conf.template | 33 ++ .../templates/routes/nvidia.conf.template | 31 ++ .../templates/routes/openai.conf.template | 21 ++ .../kustomization.yaml | 170 ++++++++++ .../local-llama3.1-70b-4bit/nginx-patch.yaml | 14 + .../remote-nim-all/kustomization.yaml | 102 ++++++ .../overlays/remote-nim-all/nginx-patch.yaml | 16 + 35 files changed, 2495 insertions(+) create mode 100644 kustomize/README.md create mode 100644 kustomize/base/anyuid-scc-permission.yaml create mode 100644 kustomize/base/excludes.json create mode 100644 kustomize/base/exploit-iq-config.yml create mode 100644 kustomize/base/exploit_iq_client.yaml create mode 100644 kustomize/base/exploit_iq_client_db.yaml create mode 100644 kustomize/base/exploit_iq_service.yaml create mode 100644 kustomize/base/image-pull-secret-permission.yaml create mode 100644 kustomize/base/includes.json create mode 100644 kustomize/base/ips-patch-client.json create mode 100644 kustomize/base/ips-patch.json create mode 100644 kustomize/base/kustomization.yaml create mode 100644 kustomize/base/nginx.yaml create mode 100644 kustomize/base/nginx/nginx_cache.conf create mode 100644 kustomize/base/nginx/nginx_ssl.conf create mode 100644 kustomize/base/nginx/templates/routes/intel.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nemo.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nim.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nvidia.conf.template create mode 100644 kustomize/base/nginx/templates/routes/openai.conf.template create mode 100644 kustomize/base/nginx/templates/variables/template-variables.conf.template create mode 100644 kustomize/base/service-account.yaml create mode 100644 kustomize/config-http-openai-local.yml create mode 100644 kustomize/img.png create mode 100644 kustomize/network-policy.yaml create mode 100644 kustomize/overlays/batch-processing/kustomization.yaml create mode 100644 kustomize/overlays/batch-processing/nginx/templates/routes/intel.conf.template create mode 100644 kustomize/overlays/batch-processing/nginx/templates/routes/nemo.conf.template create mode 100644 kustomize/overlays/batch-processing/nginx/templates/routes/nim.conf.template create mode 100644 kustomize/overlays/batch-processing/nginx/templates/routes/nvidia.conf.template create mode 100644 kustomize/overlays/batch-processing/nginx/templates/routes/openai.conf.template create mode 100644 kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml create mode 100644 kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml create mode 100644 kustomize/overlays/remote-nim-all/kustomization.yaml create mode 100644 kustomize/overlays/remote-nim-all/nginx-patch.yaml diff --git a/kustomize/README.md b/kustomize/README.md new file mode 100644 index 000000000..73c28c985 --- /dev/null +++ b/kustomize/README.md @@ -0,0 +1,290 @@ +# Procedure to Deploy + +> [!NOTE] +> Sections 1-6 and 11 are common to all deployment overlays + +1. Create a `base/secrets.env` file containing the API keys for external services `Agent Morpheus` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. + +```shell +cat > base/secrets.env << EOF +nvd_api_key=you_api_key +serpapi_api_key=your_api_key +tavily_api_key=your_api_key +nvidia_api_key=your_api_key +ghsa_api_key=your_api_key +ngc_api_key=your_api_key +EOF +``` + +2. If a namespace does not exist, create one: + +```shell +export YOUR_NAMESPACE_NAME=yourNamespaceNameHere +oc new-project $YOUR_NAMESPACE_NAME +``` + +3. Create an image pull secret to authorize pulling the `Agent Morpheus` container image: + +```shell +oc create secret generic morpheus-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +``` + +4. Create an image pull secret to authorize pulling container images from [NVIDIA GPU Cloud](https://catalog.ngc.nvidia.com) registry (`nvcr.io`): + +```shell +oc create secret generic ngc-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +``` + +5. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [Agent Morpheus Client](./base/agent_morpheus_client.yaml) configuration. + +Replace `some-long-secret-used-by-the-oauth-client` with a secure, unique secret + +```shell +export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" +``` + +```shell +cat > base/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +EOF +``` + +6. Update `Agent Morpheus` configuration file with the correct callback URL for the client service. + +```shell +export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" +find . -type f -name 'exploit-iq-config.json' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + +``` + +7. Deploy `Agent Morpheus` and `Agent Morpheus Client` using default settings (local embeddings and remote NVIDIA NIM LLM) + +```shell +oc kustomize base | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +8. Alternatively, to deploy `Agent Morpheus` with a self-hosted LLM or fully remote service, run one of the following commands: + +```shell +# Deploy agent morpheus with local llama3.1 LLM +oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +# Deploy agent morpheus with remote llama-3.1-70b-4bit LLM +oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +9. If you have not yet deployed the self-hosted embedding model and one of the LLM options above, first fetch the latest version of the `Agent Morpheus` Git submodule. + +```shell +git submodule update --init --recursive --merge +# Or if the local submodule sha is not updated to latest one, you can fetch it from remote tracking branch +git submodule update --remote --merge --recursive +``` + +10. Follow the instructions to prepare the model + 1. [Open instruction from local repository](../agent-morpheus-models/README.md) + 2. [Open instruction in browser](https://github.com/RHEcosystemAppEng/agent-morpheus-models) + +11. If it doesn't already exist, Create the `OAuthClient` using the secret from step 5 and generated route + +```bash +oc create -f - < overlays/developer/langsmith-secret.env +``` + +14. Deploy `Agent Morpheus` with developer overlay configuration. + +```shell +kustomize build overlays/developer/ | oc apply -f - +``` + +15. In a new terminal access the `Agent Morpheus` pod and run the application manually to confirm it starts. + +```shell +# enter agent morpheus pod in a new bash shell process +oc exec -it $(oc get pods -o=name -l component=exploit-iq ) -- bash +# run agent morpheus inside the pod +eval $EXPLOIT_IQ_PYTHON_COMMAND +``` + +Output: + +```shell +Http Server Started for POST at 0.0.0.0:8080 + /scan. +====Pipeline Pre-build==== +====Pre-Building Segment: linear_segment_0==== +====Pre-Building Segment Complete!==== +====Pipeline Pre-build Complete!==== +====Registering Pipeline==== +====Building Pipeline==== +====Building Pipeline Complete!==== +Source: 0 messages [00:00, ? messages====Registering Pipeline Complete!==== +====Starting Pipeline==== messages/s] +====Pipeline Started==== +====Building Segment: linear_segment_0==== +Added source: )> + └─> src.AgentMorpheusInput +...omitted for brevity +====Building Segment Complete!==== +Source: 0 messages [00:13, ? messages/s] +LLM: 0 messages [00:13, ? messages/s] +``` + +16. Modify the `Agent Morpheus` code in your local repository and use `git diff` to ensure only intended changes are present in your working directory. + +```shell +git diff +``` + +Expected Output example: +![img.png](img.png) + +17. Use `oc rsync` to copy local changes into the `exploit-iq` pod. + +```shell +# you must be in some path in the repo so it will work +oc rsync $(git rev-parse --show-toplevel)/ $(oc get pods -o=name -l component=exploit-iq ):. +``` + +18. Return to the terminal from section 15 and press `CTRL+C` to stop the agent. +Expected output: + +```shell +Stopping pipeline. Please wait... Press Ctrl+C again to kill. +Source: 0 messages [11:33, ? messages====Stopping Pipeline==== +Stopping from-cve-http, ? messages/s] +====Pipeline Stopped==== +Source[Complete]: 0 messages [00:00, ? messages/s] +LLM[Complete]: 0 messages [00:00, ? messages/s] +====Pipeline Complete==== +Total time: 701.99 sec +Pipeline runtime: 694.18 sec +``` + +19. In the same pod shell, restart the agent to load synchronized changes. + +```shell +eval $EXPLOIT_IQ_PYTHON_COMMAND +``` + +Send requests to the agent to test your modifications. Repeat steps steps 16-19 for iterative development. + +20. Delete the developer overlay resources when finished. + +```shell +# Delete all resources but keep all data saved in PVCs +kustomize build overlays/developer/ | oc delete -l purpose!=persistent -f - +# Or, Delete Everything +kustomize build overlays/developer/ | oc delete -f - +``` + +## Batch-processing Deployment Overlay + +When running in batch mode, it is important to use an optimized configuration that adjusts the following settings: + +- The maximum number of concurrent active requests in the queue. +- The maximum timeout (in minutes) before a report transitions from `sent` to expired if Agent Morpheus processing is not completed within the specified interval. +- Compatibility with a self-hosted LLM that does not enforce rate limits, instead of relying on the NVIDIA-managed service, which may have limited API key credits. +- Disabling LLM caching in NGINX to ensure repeated inputs receive fresh responses, which is essential for batch analysis workflows. + +To apply this configuration, use the batch-processing overlay instead of the default deployment overlays + +```shell +kustomize build overlays/batch-processing/ | oc apply -f - +``` + +## Transitive Support + +Transitive support enhances the agent by integrating a tool that identifies whether a specific function in a given package—either transitive or direct—is called or reachable from the application codebase. The tool returns a tuple in the format `(bool, list[Documents])`: + +- The first element (`bool`) indicates whether such a call path exists. +- The second element (list[Documents]) provides the call hierarchy — a sequence of document-level function references that trace the path from the application code to the vulnerable function within the specified package. + +## Tracing + +By default, tracing is enabled for Agent Morpheus and configured to publish traces to a Jaeger instance that is deployed alongside the application. You can disable tracing or modify its configuration by updating the values in [base/agent_morpheus_tracing.env](base/agent_morpheus_tracing.env): + +```bash +# Enable/disable tracing +EXPLOIT_IQ_TRACING_ENABLED=true + +# Enable/disable exporting traces to Jaeger +EXPLOIT_IQ_JAEGER_ENABLED=true + +# Enable/disable logging traces to the console +EXPLOIT_IQ_CONSOLE_SPANS_ENABLED=false + +# Jaeger endpoint (modify to use an external Jaeger instance) +EXPLOIT_IQ_JAEGER_ENDPOINT=http://agent-morpheus-jaeger-service:4317 + +# Service name to identify Agent Morpheus in Jaeger traces +EXPLOIT_IQ_SERVICE_NAME=agent-morpheus +``` + +### Configuration Options + +- **EXPLOIT_IQ_TRACING_ENABLED**: Master switch to enable or disable all tracing functionality +- **EXPLOIT_IQ_JAEGER_ENABLED**: Controls whether traces are exported to Jaeger +- **EXPLOIT_IQ_CONSOLE_SPANS_ENABLED**: When enabled, traces are logged to the application console +- **EXPLOIT_IQ_JAEGER_ENDPOINT**: Jaeger collector endpoint URL (use this to connect to an external Jaeger instance rather than the one deployed with Agent Morpheus) +- **EXPLOIT_IQ_SERVICE_NAME**: Service identifier that appears in Jaeger's service list + +### Configuration + +1. By default, this feature is disabled. To enable it, set the `transitive_search_tool` key in the general section of the Agent Morpheus configuration. For example: + +```json +{ + "general": { + "cache_dir": null, + "base_vdb_dir": "/morpheus-data/vdbs", + "base_git_dir": "/morpheus-data/repos", + "max_retries": 5, + "transitive_code_search": true, + "code_search_tool": true, + "model_max_batch_size": 64, + "pipeline_batch_size": 128, + "use_uvloop": true, + "use_dependency_checker": false, + "retry_on_client_errors": false, + "generate_cvss_score": true, + "generate_intel_score": true, + "intel_low_score": 51, + "insist_analysis": false + } +} +``` + +**Note: This is not a code understanding tool. It must be used in conjunction with either lexical code search (`code_search_tool=true`) or VDB embedding-based code search. While it works effectively with both methods, it requires access to all transitive package code. As a result, generating VDB embeddings may be time-consuming if `code_search_tool=false`** + +2. Because the transitive code search tool requires access to the code of transitive packages, both [includes.json](./base/includes.json) and [excludes.json](./base/excludes.json) are included in all deployments, without changing the internal client configuration. Therefore, if you would like to run agent morpheus without `transitive_code_search=true` , then customize these files accordingly. For example, add `"**/vendor/**/*"` to list of Go in [excludes.json](./base/excludes.json) diff --git a/kustomize/base/anyuid-scc-permission.yaml b/kustomize/base/anyuid-scc-permission.yaml new file mode 100644 index 000000000..207d0f509 --- /dev/null +++ b/kustomize/base/anyuid-scc-permission.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:openshift:scc:anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: exploit-iq-sa diff --git a/kustomize/base/excludes.json b/kustomize/base/excludes.json new file mode 100644 index 000000000..e9fc1320f --- /dev/null +++ b/kustomize/base/excludes.json @@ -0,0 +1,62 @@ +{ + "Go": [ + "go.mod", + "go.sum", + "**/*test*.go" + ], + "Java": [ + "target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle" + ], + "JavaScript": [ + "node_modules/**/*", + "dist/**/*", + "build/**/*", + "test/**/*", + "tests/**/*", + "example/**/*", + "examples/**/*", + "package.json", + "package-lock.json", + "yarn.lock" + ], + "TypeScript": [ + "node_modules/**/*", + "dist/**/*", + "build/**/*", + "test/**/*", + "tests/**/*", + "example/**/*", + "examples/**/*", + "package.json", + "package-lock.json", + "yarn.lock" + ], + "Python": [ + "tests/**/*", + "test/**/*", + "venv/**/*", + ".venv/**/*", + "env/**/*", + "build/**/*", + "dist/**/*", + ".mypy_cache/**/*", + ".pytest_cache/**/*", + "__pycache__/**/*", + "*.pyc", + "*.pyo", + "*.pyd", + "requirements.txt", + "Pipfile", + "Pipfile.lock" + ] +} \ No newline at end of file diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml new file mode 100644 index 000000000..61f4ff56f --- /dev/null +++ b/kustomize/base/exploit-iq-config.yml @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${PHOENIX_TRACING_URL:-http://localhost:6006/v1/traces} + project: PROJECT_NAME_PLACEHOLDER + +functions: + cve_generate_vdbs: + _type: cve_generate_vdbs + agent_name: cve_agent_executor # Used to determine which tools are enabled + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + retry_on_client_errors: false + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: CALLBACK_URL_PLACEHOLDER/vulnerabilities/{vuln_id}/comments + token_path: /var/run/secrets/kubernetes.io/serviceaccount/token + verify_path: /app/certs/service-ca.crt + + cve_process_sbom: + _type: cve_process_sbom + cve_check_vuln_deps : + _type: cve_check_vuln_deps + skip: true + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Transitive code search tool: + _type: transitive_code_search + enable_transitive_search: true + Calling Function Name Extractor: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Container Image Code QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Container Image Developer Guide QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Lexical Search Container Image Code QA System: + _type: lexical_code_search + top_k: 5 + Internet Search: + _type: serp_wrapper + max_retries: 5 + + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Internet Search + - Transitive code search tool + - Calling Function Name Extractor + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_http_output: + _type: cve_http_output + url: CALLBACK_URL_PLACEHOLDER + endpoint: /reports + +llms: + checklist_llm: + _type: ${LLM_TYPE_CHECKLIST:-nim} + api_key: ${LLM_API_KEY_CHECKLIST:-"EMPTY"} + base_url: ${CHECKLIST_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: ${LLM_TYPE_VDB_CODE_RETRIEVER:-nim} + api_key: ${LLM_API_KEY_CODE_VDB_RETRIEVER:-"EMPTY"} + base_url: ${CODE_VDB_RETRIEVER_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: ${LLM_TYPE_VDB_DOC_RETRIEVER:-nim} + api_key: ${LLM_API_KEY_DOC_VDB_RETRIEVER:-"EMPTY"} + base_url: ${DOC_VDB_RETRIEVER_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: ${LLM_TYPE_AGENT_EXECUTOR:-nim} + api_key: ${LLM_API_KEY_AGENT_EXECUTOR:-"EMPTY"} + base_url: ${AGENT_EXECUTOR_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + summarize_llm: + _type: ${LLM_TYPE_SUMMARIZE:-nim} + api_key: ${LLM_API_KEY_SUMMARIZE:-"EMPTY"} + base_url: ${SUMMARIZE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: ${LLM_TYPE_JUSTIFY:-nim} + api_key: ${LLM_API_KEY_JUSTIFY:-"EMPTY"} + base_url: ${JUSTIFY_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_generate_vdbs_name: cve_generate_vdbs + cve_fetch_intel_name: cve_fetch_intel + cve_process_sbom_name: cve_process_sbom + cve_check_vuln_deps_name: cve_check_vuln_deps + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml new file mode 100644 index 000000000..fced7d334 --- /dev/null +++ b/kustomize/base/exploit_iq_client.yaml @@ -0,0 +1,142 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client + spec: + imagePullSecrets: [] + serviceAccountName: exploit-iq-client-sa + containers: + - name: exploit-iq-client + args: + - ./application + - -Dquarkus.http.host=0.0.0.0 + - -Dquarkus.log.category."com.redhat.ecosystemappeng.morpheus".level=DEBUG + image: quay.io/ecosystem-appeng/exploit-iq-client:latest + imagePullPolicy: Always + ports: + - name: http + protocol: TCP + containerPort: 8080 + env: + - name: QUARKUS_REST-CLIENT_MORPHEUS_URL + value: http://nginx-cache:8080/scan + - name: QUARKUS_MONGODB_HOSTS + value: exploit-iq-client-db:27017 + - name: QUARKUS_MONGODB_DATABASE + value: exploit-iq-client + - name: OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: client-secret + - name: OPENSHIFT_DOMAIN + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: openshift-domain + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: QUARKUS_HTTP_SSL_CERTIFICATE_FILES + value: /etc/tls/tls.crt + - name: QUARKUS_HTTP_SSL_CERTIFICATE_KEY-FILES + value: /etc/tls/tls.key + - name: QUARKUS_HTTP_SSL-PORT + value: "8443" + - name: QUARKUS_HTTP_INSECURE-REQUESTS + value: disabled + - name: QUARKUS_HTTP_SSL_CERTIFICATE_RELOAD-PERIOD + value: 30m + - name: MORPHEUS_UI_INCLUDES_PATH + value: /config/includes.json + - name: MORPHEUS_UI_EXCLUDES_PATH + value: /config/excludes.json + + + volumeMounts: + - name: tls-certs + mountPath: /etc/tls + readOnly: true + + - name: includes-excludes + mountPath: /config + readOnly: true + + volumes: + - name: tls-certs + secret: + secretName: exploit-iq-client-tls + - name: includes-excludes + configMap: + name: includes-excludes-json +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client + annotations: + service.beta.openshift.io/serving-cert-secret-name: exploit-iq-client-tls +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + - name: metrics + protocol: TCP + port: 9000 + targetPort: 9000 + selector: + app: exploit-iq + component: exploit-iq-client +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq-client + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: reencrypt + port: + targetPort: https + to: + kind: Service + name: exploit-iq-client +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: exploit-iq +spec: + endpoints: + - interval: 30s + path: /q/metrics + port: metrics + scheme: http + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client diff --git a/kustomize/base/exploit_iq_client_db.yaml b/kustomize/base/exploit_iq_client_db.yaml new file mode 100644 index 000000000..f29144e0c --- /dev/null +++ b/kustomize/base/exploit_iq_client_db.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client-db + labels: + app: exploit-iq + component: exploit-iq-client-db +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client-db + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client-db + spec: + containers: + - name: mongodb + image: mongodb/mongodb-community-server:8.0.3-ubi8 + imagePullPolicy: IfNotPresent + ports: + - name: db + protocol: TCP + containerPort: 27017 + volumeMounts: + - name: data + mountPath: /data/db + volumes: + - name: data + persistentVolumeClaim: + claimName: exploit-iq-client-data +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-client-db + labels: + app: exploit-iq + component: exploit-iq-client-db +spec: + ports: + - name: db + port: 27017 + protocol: TCP + targetPort: 27017 + selector: + app: exploit-iq + component: exploit-iq-client-db +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-client-data + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Mi diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml new file mode 100644 index 000000000..d1ef0fae9 --- /dev/null +++ b/kustomize/base/exploit_iq_service.yaml @@ -0,0 +1,155 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + nodeSelector: + nvidia.com/gpu.deploy.driver: "true" + imagePullSecrets: [] + tolerations: + - key: p4-gpu + operator: Exists + effect: NoSchedule + serviceAccountName: exploit-iq-sa + containers: + - name: exploit-iq + image: quay.io/ecosystem-appeng/exploit-iq:latest + imagePullPolicy: Always + workingDir: /workspace/ + args: + - "aiq" + - "--log-level" + - "debug" + - "serve" + - "pipeline" + - "--config_file=" + - "/configs/exploit-iq-config.json" + - "--host" + - "0.0.0.0" + - "--port" + - "8080" + + securityContext: + runAsUser: 0 + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "8Gi" + cpu: "1000m" + nvidia.com/gpu: "1" + requests: + memory: "1Gi" + cpu: "1000m" + nvidia.com/gpu: "1" + env: + - name: SERPAPI_API_KEY + value: EXPLOIT_IQ + - name: NVD_API_KEY + value: EXPLOIT_IQ + - name: NVIDIA_API_KEY + value: EXPLOIT_IQ + - name: GHSA_API_KEY + value: EXPLOIT_IQ + - name: OPENAI_API_KEY + value: EXPLOIT_IQ + - name: NGC_API_KEY + value: EXPLOIT_IQ + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cwe-details + - name: DEPSDEV_BASE_URL + value: http://nginx-cache:8080/depsdev + - name: FIRST_BASE_URL + value: http://nginx-cache:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache:8080/ghsa + - name: NGC_API_BASE + value: http://nginx-cache:8080/nemo/v1 + - name: NIM_EMBED_BASE_URL + value: http://nginx-cache:8080/nim_embed/v1 + - name: NVD_BASE_URL + value: http://nginx-cache:8080/nvd + - name: NVD_API_KEY_HEADER + value: api-key + - name: NVIDIA_API_BASE + value: http://nginx-cache:8080/nim_llm/v1 + - name: RHSA_BASE_URL + value: http://nginx-cache:8080/rhsa + - name: SERPAPI_BASE_URL + value: http://nginx-cache:8080/serpapi + - name: UBUNTU_BASE_URL + value: http://nginx-cache:8080/ubuntu + - name: ENABLE_EXTENDED_JS_PARSERS + value: "True" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config + mountPath: /configs + - name: cache + mountPath: /morpheus-data + - name: ca-bundle + mountPath: /app/certs + readOnly: true + volumes: + - name: config + configMap: + name: exploit-iq-config + - name: cache + persistentVolumeClaim: + claimName: exploit-iq-data + - name: ca-bundle + configMap: + name: openshift-service-ca.crt +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: exploit-iq + component: exploit-iq +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-data + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi diff --git a/kustomize/base/image-pull-secret-permission.yaml b/kustomize/base/image-pull-secret-permission.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/kustomize/base/includes.json b/kustomize/base/includes.json new file mode 100644 index 000000000..3ccc429e1 --- /dev/null +++ b/kustomize/base/includes.json @@ -0,0 +1,70 @@ +{ + "Go": [ + "**/*.go" + ], + "Python": [ + "**/*.py", + "pyproject.toml", + "setup.py", + "setup.cfg" + ], + "Java": [ + "**/*.java", + "settings.gradle", + "src/main/**/*" + ], + "JavaScript": [ + "**/*.js", + "**/*.jsx", + "webpack.config.js", + "rollup.config.js", + "babel.config.js", + ".babelrc", + ".eslintrc.js", + ".eslintrc.json", + "tsconfig.json", + "*.config.js", + "*.config.json", + "public/**/*", + "src/**/*" + ], + "TypeScript": [ + "**/*.ts", + "**/*.tsx", + "tsconfig.json", + "tsconfig.*.json", + "webpack.config.js", + "webpack.config.ts", + "rollup.config.js", + "rollup.config.ts", + "babel.config.js", + ".babelrc", + ".eslintrc.js", + ".eslintrc.json", + "*.config.js", + "*.config.ts", + "*.json", + "src/**/*", + "public/**/*", + "assets/**/*" + ], + "Dockerfile": [ + "Dockerfile*", + "docker-compose.yml", + "*.dockerfile", + "*.dockerignore", + "docker-compose.*.yml", + "*.sh", + "scripts/**/*", + "*.env", + "*.yaml", + "*.yml", + "*.json", + "config/**/*", + "conf.d/**/*" + ], + "Docs": [ + "**/*.md", + "docs/**/*.rst" + ] +} \ No newline at end of file diff --git a/kustomize/base/ips-patch-client.json b/kustomize/base/ips-patch-client.json new file mode 100644 index 000000000..ccaaf1aa0 --- /dev/null +++ b/kustomize/base/ips-patch-client.json @@ -0,0 +1,6 @@ +[{ + "op": "add", + "path": "/spec/template/spec/imagePullSecrets/0", + "value": {"name": "morpheus-client-pull-secret"} +} +] \ No newline at end of file diff --git a/kustomize/base/ips-patch.json b/kustomize/base/ips-patch.json new file mode 100644 index 000000000..cf4d87c13 --- /dev/null +++ b/kustomize/base/ips-patch.json @@ -0,0 +1,6 @@ +[{ + "op": "add", + "path": "/spec/template/spec/imagePullSecrets/0", + "value": {"name": "morpheus-pull-secret"} +} +] \ No newline at end of file diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml new file mode 100644 index 000000000..41ff1768d --- /dev/null +++ b/kustomize/base/kustomization.yaml @@ -0,0 +1,69 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - service-account.yaml + - anyuid-scc-permission.yaml + - nginx.yaml + - exploit_iq_service.yaml + - exploit_iq_client_db.yaml + - exploit_iq_client.yaml + + +commonAnnotations: + deployment-variant: base + +generatorOptions: + disableNameSuffixHash: false + +secretGenerator: + - name: exploit-iq-secret + envs: + - secrets.env + + - name: oauth-client-secret + envs: + - oauth-secrets.env + +configMapGenerator: + - name: nginx-cache-config + files: + - nginx.conf=nginx/nginx_cache.conf + - name: nginx-cache-variables + files: + - nginx/templates/variables/template-variables.conf.template + - name: nginx-cache-routes + files: + - nginx/templates/routes/intel.conf.template + - nginx/templates/routes/nemo.conf.template + - nginx/templates/routes/nim.conf.template + - nginx/templates/routes/nvidia.conf.template + - nginx/templates/routes/openai.conf.template + - name: exploit-iq-config + files: + - exploit-iq-config.yml + + - name: includes-excludes-json + files: + - excludes.json + - includes.json + +patches: + - path: ips-patch.json + + target: + name: exploit-iq + kind: Deployment + + - path: ips-patch-client.json + + target: + name: exploit-iq-client + kind: Deployment + +images: + - name: quay.io/ecosystem-appeng/exploit-iq + newTag: latest + + - name: quay.io/ecosystem-appeng/exploit-iq-client + newTag: latest diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml new file mode 100644 index 000000000..5b36c4243 --- /dev/null +++ b/kustomize/base/nginx.yaml @@ -0,0 +1,148 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache + labels: + app: exploit-iq + component: nginx-cache +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: nginx-cache + template: + metadata: + labels: + app: exploit-iq + component: nginx-cache + spec: + imagePullSecrets: [] + containers: + - name: nginx + image: docker.io/nginxinc/nginx-unprivileged:1.27 + imagePullPolicy: IfNotPresent + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "100Mi" + cpu: "100m" + env: + - name: SERPAPI_API_KEY + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: exploit-iq-secret + - name: NVD_API_KEY + valueFrom: + secretKeyRef: + key: nvd_api_key + name: exploit-iq-secret + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: exploit-iq-secret + - name: GHSA_API_KEY + valueFrom: + secretKeyRef: + key: ghsa_api_key + name: exploit-iq-secret + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: exploit-iq-secret + - name: NGC_API_KEY + value: "" + - name: NGC_ORG_ID + value: "" + - name: NGINX_UPSTREAM_NVAI + value: https://api.nvcf.nvidia.com + - name: NGINX_UPSTREAM_NIM_LLM + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://nim-embed.agent-morpheus-models.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_OPENAI + value: https://integrate.api.nvidia.com + volumeMounts: + - name: routes + mountPath: /etc/nginx/templates/routes + - name: variables + mountPath: /etc/nginx/templates/variables + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: extra-config + mountPath: /etc/nginx/conf.d + - name: cache + mountPath: /server_cache + volumes: + - name: config + configMap: + name: nginx-cache-config + - name: extra-config + emptyDir: {} + - name: routes + configMap: + name: nginx-cache-routes + - name: variables + configMap: + name: nginx-cache-variables + - name: cache + persistentVolumeClaim: + claimName: nginx-cache-server +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-cache + labels: + app: exploit-iq + component: nginx-cache +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: exploit-iq + component: nginx-cache +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8080 + to: + kind: Service + name: nginx-cache +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-cache-server + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf new file mode 100644 index 000000000..5c1153373 --- /dev/null +++ b/kustomize/base/nginx/nginx_cache.conf @@ -0,0 +1,123 @@ +pid /tmp/nginx.pid; + +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + proxy_ssl_server_name on; + + proxy_cache_path /server_cache/llm levels=1:2 keys_zone=llm_cache:10m max_size=20g inactive=14d use_temp_path=off; + + proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; + + log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + log_format cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format no_cache_log '[$time_local] traceId: $http_traceId - (BYPASSED) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format mirror_log '[$time_local] traceId: $http_traceId - (MIRROR) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format nvai_cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; + + include /etc/nginx/conf.d/variables/*.conf; + + map $http_cache_control $cache_bypass { + no-cache 1; + } + + # Log to stdout + access_log /dev/stdout cache_log; + + error_log /dev/stdout info; + + client_max_body_size 1G; + + server { + listen 8080; + server_name localhost; + + proxy_http_version 1.1; + + # Headers to Add + # proxy_set_header Host $host; + proxy_set_header Connection ''; + + # Headers to Remove + proxy_ignore_headers Cache-Control; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + + # Proxy Buffer Config + proxy_busy_buffers_size 1024k; + proxy_buffers 4 512k; + proxy_buffer_size 1024k; + + # Proxy validity + proxy_cache_valid 200 202 14d; + proxy_read_timeout 8m; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_bypass $cache_bypass; + + set $http_authorization_present '[NOT PROVIDED]'; # Default to '[NOT PROVIDED]' + + if ($http_authorization) { + set $http_authorization_present '[REDACTED]'; # Set to '[REDACTED]' when the Authorization header is present + } + + # Configure a resolver to use for DNS resolution. This uses the Docker DNS resolver + # See https://tenzer.dk/nginx-with-dynamic-upstreams/ for why this is necessary + # When considering what the "base_url" should be, consider the following: + # - The base_url should be the unchangable part of the URL for any request tho that API + # - If the API uses versioning, the version should be included in the base_url + # - If the API is a subpath of a larger API, the base_url should be the path to the API + # - Examples: + # - GET `https://api.first.org/data/v1/epss` => base_url=`https://api.first.org/data/v1` + # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` + + # resolver 127.0.0.11 [::1]:5353 valid=60s; + + # rewrite_log on; + + ################ Docker Compose Services ################# + + location /scan { + proxy_pass http://exploit-iq:8080; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + # Include any additional routes from the routes directory + include /etc/nginx/conf.d/routes/*.conf; + + + ################### Redirect Handling #################### + + location @handle_redirects { + # store the current state of the world so we can reuse it in a minute + # We need to capture these values now, because as soon as we invoke + # the proxy_* directives, these will disappear + set $original_uri $uri; + set $orig_loc $upstream_http_location; + + # nginx goes to fetch the value from the upstream Location header + proxy_pass $orig_loc; + proxy_cache llm_cache; + + # But we store the result with the cache key of the original request URI + # so that future clients don't need to follow the redirect too + proxy_cache_key $original_uri; + proxy_cache_valid 200 206 14d; + } + } +} diff --git a/kustomize/base/nginx/nginx_ssl.conf b/kustomize/base/nginx/nginx_ssl.conf new file mode 100644 index 000000000..edec88a71 --- /dev/null +++ b/kustomize/base/nginx/nginx_ssl.conf @@ -0,0 +1,24 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/key.pem; + + + location / { + proxy_pass http://nginx-cache:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template new file mode 100644 index 000000000..a3389c38d --- /dev/null +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -0,0 +1,110 @@ +####################### Intel APIs ####################### + +location /serpapi { + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass https://serpapi.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nvd { + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass https://services.nvd.nist.gov; + proxy_set_header apiKey $nvd_http_api_key; + proxy_cache intel_cache; + proxy_cache_valid 200 202 365d; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cve-details { + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass https://www.cvedetails.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cwe-details { + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass https://cwe.mitre.org; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; +} + +location /first { + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass https://api.first.org; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /recf { + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass https://api.recordedfuture.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ghsa { + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass https://api.github.com; + proxy_set_header Authorization $ghsa_http_authorization; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /rhsa { + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass https://access.redhat.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ubuntu { + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass https://ubuntu.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /depsdev { + rewrite ^ $request_uri; + rewrite ^\/depsdev\/(.*) $1 break; + return 400; + proxy_pass https://api.deps.dev/$uri; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 valid=60s; +} diff --git a/kustomize/base/nginx/templates/routes/nemo.conf.template b/kustomize/base/nginx/templates/routes/nemo.conf.template new file mode 100644 index 000000000..117dd7b4d --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nemo.conf.template @@ -0,0 +1,22 @@ +location /nemo { + + location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://api.llm.ngc.nvidia.com; + proxy_set_header Connection ''; + proxy_set_header Authorization $nemo_http_authorization; + proxy_set_header Organization-ID $nemo_http_organization_id; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nemo/v1 { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://api.llm.ngc.nvidia.com; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/nim.conf.template b/kustomize/base/nginx/templates/routes/nim.conf.template new file mode 100644 index 000000000..13bd6844a --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nim.conf.template @@ -0,0 +1,39 @@ + +location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_llm/v1 { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} + + +location ~* ^/nim_embed/v1/embeddings$ { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_embed/v1 { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} diff --git a/kustomize/base/nginx/templates/routes/nvidia.conf.template b/kustomize/base/nginx/templates/routes/nvidia.conf.template new file mode 100644 index 000000000..0f6806629 --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nvidia.conf.template @@ -0,0 +1,45 @@ + +location /nvai { + + location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + + # Only cache 200/202, not 404. + proxy_cache_valid 200 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + + access_log /dev/stdout nvai_cache_log; + access_log /var/log/nginx/access.log nvai_cache_log; + } + + location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + + # Only cache 200. Not 202 as it will cause a never ending loop + proxy_cache_valid 200 202 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + } + + location /nvai/v2 { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template new file mode 100644 index 000000000..8a5902687 --- /dev/null +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -0,0 +1,24 @@ + + +location /openai { + + location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /openai/v1 { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Authorization $openai_http_authorization; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/variables/template-variables.conf.template b/kustomize/base/nginx/templates/variables/template-variables.conf.template new file mode 100644 index 000000000..4c211724f --- /dev/null +++ b/kustomize/base/nginx/templates/variables/template-variables.conf.template @@ -0,0 +1,55 @@ +map $http_authorization $nemo_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NGC_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${NGC_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NGC_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NGC_API_KEY}'; + default $http_authorization; +} + +map $http_organization_id $nemo_http_organization_id { + 'EXPLOIT_IQ' '${NGC_ORG_ID}'; + '"EXPLOIT_IQ"' '${NGC_ORG_ID}'; + 'CYBER_DEV_DAY' '${NGC_ORG_ID}'; + '"CYBER_DEV_DAY"' '${NGC_ORG_ID}'; + default $http_organization_id; +} + +map $http_authorization $nim_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' '${NVIDIA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' '${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvai_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-EXPLOIT_IQ"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-CYBER_DEV_DAY"' 'Bearer ${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $openai_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${OPENAI_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $ghsa_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${GHSA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${GHSA_API_KEY}'; + default $http_authorization; +} + +map $http_apikey $nvd_http_api_key { + 'EXPLOIT_IQ' '${NVD_API_KEY}'; + default $http_apikey; +} + diff --git a/kustomize/base/service-account.yaml b/kustomize/base/service-account.yaml new file mode 100644 index 000000000..f86c4505e --- /dev/null +++ b/kustomize/base/service-account.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploit-iq-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploit-iq-client-sa diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml new file mode 100644 index 000000000..533d10064 --- /dev/null +++ b/kustomize/config-http-openai-local.yml @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent +# tracing: +# langsmith: +# _type: langsmith +# project: default +# api_key: ${LANGSMITH_API_KEY} + +functions: + cve_generate_vdbs: + _type: cve_generate_vdbs + agent_name: cve_agent_executor # Used to determine which tools are enabled + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + retry_on_client_errors: false + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + cve_process_sbom: + _type: cve_process_sbom + cve_check_vuln_deps : + _type: cve_check_vuln_deps + skip: true + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Transitive code search tool: + _type: transitive_code_search + enable_transitive_search: true + Calling Function Name Extractor: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Container Image Code QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Container Image Developer Guide QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Lexical Search Container Image Code QA System: + _type: lexical_code_search + top_k: 5 + Internet Search: + _type: serp_wrapper + max_retries: 5 + + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Internet Search + - Transitive code search tool + - Calling Function Name Extractor + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /reports + +llms: + checklist_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + summarize_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_generate_vdbs_name: cve_generate_vdbs + cve_fetch_intel_name: cve_fetch_intel + cve_process_sbom_name: cve_process_sbom + cve_check_vuln_deps_name: cve_check_vuln_deps + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/kustomize/img.png b/kustomize/img.png new file mode 100644 index 0000000000000000000000000000000000000000..62aa97cbbf58d540a6c7d64ffd01621dfef2bb1f GIT binary patch literal 44368 zcmeFZ2UJsA+b-(<+-?0rsR#XH8L_j2zC<;hNy0n0Z^dh|`TM2z8LI~+6y4CIX-T&Tu&Kc*PamToKjlmc!lB}%t&Ue1^d7tN*JT@`X z+0P@yvt!4O{kk`u+L7`O`agdsmZZ?w>ml$6xF zapTczQ|*N#NmRRGA6GZq-;+fu!;~vK&pgILMKCwXMn%qZLY`{n*Fl}>r5g-!AvcX$ z5qDj^!RfM&Iy!fqL!^+kRz&f>A&Uj0XeyFWDv#xd28GJbyG|pJ> zl+xWdsR3v2Xwj1Fb-cq7;fJQl1vvu-nmoct4GQxUG| z&`Y+<0d7kRvh6y)RhNgYzH}6dQlI#&1;~o_>W$`$!KQUo3*Irr)(HtEp$IiG{IWS$pIcH%5RejHGotn z8F=v9;^o;qr%AJ#%09af_fFAxdQ#GG(zo}2?xov|GjHvBybykYI2<~6^-OGwOg;B5 z_!E(pw8lTyoug_z*x&C3RGTgtc-C6){h*bTL&j50*gD2Otpv4s1rocn`3o=HS?+h^)1>0 zn({v1+xDeGpqbX-Q2pxwAKCHovKgVgjbwt5sjGg{mM@P?mJP<5c3n2YoT_Igz{FNe z$NvGx3y(GtQ(f=cMZtsuy&_Cs%wWnZCj=+elj!SrbN{d(_1_O4{?I+?|2xC)j;P6p z=mYzW2Hz3wqe1y&_$ise z(21oz;gp5~Q&rPru$ynzqhoSXGXUerRrGw5ezf;~DcnUQ;gUS{3Enn>Nl;agf6>xo zhJiZ;NA0J1C!*@urQ1g)JXojX-$297Q{V*E$NhckbQ0JGO8pMveJ4->w z3HiW38rRjk3Z+py*P}wCoMjQ|IUG0g8P=C#tGQ7#q3{5+xW;R+q%>qKQ+^tVb1hsz zhB24XuxXs}!*#SeWFNsd9_UmvA$Ry1Gm=y7&puwgMg2Zg>NE=l0C}{I3Dxu1teJRP0HO$W> z$`@AoA7Yjs&wQD)(Ke`o-RiZnqljpU@sD$?u+@CMC_dF1uRQR|Mr}yhnq|ZsB2-7@ zh^Qf4qcv3LM)CT7w5>b8NA{*06WneZ1r1r77^k*Oi;QE|2)O!d<04q_ZqXHHYm5{i z{F;cYEepsmgNRG*;nR2&#F!urX^5yzmahq>^sHs>>LG7&x>a$JQ-S2jbR)FJP2|+X z5b^U%dd|kwQK&o1a2+;I*SZ}KjMeN;WYkoWdCaF)Cf>oXVZt}>_e_(_;WJKN%8N)z z$g?ecIIgbd>>dKV`V=Mtq8;ZQ)j-b^H zFu3Fdozt{6k0TM0 z%8m|Dbtp{s?ZE1de%gJcp*P1o0}vrMjc;rFfE4e+n3o7(nfQ(_g}Uy32Cc6QA);v2 zq?)lF&39WOS8wN>YD+~&w+yxOWppTrgMzcDAl#9*K}@v*Y`OslqELPNe4QgpDavn= z+5~}ANwzi5eg$KT?ED3};?3~|=`hm%{cW8VxdP`Eue*O6O*qgOw)SLdZDb+Os3bDN zs%WcNROG{+wF?)mNzA=T66S7^jdRp7DHMd&H-HO5o;TU9Ygu0P1aaiqtn7{6K#l>; zR|pr#;$g9_p8#&MtAp5C6S$)sp&HQj(rp_1*39?INV8Zw{~pHbC(AK|1Kc3GAa*~z zzuP@^@UM8w)yAd@e9YaiNaB3d9#5Z?K}?;Y7A~W*NwNI2@32pX8_}|7FDLr-{r254 zxMkBNp<{q3Rqd29!o3KX#=iZYp?3^wwN&+@PGExC^;7L|N?jx+B=m(NfpHo)4oLTH%iAFfx^B&FWv!)`?xJT$ z+*q~eRF8%)EFqIaJ>tR~qS%s|^3_O5w$d%6+rIU5}>X_{UzT4Ntj>3kLvmUKyf6Wz}YLH^-_A@?0fo~Kkmp$b_%Efrvu%HwY;Kzf)n zb8pg4boyR4Jg|3xX8?##9(1F5_KA(M63%dnlEnHc{Vc}gOYHEalyBwkEnUkw9!i3a ze}JkGHvh@Z@|eNNxZK#I@Ox68?+LTjNxhUY7tc-FnRI^QO{evcj!|O9j7Vhj^OdVL z=|u{#yC}CR({b9A2Q_ba;u1vLp=sT@UE}1*x12x6L`6m(sTZ{bsE}E|qYmb-M7`kE z(X+6*Ri%>$_kiJmA=>U^>-+iQcd{aufOH#RUe!p2m{uccX`D6e4TBVlWdxvh$)iNPmH~Ry=X^_*OKc4^zjgP}FdbP93XY$m@`I zfF+dIi9sfS5KKv*3l}EL@ok=?4OVma{dYcJlF$0^**@oVD}}@ zV~1sfy0n{tx;0VHa(;+)`-ln293eW&gJca4F8Tk=#_~ z7mmlDU5L*kOisk47er-3D?%c*eb7%Z!g4E? zj0Y6M8y2@lvsT|rRPO7vRlM2Gue{k1Vldhg;?=5ujU^?9;0kl_BFUJ-Q^a?s4IH9McV3O3)x{cjgtwCdv`Ne@S{FS24OJh3G)cD7XODN z7jC#-d}MUKneMLiB@TaVA`&52e1(iXsj;+Ekz>qDJ+9W=g>i3H7kJRzyKlhM((=(j z;2D~jdG2TG7P7?L3QBPsYXu)+g|oQne7R?@Y{0gEsxic8o0gpi~JSsd~I3xgQCy=$Y(r z-TGv#=91OR)QYaphtty{tGbtR#uVm;Z#e53sG3^lrr$G2Ijb(z398*Mzk1N#{cEGs zj9pjdeW2>C$x9iPsl4Gpr-dqYdJ@N1J|qO!yk#*3j%pUdc@#&OnVxVm)l&@8KB3BP z<1i9#79){uP!0gX`QDGz7Y=1}QEpV{C3VqBvu?Z>?JDC^4371$H%FwsbK?)tk{_FX3FqPZGx?c{*?2=$ATQA)@fh4 zb0IZ6rGo~2M8nO#g=)6sc|#6tLo_+<2JD1;efut zh4*qo!G@mU)y5EZyI!Yh%^7II{IgoVBw)QAUl-M-YyHLHYrsyEeT2Dp6(f!Ew$7-^ zG?t{N^ShMd8vaViL``aipkWHNs%E`k=nJ=#9 z?AH->@0&Aad>92d+^Xmkk4STQ3ud|MGoRH4qTVTNS%!EF65^(O74h{-+X3~ann2;f zG6yjs&I8Rdh0R9Zi4s zXTxTDz9G%Qx~L3(Bqw-(WzDBTA*_t-m`Y=RAjLg1)?Oh;h%nwCt4F-Cx9L{4&ZghT zj4EPRiz*kvsTDJ>wUlB!o21=W?qv8Kt6&Eu43cJwpEtr54#?bjdk`v4{>wu&(9O?Z zd#%EWwNY$|^SL)jIV1x%d;J-=89H1V#HJR?=f2(WL5c=3su4e(pI1e(h$PzbdD?%1C%PQZ&bGovV9JjF-wDg4E$d( zvrSD`&4!lD$g*b=#w(b9`LI(*0W)8_FgNIdd@plqqK-4QKMn60vPiIj%Anp!+XDkM z%Lw_EkUj;_vj3_{P;b-feH{SDyC6OpYZBLQ$uZW_+e&eAAN|A-YLKAdz3%r3?Jh#F z11hecp@Hssxa((WX2@$u+m?Yk=CfDFX7JA{JAU+K0=CRpwpp^;$fnn!u>0*+jY72` z0B)iF;(@BL0de#*J(um^(>aT-9oVCg3y{0NbUnUo1GVZeq(_RiDIo*%z;XOcQ_ZVV zfA2t_v1E9R)W@7-&Q~vlgRLiBDH!5J|EOvjq|0V0KEMKVSvF$1G(IxM=vpTdS7~V& zUaB}YRaXE09tDZ0aGrX)*mL2nugCX7yJN?;4VMvVe2xi-x)S_QbLozmyJK9Qb8e;T zv_GQp@!;cxdWI1LTa<}Y~S@<{GviKYyemZT)QZ|%>(bDi5w2w;GTiu@a2JT5i6 z?DNyZBA&}>Qwr5OOZpa{S+v>vd2G#s6?zGo8ENT}FT3q92N*Bt?|ilssW5;Fg-fy> z*=^h=m8WdtSY8w{pMY!AsEDbg!U4gv$$BI3uy5sC^}4Uv_L~!rpNKWa3^bmpbcYVj z=3N6Et{oCAP(N95WB<;U(vG!q4+vt*(ry^qtCq7h2+5TL?px2rWDfo;h_i(HQ!z$3 z!2~XR7x~H!%Y?Wl1#k1HWJ|DO8W0^!#{$oF!KC|Ekns87B=~K19zO}(=jxY($z%AszfmYDOK&7REia7a zYbb-_L+M@x3jWN}trGH&3BO}IoZGzs`~zp-yr(Guh(0r_Bvw1VeCv))4|fGEU(?q%L%l}PflyOBMIjyp(!7>F9G46l_NwSYWZK^7@H{1GaE$xnnD`jdH zDIhR*s^L<_?2U9~_OnW#wP)UZgvoR>>^7eys#Rg^DD-cDy2tpOi}KOz~BfG*b%PvP3mvVy;mp^=q zYyw9p0Bn=DQOk-oDd{bSF}vd}>`s_lmMO0ppZi^i?A+?n38$%jtgOOrBddH&(P=fq zjD`E#`-qkv^RX2081jX__w!&&^Q(SiF&>&l|3PnKx)I8mLmrOt>&qXejkXWeds{`(qxI{X)Ie+S~q|9gPhAgpR=WneU2)OKx9 zi0Jq59~%E>MRNSWtmMhljZs!~M3ocmlB+KetB79#?Pv9WbdTyh(9Hq`t=*)3{rMgi z7Nz{9QO!}i=2U~rOGjm>Q@nll>mr$+2Z0ml%Ecd=!j5eW(}O3^5_3{1WirQ0r7+Ob zo#DZd>H)pg9M5@^LTQet{k#i=1}L&%w7s3%NCbNZ_Q|w*-pG?t!XPCBd!Z#s>BjR7 zIzceSpjV{4z8bfoDule@lxW!9o^fjB)!r6~U6`7rN{_tMnbC4F-xl{FB^mUP@Qm7X z8J#*loyFhA)s}~5=M$&Ry6FqLHHewdiTAjCZ<$zBu`a4DhtE4JL78I%^`fe>>bgI1 z@)SA;R5`NyJMStgO;ly>PsgM-geILzhFaI|9u-Y$8mnNtAkUWB3we2(&@!&zG#|gs)<-j98Nrf>LN_ky2{1Q3%QR{FrE19EED5V`Gq$_Ybih*w(y2> z4HKcQE~~DNkldDm%P3^7ZIvumB0Hl;|smVF0w!Wd*8}k zn+vmf^9J|#O?PS8w*2meeO^1C?v>JmseXpCZ-}!EqP=UC0!|e>wCWUv=%*dtXg8B9 zbK%2|u;+Injb+V7AJ(jAW}2JU+0x$P2%i`?_iI#-oyFC@ax8eKK)#w%)!#&#)m7RG zg@|^RnS$!y_D!s0%7&d)4QizqSIS~<_CC}xgP~-wsmI(SyD647F*e+;3kj+<$1UY` z22tQLAv(=140;F`HpR8#{u4GOpf8MUztVg%)e4T=TA6*X9fS&13`U;6kl)G?!_pR~ zN^_U00z0-I(`Q;@eFSz-ptE6~Q7vlXut3(p-n`m~O12~_vY_uj_~xbm3BK9%R{d&@ zJ&7rwoj_^_t92RgmV#_JkUqpMUehCg@NNGg5AE0iQgkjD{mT6^iZsCczO=1Xoxit$ zDQdWMP8n$u(NeZM_>M~OO0TT#n)1@ln1o6B76+5gDN9`QgDbDxGms_=sa12XdFqy> zpVd3lo*^8S7rhGHxOnV20;jV1p~Pr%t+nF`!9-Ip#->#Ls~UZpqjbPU06IZ+JupaY zO8_8N!le!4#0=hvdYazCuiCVa-@BI!(aCSF(389C6=aVU{OE*dMJ|}(4j`m4BK_P5 z4}Aq~k-e8JLfRI<0_T3G(;;g_B!`~=GHBv%Uf4ngi(QF?c+Q1*P+A~u^~%=o`nd%r zqSnME5!LjS$8lqwy2vfe1p$i{QlpK`m}x{uuJ!GW(_8+cpXNwiAcXcrsGM)qP0gaRpBMQxe}VIMj&`a0KIm?bn_X(KX4iDb!btt*!ka+lts-8NlQ;O?Os4^NwC!e$8wIr z;?l!LK#w-~Xa5yb`PTJ(&YZ%p>}l&rjFA~TcLUw}M>eivyj6dL-U`5|$wq9}>X}_z z5`5M1ja#)W4H21Xh#;gP5C3~o1*p@4>F~UI3VX@l8^r}{*;rK z<&4xBB@4;N2UN^1NkzCmu5D&Gq=(r@M8-V15L2l2diC6ZO6c8nE>Ahv>mTKjozvr$ zgjI*x;C$MZmzL<+GwWOZ>F~bk`8>!s)b<+Gw0@2EYC%jTKPFW`USpJ=`x%r#N$5EW z*>F3L1VC(w2!&Hqi>T$qOW^ZpiPOc~yz*PiPqbGBoAdg!NfM#4pA1Ml?Jt{F zj`Z26<5w*rSnPHMy`p5_bLP9~)IrE*GZJc z920-*YWGz}ssj%npKEP^uN#KVMU<%uR0fu=wTH;Igv0o^(NkZ^L18j34SDr(1b%!e zBC2VoNV+o<0k~WkGzgu%{G1kqr}U7rLtk{ePdwc&)xmxfu`1nfk`k4%as2lc0MSs! z>d!UwFFA~WJ(roh*hJb;pd_Q-5rs@6=FZG`Kp_+tP0h-_oSr+z8aG{f9{lUw^Xm(i z(?k8(Z_DVk?0$Gec|t_At7wnOIsczX>mMF5;Y{heA$oUlnxe-T`DT7L8juTb6LSsD zuGJt;yi23f-(PK32AK~}TgR>&#z>J!hXeSk5Y0Nyqb;qR-{9_QH8a9@84dr*nKt@y zmqS4=2M>8I1%oPBN&{5DsqBQ&>g>`K;F-MJ<1rnBD@4VCo6yC%TzAlxyT03 z{EdI6(!H;lC1fqwyXiJkRL@wz93;0xrI-HZ^T4iG{mO8?2Y(>|x4sd8A9}tLfVYd_ z@s~$_X6cp41cIhS-3d#~C4Q0fH+6L%eO7Z}+XS<3xdX=3&Z6d>MR;c<1xaMSnR`M; zY5xq{w)QXR@qddmB#MS()am)JkPO_0=Y;P2o4 zBV3-3>>>a-(ob>36s`9}>+SQlJ@r2f7vC5bj0$3oz1CALUX-_eCZm()Bje3Ap1j?z zvM#rIwq1{~l>aU??cnlq5-pl#jV8((A9KL|2Ev+CmWAq19SZ`QXiveRBUS5MDjD9+ zrIM4Mp#GN`CDfIw|IdvQl=s2NQ9jjw1q6O$*#8|t`mLUu+l}@uE-|9WF_ct@BaXJw zeXRp9(CxL8k6fqgaqz(-XnnG%7CWp~oUQu}BR&$jZ|kOW?f;lG!Vhs`#~8~fJF zM8};;^aHqQfpc&E``{xaphB7L!(rX)j2)XaEn?+9pmOPX_XVmF{@%*Uy5TFU-y|x( z^!ybPg$Z)4fMYENGKwDIMxUaNEOez*43mri#F7 zxr-T;>=p3c_>HA@#}ofn*CpyT4%e}e^4=~IMVb2)1u&sKqlWuxD2kcp_=5ZMegXPg zGJyapcIYqGO)+3>7Pz{oqTiQK#P_k|$?ktt^w!mUX$<_u(DKLdt3)5$C^XM8U=f+H5`FzE2laCmhgXqR2HO8R1AL=!>RhUQ}EAcR}aA(Qxa%LVQy1AxZxj z4A%bLV6bSg@Uu@YG^d0}( zKr&3%x)Tq~&|DfYvnX|?@wP_=(e$=hhJQN_0=3;6IArvYD4H;Il^5j!`Xy@6(s zx9|triL&;au<${=(oR5PYWSru$tv%y5m1^h6Y?U+qC#x;bL zkcRG$n5Jh~5mw4|*0uP{6;#(hl&+n6@jN z7A$B|)EEDEs@>W5tXa_gIYKyHBA-SAJ>k8r!m zn-^f<>LuH%{?s}+5Ws7WQ%3504O1)&O-;?xpDsU9a4BjC$ym1OBw1vmn|rPH>@_=Q zb9x~8_48=gW2C=PSy@Jf`@qEV47rj*%!mj8*Fq;ne35pORRe~(O_FZT;eC;xO|=iR3BZx-Z|xtv6Vdi?|S~q)Czq>vxN8C-t zcL-FcOF~!yA76|_6#4UQEwhTL4wsJZLSOvah6Ef=Z?C$sOQj+qM2CWq&J zHX8sv&$)Z*L6bsJU|koj2y<&@-dbR%OUQ1Jx4YEQTPgsu$5!GyDT_R0O)yR{)tl@Z zs4Wc4ARIYt%8WT45qLvK;a zKuAm-5TPu<0zjD)r?h+TAIz&ieohn9EoT9tw;5?&c9u7d?D^^!`Ge~hkq1jp3FFd~ z5g{D`1$wUd{D`I7RaBe%5V&ECfm!EQ4?!41*au!+R8j&6uh+G&T~-Bj>rR`&G;q?z zL{a-m_~!VlLp1?hwfxfn%nYUsM!nf7V$_)NMp^wLgEjP-13nJn3h_k>+wG`dS&6rn z%XV_FG<&|VgvQl~<8L<8VzF3LtwFE)Qx-0U&J7aK@{7(_f4gH;=r8TF>_W;a^-fXq z$&AISkG+MzQs8qwcT}YzKB&FK?3!%rw3#?QJlw)%vYzX6()hEa_+}B;ayrm8F}eDd zb@`i|)v8_Eg*HA=38xjtFZNta^p!FXgJfn!SW{}$g>qZqEWTyKId>xV!c=BSVxhhV zF?VcW>Q^$40)RiyLmi6)yr!ol+{^>e25QM>w*a-+_?Dw`aECfDW!azW9GFk00I! z9|uQ#H7`2cKjHo}0spLxnv5~pDLt1ZY8cLCMC^o*tEm)4e$FAfS5E#9R5GkHJT+6e&eFVu~+R4h9_*3;8Gx+Fd{;hYmDFIQx;u@IjEsQ;jFE}Gc52F)76mJPw7pH z*yIDMkN5%r6ezX{@9UvQi7R&OY`g+tEXlI8La4X%lIYOf>N8O?_0t6BFD1Rc9gxzk ztji4+_d2h2kG80Z0sGcqR`(SKmcS*4`*RoY)x!sRM=k-S^>NJ|cleab-ClK^Xj8|7 zU@01-DM5-#1I?ZSXyz#Ow)We`(70r8t+Z#;~2$U zXA$m;QLIpqF(6H}BL{Ph!7!gV%VIBGVr}(C)313vWjptSSu;h!EJ}qpy$CjAwH!gQ zgJeRhfoIRsS+=*L+4`Q%?2y}8Y)u(z_%k_m@F+{6Id1E#=jaUUb9jM*ilm;No@wFe zp{-YJn1un~h{$_AT^qaSEhir9ZPr?C42q3?iZ@-hc9DPUd-yIG6rZ&yaF(qE2Rij| zOy4wttK7zWKuelS5)0^y8_rr2a(&ySZ(HZ_HM#>LA`eu z@M2<3cv7%xBC&uKb+PvniyhZU2W2<%9TcHwn#s^n%K8*B1hnQni##k&Uz(grz(P5_ z@L*Puh0lv8ZVl6frPFv$l`onTbi4mE0r6$qkct`;#ehyH&sb5w%v!ySqoBM#7-9V5 zGf#T`LNkC7t29Qr;<~XIFpOz4R!RX6e@U|)or?;mnZ@`=vo7=E#~BEVBtFb0f>BUw z?{4n76alZ58H;TWjd|f5=K-(}5526i-T5+7TbNI3m=VzP*fl46Cbju52FkTb#riUP z3p+nu`;8=x3H!MAG599YiB}}UG#Gl5-05U!VC3QNe|^T-!#wQ}VJCGNoja#I1MFr# zh$+t8-|XQ|FeeQ?=@88yT^plqKWh;~=WRI|8X9}dQj9AjF1#qnWz5@M%)IcK&Au*% z%i=T=BIcg1ArDmhhV)bWKy`>=Ho}04?gHN}Wq7(A#eeP&5jr6}`q88{-(F^XaaO4M z&9)_7G!Ka@IGJg?_LUIGvjE%pMLi%;KZhrY2Bq{VDV450VJ04f)^pBqrK1H9OKZ+o z7N5Hdm!3+N4hLs#_|gahQA}%kN(;D$>2TP%d|?n6&dyy#;&Hy;yAEdIF#w14#hbOL zSHRA4T)}OD#&At)V;q)Q4Vb!(qED$CIY+ob`tN#=k$mC9z9`}z+joq)#45FJa8x7XSMa?)}t?mjvMFR=`TI#21M zjmLHce0|ZUjsdC-*;Z$R9ca%?BMIqQMb51b&Hw2umeq5A<}8jR!v4056>(=kB1*TNTN!dq;K~y`RqZi zwS!&9t(HoB9}t9Dr2bHdvo*4uJGz(`4W&*Lr3}wTWw3P3m1bWZ3f`ca=Ki$uY0t_1 zKvmd-fjeh>`_QzBn^A4ep!bMi4*i4M^138(diIF|`GrtN*|GVulNx56lQDzyOlkZH zNKpdrv~0Y>ZSa}vC_ZPlOa#kdPMEG-4V6cBziE8eu;lv=GRnNhVO!D}`~M!}zd&P} zOGu09VP6j_K80=1?RHut;80ym2@-PnTwpf0PVp|KOrh|t$n%#$4R9ll^s0*0V|sV0 zE#1(whAk=6^g%+4W6sb+ws*+Y*={_GqEjFPH6tLo#3;+D6#CMcfbEUX+}UBqO%(mp zrS4{TGg{IV%YB&z$*!*4^;|zY*EE{XKU($fTE_1@#tws@r-ZM6;!8)sEbRnDOB z_6I_u)+q<@iwN4bKMFZKZx{I+6v&pfU&}EM2pfq6O~y#!GCN-kt&EuAI_Oh*m;g9L z18BIOO$V7Q2cBLc=qpPR= z6d6v`>p{qS9F7Nq{La#UNvIiG+1BO|-|u8O*Dm1BtOJEWGO`b~-1vwj@)%lM5ZNSE#k>k4M7r0f|dO49z<#&f`zHIVH4y*6 zuV-Jd4%;*+uTXp)vhkF{dyCI3vXsq=859hgR>-+GwPB!P{&v;VTi{U!sPXkeobh=Y zb&}9}rFcE+)RwO`&>0-}SbS__QOuUjc$mlC#$Z7Eu9E(Fww@nQ-;{&m@F})#|i}y8RyJN2|a7kjYr3*#3#xKE(LDEZ7|xmp}nqnAE+hT=#X> z#&?u?#eiG-mr!n-Y=*G}r8CZ$Vr49b4=Ll_BxSU#HzAEPiy;+lg69q|mM*6}e*9a3 z>!UMkBRF705iZ^3;YxRJ1t+Rj7ZTNO%|R5)WDpFW)5~&t=`)|IpWq;q>$z*t3hq^I zXmk##M8rCh?-6h4oBA(mj0YvkX%q!;lg)Oeq(k8XR|Iu^tx=vbonOsr)o^Q~Jl_~O zoc`3dpa-#6T{X3Rng1~YEt0$sYe6fWpB40L)Y8#E;9AHDVJl#<<8Ec7=WW-^PVi2k zfn^mS^H$|+?{#@GB2$3{-Ucw)wSDehF_bh^DiKz|inUK=tD*5Jj3c);*OO>sYT|y&>+U2b*DX)F>D{THr=)reIxP5! zBt6aABe6lx)`BCk+mWu9w=RiV`ov#L-j6HC4@#?F=2x5Zo18=C7lawnu%6G9*CyIs z#f7?p{^atj@*3ATCtU*pA$BHHp(#E`OJw^{k7zFMQmn}&G8%OOQ32_u3z8#r)lO!p zb4N>f(YH?w_i^~&291Nl+OQVcds&1BaT(ANgcri9RD)pS|HmREJkW;I@WPQS#vp9w zS~b(l(k?`8TTjwSA4r>10jjT8+Y&#~QIwd!F|~D6aKUIlrU{fO8sr|>ml473v4oP{ zJnNt?z&%irZmT`3A?}-6ZYM{leM(5fwB30^`Tm~aT~;+gC72T!osk21kwZm$TXphN z#FSXeHyqI!zB*>eniMQf<~Fqg;rL2cLfS9_99qiOwG~Zio%y=)?F)ZDmyI1?)}@x0 z-o2}+pkBbXQZ~=S6VSlVmIgVW6j*q~#laBIb@j`znZSRfwE3^%GNU(=1Q}{*M z!W>M+SHjs7QM@=Lg- zTIjvV0M78&>RhmXzJ8h}2gfC3Es9-ZpKnp*e6P+~TypU2X>buFblVgXP4XKy33j9m;fJmq3xQa%_`}YUM zspg_)C+DqvbPE>rkJ8w(tktrUz>Fn*5K>MZ@v+*0{oUXJN?7J@6$bhYpdmo)dp2O{_=0LgQ}8yjM}lsqr_fycW$^JU+;krILEoa3Pd+ z_kyq*h0||_emYUVDA`&TkY5sShG~Gubf7D}fSis^1yG~h-KgsGQE2>A^|2SHT(h?1 z>~4oO+T|%>KJ6K`#1ucym+s>oJ*IqODsXGO?T?%kPh0=2E!cDC(ci|A!d9rHQzfS5 zKka%~`=tVHlCPgRdqy&Bi4X5r<9w3DOVO%H5x*s=J28Umy$^~gP@ug;2?_a_hJ{CB zq0O@cRTk*^01KCT?H`Z%r{k@d!CT4R0^_0uPh7pH%W`y(ZlQfOE9)9?X2tDc>2v>e z>ycPL(|2y{Z5&DQ@}<)H0w48Go5~LXT5@F@gm9&;`5drUSj`~9X@8LVP<9szN?5W+ zQgp{!sb`mh(>qivo4uW`4y9J{`r3z2np9=97*P)FuONIvH=0sOZ#N)&0gDiU6ge zmB)PHqt+$%Tq4KyDm)z5TDg8T7aIZ`YnD#(AV$K_%`upRcARB0=;~nPI3>BNLebq_ z_(wptrD}W%M)k~StV8?NE*IpjaQ{O9;DsmY>{zgd)d2`YTLc2AV0ZHO=?0&34|wbo zX!JqgX(CbBd_u7r(c|wOeD21`+ki{2@=9J6p_Z1czN@-x{`8qJh*97m)Jd>;s2xM_jOU6G0hP;(Fa8jYzim5e z?9_qWdw)BEbiy>a;f$+xDjt$OGR14lBC<7?u0+Qleiry;@CldWOcP&t%F0}TbW!BIJgDuEL!t0yKRVZ^cq?0ia=Sr`AJKR{i95iSErZSNo2%FV_-E(+Y-?Z4 zpytf+<0CaJa?Fz&0n9=;SB{JX=JM#427@?a(>W(ZhhAQJIr}9`J4ZTAh}i`jQpAM% z)ebd@PIyNhdt|vB8F*(48&{2%*!*#$OwA=J7)+jRz2Vy&5~$sd#6lbUVJo*XmDl+0 zZ^uc@2*E`>3q}<={BM$f{u6+Mx6^sKd63dlG}vdd@N6$dNpuS-q$poyO(EHlV@{iO z!Z_I9Smu|fAmu}^2PJHX-@MF9T0Hv8Q`gN(o6+xmhxAUR4*OXWRKbFfMq! zc18fQ@I86#izM@ zVi&l3(=JD7-~JU+p`fYD3j_ZFXFiKW-<*Q87^vf3YSq(6<4y=8ZjR?!h5LDI?R#y_ zV_4-oGa>QnX?ONA5Iq9XUg{JgZsrB3s3Hmu-dzJ;MsAwU2xf6NKr1BhR(yMmk2>q# zNk5QS9$9PrNYGZK`S>iwadKPCFqpOPmfy8#;$auDhgw-?vI_3KaSwmJabC-AO{Lp^-cM# zH~Nrs|Ki1TjMeFaC|&Pr&GS7Ud+zrB_3C(C;?-4E?c*0zwclZeAH0NjsVms4E6tz{ zK8yuggalbxS;0tw6kB!NEZF$YbtwgLg~21H#ZN{MoNfD-M~RIZ5~t>#C+~Cx91n zd!L*uZZ>Hy*(RVMpwDac4IN&yr8J7p0C^bOmLKaR8O*+=)s!D7n?#u0=EUBlI*6_D z89UweII)w}>$g1p7gZIo)0CwkZhc$U1o?)H>U`oW>sUtHcj%vkwSDJq!D!~TeH<;F zFvJKH!mTFZ?n(G^3ZCKRys7=ZbTAd--zm{NihpL3~M&wj$VI0X$YB^bN z=8LexEzh+w2d* zUHPJ~y2+|TX_F$1aBU3k{w`ZP76FF9$sy5Sh${gr#<@ah`w zp550M|9OBzF31wl^668*9yt=Sfro#2%b}YSpEM-8RCq?sdrCr=vl*myR<3fWy7~j1OupMe*+B*Nb!&>T2-TQo z+IrLj3)OXsHBh5c(G;I^N}ff+&3;(b--lICA{x>rM|Lg) z>)qEL*i|E+K-@mKCnDurRk}K2h%Tb|%->$&)!UTGzsCt_o{OXc?#3doSe@6nSZ6N` znS6b&m!VwtCA>`_Pv2zgc|LDe_?@@bBQ0)`hM)Q}Vnl>9FnBZP&?5=7?5a=+r{}1I zAz&?0VjY8!rA5?2``PlmvPdgUw!UFC=hWA@gKE2aT0eplWUL>5C=+$Ovl!IW1p}S9 zHZMbYcJuhFS~A}SMWuwvij&ho@}V3uZm>P7KE@{HGwtK8z@;9J%*Vjv#uBp5`UQ46 zu=U0}4GY_Ht59`KH&yn(#LFzj(;8&*{^bR~^lXC_wnY#rpM9Mf@=9p6Oa1;U-YJu3 z7cvYx<}%gTaEftVpi;8>^OucB^4>vRk%BH_vy1cMC#oT~360mk9rJOhn88S|%gXVu z5=aGza{{}hl|PR)>TW{J-Q_mJtd~!8Cc)OGUt#fo@+gB|vIyw)P2Ja;Wp^X0Pkd@F zlV@%Vt7Ww%iSX||;<$BvWV|M3&mY&`$+*AX>c5uNhS=}2xXah?%+_??%RKw>t)irj zI{G~CzRloM$|be#r%S}H8-e*JPPE5mM#Go)skWBA2Q`DJ*@yPoW0U_F2Xo8*7uqx3 zX{=K-)EXiQxyG_JI{wM;5Ec%IiQ6rlDr^Z^dWgjoWhKUow5hXpFg&B*%|r^0M)wY9 zw3)1Ub{>RGp3X|>s#6Q=JZ>iefE&G$HPgPZTSW>RAgs|7E&1Vkoj?xzo;UD)Bu3eg zO|Asxu$+(Cv1vo|$T78X$f17Z7){Q~VQ$KdAyeqseN`>LBSyp`3~wi_ZFFusV-V;R zNI7tObwZ;leMua4{sD_MclkD9nCXrcJY!D zIK$wQ2uF$Gni6w9(CE{DqwYPUnoy&zL3=|)M8yUXQ4kRVf^>)?g49S?5JR~Lf)Hr} zk|HY7K?Fo;kuF_&Pozdc2)#>$Py-=`KuFKT>wV{&Su^whn?EEgEBwfl^PF?`K6{_c z%-F4lRP~<27oskyHWXtzet%QnJg!bKP(wr#@FBd{sYeLk=WXeqrtj|yHGeIaf~Byl zBCsDM{;ez7dGpXLKksZQJXdvhr=Vv=ZN)fMcd7Tg>^SlhfYWrQg2C24_tR$=PGo&~ z*egD7;u&lQPzN8S4FUg}Y`RS`>{1pn%(`Bnpy7M0c&)P%wzd81mrTibTZT2MgEUb7 zMcb#sO4^jE&j_#AhJzXvK6-~(a2GWlGfn1n!Q(UHdej8B4{A*_maJT3kPNS(a?zeG+(8i!bS*0eb9eq!CNe|>E=r!8DoFV~ zk)SgT+mu^Bw|*m30;)Q6_mM2(2g>y&=h-q&@3-x~xmBl3E={30D-xoB4loeji zN{d$H$c#eN?;)T0u9wA3>}Q!Ls!g~bcbbZ|jf0fnq_%R*r$gZ>;?C}{bA)zqPGGgo zveZ?PxSOz|7mtKhbq@fUn5~E+10Gu|vj9@77OkBx353^#}I@On%K~J$l`X ziS>QbaxE&$Py4dzWe6|wl^SwMI8#W7u(I&eJz#p^c*#UKOJ1aPQEX&?*@wVAIOneB z9458*V7%JUcKkTv(M%)#OAnH|tn&b-SWc+9PQBGpK&libjR~7H} z`-!{k#ttm5C_ZJfmzd?-3i9|GCfQIIcALDI$(35U_kIk%FQ)?FyXQ6Ce4*EKCR9Mk z;J2^uvBbojpy#9=8quYjdCceq+qRmir%djFd^ah@pe4U^Eou?7z{W+X+W{O@B)f(p zTU~CVfnazdSLo+@+U3v-G?9(%8&l|tjqAuLsC1uWMuFn&xUC4_Le8NkvRNTnD~yIY z{Vo0pc9xXTpqb*okkSVod?o|uQBvY#MY|}=j;{jHN{1@8h{n_Sra|VE8#_n_rGs9e zWRdW$2zClhb!7_6qf-YFja&h1Xln}oaB(d>G88caUK0J%-_Uf0BP4q5H%CtskxLUH zK;m$n>mX`sJ|(f>Lv!`P;6AT8it#C)1@Gdanz?1_!SvNQ2hZS?x-ucjThbMiKB`y# zR!2Dy_oT*1n14H$bqe`Rn59yam48_}xap~cx7@r#d+nljJ-i%Hhf$CDMtpc!O3uF# zTDRf;wbg)7-)|X_-wsxMR;G!#NqTVLaIzC81*aImk}UZmP2AObWsUv(T;Q|o&b{A7 zgAmQ_Q$N#J230ri)={Op447Z~(qnwQ0~Y~L+7jG=Bb|XIKi3RHsb{rnz(&kI_ELA_ zSN@D~5B6cR*~bHE4V9My2HQ+$JkB-v?9#rpPq&oyG>@!pC2jVeF}V)pW{Wf|W*eHF z@&pSJM#EhX*pRw@Qv8;W^i1_ez8755|IOaiw^}UPI1xBEPIwGcQWbVtRDJQW06TC3 zK5QeODgRw2Rz<2EE$WK17YXwJq9cPG zi*(F6s=ZXI`SNU3+c!B#EKVHV@l(4(vpI~LFbR$lYJwY-k#1OKdEm2E&Y?kJ2K$N4 z3&0WZB;*Rl{I#iQ*Y7#KaP0*9m{MEIDO-ayOcl@ftk2yMkT@iHJizVH_99bK=%<^0 z&O)Twyb;AENgvQaU%n%-@q_Z-GOSb@ngz9g(sUPD$ksnt>wxNrR!TEZKC4&nEM}&a zYIbk=I1?Yp)nyxzWRJx`5K|B}9c=gP8N3C{kYW6qCK!V)7VwE}LEspC|OM#4?C_FfqB3Xkv#nXk! zKj+F z@BGJB>m3<|++|Jac26^>f4-%EsOmY{@;0;cM~`AmT%40L^MsW6_UzBpdC1mcsZOM* zB+(hRwr>W7%ZYtA2#7TcGHa=i1ly!Ywd4$4M}h@1QkljK-2Gp=ZYsZpkpq#_ZUzHq z4rLLjJ0HO7j_OZ5n@(+x>227cB~^ajoB|(qp6?mgOXWl}l~xy}9`GMQ0@ z_9PEScI;^_@~Xe496?Q!or9l>`c<`bi=Hh@b1y$%v_0KEztB4)n!P*KU~EFfe607m z0k8M@=G6GgbsG+Qk^Go<)Z-b=WxiE#QwSbCV_i|XXR$lb%d>TwD;S>XxS<tO|C^Q6g0*a zP$3MtiKsQ{AYkiswq{HppJ<{S`KD@l5I}A}PR*CHoc3(MI&Cu0T{^ zZf%1<`_KUb-jX-|17P|yqb+uOt)S(H@nxpuWTYoK`yy@k{lC!I?mZjmFQu^ILE+{} z#H5q-(F|x?SdhO2lqfZ09CX~meZ=rEA2@r|8UU;tNg7fBF})TBBfGR-GXR1ORLbUZbZ66CGkCd)a`nV`S>dV0#qRc37HJ53}U?e z45A;fOi>Abux!N;IzpH)qU+(iTecMgykrj@1Bv!;$9u92DYF%93vSsKwH_0~NM07B zE=soc+-+Gs(j}@kk)py&Viv@S=1iZ1239m(iD=$(Q`pdQ5^lZ4p11P)I!mp4%9}VU z_2{9~+{)f3<;L8s=vr>==0eIzjWF(Z8s<1ge9{(7Z^qL^<8T$QT>5AYN(C7?jIIp> zHICZD>Wx+)`?J`C4AV*ZJd_=JlT&;`%tdn9KBZs~9X^mN{fz!6bveTda6q$Vd=fil zmIA56jO*xN@y%1X3FIQd28`QLK`?;h*p`SnVLxAfj zM+4o-j8Kq1qYu81segZRBbT(e>miPDuo-bZ?EA9jKT?X+=PI*eW|5+U`?~R%&`38_ z_9a(c-%x?j!LY_8OiG|XkqBG>xrJ^Cxc(Zbi8hSEq`I>m;L@}sU{>6YkZj}BZ^^Z1 z!_s_$=hV&0WNN2I)DxsklrP=UOmIJBGxxIx@AS(=s5@Q2iq7$1%nZ1EnrlaxOlwD43WfXBYO1_nl=k9Uz&^%YYerHpt*?9B;6oxUR<*j0 z6Y&z_*xu-YpQ;fRL*EJJ5TxtuD@U$ zNhOVDzxBItu}-7~*!48nIFis7#Q)a&HZ#(vEH`lRW-~$#+#OrDo#spLMT^8sQ?~ky zz@)@?=;FPj&;}6#P_)<#5nhqj)Sp|sWDV8Fe>I-|LM}>{^35aZ4>Z-QsK2oM>3U%A z;n%XGbK|cHgQo``pLfT+Qeqw{3E$uEh?bU>e(mU=EiF_U4se{Gxv6)kblWYgy(m*c zQ#?j@Rm=vSb~i&*@5?7~Ksq`$8qf#5)&)zSSnfZ)nY1`bUXIdcj-7_YG3mXKY5WaJ z<4};I?_T?!SKr85ac%_8_4rLQ=U0hWp)Zroe*gcInJZ;Cl1dj`>sjypeG^DRY7TQ5>==QU;-2Qy%v_7*8q(>0mc} z;-S%=tVo7HF0Do`C~L?CRxglK9C*~Fj+RxbB@~<|yha9KqX{HGteXQPkD6BPV1!4q z?{75XoHkqKro|ih1VbcN>c1kjv?SKSb)aF9PRK$yEDi)E6|7>M`RFWE)J-L85Ei}> zB}kw9zH4Tnp>EW})}AevuY*BF5DYv(GTbdn!#GDYv3SliX#U*uZ1O?VX{mFVC+h?V z5mkS8$VuyCa~O#leBJPvq{cp}OtG7OlUNIpf3wqo?=NVU<-xHhU6k@nrzeZo9~Hcz z{AznxE!qi`ihU=4u9-GGsl(V6h#r5LOnd#Q`mMB=P%SJ9iUm5lv;ikx#fDS-=dRU$ z5FL>bk+^9^Y7~JFd)*rBB<>R3WjUBqTd)XCD(h-32A9x+8N2ujw>q!~)BEH-%+>Q< z5c9LN2e8)(EBAkxXkmMEet!N!HhTLfL>Ki;e}=&8L0VxS3w;i60@}0uc2ilm;i;OB zFi7neE(u3V3=(@aYB8;Py3h4}_R}EOR<{o+CElVhu9;Qmn_NEF7BG&WoPOAcZcXdZ zd}cW(Tf;A7VS)xr7`-We<&*b+9w#K2qww;lsJLU0$EXG`1FXX({$HA< zzV-*dQ!NNQC!#|*dJcPwu}R5b@&!;l`E3FwoU=vZ^Q*z0AP3$@ke)iD_lw7rVe+=( zIUUqjNR_G@< zF{|{kZSb03>oLEbj7yowndGMh%e&^GUmFH)Za$O$U3T}sUdbOVs)79WvNxRK-uC>W z_q&0r%G{70%i3O1Mo*-{umE9U&bQyJ z@8@#;A5WxPy}9lB=?@_+j`~U%vaj%c2+p~KZG81?{s(gy`}^;>VxaPC znAwGHv-CSKftR0Ibcu&EFwmA*#p@fNcT1x~V#)aC*3}hZEiKG$)8HAhyVQkjzsBrHiCS@yv{_+utF{oKXDVp$p<-eZS#fBJ#h#_rnD;{%zn|?H)Jn>IBFP&^Kac*Y6P>~J-F7L*K%Ri?Ok#i= zE`iN>yIs%ZXok*8@gozdmS4Z?1(tU#Bo?d+T_cT zYEJrf{0M`)G{PvvKmKo=SZShvQUh;g>sWpLC6W~L%S~|MUW$U#u^%!WBX1tD^Q**o z1yv`IcJ^<|UX#KY3#T4?b%FwFXHhw<@~`jj!kUDyd~IM#v5O>M;%il1Y8wyK92D6# zJAF*+@nh@Yx>;aa(bLjCSBbXU3I#h-tQd&W?ro@;96cxbl6$H#z6cK(v2xCCI0%A5 zJCWr;tliJocVBK)K4|OY5c!$SImQ^@iF-bG;iap0TgI6SpQ4rhu^lc0&hi1Ed};Pl~Ri_%61I)PY10%UG6G8J!Cs2Qk=y=e zpLam&YG$K;zogs(Z;2af65Pak=3ed?N@kSw@zzVvtC@$RVCcVVmZ__q6k)R@Y}CsXsKEN-%jeMrDH_Qe!*tD1Tot&v!lU+t+D_-3zV( z^s%JKYlXaArZLO7B=W*Gq=Q95ov$6Mog0fivD|mCh}L5OaoK}(u#8|Y)8-?;v0oao zZyQMVT{4+YotD0^KpTk%%II7gq`&Wq$8!@;YVZjVhz@22V9udNyEAvrzlr`G>w&>! z*xz}}#G$^Zw@%!PBC?IxhF+)pu8%sSqGrn_1(yZWPziBD7Mwi#yRLY>C{U!b&H@32 zM&@ni(Gr<>D9dnZe<3`N{+5<9hlesPIJtzThxK0$fyfcR>M}@lPSG=VG=tX-wX<>b zQtry5C(PZlTTYt06$O+*RAZh9Rad|9q{c~%a`xy()<{@Jb zifX{^B9q%elhZLgZym*(c>`n95jrevD3>VR8E>G!bG`1ig4G;XJL+$KyL>i6AY<|8 zA<$|RFO?3`@cg2*qT3{>7wj~}rV+*T*8BQY^F)-WDXeexJL(BbsVccC0Rhi}pj zpUyJs#U6L&w9z|EhNAS@ezy)D%ATVBw!ZkXb-6sJbW^qTNhdzo;c{d?ptUMHh?&qW z#g4v$RE9SphWrGaMss9afF1X84Zyv6_{0<`cp*1?cq8!QRy-VQ;=n$V@N~e4tzi+l zH4y1k2;cR&)cq?)IVER+t$FXYC`pd?)A75j%vDpR1j#Lw74k7MVMK=+W(MHobO+Yd z2F0(|*b;!69zJU6uab#mV)`cskOZQG{#I0@hLUg| z4$G^JS3+!YB;WE05a>__Z$qG%VjR%SyyGRXm^90Z+m{~0z0>~ca564=9HYxNB-oqq z!l|4KaPG;m%`jTk`f^E!M!QydouiQh_JiW){*0P&Q7-}LmY+)|1X+YWt6E6ov@@@0 zxtDc~I!a{f+01cF(m{E2%8#!e0T0*)Bl5BqwSOrdVOJraEK6L&-G4siR37BnL1MI6 zWahgoKWb~|Y`se9vYjx!dvjqyY7b&o1GeZ@cVxL)#bUgEyJ(jRNO~GM&eIbMs<3lSq{NUjaH3l~ADQZsH#lg2Fxz-N9rF9yWD-eAK*kB1$ zt9pKBha{zGc?E%5^LM*h9!=|AE43^oa9X={TLBQi@#BXNy3K)m2Y4!z826cv9ZzP+FW=A71A_oOBKl6jKauCe!475$v)Nojptc{nEW(Jk> ztP;Y-0)*K(B5Yo5=wQ9c{2h+;2xJm6PZG-zSVu9YeX?{yZZ2$9CGQ<|89YY+7-2GZ z4x+M6A5O&94I>aNe#^i{Yn{Ze57vPfyX!t}TJw;D{U_no4_^`OsRMq2{_$`8S1@~( z4*mX8x>H0jw*DjZQ)TlJDA9vLePfZN4#D&A{31~OGboU{VTAQms}N2%6KkdFs=|JMd|A34om zi#8)XbmH34Lkk2CKepJtMZI}{H-XF(OM1X&YTIqPox=yL^dVoLeD!1SC)ry^($B3G zdzyau7L9pTEOY9H!iQ*=8o^U?U_|%~kXRDvzYrEp`Ep$mZg{k5Rg&%smzg95DsR=P z;RoiC7g}M{Gx(Y@M_aSTz_n$5KMAa0Eg*(FCXUS*yB z_DbIK(%>s$hl~40H6v_3DY1M&)X|81$4jFO(~6XnY84tlum7( zt$B)E4T@jXMK$<{9RuSI+IpevUmEnv(Zl-=a04x`Ggs|J*8e1G{m%#Rwfq{H+CA{@ zH~E^iPVzRSZiF!O_QZZrZ1t~{>w9VOCoFnabIUQE+WclRM;eRLJA)defQb+pcp)d3 zyFKqf3%m8&|9&-UJ-TpEMm zLB|TYzFeuF3}BcGRV|9No?E1@b zOZ-cS-kE;UBcF1skAR|;2|}{YgfS>5m;MoT$odV^id6APoOA0h=&Vlv$1$r)5MN&M zRnKz|Z}LPd?#z`hi1o4leUsfbbJ8R)dL92*Dhm(UuD&DRxh(r2bY&U6NT`|oEwJ_R z=Pcd%BOjeGnEA_+E8Nnm6QDv_Eb}!M`ccFu-rkPFD}o~nX)UvNAb-%#!XLCV1RH}0 zaO?B;^nT2|qu}dPfJ;6bW?*?d$N;*IBRb#FNjAy(}kf2O0g%2F|ZHeC+agbyqk@+ZCQ|GPki|ld~P) z&;pd;phi<%+Nb%w-W?^*`>1_v6=!shOhx+5O{-CY=oLXF1Q%*6BEPs-tEOBTo&7h1 zH*nZns4x98weh$O*Yliz_Nk0j&)HtD2zv(QxXJtLczU)&Q@a=W7}Knyc6OU|WoEcu zaFLHfub_C>R0n`bwk60lzjex`Sb(++_a7!jMvOPl9mywW{SA!`#!WoBaLZJRM4pd9 zV)#98A`2(59cCtjcukM+#>IQb)1U?dGghKmDo#i*SOn!do~ijSxENzQ^dbGG{GK)! zih+u|jakYEGkzrDGSJ8cekLykgL*KKpl=W($QWWnw*bp*sGK>-Zi-6|!hMMo;Z@1b zW@MpPrLnl?J<$}0kmyyhFaI$uJsG$#GrO5|5edEO>5g^cf-wgBXS!^9wVQ>=ZEgQ$;V@^>jMmWr~@@}sl=^#{Tn zwWnE$|H0_L50ytmHU7>G{(JfI>i-sRL413;b2#H#KU$^tUsfplY8RgqlJ!m7b#LOt zjmy0%8}X##x^Ys~_zEx(L}Ri#X*qyvU8)0vvvf(&dkw^U4PIxh)mG1xyJP#oEr?{m zzlUI6yHVF)h(0&AdB%O!sLCtyIMUsbv%VvSPY^{7l*vn>4}ba^tXOe)3TpoOxeda@ zk#KwWr7Yzs5jqJIOh%QbsVXeX!*fZrA|8?vhjbZVs?oaCY@Vk0TRm*?E&SHeJM)wbpz_t|3}) zhMw7PuSE+r{LJ%>Dt9$>2iaYO&eGRAo#W#>%^(*$>O2|OhC$I8=!@eXjmPlpz4kvx-5J(b=V%VJs=Tf6!Z9^kO&Y^d#dD0G zomti?qRHAUSc<}Hu!l=H{WCnd&3Nw%ZBLYL)!hGb<%RXwbrE5jTT3-mqm0t9XK@7i z=K9Si@}nOE;uWW^E>!7EN+<9?MiO_*s`}?wVxYBy)x=rr^m1#$`I5<|kAJ_#dR?$Qso`O_s>Kr)l+kFEkxZCr{-n&UpTmLz}_io_$4gL;T zVuQyg&vIs)cpOxy?C__pT;>I2Z{_0C9hdeV4F1e3HxDkyWDnN6f8ua<_=+Ipyps0! zPkS!3(3ExuEdC5`TB78&D7=`wH7W+zz7pRjwsXEyUTuvTu&#VmJ!{aGQLg^sDHe6! ztu!Uzk_0Ka2p_FLc#k5?mZ z);dLi;$o4({_6M%ngbE<*-bzR>03QtB4AA9rc&;b6L-#Q%8zz%MTo%l@|I<<#BT>5ESs!>q`g@Ge1ai{gv#!&AhQNBj~z3LjN5l6Bih73ja;hibm_)q z&cY|!mj3au5*4q{RXUPl0ROG+WaV_MWwpw($>J6UUXIqMry zRVm-zt`xUCXMYui<9@%xM0D9^otRo2hu{mCOFeX+sUlK~phyUOfTPupJL?CtS@GUbKz;YKV zg_`0IJGnw6OCS?hxAE_GXjo^g7R~Wadf1Y3=02%8Kynj9Hy$74$i{+YxHT*LD|t-a z6imV9b&}g~k&bpwpY@geu_78Y2)q?`2AVN77you-1Y9|di&tC}**^-oJGQ1?IQ4Ie zt~BSI1`5K)Q%9)*|K z-f)1Gj_1km8dam&iqaYu!#-vpmfp~5Cs3aSBH&jzUyO209FmHG?4M07kchNEV5#OO zc_XXIR%dIS0P~@m?=$!^d?Sa}nb4h6t7@uQZ_Y}?d=LyMioO`I)7t&LImc_}=7}kB zFAUg<8yvcvYff01=Zhsh)qBbZKdVe>W+8t>CWFVt+HE4T-3e)7bn_1W7TsnsO;ug*4mPIg&uA-&@Xpu#tMp>o7RGa zE`s&;cv|5fT)#^Qih1jOLDvI9qQ9FP+D)QA>w=_G2z9*NFf)EoX2W!b%C)j5f$tHK zQ9!?IT22?tr;s|Q7x#om^?r$JGP5o?lWZFnHTyT7{~$dn z4r#$NTJf9$p5Mll)BN6plAixBMiZU~9Kp@}iO4(>sGQJVirAS2*UBxIL!BSbn)F5` z=QCX1USAZOy}7D5F6#n940hI8>0mgCp_}knVB&-dG04tiy}UZTf`_LxoCp_OU4OpB z-9@5wdU8&lfr!&mx(i7B`Ccmj$&xhsl77yWDc3Fduhh`kIxF;|bCh9*Z94Al!7pv6RrNh;=1%Kx&NnzCj z8C_93Y>>&!=r&W^weM$G(a%^!y3S(xDb;fk5O?(x$bT;W6n1_1q0CzJ-K7!Yh4VN{ z@KzXlHYPN`i;h|FvRDCB4%tqvvWCM9E=q93sP`@_4uL#BHC*CC8RUz((6MqaYgiLw zxnf;ge5lSaKGX0Ch&u4$Upg~a2HD#5og&FO-e=Vp5l8j;|G!v z)mx9G_gc=2weL0MgfU)DNA&z-SV1IT5ttatRT9@k%$Q zkSb%8gf87lND4Y-qd_qqP#jnM!4fZ`sbk|4)ON#A&B+NkTjfm=GsTwvsh4{T$bm4K z#fGNU&o#dvSrt+1)h99*{T2zka{_s}6N^R~=sR1+E)tZ5MvN7D9MiBBAO1JgZeQtY zzg+~iw-&b*v?^*0i%xEPO&`br<WFy*~q!OQ?F!zc9$Obr$r?z*mic zUh_xZmR|TQG|xIEA0c1*hy4oYj>PeEU*6_!Q2MGs9ozH|0_X2gUaN{bvAE4#7LtlC z3AC|!U5X1SM5R4Z2c>3%j;f!#VV3IkZ~Hr;-_-x{PN0F?oUVDR%~lx4k*zBWh8sdb zdGh)=`_ba)h>oybRMDhG!gUinOYhF-So`5>XZHLdwRt9W^!n(>Ikw|V2wSD<8vqd zaeNYM`ro`n>;FTZJXls1UGeQbsBa&qZJ2PGpP|0M1WX~RcQ*MfR=#s7C;Lnjx#*hz zy+pCz)!oZlk3W4CZ=WRGdRe_y*YJlh(dzH|i;Qu`JmMhu*C=9=yFG6(tudoFrIzQ6@;M9GnT6O1T^PwD=fTQ`eN=o$Z+v9jiVWQKSt~d0k|2@w zwc^EAd%IuAfgHEH32J#Ncg(n~Il@a3KP z9B%Q(XG^btvh1-|pkX&914C=yfBJ9`ZS{ z9us^I@Hzb9ovU=++eMfTGe{t|v{uK0HU4AUIKZn~+`g%6LEG{%a6XrtNCc9A=B|{D zxW2vH;C>dcthZ_R53Fth0~~0$VFW@Y@_njxxrfS zz8*dqwtG}kpx40jWBePXHG_f%vx|+J_?LwrJT;)_HzGrs$va<<71mn01bMXyolai} zNUY_m{L4)K9ckndP{SAt=0SgwDt3aO3~=(8RYvPAmItA|ov%7$89|Cr!^n?|YoN4M z`dj|l+dLfG%y`hf)ExsIftYHJoO&#%hv$t}I>5ox$&DC!v_nry7We*BCIXw{Jaco} z8-@A-cvoF&$cd3Dzb__QWXW?d{*wf#XCZ@J3x+eVeTA96V~3o8q25ia#ZeG{6PvwG z0i&zVfNvrvF>ZMPs(8b6qvo9cU~H)psE(N@hO~g~ng2Hp2r<%f$(V*!>r`ngouVgl zUWHQ*uAgCL&70-&@PF_BeFQw=MheE0smlt!zwDo%Qidq|!TtV{>T5E1S8safGIP3&yrpT_0VCU{M(Xe*MFDD|ha9rqpmto&ndJ9QsXEHPkF zQ~!qscIKwV(b}o6GxC3j9fpxntLr&L-8!|SaI9&s+m#j}5=W^YG!VFa&YblT4s5=K zg<1KJjVOyQE_QAQ>=xXznYvVqs)cLEl)Ind2NdNRI*dl;gQn)cED6IDe_2Qm@VMq? z4Z+Sj<;|;uB9Y=O+*> zeW{q?@?Na~R-x~6?q#&w)ON-TJbWAvwBWG2Qo=bk3sns?@C|RI7h_QvFK`C0G8WeN z;wksQHe4m^jtG|3G41~Q^@RE_d}7EK~^ zm`AMA6Ych8#3X@lO<}|bQ|2IPi1R(=z+ARAfA9v?xufX%d^ds zutJvv<_z;x(~YTdWA?=C#!+U>I8v9>$+^VXS)#n%;N7z7GlD(1;+BmZ%!Q-_@R@4D zPTgelmE!2xV+H_GOiowV^(KzhxskF1K^_nv*vJ2~?4X&rqyNrb?a_yG$yUunclhOI z7URJQ2|Gtv%+p$GFqaO#4pRg?UwhMH_)_V5Hj(}+D4lqN4!KYXd@HM4S zn;X0WabSv1g4d8B($XyZd=1nHE|$y5NdH@M9x{Qn2H-Tx=9451WvHZ6y(LheG-jBH zF4%bdA@Hhq^OF|8ANL9;@8l_Mn<)!XO65>v+yuHI9!Iy>D!25pw07rU7;();R9*&3 zpqNlcBnTRm1_tLp855?_i3RJ!l=W1J zE&n^x($Kl9`0lruy*=nLJ2*l}dWmvBx0t830cLSjYaQ||w>}avUWKg4YgRh8johtw z=%c*J&E5R$G%JBrgqyM=`5uw@t)XvGRF@d$!{O&Ly}D|Dm;&?nl)8Sy9{W@Qi3@fQ zoiW5C$@}_)CU1-_?lrSI+KauW4!UZ`b}K^6?yxIA<8Bz#AtB7U@f7AMH%U>qG}^V? z{K(NYnBUgSe+=HwV8X>kv;Sq_-pk1QbKkmpWT!-&G*EcaMI8m!^g?3FxoR(TS!45h z(H`o2f=`L7WChAGN?vW;3UcEsrnFAo~9gue)5! z|I>yooR@G(AmgT$zZbSl&h)Z!`c(5hB1fjFf)|NsR^vXF;fxc=*oVl2Q%hJ<_#}@n z`9w#5Z)&=`^{VJU!Moi1!~B&4{ms*0*8ckKhi5Z`8`6yMIh=5} z?ODz_GCsX+b@rjZ6t~H?jjOILs<5@ZgZQ#=s(B4;eDPjv3;xGk$O$DpCivAv%B%Vi zB3yYsk5K|xLmEk+L0Z;VUJ}9fG3?bn$*zHz%T)A3Lju8Q2IcG_;=QE56RK$U~5+!{rG&1idffBWY4&T zJ0gFFeR=Oaf@JI3nC4wYR3vNR4Dw}+7~nOtsViJB0u_^t%sPg~HwP<4f6v-q{{wN9~ZlHD#VDdlWxcTa9+kT`cl zQu??F+{#=*G}y`WrO!=eR!HjQA^rh$WFBa2tU221c;rw~(*P^zkdV-@#4@`CvWE2C z&=798bNB>Rof!0S>bIh}Ned^Ep4SKJ8lu>@4{>{Z*lw4(4^~lGn;g5wjj(|c&4>zn zR=-VVd~@Xi+IGClub{KOKS;A9YU)}MB4QFO>b@(1KRbs@NtdJL_4z-bWm#WWce24F z{R7_184tCf9=nRAi~O@m|5S+agxz^f|L)EZy-)$uKX6-+Jlpn5t<1}qdKqlWVn4mQ z#=PKpkn)i|0dVJ?y*=(5d=3YS<~!um|MJ{0N=yIgBEHBPbGmDIlP~c{z5(}1gY`~z zij4=ZNWAI7D(bM0tLl*S!(4Q)jI(9X8EUTr)cZF5zHry(r^|_MfHectIW$f029oUb z0L?mp3lFEv50*7(%I9*T@lBz&aZPgX$o2hXK*`LjkRiMyGypbtt(I~d^WVhd|guHr%y#gA|~mpwNlx; zEf~8GjCr@Oup~cZe(d;_f%Dm%JO2i=TuHFS^L$yWr%wQ~ksFFfR66FcH*VsBV+79` zq-mdw;mfe=H%>_oRekt)>IQEq&0OrP^#9sdz$xXPTDA8agAf6c%@{O1j2mBr|KsJw zdSD7t0&}isaW=P7CL!M-S217vc#8w@dU}wR*luiMH#rY^NuMvmPDj3QqQ+Hz=X>|- zMEE~0-#LdmIJ(vjPl*gFL`)K*!?|M%WDJ+dO^nAgncrC~0FOzC&J2RpQZWJS9c5Z# zRdcumOc}3}$uhxvvg_}&Hx9;JS>vxf@Fbs{ z_X1BmS3{&t#|aIv1F(TvTxZyaV9zd$KbvNSa9}Z#H#l8qz#?v=6H9?`);EeoJh%`s zNHLw_if^Qp0WVIn!}#nJHARQ_|LxPh*+%!}{~IriD~tghB3MpCqCn7%(JiNcmM_{! zPpgCfn+nBY!pw3`*_WIB>YR}K+%0&a&O;In(}=AY*L!XLf^TP4=pTC(b@x2;RDC-> zW|kILw5;d&d38@_e5?!1cV9Waxh+ZkV|I)-drvQIZWnW07tI)m?!vdCke%^NVV)7BV#spU6!vEYzBct#hm zBFk$wO;Lb7YAK@Hr8jQx0ed4p4;Hp%Y6gD6K4?DC12ubc*l}^i$N#n{#CHHj7tZ%C z^#8$r&9+?C1RIKk?(>x${Y8Hzs)i{XPQCZO8=B-ewPo0>R;7LhGaD74Mse48oNYG6 zvtSv=TzJx`a57fZ^TNa#aQu-`oN4-YI-v7c=XU?yCa`90G%DLw$+ zeyX8je?sMh`S-QXxL!qApzjG^5uGIsVc;lO4Sk0LTqIXXcl6QtAoXa6Qbkz z@l1fyiQ(>-?^0}QPqq-*XK|2aWFZ$Zw0#OxcCrQbGM8$cN+PCPGP$NET_;DaKpEn} zZzOvubZT9{Ob1F?=c4$ebb%9KJx=>;YuAG$FLMfgPI>gigusbDdZ0MQYaP3PmM+fM zhjfH1bMpvw_Ye&nm|-O5{25Si)OYIHIRW!Y8eblF3T46RAf)wkow;pWVRydaPh(`T zP}CAog14+&z{vTQt}vP@-c>5pbF^eKwu!LR@5Q}4cnd~kL6!k4tQ@}3hA-EnUY-uc zL1JcM;MSM%Ie+7JlfbupD2>N}QKrAvnCDn{-Q_?2$1bT0y23CT>;S~SQ=D{CZ!0t8 zLv!=#;6=Q8*kw6cY*awYsSow9y^{Y?00XLS`6%L%QUOcXk z){e6Ao8uo(ojb>k@|_5WO_hx}yoTygb_W z`TzW_8Gsk&|M^>GNs<3lv3KXFwMaeWPI814uiGM=vJHtS0V&SwMF-jzE9Mk2|wS(CR>(A0uQ9ImTp};qBRXPL{)7 zCWYl&wH9gDZ*N?wJ$YIIKfxgV5?xMjnv(6+)EH4GJ_x)j-fVmpDUJ=71(QT{d;1VE zTJ@paEa*VfmC?~qWeVy!Td6@DkV^>p%V8NrZQ@b>SZHD`+nL39dkk49kfHc^${jS3YDCfD!rpuI#-KYP`sh9)&+3#tO&VeO@T9rIPB4hJN z#(##>gqw$t*uBy3Qhe^Dv>`I+gY@@$z~plSdT;%*494+`uGv4lkU7)oAL^rjTHl;6 z1-(jGI+$25zDkyV-JOYhN{E~!YkKnd)>b-jQhOLw;IaD%3KPb{y-`r49uffq9YZo& zp1!L)D7xP)JA`_AlMHPSv1L{ixJaXk_%I_s-J4gD9tg8ZHph1ubZ#UiesX9D&UA`{ggG4_Z1_*%oVA1}4H!A7fz5Kg&L#d* zh0ZYH?LT$4jdIB}B%hFq`3~mrW2sYz@SOdgC`|&?eWHNk3C4E(!9q8``HS}CzzfK7 zh>Ov-Iya4&kM&qVd*G`Z!xbyy`nUk;tG*1rJ5kDu{eQG}D3OFc zgpiPgEUz8#ZtuQ#?m6%M*Qay3`%8CK{i^D#`qfVjDO=q%hRHg!&G8)br;S9X2Qm{f z@eWh4Tb-^z_(cD@OS4-*Jt4HIr2krQklvsOke8u0GF)EcBT))NunriZY1_}j!phLB zMF9vs#a*uJI!G!!7%X#;XvLx*wG~#Pk0wIc&wOqznne`5PwqAT7Nh~qILbgDrSbR5ry(dr0MS~9UF6Fht%-*IPPfkQ(yc%b0e|ofr%^) z-eVJ*ajMAfdp~G7MzCHj`j#9F#W8g(`)K#EaQPO$!X+}YJ5cts(9)9Y^!Cor$&W-u zO`gLE<^FbEzWnryarw$@Xbk3;rgJpAn2ASa&E*L$HYc>fhE zEhg-#cqe8(49Ag0LnNbO( z>K_riWJaaDWn5E@AWaV{{Bhh0bnfq(Q$1s=b*?$XIZB{(Lh@EO=nz?__4Qb&5o*pt zl#Bf=T6xSNJKB>&vu8NRKuy|RpMWirRt|egaQG4ICeboe5a&Zej;sG>1dN9*WUXqn z0yflp#X;!Muk?cvI)i^spZbmpyC8)sfeHjB1#{>;bSpa**l6P;?J>kCL)`^u**=Zg zWx7F)P!+b zz9wA)%6`^0pLbkpbf6)-tgY{?=h~dL-1B4O2yO3ND9Z<|ABdRT6m1WmLt2v#6g13+ z{_Z;*o=k5{4y4;Oe|aHT8ExpJr#LU^m!8}5`64p&rqjAFiB%T-s~=Vjy6>6}#30pb zmQ4rP?+=ohK$UgsmX9qjb3MTwPeF=l7En8J_P#!dZHp<1}X z-;hSMg*dZ(`(@L8$A7%L=U$R3rva0HS*E_pPUh~iXk*cO=y+QUP*#20 z^K{hsJ%D-Ou?R90XRO>Lauy-a17(v#un$0U-@Ys$AQy&fL^HQ`rDN{!d^T z=6ICi<78)((Gt$dy=wRZalBM%1}Hb_<-0#19m4NRE*(3TJUWz6y=eXQ@u@dNw3-37 zM*EX?*^1gJ&5?I=X~P4~X@uXe8zAOOs@mcYbJX8N1F1|^&C*uB(fNjX6QDu2xo>gz zqE#wmFcK2cHz23k%^7A!qkhA(FG%K=M#pW{WYAPPz$`IL9iy|Bvjb47dv=&MB*@)m z5ULMH#3>E^2{zF$@$-TM)x@X7Td{k#aWk-*BGh4>aJHwPI*|Wf5J+O%hSwVz^1jt5?eDdL>x;yLRM%pXcF%Jo&h zVg9(hGU^|?5bdW%4M8Q&Cq{}yi&pVngI4$Q22T9eEAw87@qKhHMrM+$5n`YDmm~IW zUp_z9R?dz~8O$)aRgaW&J&>%B|8jGTeikpA)X@qbu^v)K+f5z+XlyqIqcurT{UqmoGmw${+)l6Wy`5!fmtf~qw$@X4f)u+Iaq9^%6Q88q_FmsgiN z95*wX<81;C-F^XkP@Nk3ne;WVAu4|y&ffzrt(bmUhQEb6bHhm~OI$K1S3(>?JBN@% zfjw}kilz+9NIGo+j(HEG%gky3hX%sq%DoMkNLPuMp<|Y%+b(57Zi%k|CAJ9LRyF5FOwLT`j(QKA5(=}k5;{&)!YSh zESp{NSh`7+LZsZE;uOl>v&<`^gb@186>3?=l;q&-?zt7_=vxmDfT#Q@wyVA9aFvnc zDB5#iJE&Jloy8{&-Or5B6DvL}7csDr0tUpp-7`Yia2AR|>!9Q5^aeLEPxb|Gu%H^X zA56^rd{SX*Uzg*y=ve2%U@a^ z*Iqq$9)53)RQGr0M%?PTqn+FTQT`scYjUswQeS=4}nOZQ55jkaN)j{fWEE=>CTTvtSDjOI<8&jHa~JIP<-)#t9ZFpVkemzh^^+ zx1b$MJ98>-Oz9&NAvZKgg=7f*J&3)cbQ+JSna`^);t(lZ&LF2x!0D5XXZWc4#7PB% zu(dH1;>(v*K&n0tfuT!YAmy2iUOUM+(?Z~b9+6F#yribEY+1LKqzl)IGx5p250T>t z*7*^O2Hl6XoK|sUS&6}Xbx6|nS5ay1Z@^eDcIu%O*Bxz#I$`plne0?@NpoPBfLC|0 z-x5Bl28@?`GgC1oNlH7s8HXVWJwA+qq%{HWZKlfkw=qJb?_C-g9twcgl(WRPdH1uPi3E!4R)q|kbtVd;F)QR zfq6k9S=(o#IWSEaXJ^uJU3wNq#whE|Y~*pyk+;+6?B~w$?6RYab)z2*DA()NJm| zs>t1eWb4|phAiS(*D8Z?b$4%jH!P{(UIl<@J0&IT9p1h-eRAp`@|jKZlk@tMQ>qU( z%vn7P)8~HsAj#NG;p5z>7icP zPzAeENHQfoR59YNCZS~61GvC71;_NPGb{AS56?XjRl5dzd5{^~R{_l%{%_V$Zj(po^y zeIwLTOlw(J!75IriMDPV#y@;9CHcGSr>F0tVKC0n zOi%AV!9LFziRx*yxGi;E+Ps;Zf7Z!U7J)W#s*}tXb2IKyM#M#Y80MGK9!{V5Ou13{ zZZj0>;QVzB^5mp@atflAT$CQKp0=z(9vB$7n96Qz=hb>W#JCUKHqnpZle-pfZGuEA zFPG*d27gR<>E8p4u82cwYhaVxCIy^S{aL-?OQ4cOR++UdXvWP5LE-QwY*~##OEASX zTPlFR1bw&8;<10<>i$2Oapr!X!558TS6M^uzv-wmYG8PJblmD&g_>PTptSzV3;R6M zIoXbFJzcFGg5px$`=9D0YrD1O%?uLWaD~iLopK$%_-P7^PgKNomxqPor&4L<<;owU zmTbi^A7TFv=H*>7Wjrg4Jr!`~oH6v+Xf^-+7-&~f`h*X$>|&5W3|+!SSzWm9({oxc zDHf8tqYq%a?(V-ZAv*i?iwHU(UR_Uv?(0pssR2ZtmB>I7yV*fC|604qVzKAwhy-QZ zJJBZl43tClo%8~!^8&%x?7_>&{T<(ecXgAFt|Z^I0*p!bh9oVUCP$~(?&;zz`9lh{HzBY}x!XEPY zN>R@`_j>0=B5 zt2IJU%7IL6t*Wm$i&4uR=3B}VeM78`AY*|uN%bYe-j`qBQXTQ>UtxS`+8NJ@N+~a> zG8gHqC-WSp^W(zcYT|OpJeud)%Sy%X`bQ)(`X?j`{)R-a*O2Jr|Aa)4a;>*Z);)(B zq@@+;)CnHuv}sjqJ1k*blt}{Y54!&?t$*q}gX+^f&9^Jn~u6=(+uCJ+REpROk!w;S+g!dh$ zD-WBM=*i*QMNEshK*Lh?%PxfA`F$1f6zyFRP@S#8nE3LUgtH^T!wT7b()~|5%CgNi zHW57fUyOIUjMA3Fjeb<@XOEW(oyh(82CMS8_HfP=u35kz)xKuFQgA$E>&K6+U;{cdiDp=Olya~~;d~cLR}+-rgYdPu zZ>K>HV4`LtZ+i@%$+bdYi##|`H7lfL$jB6?*{;QCU{E8*&FNmhxZfH%d6u*oj3*Tg zGs;%tzVMc>pE?j>?dnNnTF^|l(StHK#u|gy!9gT-98oP=>xsVCmCAityFMVFJJy2@ z7DTy$Qu{pk*vQ!0Jo1}P;dyW@bu!23VnH9dy>wpS%}7_r?a#et=fAh{0M-+&jMJv; z&928i&+4)~`SxISb%dR4aGa-P6ACzu1Ka6aBS>!9*@;#<4#%$+@M7+(r}qG${4qw+ zZDh(~PSo(nOiZaz;T%UwN(ie7kCFCOg*%v&+p)8cx-3bX*lUhkRbgG1rGa|3BhpSb z(NccxhOLPnRMReQ*Mk^5@$x3G$HzbY`p#I_qlWz9zVAL3Q6q)|#jwG8;JchDW6;(U z+;P!MSf&0J&**J7utqNp5E+fT601|XWenqTIK}0imp9U$8e* zuAD2McJ|W*z9Y&XTOh)Bm#RTsP;HFv0a~ANSm5BZUZyeUcUJa* zSzJoIKjH~n7w72xWhKlAZcYjEIs%66?fy81NjI;6(`lJ8>zVP*gyl}Ydv46Bi9j|f zqXsWg!5Jq|>}Oo&c$X|?gx(K0&V<3Tw_%SavJACdo8mM?vm`H5jlV01)8jnjnnPi+ zsNOi^=n5^PhjS^}EWRB#eV9zyb8&x`Sz8*n=2>Wfg~s*haZyb=mYQM!xbWIk`k%;- z{dcju;2*L3#lOaGhmg4}sCo>sUSX@G&KlS`L_j&@%&R5NB)BRfe_Zu97v9Q)UIo8O z37eKOlvUMJ$hNVxun<^r@YCUvyQ^N?ORs-7TyyEkg9il#xfa0IBC`gy@1DkIr-P5} zJ5PTP3#E^4AA`5t-`perJ=TNl|66!yppQ5(&8XTu{OtP4x2Re3vyfVplC7r~E1Bk@#SuuS1|CsjV~;>({Uwv3 zvD(v%+|p#*e!E37Kofi71!Xc>)C%S-291=Fj#VcZ&&#{HPjq9{S08t@ z6Y#kDh?4Lneif-Y9`mr=bF!nY$`+mS&1XFs{%RZx6eQ0oKltUB+8Hi|XIvJ@K!?b1AxpeMSq-=8q|@(AJpBL$5-#T4@c zbW{)Br=+?*if?Z~P0nJtB{IxBf3AGDk;7z%W_lu1!OKX{`Q)9z_tiHW2iVRYCv@4j z`_*f#uK0Jqiz*-&+TwMmrJNAV#lZp38S)rvKVuzZ@eS_}#Y{JhWZ1?yiv`5FbAG{*txzS!2v@{dwpGMMsyw zRMK3>#6Ue_e3v~SO5OddD)E}%lCUKk@E2B3MZSNRc}sGux=1XfiT1=DlrIP^1) zySH}ns;8<-O7p=y_&#*U z`jAxO5^6o$q_&}75g8<9%9jK3EeOAA7-v&|NP1-d+wTL(7mmd2$ul-^*8FHLwYlOd zj?1gKF Date: Fri, 15 Aug 2025 02:04:51 +0300 Subject: [PATCH 025/286] fix: add catch_pipeline_errors decorator to several function and propagate trace_id to more places Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_checklist.py | 5 +++-- src/vuln_analysis/register.py | 22 ++++++++++++++----- .../tools/transitive_code_search.py | 2 +- src/vuln_analysis/utils/data_utils.py | 4 ++-- src/vuln_analysis/utils/llm_engine_utils.py | 2 +- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 1675aa0cc..99dff2d67 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -23,10 +23,10 @@ from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig from pydantic import Field - from vuln_analysis.utils import data_utils +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id -logger = logging.getLogger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"): @@ -62,6 +62,7 @@ async def generate_checklist_for_cve(cve_intel): return cve_intel["vuln_id"], checklist[0] async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) intel_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe(state.cve_intel) workflow_cve_intel = intel_df.to_dict(orient='records') diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 416db7d29..08852b0c1 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -44,11 +44,13 @@ from vuln_analysis.tools import transitive_code_search from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp +from vuln_analysis.utils.error_handling_decorator import catch_pipeline_errors_async # pylint: enable=unused-import -from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output +from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input +from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id -logger = logging.getLogger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): @@ -66,8 +68,8 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): cve_justify_name: str = Field(description="Function to generate justifications for each CVE") cve_output_config_name: str | None = Field(default=None, description="Function to output workflow results " - "(e.g. cve_file_output, cve_http_output). " - " If None, only prints to console") + "(e.g. cve_file_output, cve_http_output). " + " If None, only prints to console") description: str = Field(default="Vulnerability analysis for container security workflow", description="Workflow function description") @@ -92,31 +94,38 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None # Define langgraph node functions + @catch_pipeline_errors_async async def add_start_time_node(state: AgentMorpheusInput) -> AgentMorpheusInput: """Adds the start time to the input""" state.scan.started_at = datetime.now().isoformat() return state + @catch_pipeline_errors_async async def generate_vdbs_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: """Generates VDBs based on CVE input""" return await cve_generate_vdbs_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Fetch intel for CVE input""" return await cve_fetch_intel_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def calculate_intel_score_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Calculate score for intel source""" return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async async def process_sbom_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Process SBOMs for CVE input""" return await cve_process_sbom_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def check_vuln_deps_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Check for vulnerable dependencies""" @@ -142,11 +151,13 @@ async def justify_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineSt return await cve_justify_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def add_completed_time_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Adds the completed time to the output""" state.input.scan.completed_at = datetime.now().isoformat() return state + @catch_pipeline_errors_async async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Outputs results using configured output function""" @@ -167,8 +178,9 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput subgraph_builder.add_edge("summarize", "justify") subgraph = subgraph_builder.compile() + @catch_pipeline_errors_async async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): - + trace_id.set(message.input.scan.id) subgraph_input = preprocess_engine_input(message) subgraph_input = finalize_preprocess_engine_input(message, subgraph_input, builder) if len(subgraph_input.cve_intel) > 0: diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index f2c54bf0f..43a9a5b3b 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from vuln_analysis.functions.cve_agent import ctx_state from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from aiq.builder.builder import Builder @@ -96,6 +95,7 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> list: + state: AgentMorpheusEngineState = ctx_state.get() si = state.original_input.input.image.source_info documents_embedder = DocumentEmbedding(embedding=None) diff --git a/src/vuln_analysis/utils/data_utils.py b/src/vuln_analysis/utils/data_utils.py index 7fca6c937..1acab1197 100644 --- a/src/vuln_analysis/utils/data_utils.py +++ b/src/vuln_analysis/utils/data_utils.py @@ -24,8 +24,8 @@ from pydantic.v1 import BaseModel as BaseModelV1 from vuln_analysis.data_models.cve_intel import CveIntel - -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) def to_serializable_object(obj: typing.Any): diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index d20a2164c..87735f6b2 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import typing from ordered_set import OrderedSet @@ -192,6 +191,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, trace_id.set(message.input.scan.id) vulnerable_dependencies: list[VulnerableDependencies] | None = message.info.vulnerable_dependencies + if vulnerable_dependencies is None: vdc_skipped_vulns = [] else: From f10c64988288cfcc6c50c88f3bdfd3c84bf22426 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 21 Aug 2025 14:57:03 +0300 Subject: [PATCH 026/286] fix: fix regression of agent loop stage' performance Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index d3be34f7d..610521706 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -26,6 +26,7 @@ from langchain.agents import AgentExecutor from langchain.agents import create_react_agent from langchain.agents.agent import RunnableAgent +from langchain.agents.mrkl.output_parser import MRKLOutputParser from langchain_core.exceptions import OutputParserException from langchain_core.prompts import PromptTemplate from pydantic import Field @@ -88,6 +89,7 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, agent = create_react_agent(llm=llm, tools=tools, prompt=prompt, + output_parser=MRKLOutputParser() stop_sequence=["\nObservation:", "\n\tObservation:"]) agent_executor = AgentExecutor( From 6657af8706923a3d2cb64bda7671bf31c911a201 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 21 Aug 2025 17:13:19 +0300 Subject: [PATCH 027/286] fix: all ocp deployments variants should have a token bearer auth Signed-off-by: Zvi Grinberg --- kustomize/base/exploit-iq-config.yml | 4 ++ .../functions/cve_http_output.py | 41 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 61f4ff56f..e7bf351ea 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -105,6 +105,10 @@ functions: _type: cve_http_output url: CALLBACK_URL_PLACEHOLDER endpoint: /reports + authentication: + auth_type: bearer + token_path: /var/run/secrets/kubernetes.io/serviceaccount/token + verify_path: /app/certs/service-ca.crt llms: checklist_llm: diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index e7d071678..d84068dab 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import base64 import logging from http import HTTPStatus @@ -27,12 +27,19 @@ logger = LoggingFactory.get_agent_logger(__name__) + class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): """ Defines a function that sends CVE workflow output to HTTP endpoint. """ url: str = Field(description="URL to send CVE workflow output") endpoint: str = Field(description="Endpoint to send CVE workflow output") + auth_type: str = Field(default="disabled", description="Type of auth - bearer, basic or disabled") + token: str = Field(description="Token to authenticate when sending CVE workflow output") + token_path: str | None = Field(default=None, description="Path to token file containing auth token") + verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") + username: str | None = Field(default=None, description="Username to authenticate with for basic auth") + password: str | None = Field(default=None, description="Password to authenticate with for basic auth") @register_function(config_type=CVEHttpOutputConfig) @@ -45,10 +52,16 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: trace_id.set(message.input.scan.id) model_json = message.model_dump_json(by_alias=True) url = config.url + config.endpoint - headers = {'Content-type': 'application/json'} + headers = {'Content-type': 'application/json', 'traceId': trace_id.get()} + auth_header = get_auth_header(config.token_path) + if auth_header is not None: + headers['Authorization'] = auth_header + verify = True + if config.verify_path: + verify = config.verify_path try: http_utils.request_with_retry(request_kwargs={ - "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers + "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers, "verify": verify }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) except Exception as e: logger.error('Unable to send output response to %s. Error: %s', url, e) @@ -60,3 +73,25 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: yield FunctionInfo.from_fn(_arun, input_schema=AgentMorpheusOutput, description=("Sends CVE workflow output to HTTP endpoint.")) + + +def get_auth_header(http_config: CVEHttpOutputConfig | None) -> str | None: + match http_config.auth_type: + case "basic": + if http_config.username and http_config.password: + credentials = f"{http_config.username}:{http_config.password}" + encoded_creds = base64.b64encode(credentials.encode('utf-8')).decode('utf-8') + return f"Basic {encoded_creds}" + return None + case "bearer": + if http_config.token: + return f"Bearer {http_config.token}" + elif http_config.token_path: + try: + with open(http_config.token_path, 'r') as file: + return f"Bearer {file.read().strip()}" + except Exception as e: + logger.warn(f"Unable to read OAuth token: {e}") + return None + case None: + return None From e1d8b681096d4e66e0221aa1e76e069990a3465c Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 21 Aug 2025 14:57:03 +0300 Subject: [PATCH 028/286] fix: fix regression of agent loop stage' performance Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index d3be34f7d..d1971174c 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -26,6 +26,7 @@ from langchain.agents import AgentExecutor from langchain.agents import create_react_agent from langchain.agents.agent import RunnableAgent +from langchain.agents.mrkl.output_parser import MRKLOutputParser from langchain_core.exceptions import OutputParserException from langchain_core.prompts import PromptTemplate from pydantic import Field @@ -88,6 +89,7 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, agent = create_react_agent(llm=llm, tools=tools, prompt=prompt, + output_parser=MRKLOutputParser(), stop_sequence=["\nObservation:", "\n\tObservation:"]) agent_executor = AgentExecutor( From db477a94f47941577ce9268345db18ca2feffbb6 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 22 Aug 2025 15:49:32 +0300 Subject: [PATCH 029/286] chore: change the image tag to the new version Signed-off-by: Zvi Grinberg --- kustomize/base/exploit_iq_service.yaml | 2 +- .../kustomization.yaml | 0 .../nginx-patch.yaml | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename kustomize/overlays/{local-llama3.1-70b-4bit => self-hosted-llama3.1-70b-4bit}/kustomization.yaml (100%) rename kustomize/overlays/{local-llama3.1-70b-4bit => self-hosted-llama3.1-70b-4bit}/nginx-patch.yaml (100%) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index d1ef0fae9..721b60aaa 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -29,7 +29,7 @@ spec: serviceAccountName: exploit-iq-sa containers: - name: exploit-iq - image: quay.io/ecosystem-appeng/exploit-iq:latest + image: quay.io/ecosystem-appeng/agent-morpheus-rh:nat imagePullPolicy: Always workingDir: /workspace/ args: diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml similarity index 100% rename from kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml rename to kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml similarity index 100% rename from kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml rename to kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml From 4c90caf8d58e922653d690d726179c7eec0a2e10 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 22 Aug 2025 15:50:30 +0300 Subject: [PATCH 030/286] docs: adjust and refactor deployment documentation to new version Signed-off-by: Zvi Grinberg --- kustomize/README.md | 224 ++++++-------------------------------------- 1 file changed, 27 insertions(+), 197 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 73c28c985..76725b40a 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,18 +1,13 @@ # Procedure to Deploy -> [!NOTE] -> Sections 1-6 and 11 are common to all deployment overlays - 1. Create a `base/secrets.env` file containing the API keys for external services `Agent Morpheus` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. ```shell cat > base/secrets.env << EOF nvd_api_key=you_api_key serpapi_api_key=your_api_key -tavily_api_key=your_api_key nvidia_api_key=your_api_key ghsa_api_key=your_api_key -ngc_api_key=your_api_key EOF ``` @@ -26,23 +21,18 @@ oc new-project $YOUR_NAMESPACE_NAME 3. Create an image pull secret to authorize pulling the `Agent Morpheus` container image: ```shell -oc create secret generic morpheus-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson -``` - -4. Create an image pull secret to authorize pulling container images from [NVIDIA GPU Cloud](https://catalog.ngc.nvidia.com) registry (`nvcr.io`): - -```shell -oc create secret generic ngc-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +oc create secret generic exploit-iq-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -5. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [Agent Morpheus Client](./base/agent_morpheus_client.yaml) configuration. +4. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [Agent Morpheus Client](./base/agent_morpheus_client.yaml) configuration. -Replace `some-long-secret-used-by-the-oauth-client` with a secure, unique secret +Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret ```shell export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" ``` + ```shell cat > base/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET @@ -50,41 +40,30 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -6. Update `Agent Morpheus` configuration file with the correct callback URL for the client service. +5. Update `ExploitIQ` configuration file with the correct callback URL for the client service. ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.json' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` - -7. Deploy `Agent Morpheus` and `Agent Morpheus Client` using default settings (local embeddings and remote NVIDIA NIM LLM) +>[!IMPORTANT] +You should only run one of 6,7 steps, depends on if you want to run the service with a self hosted LLM or not. +6. To deploy `EXploitIQ` with a self-hosted LLM , run: ```shell -oc kustomize base | oc apply -f - -n $YOUR_NAMESPACE_NAME -``` - -8. Alternatively, to deploy `Agent Morpheus` with a self-hosted LLM or fully remote service, run one of the following commands: +# Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM +oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME -```shell -# Deploy agent morpheus with local llama3.1 LLM -oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME -# Deploy agent morpheus with remote llama-3.1-70b-4bit LLM -oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -9. If you have not yet deployed the self-hosted embedding model and one of the LLM options above, first fetch the latest version of the `Agent Morpheus` Git submodule. - +7. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: ```shell -git submodule update --init --recursive --merge -# Or if the local submodule sha is not updated to latest one, you can fetch it from remote tracking branch -git submodule update --remote --merge --recursive +# Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM +oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` - -10. Follow the instructions to prepare the model - 1. [Open instruction from local repository](../agent-morpheus-models/README.md) - 2. [Open instruction in browser](https://github.com/RHEcosystemAppEng/agent-morpheus-models) - -11. If it doesn't already exist, Create the `OAuthClient` using the secret from step 5 and generated route +>[!WARNING] +Without Completing the following step with the correct secret from step 4, authentication and logging into the UI App will fail!. +8. If it doesn't already exist, Create the `OAuthClient` Custom Resource using the secret (from step 4) and generated route ```bash oc create -f - < overlays/developer/langsmith-secret.env -``` - -14. Deploy `Agent Morpheus` with developer overlay configuration. - -```shell -kustomize build overlays/developer/ | oc apply -f - -``` - -15. In a new terminal access the `Agent Morpheus` pod and run the application manually to confirm it starts. - -```shell -# enter agent morpheus pod in a new bash shell process -oc exec -it $(oc get pods -o=name -l component=exploit-iq ) -- bash -# run agent morpheus inside the pod -eval $EXPLOIT_IQ_PYTHON_COMMAND -``` - -Output: - -```shell -Http Server Started for POST at 0.0.0.0:8080 - /scan. -====Pipeline Pre-build==== -====Pre-Building Segment: linear_segment_0==== -====Pre-Building Segment Complete!==== -====Pipeline Pre-build Complete!==== -====Registering Pipeline==== -====Building Pipeline==== -====Building Pipeline Complete!==== -Source: 0 messages [00:00, ? messages====Registering Pipeline Complete!==== -====Starting Pipeline==== messages/s] -====Pipeline Started==== -====Building Segment: linear_segment_0==== -Added source: )> - └─> src.AgentMorpheusInput -...omitted for brevity -====Building Segment Complete!==== -Source: 0 messages [00:13, ? messages/s] -LLM: 0 messages [00:13, ? messages/s] -``` - -16. Modify the `Agent Morpheus` code in your local repository and use `git diff` to ensure only intended changes are present in your working directory. - -```shell -git diff -``` - -Expected Output example: -![img.png](img.png) - -17. Use `oc rsync` to copy local changes into the `exploit-iq` pod. - -```shell -# you must be in some path in the repo so it will work -oc rsync $(git rev-parse --show-toplevel)/ $(oc get pods -o=name -l component=exploit-iq ):. -``` - -18. Return to the terminal from section 15 and press `CTRL+C` to stop the agent. -Expected output: - -```shell -Stopping pipeline. Please wait... Press Ctrl+C again to kill. -Source: 0 messages [11:33, ? messages====Stopping Pipeline==== -Stopping from-cve-http, ? messages/s] -====Pipeline Stopped==== -Source[Complete]: 0 messages [00:00, ? messages/s] -LLM[Complete]: 0 messages [00:00, ? messages/s] -====Pipeline Complete==== -Total time: 701.99 sec -Pipeline runtime: 694.18 sec -``` - -19. In the same pod shell, restart the agent to load synchronized changes. - ```shell -eval $EXPLOIT_IQ_PYTHON_COMMAND +cat > base/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +EOF ``` -Send requests to the agent to test your modifications. Repeat steps steps 16-19 for iterative development. -20. Delete the developer overlay resources when finished. +9. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: ```shell +DEPLOYMENT_VARIANT_NAME=remote-nim-all +#DEPLOYMENT_VARIANT_NAME=self-hosted-llama3.1-70b-4bit # Delete all resources but keep all data saved in PVCs -kustomize build overlays/developer/ | oc delete -l purpose!=persistent -f - +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=persistent -f - # Or, Delete Everything -kustomize build overlays/developer/ | oc delete -f - +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - ``` ## Batch-processing Deployment Overlay @@ -213,7 +109,7 @@ kustomize build overlays/developer/ | oc delete -f - When running in batch mode, it is important to use an optimized configuration that adjusts the following settings: - The maximum number of concurrent active requests in the queue. -- The maximum timeout (in minutes) before a report transitions from `sent` to expired if Agent Morpheus processing is not completed within the specified interval. +- The maximum timeout (in minutes) before a report transitions from `sent` to expired if ExploitIQ processing is not completed within the specified interval. - Compatibility with a self-hosted LLM that does not enforce rate limits, instead of relying on the NVIDIA-managed service, which may have limited API key credits. - Disabling LLM caching in NGINX to ensure repeated inputs receive fresh responses, which is essential for batch analysis workflows. @@ -222,69 +118,3 @@ To apply this configuration, use the batch-processing overlay instead of the def ```shell kustomize build overlays/batch-processing/ | oc apply -f - ``` - -## Transitive Support - -Transitive support enhances the agent by integrating a tool that identifies whether a specific function in a given package—either transitive or direct—is called or reachable from the application codebase. The tool returns a tuple in the format `(bool, list[Documents])`: - -- The first element (`bool`) indicates whether such a call path exists. -- The second element (list[Documents]) provides the call hierarchy — a sequence of document-level function references that trace the path from the application code to the vulnerable function within the specified package. - -## Tracing - -By default, tracing is enabled for Agent Morpheus and configured to publish traces to a Jaeger instance that is deployed alongside the application. You can disable tracing or modify its configuration by updating the values in [base/agent_morpheus_tracing.env](base/agent_morpheus_tracing.env): - -```bash -# Enable/disable tracing -EXPLOIT_IQ_TRACING_ENABLED=true - -# Enable/disable exporting traces to Jaeger -EXPLOIT_IQ_JAEGER_ENABLED=true - -# Enable/disable logging traces to the console -EXPLOIT_IQ_CONSOLE_SPANS_ENABLED=false - -# Jaeger endpoint (modify to use an external Jaeger instance) -EXPLOIT_IQ_JAEGER_ENDPOINT=http://agent-morpheus-jaeger-service:4317 - -# Service name to identify Agent Morpheus in Jaeger traces -EXPLOIT_IQ_SERVICE_NAME=agent-morpheus -``` - -### Configuration Options - -- **EXPLOIT_IQ_TRACING_ENABLED**: Master switch to enable or disable all tracing functionality -- **EXPLOIT_IQ_JAEGER_ENABLED**: Controls whether traces are exported to Jaeger -- **EXPLOIT_IQ_CONSOLE_SPANS_ENABLED**: When enabled, traces are logged to the application console -- **EXPLOIT_IQ_JAEGER_ENDPOINT**: Jaeger collector endpoint URL (use this to connect to an external Jaeger instance rather than the one deployed with Agent Morpheus) -- **EXPLOIT_IQ_SERVICE_NAME**: Service identifier that appears in Jaeger's service list - -### Configuration - -1. By default, this feature is disabled. To enable it, set the `transitive_search_tool` key in the general section of the Agent Morpheus configuration. For example: - -```json -{ - "general": { - "cache_dir": null, - "base_vdb_dir": "/morpheus-data/vdbs", - "base_git_dir": "/morpheus-data/repos", - "max_retries": 5, - "transitive_code_search": true, - "code_search_tool": true, - "model_max_batch_size": 64, - "pipeline_batch_size": 128, - "use_uvloop": true, - "use_dependency_checker": false, - "retry_on_client_errors": false, - "generate_cvss_score": true, - "generate_intel_score": true, - "intel_low_score": 51, - "insist_analysis": false - } -} -``` - -**Note: This is not a code understanding tool. It must be used in conjunction with either lexical code search (`code_search_tool=true`) or VDB embedding-based code search. While it works effectively with both methods, it requires access to all transitive package code. As a result, generating VDB embeddings may be time-consuming if `code_search_tool=false`** - -2. Because the transitive code search tool requires access to the code of transitive packages, both [includes.json](./base/includes.json) and [excludes.json](./base/excludes.json) are included in all deployments, without changing the internal client configuration. Therefore, if you would like to run agent morpheus without `transitive_code_search=true` , then customize these files accordingly. For example, add `"**/vendor/**/*"` to list of Go in [excludes.json](./base/excludes.json) From 354a69c7c6755ac9142b34795add290b2522d0a9 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 24 Aug 2025 03:21:13 +0300 Subject: [PATCH 031/286] chore: fix deployments problems and configurations Signed-off-by: Zvi Grinberg --- kustomize/README.md | 2 +- kustomize/base/exploit-iq-config.yml | 39 ++++-- kustomize/base/exploit_iq_client.yaml | 2 +- kustomize/base/exploit_iq_service.yaml | 6 +- kustomize/base/ips-patch-client.json | 4 +- kustomize/base/ips-patch.json | 4 +- kustomize/base/kustomization.yaml | 8 +- kustomize/base/nginx.yaml | 11 +- .../templates/routes/intel.conf.template | 2 - .../template-variables.conf.template | 13 -- kustomize/config-http-openai-local.yml | 16 +++ .../remote-nim-all/kustomization.yaml | 63 ++++------ .../kustomization.yaml | 118 ++++++------------ 13 files changed, 118 insertions(+), 170 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 76725b40a..1187ad58d 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -21,7 +21,7 @@ oc new-project $YOUR_NAMESPACE_NAME 3. Create an image pull secret to authorize pulling the `Agent Morpheus` container image: ```shell -oc create secret generic exploit-iq-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +oc create secret generic exploit-iq--pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` 4. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [Agent Morpheus Client](./base/agent_morpheus_client.yaml) configuration. diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index e7bf351ea..87976ffdc 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -16,12 +16,12 @@ general: use_uvloop: true - telemetry: - tracing: - phoenix: - _type: phoenix - endpoint: ${PHOENIX_TRACING_URL:-http://localhost:6006/v1/traces} - project: PROJECT_NAME_PLACEHOLDER +# telemetry: +# tracing: +# phoenix: +# _type: phoenix +# endpoint: ${PHOENIX_TRACING_URL:-http://localhost:6006/v1/traces} +# project: PROJECT_NAME_PLACEHOLDER functions: cve_generate_vdbs: @@ -40,7 +40,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: CALLBACK_URL_PLACEHOLDER/vulnerabilities/{vuln_id}/comments + endpoint: https://exploit-iq-client.exploit-iq-nat.svc:8443/vulnerabilities/{vuln_id}/comments token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt @@ -103,12 +103,17 @@ functions: llm_name: justify_llm cve_http_output: _type: cve_http_output - url: CALLBACK_URL_PLACEHOLDER + url: https://exploit-iq-client.exploit-iq-nat.svc:8443 endpoint: /reports - authentication: - auth_type: bearer - token_path: /var/run/secrets/kubernetes.io/serviceaccount/token - verify_path: /app/certs/service-ca.crt + auth_type: bearer + token_path: /var/run/secrets/kubernetes.io/serviceaccount/token + verify_path: /app/certs/service-ca.crt + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false llms: checklist_llm: @@ -159,6 +164,15 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + + intel_source_score_llm: + _type: ${LLM_TYPE_INTEL_SOURCE_SCORE:-nim} + api_key: $LLM_API_KEY_INTEL_SOURCE_SCORE:-"EMPTY"} + base_url: ${INTEL_SOURCE_SCORE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${INTEL_SOURCE_SCORE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -172,6 +186,7 @@ workflow: _type: cve_agent cve_generate_vdbs_name: cve_generate_vdbs cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index fced7d334..c73d4350d 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -27,7 +27,7 @@ spec: - ./application - -Dquarkus.http.host=0.0.0.0 - -Dquarkus.log.category."com.redhat.ecosystemappeng.morpheus".level=DEBUG - image: quay.io/ecosystem-appeng/exploit-iq-client:latest + image: quay.io/ecosystem-appeng/agent-morpheus-client:latest imagePullPolicy: Always ports: - name: http diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 721b60aaa..1388191aa 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -37,9 +37,7 @@ spec: - "--log-level" - "debug" - "serve" - - "pipeline" - - "--config_file=" - - "/configs/exploit-iq-config.json" + - "--config_file=/configs/exploit-iq-config.yml" - "--host" - "0.0.0.0" - "--port" @@ -89,8 +87,6 @@ spec: value: http://nginx-cache:8080/nim_embed/v1 - name: NVD_BASE_URL value: http://nginx-cache:8080/nvd - - name: NVD_API_KEY_HEADER - value: api-key - name: NVIDIA_API_BASE value: http://nginx-cache:8080/nim_llm/v1 - name: RHSA_BASE_URL diff --git a/kustomize/base/ips-patch-client.json b/kustomize/base/ips-patch-client.json index ccaaf1aa0..e2b53b801 100644 --- a/kustomize/base/ips-patch-client.json +++ b/kustomize/base/ips-patch-client.json @@ -1,6 +1,6 @@ [{ "op": "add", "path": "/spec/template/spec/imagePullSecrets/0", - "value": {"name": "morpheus-client-pull-secret"} + "value": {"name": "exploit-iq-pull-secret"} } -] \ No newline at end of file +] diff --git a/kustomize/base/ips-patch.json b/kustomize/base/ips-patch.json index cf4d87c13..e2b53b801 100644 --- a/kustomize/base/ips-patch.json +++ b/kustomize/base/ips-patch.json @@ -1,6 +1,6 @@ [{ "op": "add", "path": "/spec/template/spec/imagePullSecrets/0", - "value": {"name": "morpheus-pull-secret"} + "value": {"name": "exploit-iq-pull-secret"} } -] \ No newline at end of file +] diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 41ff1768d..fddd67d64 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -62,8 +62,8 @@ patches: kind: Deployment images: - - name: quay.io/ecosystem-appeng/exploit-iq - newTag: latest + - name: quay.io/ecosystem-appeng/agent-morpheus-rh + newTag: nat - - name: quay.io/ecosystem-appeng/exploit-iq-client - newTag: latest + - name: quay.io/ecosystem-appeng/agent-morpheus-client + newTag: on-pr-809af874e075a1fb1f2ca5c2d94a08065e675823 diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index 5b36c4243..68e9922bb 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -41,21 +41,12 @@ spec: secretKeyRef: key: serpapi_api_key name: exploit-iq-secret - - name: NVD_API_KEY - valueFrom: - secretKeyRef: - key: nvd_api_key - name: exploit-iq-secret - name: NVIDIA_API_KEY valueFrom: secretKeyRef: key: nvidia_api_key name: exploit-iq-secret - - name: GHSA_API_KEY - valueFrom: - secretKeyRef: - key: ghsa_api_key - name: exploit-iq-secret + - name: OPENAI_API_KEY valueFrom: secretKeyRef: diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index a3389c38d..2e40ccf85 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -13,7 +13,6 @@ location /serpapi { location /nvd { rewrite ^\/nvd(\/.*)$ /rest$1 break; proxy_pass https://services.nvd.nist.gov; - proxy_set_header apiKey $nvd_http_api_key; proxy_cache intel_cache; proxy_cache_valid 200 202 365d; proxy_cache_methods GET; @@ -67,7 +66,6 @@ location /recf { location /ghsa { rewrite ^\/ghsa(\/.*)$ $1 break; proxy_pass https://api.github.com; - proxy_set_header Authorization $ghsa_http_authorization; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; diff --git a/kustomize/base/nginx/templates/variables/template-variables.conf.template b/kustomize/base/nginx/templates/variables/template-variables.conf.template index 4c211724f..4330bc950 100644 --- a/kustomize/base/nginx/templates/variables/template-variables.conf.template +++ b/kustomize/base/nginx/templates/variables/template-variables.conf.template @@ -40,16 +40,3 @@ map $http_authorization $openai_http_authorization { default $http_authorization; } -map $http_authorization $ghsa_http_authorization { - 'Bearer EXPLOIT_IQ' 'Bearer ${GHSA_API_KEY}'; - 'Bearer "EXPLOIT_IQ"' 'Bearer ${GHSA_API_KEY}'; - 'Bearer CYBER_DEV_DAY' 'Bearer ${GHSA_API_KEY}'; - 'Bearer "CYBER_DEV_DAY"' 'Bearer ${GHSA_API_KEY}'; - default $http_authorization; -} - -map $http_apikey $nvd_http_api_key { - 'EXPLOIT_IQ' '${NVD_API_KEY}'; - default $http_apikey; -} - diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 533d10064..b6dffde64 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -107,6 +107,12 @@ functions: _type: cve_http_output url: http://localhost:8080 endpoint: /reports + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false llms: checklist_llm: @@ -158,6 +164,15 @@ llms: max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + embedders: nim_embedder: _type: nim @@ -170,6 +185,7 @@ workflow: _type: cve_agent cve_generate_vdbs_name: cve_generate_vdbs cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist diff --git a/kustomize/overlays/remote-nim-all/kustomization.yaml b/kustomize/overlays/remote-nim-all/kustomization.yaml index 7b9ecc5d7..ff9cf1d90 100644 --- a/kustomize/overlays/remote-nim-all/kustomization.yaml +++ b/kustomize/overlays/remote-nim-all/kustomization.yaml @@ -18,9 +18,8 @@ patchesStrategicMerge: app: exploit-iq component: exploit-iq annotations: - api-key: "EXPLOIT_IQ" - nim-llm-baseurl: http://nginx-cache:8080/nim_llm/v1 - model-name: hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 + api-key: &api-key "EXPLOIT_IQ" + nim-llm-baseurl: &nim-llm-baseurl "http://nginx-cache:8080/nim_llm/v1" spec: selector: matchLabels: @@ -33,70 +32,52 @@ patchesStrategicMerge: component: exploit-iq spec: containers: - - name: exploit-iq-service + - name: exploit-iq imagePullPolicy: Always workingDir: /workspace/ env: - name: LLM_API_KEY_CHECKLIST - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_CODE_VDB_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_DOC_VDB_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_AGENT_EXECUTOR - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_SUMMARIZE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_JUSTIFY - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key + + - name: LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key - name: CHECKLIST_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] + value: *nim-llm-baseurl - name: CODE_VDB_RETRIEVER_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] + value: *nim-llm-baseurl - name: DOC_VDB_RETRIEVER_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] + value: *nim-llm-baseurl - name: AGENT_EXECUTOR_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] + value: *nim-llm-baseurl - name: SUMMARIZE_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] + value: *nim-llm-baseurl - name: JUSTIFY_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['nim-llm-baseurl'] - + value: *nim-llm-baseurl + + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *nim-llm-baseurl + diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml index a0abee24c..9446d3e2c 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -18,10 +18,10 @@ patchesStrategicMerge: app: exploit-iq component: exploit-iq annotations: - llm-type: openai - api-key: "EMPTY" - openai-llm-baseurl: http://nginx-cache:8080/openai/v1 - model-name: hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 + llm-type: &llm-type openai + api-key: &api-key "EMPTY" + openai-llm-baseurl: &openai-llm-baseurl http://nginx-cache:8080/openai/v1 + model-name: &model-name hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 spec: selector: matchLabels: @@ -39,131 +39,95 @@ patchesStrategicMerge: workingDir: /workspace/ env: - name: LLM_TYPE_CHECKLIST - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type - name: LLM_TYPE_VDB_CODE_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type - name: LLM_TYPE_VDB_DOC_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type - name: LLM_TYPE_AGENT_EXECUTOR - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type - name: LLM_TYPE_SUMMARIZE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type - name: LLM_TYPE_JUSTIFY - valueFrom: - fieldRef: - fieldPath: metadata.annotations['llm-type'] + value: *llm-type + + - name: LLM_TYPE_INTEL_SOURCE_SCORE + value: *llm-type - name: LLM_API_KEY_CHECKLIST - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_CODE_VDB_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_DOC_VDB_RETRIEVER - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_AGENT_EXECUTOR - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_SUMMARIZE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key - name: LLM_API_KEY_JUSTIFY - valueFrom: - fieldRef: - fieldPath: metadata.annotations['api-key'] + value: *api-key + + - name: $LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key - name: CHECKLIST_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl - name: CODE_VDB_RETRIEVER_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl - name: DOC_VDB_RETRIEVER_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl - name: AGENT_EXECUTOR_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl - name: SUMMARIZE_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl - name: JUSTIFY_LLM_API_BASE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['openai-llm-baseurl'] + value: *openai-llm-baseurl + + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *openai-llm-baseurl - name: CHECKLIST_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] + value: *model-name - name: CODE_VDB_RETRIEVER_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] + value: *model-name - name: DOC_VDB_RETRIEVER_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] - + value: *model-name + - name: AGENT_EXECUTOR_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] + value: *model-name - name: SUMMARIZE_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] + value: *model-name - name: JUSTIFY_MODEL_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['model-name'] + value: *model-name + + - name: INTEL_SOURCE_SCORE_MODEL_NAME + value: *model-name From 634d3928bf5fbe93aa269e86ba63a6296d4640f5 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 24 Aug 2025 03:22:20 +0300 Subject: [PATCH 032/286] fix: avoid pipeline startup crash Signed-off-by: Zvi Grinberg --- .gitignore | 3 +++ src/vuln_analysis/functions/cve_http_output.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa5db811f..6e527f541 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,9 @@ ENV/ env.bak/ venv.bak/ +# Env files +*.env + # Spyder project settings .spyderproject .spyproject diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index d84068dab..6c38d0594 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -35,7 +35,7 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): url: str = Field(description="URL to send CVE workflow output") endpoint: str = Field(description="Endpoint to send CVE workflow output") auth_type: str = Field(default="disabled", description="Type of auth - bearer, basic or disabled") - token: str = Field(description="Token to authenticate when sending CVE workflow output") + token: str | None = Field(default=None, description="Token to authenticate when sending CVE workflow output") token_path: str | None = Field(default=None, description="Path to token file containing auth token") verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") username: str | None = Field(default=None, description="Username to authenticate with for basic auth") From 333f3db45280afb59b0ea5c79b0c5a4f266aac15 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 08:18:57 +0300 Subject: [PATCH 033/286] chore: adapt ci builds pipelineruns for ExploitIQ on NAT Signed-off-by: Zvi Grinberg --- .dockerignore | 21 ++ .tekton/on-pull-request.yaml | 159 +++++++++++++++ .tekton/on-push.yaml | 153 +++++++++++++++ .tekton/on-tag.yaml | 182 ++++++++++++++++++ .../pvcs/buildah-containers-storage-pvc.yaml | 14 ++ .tekton/pvcs/buildah-temp-storage-pvc.yaml | 14 ++ .tekton/tasks/buildah-task.yaml | 168 ++++++++++++++++ kustomize/base/exploit_iq_client_db.yaml | 2 +- 8 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .tekton/on-pull-request.yaml create mode 100644 .tekton/on-push.yaml create mode 100644 .tekton/on-tag.yaml create mode 100644 .tekton/pvcs/buildah-containers-storage-pvc.yaml create mode 100644 .tekton/pvcs/buildah-temp-storage-pvc.yaml create mode 100644 .tekton/tasks/buildah-task.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..5c2354a19 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# Folders to ignore +.vscode +**/__pycache__ +**/*.ipynb_checkpoints +**/*Dockerfile* +.tmp + +# Ignore any build folder +./build* +# Ignore the build cache +./.cache + +# Ignore specific files +docker/commands.sh +*.egg-info/ +kafka-docker +# Ignore env files +**/*.env + +#Ignore the deployment configuration +./kustomize/ diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml new file mode 100644 index 000000000..e6857566e --- /dev/null +++ b/.tekton/on-pull-request.yaml @@ -0,0 +1,159 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-pr + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[pull_request]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[main, rh-aiq-main]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: image-expires-after + value: 5d + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:on-pr-{{revision}} + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + - name: image-expires-after + default: "" + description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: buildah-pvc + runAfter: + - fetch-repository + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=quay.expires-after=$(params.image-expires-after) + --label=agent_morpheus_vcs_branch={{source_branch}} + --label=agent_morpheus_vcs_build_commit_id={{revision}} + --label=agent_morpheus_vcs_repo={{repo_url}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: ruben-morpheus + workspaces: + - name: source + workspace: source + + - name: dockerconfig + workspace: dockerconfig-ws + + - name: buildah-storage-dir + workspace: buildah-storage-dir + + - name: buildah-temp-cache + workspace: buildah-temp-cache + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml new file mode 100644 index 000000000..f7c785d81 --- /dev/null +++ b/.tekton/on-push.yaml @@ -0,0 +1,153 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-push + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[rh-aiq-main]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:nat + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: buildah-pvc + runAfter: + - fetch-repository + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=agent_morpheus_vcs_branch={{source_branch}} + --label=agent_morpheus_vcs_build_commit_id={{revision}} + --label=agent_morpheus_vcs_repo={{repo_url}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: ruben-morpheus + workspaces: + - name: source + workspace: source + + - name: dockerconfig + workspace: dockerconfig-ws + + - name: buildah-storage-dir + workspace: buildah-storage-dir + + - name: buildah-temp-cache + workspace: buildah-temp-cache + + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml new file mode 100644 index 000000000..5cf68adc5 --- /dev/null +++ b/.tekton/on-tag.yaml @@ -0,0 +1,182 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-tag + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[refs/tags/*]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: 'quay.io/ecosystem-appeng/agent-morpheus-rh' + - name: tag-name + value: "{{ target_branch }}" + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: tag-name + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: format-image-name + params: + - name: image_name + value: $(params.output-image) + - name: tag_name + value: $(params.tag-name) + taskSpec: + params: + - name: image_name + type: string + - name: tag_name + type: string + results: + - name: target-image + steps: + - name: format-image + image: registry.redhat.io/ubi9/ubi-minimal:9.4 + script: | + #!/usr/bin/env bash + IMAGE="$(params.image_name):$(echo $(params.tag_name) | sed 's|refs/tags/||')" + echo "Building image: $IMAGE" + + # Set the formatted image as an output + echo -n "${IMAGE}" > "$(results.target-image.path)" + - name: buildah-pvc + runAfter: + - fetch-repository + - format-image-name + params: + - name: IMAGE + value: $(tasks.format-image-name.results.target-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=agent_morpheus_vcs_branch={{source_branch}} + --label=agent_morpheus_vcs_build_commit_id={{revision}} + --label=agent_morpheus_vcs_repo={{repo_url}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: ruben-morpheus + workspaces: + - name: source + workspace: source + + - name: dockerconfig + workspace: dockerconfig-ws + + - name: buildah-storage-dir + workspace: buildah-storage-dir + + - name: buildah-temp-cache + workspace: buildah-temp-cache + + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/pvcs/buildah-containers-storage-pvc.yaml b/.tekton/pvcs/buildah-containers-storage-pvc.yaml new file mode 100644 index 000000000..667ffab33 --- /dev/null +++ b/.tekton/pvcs/buildah-containers-storage-pvc.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc-buildah-storage +spec: + accessModes: + - ReadWriteOnce + resources: + limits: + storage: 150Gi + requests: + storage: 150Gi + storageClassName: gp3-csi + diff --git a/.tekton/pvcs/buildah-temp-storage-pvc.yaml b/.tekton/pvcs/buildah-temp-storage-pvc.yaml new file mode 100644 index 000000000..53580d960 --- /dev/null +++ b/.tekton/pvcs/buildah-temp-storage-pvc.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc-buildah-temp-storage +spec: + accessModes: + - ReadWriteOnce + resources: + limits: + storage: 60Gi + requests: + storage: 60Gi + storageClassName: gp3-csi + diff --git a/.tekton/tasks/buildah-task.yaml b/.tekton/tasks/buildah-task.yaml new file mode 100644 index 000000000..60f9a43a7 --- /dev/null +++ b/.tekton/tasks/buildah-task.yaml @@ -0,0 +1,168 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: buildah-pvc + namespace: ruben-morpheus + +spec: + description: | + Buildah task builds source into a container image and + then pushes it to a container registry. + params: + - description: | + Fully qualified container image name to be built by buildah. + name: IMAGE + type: string + - default: ./Dockerfile + description: | + Path to the `Dockerfile` (or `Containerfile`) relative to the `source` workspace. + name: DOCKERFILE + type: string + - default: + - '' + description: | + Dockerfile build arguments, array of key=value + name: BUILD_ARGS + type: array + - default: . + description: | + Path to the directory to use as context. + name: CONTEXT + type: string + - default: overlay + description: | + Set buildah storage driver to reflect the currrent cluster node's + settings. + name: STORAGE_DRIVER + type: string + - default: oci + description: 'The format of the built container, oci or docker' + name: FORMAT + type: string + - default: '' + description: | + Extra parameters passed for the build command when building images. + name: BUILD_EXTRA_ARGS + type: string + - default: '' + description: | + Extra parameters passed for the push command when pushing images. + name: PUSH_EXTRA_ARGS + type: string + - default: 'false' + description: | + Skip pushing the image to the container registry. + name: SKIP_PUSH + type: string + - default: 'true' + description: | + Sets the TLS verification flag, `true` is recommended. + name: TLS_VERIFY + type: string + - default: 'false' + description: | + Turns on verbose logging, all commands executed will be printed out. + name: VERBOSE + type: string + results: + - description: | + Fully qualified image name. + name: IMAGE_URL + type: string + - description: | + Digest of the image just built. + name: IMAGE_DIGEST + type: string + stepTemplate: + computeResources: + requests: + memory: 5Gi + cpu: 1 + limits: + memory: 10Gi + cpu: 2 + env: + - name: PARAMS_IMAGE + value: $(params.IMAGE) + - name: PARAMS_CONTEXT + value: $(params.CONTEXT) + - name: PARAMS_DOCKERFILE + value: $(params.DOCKERFILE) + - name: PARAMS_FORMAT + value: $(params.FORMAT) + - name: PARAMS_STORAGE_DRIVER + value: $(params.STORAGE_DRIVER) + - name: PARAMS_BUILD_EXTRA_ARGS + value: $(params.BUILD_EXTRA_ARGS) + - name: PARAMS_PUSH_EXTRA_ARGS + value: $(params.PUSH_EXTRA_ARGS) + - name: PARAMS_SKIP_PUSH + value: $(params.SKIP_PUSH) + - name: PARAMS_TLS_VERIFY + value: $(params.TLS_VERIFY) + - name: PARAMS_VERBOSE + value: $(params.VERBOSE) + - name: WORKSPACES_SOURCE_BOUND + value: $(workspaces.source.bound) + - name: WORKSPACES_SOURCE_PATH + value: $(workspaces.source.path) + - name: WORKSPACES_DOCKERCONFIG_BOUND + value: $(workspaces.dockerconfig.bound) + - name: WORKSPACES_DOCKERCONFIG_PATH + value: $(workspaces.dockerconfig.path) + - name: WORKSPACES_RHEL_ENTITLEMENT_BOUND + value: $(workspaces.rhel-entitlement.bound) + - name: WORKSPACES_RHEL_ENTITLEMENT_PATH + value: $(workspaces.rhel-entitlement.path) + - name: RESULTS_IMAGE_URL_PATH + value: $(results.IMAGE_URL.path) + - name: RESULTS_IMAGE_DIGEST_PATH + value: $(results.IMAGE_DIGEST.path) + steps: + - args: + - '$(params.BUILD_ARGS[*])' + computeResources: {} + image: 'registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e' + name: build + script: | + set -e + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAojCiMgV3JhcHBlciBhcm91bmQgImJ1aWxkYWggYnVkIiB0byBidWlsZCBhbmQgcHVzaCBhIGNvbnRhaW5lciBpbWFnZSBiYXNlZCBvbiBhIERvY2tlcmZpbGUuCiMKCnNob3B0IC1zIGluaGVyaXRfZXJyZXhpdApzZXQgLWV1IC1vIHBpcGVmYWlsCgpzb3VyY2UgIiQoZGlybmFtZSAke0JBU0hfU09VUkNFWzBdfSkvY29tbW9uLnNoIgpzb3VyY2UgIiQoZGlybmFtZSAke0JBU0hfU09VUkNFWzBdfSkvYnVpbGRhaC1jb21tb24uc2giCgpmdW5jdGlvbiBfYnVpbGRhaCgpIHsKICAgIGJ1aWxkYWggXAogICAgICAgIC0tc3RvcmFnZS1kcml2ZXI9IiR7UEFSQU1TX1NUT1JBR0VfRFJJVkVSfSIgXAogICAgICAgIC0tdGxzLXZlcmlmeT0iJHtQQVJBTVNfVExTX1ZFUklGWX0iIFwKICAgICAgICAiJEAiCn0KCiMKIyBQcmVwYXJlCiMKCiMgbWFraW5nIHN1cmUgdGhlIHJlcXVpcmVkIHdvcmtzcGFjZSAic291cmNlIiBpcyBib3VuZGVkLCB3aGljaCBtZWFucyBpdHMgdm9sdW1lIGlzIGN1cnJlbnRseSBtb3VudGVkCiMgYW5kIHJlYWR5IHRvIHVzZQpwaGFzZSAiSW5zcGVjdGluZyBzb3VyY2Ugd29ya3NwYWNlICcke1dPUktTUEFDRVNfU09VUkNFX1BBVEh9JyAoUFdEPScke1BXRH0nKSIKW1sgIiR7V09SS1NQQUNFU19TT1VSQ0VfQk9VTkR9IiAhPSAidHJ1ZSIgXV0gJiYKICAgIGZhaWwgIldvcmtzcGFjZSAnc291cmNlJyBpcyBub3QgYm91bmRlZCIKCnBoYXNlICJBc3NlcnRpbmcgdGhlIGRvY2tlcmZpbGUvY29udGFpbmVyZmlsZSAnJHtET0NLRVJGSUxFX0ZVTEx9JyBleGlzdHMiCltbICEgLWYgIiR7RE9DS0VSRklMRV9GVUxMfSIgXV0gJiYKICAgIGZhaWwgIkRvY2tlcmZpbGUgbm90IGZvdW5kIGF0OiAnJHtET0NLRVJGSUxFX0ZVTEx9JyIKCnBoYXNlICJJbnNwZWN0aW5nIGNvbnRleHQgJyR7UEFSQU1TX0NPTlRFWFR9JyIKW1sgISAtZCAiJHtQQVJBTVNfQ09OVEVYVH0iIF1dICYmCiAgICBmYWlsICJDT05URVhUIHBhcmFtIGlzIG5vdCBmb3VuZCBhdCAnJHtQQVJBTVNfQ09OVEVYVH0nLCBvbiBzb3VyY2Ugd29ya3NwYWNlIgoKcGhhc2UgIkJ1aWxkaW5nIGJ1aWxkIGFyZ3MiCkJVSUxEX0FSR1M9KCkKZm9yIGJ1aWxkYXJnIGluICIkQCI7IGRvCiAgICBCVUlMRF9BUkdTKz0oIi0tYnVpbGQtYXJnPSRidWlsZGFyZyIpCmRvbmUKCiMgSGFuZGxlIG9wdGlvbmFsIGRvY2tlcmNvbmZpZyBzZWNyZXQKaWYgW1sgIiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfQk9VTkR9IiA9PSAidHJ1ZSIgXV07IHRoZW4KCiAgICAjIGlmIGNvbmZpZy5qc29uIGV4aXN0cyBhdCB3b3Jrc3BhY2Ugcm9vdCwgd2UgdXNlIHRoYXQKICAgIGlmIHRlc3QgLWYgIiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSH0vY29uZmlnLmpzb24iOyB0aGVuCiAgICAgICAgZXhwb3J0IERPQ0tFUl9DT05GSUc9IiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSH0iCgogICAgICAgICMgZWxzZSB3ZSBsb29rIGZvciAuZG9ja2VyY29uZmlnanNvbiBhdCB0aGUgcm9vdAogICAgZWxpZiB0ZXN0IC1mICIke1dPUktTUEFDRVNfRE9DS0VSQ09ORklHX1BBVEh9Ly5kb2NrZXJjb25maWdqc29uIjsgdGhlbgogICAgICAgICMgZW5zdXJlIC5kb2NrZXIgZXhpc3QgYmVmb3JlIHRoZSBjb3B5aW5nIHRoZSBjb250ZW50CiAgICAgICAgaWYgWyAhIC1kICIkSE9NRS8uZG9ja2VyIiBdOyB0aGVuCiAgICAgICAgICAgbWtkaXIgLXAgIiRIT01FLy5kb2NrZXIiCiAgICAgICAgZmkKICAgICAgICBjcCAiJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19QQVRIfS8uZG9ja2VyY29uZmlnanNvbiIgIiRIT01FLy5kb2NrZXIvY29uZmlnLmpzb24iCiAgICAgICAgZXhwb3J0IERPQ0tFUl9DT05GSUc9IiRIT01FLy5kb2NrZXIiCgogICAgICAgICMgbmVlZCB0byBlcnJvciBvdXQgaWYgbmVpdGhlciBmaWxlcyBhcmUgcHJlc2VudAogICAgZWxzZQogICAgICAgIGVjaG8gIm5laXRoZXIgJ2NvbmZpZy5qc29uJyBub3IgJy5kb2NrZXJjb25maWdqc29uJyBmb3VuZCBhdCB3b3Jrc3BhY2Ugcm9vdCIKICAgICAgICBleGl0IDEKICAgIGZpCmZpCgpFTlRJVExFTUVOVF9WT0xVTUU9IiIKaWYgW1sgIiR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX0JPVU5EfSIgPT0gInRydWUiIF1dOyB0aGVuCiAgICBFTlRJVExFTUVOVF9WT0xVTUU9Ii0tdm9sdW1lICR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEh9Oi9ldGMvcGtpL2VudGl0bGVtZW50OnJvIgpmaQoKIwojIEJ1aWxkCiMKCnBoYXNlICJCdWlsZGluZyAnJHtQQVJBTVNfSU1BR0V9JyBiYXNlZCBvbiAnJHtET0NLRVJGSUxFX0ZVTEx9JyIKCltbIC1uICIke1BBUkFNU19CVUlMRF9FWFRSQV9BUkdTfSIgXV0gJiYKICAgIHBoYXNlICJFeHRyYSAnYnVpbGRhaCBidWQnIGFyZ3VtZW50cyBpbmZvcm1lZDogJyR7UEFSQU1TX0JVSUxEX0VYVFJBX0FSR1N9JyIKCiMgUHJvY2VzcyBCVUlMRF9FWFRSQV9BUkdTCmJ1aWxkX2V4dHJhX2FyZ3NfdG1wPSQoZWNobyAiJHtQQVJBTVNfQlVJTERfRVhUUkFfQVJHUzotfSIgfCB4YXJncyAtbjEpCmlmIFtbIC1uICIkYnVpbGRfZXh0cmFfYXJnc190bXAiIF1dOyB0aGVuCiAgICByZWFkYXJyYXkgLXQgYnVpbGRfZXh0cmFfYXJncyA8PDwgIiRidWlsZF9leHRyYV9hcmdzX3RtcCIKZWxzZQogICAgYnVpbGRfZXh0cmFfYXJncz0oKSAjIEVtcHR5IGFycmF5IGlmIG5vIGV4dHJhIGFyZ3MKZmkKCl9idWlsZGFoIGJ1ZCAiJHtidWlsZF9leHRyYV9hcmdzW0BdfSIgXAogICAgJEVOVElUTEVNRU5UX1ZPTFVNRSBcCiAgICAiJHtCVUlMRF9BUkdTW0BdfSIgXAogICAgLS1maWxlPSIke0RPQ0tFUkZJTEVfRlVMTH0iIFwKICAgIC0tdGFnPSIke1BBUkFNU19JTUFHRX0iIFwKICAgICIke1BBUkFNU19DT05URVhUfSIKCmlmIFtbICIke1BBUkFNU19TS0lQX1BVU0h9IiA9PSAidHJ1ZSIgXV07IHRoZW4KICAgIHBoYXNlICJTa2lwcGluZyBwdXNoaW5nICcke1BBUkFNU19JTUFHRX0nIHRvIHRoZSBjb250YWluZXIgcmVnaXN0cnkhIgogICAgZXhpdCAwCmZpCgojCiMgUHVzaAojCgpwaGFzZSAiUHVzaGluZyAnJHtQQVJBTVNfSU1BR0V9JyB0byB0aGUgY29udGFpbmVyIHJlZ2lzdHJ5IgoKW1sgLW4gIiR7UEFSQU1TX1BVU0hfRVhUUkFfQVJHU30iIF1dICYmCiAgICBwaGFzZSAiRXh0cmEgJ2J1aWxkYWggcHVzaCcgYXJndW1lbnRzIGluZm9ybWVkOiAnJHtQQVJBTVNfUFVTSF9FWFRSQV9BUkdTfSciCgojIHRlbXBvcmFyeSBmaWxlIHRvIHN0b3JlIHRoZSBpbWFnZSBkaWdlc3QsIGluZm9ybWF0aW9uIG9ubHkgb2J0YWluZWQgYWZ0ZXIgcHVzaGluZyB0aGUgaW1hZ2UgdG8gdGhlCiMgY29udGFpbmVyIHJlZ2lzdHJ5CmRlY2xhcmUgLXIgZGlnZXN0X2ZpbGU9Ii90bXAvYnVpbGRhaC1kaWdlc3QudHh0IgoKIyBQcm9jZXNzIFBVU0hfRVhUUkFfQVJHUwpwdXNoX2V4dHJhX2FyZ3NfdG1wPSQoZWNobyAiJHtQQVJBTVNfUFVTSF9FWFRSQV9BUkdTOi19IiB8IHhhcmdzIC1uMSkKaWYgW1sgLW4gIiRwdXNoX2V4dHJhX2FyZ3NfdG1wIiBdXTsgdGhlbgogICAgcmVhZGFycmF5IC10IHB1c2hfZXh0cmFfYXJncyA8PDwgIiRwdXNoX2V4dHJhX2FyZ3NfdG1wIgplbHNlCiAgICBwdXNoX2V4dHJhX2FyZ3M9KCkgIyBFbXB0eSBhcnJheSBpZiBubyBleHRyYSBhcmdzCmZpCgpfYnVpbGRhaCBwdXNoICIke3B1c2hfZXh0cmFfYXJnc1tAXX0iIFwKICAgIC0tZGlnZXN0ZmlsZT0iJHtkaWdlc3RfZmlsZX0iIFwKICAgICIke1BBUkFNU19JTUFHRX0iIFwKICAgICJkb2NrZXI6Ly8ke1BBUkFNU19JTUFHRX0iCgojCiMgUmVzdWx0cwojCgpwaGFzZSAiSW5zcGVjdGluZyBkaWdlc3QgcmVwb3J0ICgnJHtkaWdlc3RfZmlsZX0nKSIKCltbICEgLXIgIiR7ZGlnZXN0X2ZpbGV9IiBdXSAmJgogICAgZmFpbCAiVW5hYmxlIHRvIGZpbmQgZGlnZXN0LWZpbGUgYXQgJyR7ZGlnZXN0X2ZpbGV9JyIKCmRlY2xhcmUgLXIgZGlnZXN0X3N1bT0iJChjYXQgJHtkaWdlc3RfZmlsZX0pIgoKW1sgLXogIiR7ZGlnZXN0X3N1bX0iIF1dICYmCiAgICBmYWlsICJEaWdlc3QgZmlsZSAnJHtkaWdlc3RfZmlsZX0nIGlzIGVtcHR5ISIKCnBoYXNlICJTdWNjZXNzZnVseSBidWlsdCBjb250YWluZXIgaW1hZ2UgJyR7UEFSQU1TX0lNQUdFfScgKCcke2RpZ2VzdF9zdW19JykiCmVjaG8gLW4gIiR7UEFSQU1TX0lNQUdFfSIgfCB0ZWUgJHtSRVNVTFRTX0lNQUdFX1VSTF9QQVRIfQplY2hvIC1uICIke2RpZ2VzdF9zdW19IiB8IHRlZSAke1JFU1VMVFNfSU1BR0VfRElHRVNUX1BBVEh9Cg==" |base64 -d >"/scripts/buildah-bud.sh" + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZGVjbGFyZSAtcnggUEFSQU1TX0lNQUdFPSIke1BBUkFNU19JTUFHRTotfSIKZGVjbGFyZSAtcnggUEFSQU1TX0RPQ0tFUkZJTEU9IiR7UEFSQU1TX0RPQ0tFUkZJTEU6LX0iCmRlY2xhcmUgLXggUEFSQU1TX0NPTlRFWFQ9IiR7UEFSQU1TX0NPTlRFWFQ6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19TVE9SQUdFX0RSSVZFUj0iJHtQQVJBTVNfU1RPUkFHRV9EUklWRVI6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19CVUlMRF9FWFRSQV9BUkdTPSIke1BBUkFNU19CVUlMRF9FWFRSQV9BUkdTOi19IgpkZWNsYXJlIC1yeCBQQVJBTVNfUFVTSF9FWFRSQV9BUkdTPSIke1BBUkFNU19QVVNIX0VYVFJBX0FSR1M6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19TS0lQX1BVU0g9IiR7UEFSQU1TX1NLSVBfUFVTSDotfSIKZGVjbGFyZSAtcnggUEFSQU1TX1RMU19WRVJJRlk9IiR7UEFSQU1TX1RMU19WRVJJRlk6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19WRVJCT1NFPSIke1BBUkFNU19WRVJCT1NFOi19IgoKZGVjbGFyZSAtcnggV09SS1NQQUNFU19TT1VSQ0VfUEFUSD0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9QQVRIOi19IgpkZWNsYXJlIC1yeCBXT1JLU1BBQ0VTX1NPVVJDRV9CT1VORD0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9CT1VORDotfSIKZGVjbGFyZSAtcnggV09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSD0iJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19QQVRIOi19IgpkZWNsYXJlIC1yeCBXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19CT1VORD0iJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19CT1VORDotfSIKZGVjbGFyZSAtcnggV09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEg9IiR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEg6LX0iCmRlY2xhcmUgLXJ4IFdPUktTUEFDRVNfUkhFTF9FTlRJVExFTUVOVF9CT1VORD0iJHtXT1JLU1BBQ0VTX1JIRUxfRU5USVRMRU1FTlRfQk9VTkQ6LX0iCgpkZWNsYXJlIC1yeCBSRVNVTFRTX0lNQUdFX0RJR0VTVF9QQVRIPSIke1JFU1VMVFNfSU1BR0VfRElHRVNUX1BBVEg6LX0iCmRlY2xhcmUgLXJ4IFJFU1VMVFNfSU1BR0VfVVJMX1BBVEg9IiR7UkVTVUxUU19JTUFHRV9VUkxfUEFUSDotfSIKCiMKIyBEb2NrZXJmaWxlCiMKCiMgZXhwb3NpbmcgdGhlIGZ1bGwgcGF0aCB0byB0aGUgY29udGFpbmVyIGZpbGUsIHdoaWNoIGJ5IGRlZmF1bHQgc2hvdWxkIGJlIHJlbGF0aXZlIHRvIHRoZSBwcmltYXJ5CiMgd29ya3NwYWNlLCB0byByZWNlaXZlIGEgZGlmZmVyZW50IGNvbnRhaW5lci1maWxlIGxvY2F0aW9uCmRlY2xhcmUgLXIgZG9ja2VyZmlsZV9vbl93cz0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9QQVRIfS8ke1BBUkFNU19ET0NLRVJGSUxFfSIKZGVjbGFyZSAteCBET0NLRVJGSUxFX0ZVTEw9IiR7RE9DS0VSRklMRV9GVUxMOi0ke2RvY2tlcmZpbGVfb25fd3N9fSIKCiMKIyBBc3NlcnRpbmcgRW52aXJvbm1lbnQKIwoKW1sgLXogIiR7RE9DS0VSRklMRV9GVUxMfSIgXV0gJiYKICAgIGZhaWwgInVuYWJsZSB0byBmaW5kIHRoZSBEb2NrZXJmaWxlLCBET0NLRVJGSUxFIG1heSBoYXZlIGFuIGluY29ycmVjdCBsb2NhdGlvbiIKCmV4cG9ydGVkX29yX2ZhaWwgXAogICAgV09SS1NQQUNFU19TT1VSQ0VfUEFUSCBcCiAgICBQQVJBTVNfSU1BR0UKCiMKIyBWZXJib3NlIE91dHB1dAojCgppZiBbWyAiJHtQQVJBTVNfVkVSQk9TRX0iID09ICJ0cnVlIiBdXTsgdGhlbgogICAgc2V0IC14CmZpCg==" |base64 -d >"/scripts/buildah-common.sh" + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKIyB0ZWt0b24ncyBob21lIGRpcmVjdG9yeQpkZWNsYXJlIC1yeCBURUtUT05fSE9NRT0iJHtURUtUT05fSE9NRTotL3Rla3Rvbi9ob21lfSIKCiMKIyBGdW5jdGlvbnMKIwoKZnVuY3Rpb24gZmFpbCgpIHsKICAgIGVjaG8gIkVSUk9SOiAkeyp9IiAyPiYxCiAgICBleGl0IDEKfQoKZnVuY3Rpb24gcGhhc2UoKSB7CiAgICBlY2hvICItLS0+IFBoYXNlOiAkeyp9Li4uIgp9CgojIGFzc2VydCBsb2NhbCB2YXJpYWJsZXMgYXJlIGV4cG9ydGVkIG9uIHRoZSBlbnZpcm9ubWVudApmdW5jdGlvbiBleHBvcnRlZF9vcl9mYWlsKCkgewogICAgZGVjbGFyZSAtYSBfcmVxdWlyZWRfdmFycz0iJHtAfSIKCiAgICBmb3IgdiBpbiAke19yZXF1aXJlZF92YXJzW0BdfTsgZG8KICAgICAgICBbWyAteiAiJHshdn0iIF1dICYmCiAgICAgICAgICAgIGZhaWwgIicke3Z9JyBlbnZpcm9ubWVudCB2YXJpYWJsZSBpcyBub3Qgc2V0ISIKICAgIGRvbmUKCiAgICByZXR1cm4gMAp9Cg==" |base64 -d >"/scripts/common.sh" + ls /scripts/buildah-*.sh; + chmod +x /scripts/buildah-*.sh;echo "Running Script /scripts/buildah-bud.sh"; + /scripts/buildah-bud.sh; + securityContext: + capabilities: + add: + - SETFCAP + volumeMounts: + - mountPath: /scripts + name: scripts-dir + workingDir: $(workspaces.source.path) + volumes: + - emptyDir: {} + name: scripts-dir + workspaces: + - description: | + Container build context, like for instnace a application source code + followed by a `Dockerfile`. + name: source + - description: | + Storage root dir for the spaces required by buildah when building images. + mountPath: /var/lib/containers/storage + name: buildah-storage-dir + + - description: | + Storage for temp spaces required by buildah cache when building images. + mountPath: /var/tmp + name: buildah-temp-cache + + + - description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson. + name: dockerconfig + optional: true + - description: An optional workspace that allows providing the entitlement keys for Buildah to access subscription. The mounted workspace contains entitlement.pem and entitlement-key.pem. + mountPath: /tmp/entitlement + name: rhel-entitlement + optional: true \ No newline at end of file diff --git a/kustomize/base/exploit_iq_client_db.yaml b/kustomize/base/exploit_iq_client_db.yaml index f29144e0c..41109b8f5 100644 --- a/kustomize/base/exploit_iq_client_db.yaml +++ b/kustomize/base/exploit_iq_client_db.yaml @@ -63,4 +63,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 200Mi + storage: 2Gi From 0a07cd1acef7df657378e0afaaedd43dd8361141 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 08:28:45 +0300 Subject: [PATCH 034/286] chore: delete redundant old pvc defs Signed-off-by: Zvi Grinberg --- .tekton/pvcs/buildah-containers-storage-pvc.yaml | 14 -------------- .tekton/pvcs/buildah-temp-storage-pvc.yaml | 14 -------------- 2 files changed, 28 deletions(-) delete mode 100644 .tekton/pvcs/buildah-containers-storage-pvc.yaml delete mode 100644 .tekton/pvcs/buildah-temp-storage-pvc.yaml diff --git a/.tekton/pvcs/buildah-containers-storage-pvc.yaml b/.tekton/pvcs/buildah-containers-storage-pvc.yaml deleted file mode 100644 index 667ffab33..000000000 --- a/.tekton/pvcs/buildah-containers-storage-pvc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: pvc-buildah-storage -spec: - accessModes: - - ReadWriteOnce - resources: - limits: - storage: 150Gi - requests: - storage: 150Gi - storageClassName: gp3-csi - diff --git a/.tekton/pvcs/buildah-temp-storage-pvc.yaml b/.tekton/pvcs/buildah-temp-storage-pvc.yaml deleted file mode 100644 index 53580d960..000000000 --- a/.tekton/pvcs/buildah-temp-storage-pvc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: pvc-buildah-temp-storage -spec: - accessModes: - - ReadWriteOnce - resources: - limits: - storage: 60Gi - requests: - storage: 60Gi - storageClassName: gp3-csi - From 949071dc4fdad6398987f923b688693039a3ee34 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 08:33:12 +0300 Subject: [PATCH 035/286] docs: change app name to the new one Signed-off-by: Zvi Grinberg --- kustomize/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 1187ad58d..494cdb02d 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,6 +1,6 @@ # Procedure to Deploy -1. Create a `base/secrets.env` file containing the API keys for external services `Agent Morpheus` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. +1. Create a `base/secrets.env` file containing the API keys for external services `ExploitIQ` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. ```shell cat > base/secrets.env << EOF @@ -18,13 +18,13 @@ export YOUR_NAMESPACE_NAME=yourNamespaceNameHere oc new-project $YOUR_NAMESPACE_NAME ``` -3. Create an image pull secret to authorize pulling the `Agent Morpheus` container image: +3. Create an image pull secret to authorize pulling the `ExploitIQ` container image: ```shell oc create secret generic exploit-iq--pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -4. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [Agent Morpheus Client](./base/agent_morpheus_client.yaml) configuration. +4. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret From c17fc0f420c826474d319acbc7c1b9b7080fbbc6 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 08:59:53 +0300 Subject: [PATCH 036/286] docs: add instructions how to install and run the agent locally Signed-off-by: Zvi Grinberg --- kustomize/README.md | 51 +++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 494cdb02d..d8e4cee2f 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,4 +1,38 @@ -# Procedure to Deploy +# Procedure to Run and Deploy + +## Install and Run Locally + +One can run ExploitIQ on his local machine ( No GPU dependency is required!), for the purpose of testing, debugging and troubleshooting problems: + +1. Install python version 3.12 by any wanted mean ( conda,pyenv, venv, poetry), and switch to that python environment. +2. Install lightweight [uv package manager](https://github.com/astral-sh/uv#installation) +3. inside the new python env, define the following environment variables: + +```shell +PYTHON_VERSION=3.12 +export CHECKLIST_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CODE_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CVE_AGENT_EXECUTOR_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export DOC_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export JUSTIFY_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export NVIDIA_API_BASE=http://YOUR_SELF_HOSTED_OPENAI_LLM_ADDRESSS/v1 +export PYTHONUNBUFFERED=1 +export SERPAPI_API_KEY=YOUR_SERPAPI_KEY +export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +``` +4. Install all dependencies ( including `aiq` binary): +```shell +# Make sure you're on the repo' root. + cd $(git rev-parse --show-toplevel) + uv venv --python ${PYTHON_VERSION} /workspace/.venv && uv sync +``` + +5. After a successfull completed installation, within the new python environment, run: +```shell +aiq --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 +``` + +## Deploy And Run On OCP 1. Create a `base/secrets.env` file containing the API keys for external services `ExploitIQ` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. @@ -103,18 +137,3 @@ kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=pers # Or, Delete Everything kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - ``` - -## Batch-processing Deployment Overlay - -When running in batch mode, it is important to use an optimized configuration that adjusts the following settings: - -- The maximum number of concurrent active requests in the queue. -- The maximum timeout (in minutes) before a report transitions from `sent` to expired if ExploitIQ processing is not completed within the specified interval. -- Compatibility with a self-hosted LLM that does not enforce rate limits, instead of relying on the NVIDIA-managed service, which may have limited API key credits. -- Disabling LLM caching in NGINX to ensure repeated inputs receive fresh responses, which is essential for batch analysis workflows. - -To apply this configuration, use the batch-processing overlay instead of the default deployment overlays - -```shell -kustomize build overlays/batch-processing/ | oc apply -f - -``` From 43ba4ac98ad8c7fbbcd536c53d5214c20db60ee3 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 15:02:18 +0300 Subject: [PATCH 037/286] chore: add Go Binary to container image Signed-off-by: Zvi Grinberg --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0f104b425..7ff062fba 100755 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,10 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && update-ca-certificates +RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24.1.linux-amd64.tar.gz \ + && tar -C /usr/local -xzf /tmp/go1.24.1.linux-amd64.tar.gz \ + && rm /tmp/go1.24.1.linux-amd64.tar.gz + # Set SSL environment variables ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt From 02bc587972419c13ff7c085b842e0da10987b42d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 15:03:12 +0300 Subject: [PATCH 038/286] chore: fix deployment configuration Signed-off-by: Zvi Grinberg --- kustomize/base/exploit-iq-config.yml | 16 ++++++++-------- kustomize/base/exploit_iq_client.yaml | 2 +- kustomize/base/exploit_iq_service.yaml | 4 ---- kustomize/base/nginx/nginx_cache.conf | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 87976ffdc..8ed6b4578 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -165,14 +165,14 @@ llms: max_tokens: 1024 top_p: 0.01 - intel_source_score_llm: - _type: ${LLM_TYPE_INTEL_SOURCE_SCORE:-nim} - api_key: $LLM_API_KEY_INTEL_SOURCE_SCORE:-"EMPTY"} - base_url: ${INTEL_SOURCE_SCORE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${INTEL_SOURCE_SCORE_MODEL_NAME:-meta/llama-3.1-70b-instruct} - temperature: 0.0 - max_tokens: 1024 - top_p: 0.01 + intel_source_score_llm: + _type: ${LLM_TYPE_INTEL_SOURCE_SCORE:-nim} + api_key: ${LLM_API_KEY_INTEL_SOURCE_SCORE:-"EMPTY"} + base_url: ${INTEL_SOURCE_SCORE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${INTEL_SOURCE_SCORE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index c73d4350d..9a60c060b 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -35,7 +35,7 @@ spec: containerPort: 8080 env: - name: QUARKUS_REST-CLIENT_MORPHEUS_URL - value: http://nginx-cache:8080/scan + value: http://nginx-cache:8080/generate - name: QUARKUS_MONGODB_HOSTS value: exploit-iq-client-db:27017 - name: QUARKUS_MONGODB_DATABASE diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 1388191aa..17847a658 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -61,12 +61,8 @@ spec: env: - name: SERPAPI_API_KEY value: EXPLOIT_IQ - - name: NVD_API_KEY - value: EXPLOIT_IQ - name: NVIDIA_API_KEY value: EXPLOIT_IQ - - name: GHSA_API_KEY - value: EXPLOIT_IQ - name: OPENAI_API_KEY value: EXPLOIT_IQ - name: NGC_API_KEY diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index 5c1153373..21ff562ee 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -89,7 +89,7 @@ http { ################ Docker Compose Services ################# - location /scan { + location /generate { proxy_pass http://exploit-iq:8080; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; From c549f729b486715518018fec190c5431de400dfb Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 15:15:59 +0300 Subject: [PATCH 039/286] refactor: change deployment labels name to exploit_iq Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 6 +++--- .tekton/on-push.yaml | 6 +++--- .tekton/on-tag.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index e6857566e..b6e244125 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -97,9 +97,9 @@ spec: --target base --layers --label=quay.expires-after=$(params.image-expires-after) - --label=agent_morpheus_vcs_branch={{source_branch}} - --label=agent_morpheus_vcs_build_commit_id={{revision}} - --label=agent_morpheus_vcs_repo={{repo_url}} + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} --format=docker taskRef: resolver: cluster diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml index f7c785d81..283b90350 100644 --- a/.tekton/on-push.yaml +++ b/.tekton/on-push.yaml @@ -91,9 +91,9 @@ spec: value: >- --target base --layers - --label=agent_morpheus_vcs_branch={{source_branch}} - --label=agent_morpheus_vcs_build_commit_id={{revision}} - --label=agent_morpheus_vcs_repo={{repo_url}} + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} --format=docker taskRef: resolver: cluster diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml index 5cf68adc5..b0a3ba181 100644 --- a/.tekton/on-tag.yaml +++ b/.tekton/on-tag.yaml @@ -119,9 +119,9 @@ spec: value: >- --target base --layers - --label=agent_morpheus_vcs_branch={{source_branch}} - --label=agent_morpheus_vcs_build_commit_id={{revision}} - --label=agent_morpheus_vcs_repo={{repo_url}} + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} --format=docker taskRef: resolver: cluster From dc0e0d199173a4e424c6b3b14b45030e5d0693a5 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 16:42:40 +0300 Subject: [PATCH 040/286] fix: make go binary to be on image path Signed-off-by: Zvi Grinberg --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7ff062fba..7f0f899e2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc # Enivronment variables for the venv -ENV PATH="/workspace/.venv/bin:$PATH" +ENV PATH="/workspace/.venv/bin:/usr/local/go/bin:$PATH" # Mark all git repos as safe to avoid git errors RUN echo $'\ From 2d2aec575a8223b2eb69bf6844994877e7062b64 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 16:55:04 +0300 Subject: [PATCH 041/286] fix: wrong parameter passing Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_http_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index 6c38d0594..c9764a9bf 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -53,7 +53,7 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: model_json = message.model_dump_json(by_alias=True) url = config.url + config.endpoint headers = {'Content-type': 'application/json', 'traceId': trace_id.get()} - auth_header = get_auth_header(config.token_path) + auth_header = get_auth_header(config) if auth_header is not None: headers['Authorization'] = auth_header verify = True From 63b6b9a2efd0c9598200c26fcb587a8740dfe2f9 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Aug 2025 18:36:24 +0300 Subject: [PATCH 042/286] chore: add phoenix tracing as side car chore: remove gpu node scheduling config from deployment Signed-off-by: Zvi Grinberg --- kustomize/base/exploit-iq-config.yml | 13 +++--- kustomize/base/exploit_iq_service.yaml | 60 ++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 8ed6b4578..e93c662df 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -16,12 +16,13 @@ general: use_uvloop: true -# telemetry: -# tracing: -# phoenix: -# _type: phoenix -# endpoint: ${PHOENIX_TRACING_URL:-http://localhost:6006/v1/traces} -# project: PROJECT_NAME_PLACEHOLDER + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent + functions: cve_generate_vdbs: diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 17847a658..b3fb803c9 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -19,15 +19,29 @@ spec: app: exploit-iq component: exploit-iq spec: - nodeSelector: - nvidia.com/gpu.deploy.driver: "true" imagePullSecrets: [] - tolerations: - - key: p4-gpu - operator: Exists - effect: NoSchedule serviceAccountName: exploit-iq-sa containers: + - name: exploit-iq-phoenix-tracing + image: quay.io/ecosystem-appeng/agent-morpheus-rh:nat + imagePullPolicy: Always + workingDir: /workspace/ + args: + - "phoenix" + - "serve" + securityContext: + runAsUser: 0 + ports: + - name: tracing + protocol: TCP + containerPort: 6006 + resources: + limits: + memory: "2Gi" + cpu: "500m" + requests: + memory: "1Gi" + cpu: "100m" - name: exploit-iq image: quay.io/ecosystem-appeng/agent-morpheus-rh:nat imagePullPolicy: Always @@ -53,11 +67,9 @@ spec: limits: memory: "8Gi" cpu: "1000m" - nvidia.com/gpu: "1" requests: memory: "1Gi" cpu: "1000m" - nvidia.com/gpu: "1" env: - name: SERPAPI_API_KEY value: EXPLOIT_IQ @@ -134,6 +146,38 @@ spec: component: exploit-iq --- apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-phoenix-tracing + labels: + app: exploit-iq + component: exploit-iq-tracing +spec: + ports: + - name: tracing + port: 6006 + protocol: TCP + targetPort: 6006 + selector: + app: exploit-iq + component: exploit-iq +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: exploit-iq + component: exploit-iq-tracing + name: exploit-iq-phoenix-tracing +spec: + port: + targetPort: tracing + to: + kind: Service + name: exploit-iq-phoenix-tracing + weight: 100 +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: exploit-iq-data From eac3e6e0b8caf8d523dfd44f6c0036da54925b3e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 18 Aug 2025 12:01:47 +0300 Subject: [PATCH 043/286] feat: generate CVSS for vulnerability analysis Signed-off-by: Ilona Shishov --- docker-compose.yml | 1 + .../configs/config-http-openai.yml | 33 +- src/vuln_analysis/configs/config-tracing.yml | 25 ++ src/vuln_analysis/configs/config.yml | 31 ++ src/vuln_analysis/data_models/output.py | 12 +- src/vuln_analysis/data_models/state.py | 3 + src/vuln_analysis/functions/cve_agent.py | 5 +- .../functions/cve_generate_cvss.py | 282 ++++++++++++++++++ src/vuln_analysis/register.py | 13 +- src/vuln_analysis/runtime_context.py | 10 + .../tools/container_image_analysis_data.py | 71 +++++ .../tools/lexical_full_search.py | 2 +- src/vuln_analysis/tools/local_vdb.py | 2 +- src/vuln_analysis/utils/llm_engine_utils.py | 43 ++- src/vuln_analysis/utils/prompting.py | 45 +++ uv.lock | 5 +- 16 files changed, 561 insertions(+), 22 deletions(-) create mode 100644 src/vuln_analysis/functions/cve_generate_cvss.py create mode 100644 src/vuln_analysis/runtime_context.py create mode 100644 src/vuln_analysis/tools/container_image_analysis_data.py diff --git a/docker-compose.yml b/docker-compose.yml index 76f8b29ca..d89b85d6b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,7 @@ services: - CODE_VDB_RETRIEVER_MODEL_NAME=${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - DOC_VDB_RETRIEVER_MODEL_NAME=${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - CVE_AGENT_EXECUTOR_MODEL_NAME=${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + - CVE_GENERATE_CVSS_MODEL_NAME=${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} - SUMMARIZE_MODEL_NAME=${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} - JUSTIFY_MODEL_NAME=${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} - EMBEDDER_MODEL_NAME=${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 3c85be747..50b030859 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -15,6 +15,12 @@ general: use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent telemetry: tracing: @@ -77,7 +83,8 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 - + Container Image Analysis Data: + _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -96,6 +103,21 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: cve_generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false cve_summarize: _type: cve_summarize llm_name: summarize_llm @@ -146,6 +168,14 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + cve_generate_cvss_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 summarize_llm: _type: openai api_key: "EMPTY" @@ -188,6 +218,7 @@ workflow: cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index fa465c43b..f261a5ba8 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -62,6 +62,8 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 + Container Image Analysis Data: + _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -77,6 +79,21 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: cve_generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false cve_summarize: _type: cve_summarize llm_name: summarize_llm @@ -124,6 +141,13 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + cve_generate_cvss_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 summarize_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -163,6 +187,7 @@ workflow: cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 6fd39d527..9e867292a 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -15,6 +15,12 @@ general: use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent functions: cve_generate_vdbs: @@ -52,6 +58,8 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 + Container Image Analysis Data: + _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -67,6 +75,21 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: cve_generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false cve_summarize: _type: cve_summarize llm_name: summarize_llm @@ -114,6 +137,13 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + cve_generate_cvss_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 summarize_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -153,6 +183,7 @@ workflow: cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index ab0a3b01f..ba262f32b 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -39,7 +39,15 @@ class ChecklistItemOutput(BaseModel): response: str intermediate_steps: list[AgentIntermediateStep] | None = None - +class CVSSOutput(BaseModel): + """ + CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image. + - vector_string: The CVSS vector string that encodes the metric values used to calculate the score. + - score: The calculated CVSS base score representing the severity of the vulnerability in the given image. + """ + vector_string: str + score: str + class JustificationOutput(BaseModel): """ Final justification for the vulnerability. @@ -63,12 +71,14 @@ class AgentMorpheusEngineOutput(BaseModel): - checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent. - summary: a short summary of the checklist inputs and responses, generated by an LLM. - justification: a JustificationOutput object containing details of the model's justification decision. + - cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability. """ vuln_id: str checklist: list[ChecklistItemOutput] summary: str justification: JustificationOutput intel_score: int + cvss: CVSSOutput | None class AgentMorpheusOutput(AgentMorpheusEngineInput): diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index 59cbb399e..3a52662a5 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -33,3 +33,6 @@ class AgentMorpheusEngineState(BaseModel): final_summaries: dict[str, str] = {} justifications: dict[str, dict[str, str]] = {} poor_quality_intel_vul: dict[str, int] = {} + cvss_results: dict[str, dict[str, str]] = {} + current_vuln_id: str | None = None + diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index d1971174c..6ea0cc882 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -14,7 +14,7 @@ # limitations under the License. import asyncio -import contextvars +from vuln_analysis.runtime_context import ctx_state import logging import typing from aiq.builder.builder import Builder @@ -36,9 +36,6 @@ logger = LoggingFactory.get_agent_logger(__name__) -ctx_state = contextvars.ContextVar("ctx_state", default="default_value") - - class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): """ Defines a function that iterates through checklist items using provided tools and gathered intel. diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py new file mode 100644 index 000000000..3f83e76e2 --- /dev/null +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from vuln_analysis.runtime_context import ctx_state +import logging +import json +import typing + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from langchain.agents import AgentExecutor +from langchain.agents import create_react_agent +from langchain.agents.agent import RunnableAgent +from langchain_core.exceptions import OutputParserException +from langchain_core.prompts import PromptTemplate +from pydantic import Field +from cvss import CVSS3 + +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.utils.prompting import get_cvss_prompt + +logger = logging.getLogger(__name__) + +OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" +CVSS_VECTOR_STRING = "vector_string" +CVSS_SCORE = "score" +DEFAULT_CVSS_METRICS_LIST = [ + { + "metric_name": "Attack Vector", + "metric_abbreviation": "AV", + "question": "Where can the attack be executed from?", + "values": [ + {"value": "Network", "abbreviation": "N", "definition": "Remote attack over the internet."}, + {"value": "Adjacent", "abbreviation": "A", "definition": "Requires local network access."}, + {"value": "Local", "abbreviation": "L", "definition": "Requires local access to the system."}, + {"value": "Physical", "abbreviation": "P", "definition": "Requires physical access."} + ] + }, + { + "metric_name": "Attack Complexity", + "metric_abbreviation": "AC", + "question": "How difficult is the attack to execute?", + "values": [ + {"value": "Low", "abbreviation": "L", "definition": "The attack is straightforward."}, + {"value": "High", "abbreviation": "H", "definition": "The attack requires special conditions."} + ] + }, + { + "metric_name": "Privileges Required", + "metric_abbreviation": "PR", + "question": "What level of access does the attacker need?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No privileges needed."}, + {"value": "Low", "abbreviation": "L", "definition": "Basic user privileges needed."}, + {"value": "High", "abbreviation": "H", "definition": "Administrative privileges needed."} + ] + }, + { + "metric_name": "User Interaction", + "metric_abbreviation": "UI", + "question": "Does the attack require user action?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No user interaction needed."}, + {"value": "Required", "abbreviation": "R", "definition": "Requires user action."} + ] + }, + { + "metric_name": "Scope", + "metric_abbreviation": "S", + "question": "Does the attack affect other components or systems?", + "values": [ + {"value": "Unchanged", "abbreviation": "U", "definition": "The impact is limited to the vulnerable system."}, + {"value": "Changed", "abbreviation": "C", "definition": "The impact extends beyond the vulnerable system."} + ] + }, + { + "metric_name": "Confidentiality Impact", + "metric_abbreviation": "C", + "question": "Does the attack expose sensitive data?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on data confidentiality."}, + {"value": "Low", "abbreviation": "L", "definition": "Partial exposure of non-sensitive data."}, + {"value": "High", "abbreviation": "H", "definition": "Full exposure of sensitive data."} + ] + }, + { + "metric_name": "Integrity Impact", + "metric_abbreviation": "I", + "question": "Can the attacker modify critical data?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on data integrity."}, + {"value": "Low", "abbreviation": "L", "definition": "Limited unauthorized modifications."}, + {"value": "High", "abbreviation": "H", "definition": "Complete control over data integrity."} + ] + }, + { + "metric_name": "Availability Impact", + "metric_abbreviation": "A", + "question": "Does the attack affect system uptime?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on system availability."}, + {"value": "Low", "abbreviation": "L", "definition": "Minor service degradation."}, + {"value": "High", "abbreviation": "H", "definition": "Complete system failure."} + ] + } +] + +class CVEGenerateCvssToolConfig(FunctionBaseConfig, name="cve_generate_cvss"): + """ + Defines a function that iterates through checklist items using provided tools and gathered intel. + """ + llm_name: str = Field(description="The LLM model to use with the CVE agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to CVE agent.") + max_concurrency: int | None = Field(ge=1, + default=None, + description="The maximum number of concurrent agent invocations.") + max_iterations: int = Field(default=10, description="The maximum number of iterations for the agent.") + prompt: str | None = Field( + default=None, + description= + "Manually set the prompt for the specific model in the configuration. The prompt can either be passed in as a " + "string of text or as a path to a text file containing the desired prompting.") + replace_exceptions: bool = Field(default=False, + description="Whether to replace exception message with custom message.") + replace_exceptions_value: str | None = Field(default=None, description="Message if replace_exceptions is true") + return_intermediate_steps: bool = Field( + default=False, + description= + "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") + verbose: bool = Field(default=False, description="Set to true for verbose output") + skip: bool = Field(default=False, + description="Whether or not the CVSS generator should be skipped.") + +def _make_parse_error_handler(is_openai: bool): + def handle_parse_error(exception: OutputParserException) -> str: + if is_openai: + logger.debug(f"Exception intercepted in notify {exception}") + exception_string = str(exception) + if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: + return (f"Check your output and make sure it conforms! Do not output an action " + f"and a final answer at the same time") + return "Check your output and make sure it conforms, use the Action/Action Input syntax" + return handle_parse_error + +async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, + state: AgentMorpheusEngineState) -> AgentExecutor: + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + is_openai = "openai" in llm.__class__.__module__.lower() + prompt = PromptTemplate.from_template(get_cvss_prompt(config.prompt, is_openai)) + + # Filter tools that are not available + tools = [ + tool for tool in tools + if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or + (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or + (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None)) + ] + + error_handler = _make_parse_error_handler(is_openai) + + agent = create_react_agent(llm=llm, + tools=tools, + prompt=prompt) + + agent_executor = AgentExecutor( + agent=agent, + tools=tools, + early_stopping_method="force", + handle_parsing_errors=error_handler, + max_iterations=config.max_iterations, + return_intermediate_steps=config.return_intermediate_steps, + verbose=config.verbose) + + # Disable streaming for accurate token counts + if isinstance(agent_executor.agent, RunnableAgent): + agent_executor.agent.stream_runnable = False + + return agent_executor + + +async def _run_for_vuln(agent: AgentExecutor, + state: AgentMorpheusEngineState, + vuln_id: str, + semaphore: asyncio.Semaphore | None) -> list: + state_copy = state.model_copy(deep=True) + state_copy.current_vuln_id = vuln_id + token = ctx_state.set(state_copy) + try: + async def _invoke_metric(metric: dict): + if semaphore: + async with semaphore: + return await agent.ainvoke({"input": metric}) + return await agent.ainvoke({"input": metric}) + + return await asyncio.gather(*(_invoke_metric(metric) for metric in DEFAULT_CVSS_METRICS_LIST)) + finally: + ctx_state.reset(token) + + +def _postprocess_results(results: list, + replace_exceptions: bool, + replace_exceptions_value: str | None) -> tuple[str, str]: + """ + Post-process per-metric results for a single CVE into a CVSS vector string and score. + Replace exceptions with placeholder values if configured. + """ + outputs: list[str] = [] + + for answer in results: + # Handle exceptions returned by the agent + # OutputParserException is not a subclass of Exception, so we need to check for it separately + if isinstance(answer, (OutputParserException, Exception)): + if replace_exceptions: + return "", replace_exceptions_value + else: + return "", f"Failed to generate a CVSS score for this analysis: {str(answer)}" + + outputs.append(answer["output"]) + + vector_string = f"CVSS:3.1/" + "/".join(outputs) + + try: + cvss_obj = CVSS3(vector_string) + score = str(cvss_obj.scores()[0]) # get base score + except Exception as e: + return vector_string, f'Failed to generate a CVSS score from vector string. Error: {e}' + + return vector_string, score + +@register_function(config_type=CVEGenerateCvssToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_generate_cvss(config: CVEGenerateCvssToolConfig, builder: Builder): + + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + + if config.skip: + logger.info("`config.skip` is set to True. Skipping CVSS generation.") + return state + + agent = await _create_agent(config, builder, state) + + results = await asyncio.gather(*( + _run_for_vuln(agent, state, vuln_id, semaphore) + for vuln_id in state.checklist_results.keys() + )) + + cvss_results = {} + + for vuln_id, result in zip(state.checklist_results.keys(), results): + vector_string, score_or_error = _postprocess_results(result, config.replace_exceptions, config.replace_exceptions_value) + + cvss_results[vuln_id] = { + CVSS_VECTOR_STRING: vector_string, + CVSS_SCORE: score_or_error + } + + state.cvss_results = cvss_results + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineState, + description=("Generates the CVSS (Common Vulnerability Scoring System) score and vector string for the vulnerability analysis results.")) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 08852b0c1..095e6b508 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -39,9 +39,11 @@ from vuln_analysis.functions import cve_justify from vuln_analysis.functions import cve_process_sbom from vuln_analysis.functions import cve_summarize +from vuln_analysis.functions import cve_generate_cvss from vuln_analysis.tools import lexical_full_search # This is actually registers the tool in the type registry of NAT! from vuln_analysis.tools import transitive_code_search +from vuln_analysis.tools import container_image_analysis_data from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp from vuln_analysis.utils.error_handling_decorator import catch_pipeline_errors_async @@ -64,6 +66,7 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): cve_check_vuln_deps_name: str = Field(description="Function name to check vulnerable dependencies") cve_checklist_name: str = Field(description="Function name to generate checklist") cve_agent_executor_name: str = Field(description="Function name to run CVE agent on checklist") + cve_generate_cvss_name: str = Field(description="Function name to generate CVSS") cve_summarize_name: str = Field(description="Function name to generate summary") cve_justify_name: str = Field(description="Function to generate justifications for each CVE") cve_output_config_name: str | None = Field(default=None, @@ -89,6 +92,7 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): cve_check_vuln_deps_fn = builder.get_function(name=config.cve_check_vuln_deps_name) cve_checklist_fn = builder.get_function(name=config.cve_checklist_name) cve_agent_executor_fn = builder.get_function(name=config.cve_agent_executor_name) + cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_summary_fn = builder.get_function(name=config.cve_summarize_name) cve_justify_fn = builder.get_function(name=config.cve_justify_name) cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None @@ -141,6 +145,11 @@ async def agent_executor_node(state: AgentMorpheusEngineState) -> AgentMorpheusE return await cve_agent_executor_fn.ainvoke(state.model_dump()) + async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Generates CVSS for the results of the execution""" + + return await cve_generate_cvss_fn.ainvoke(state.model_dump()) + async def summarize_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: """Summarizes the results of the execution""" @@ -169,12 +178,14 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput subgraph_builder = StateGraph(AgentMorpheusEngineState) subgraph_builder.add_node("checklist", checklist_node) subgraph_builder.add_node("agent_executor", agent_executor_node) + subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_node("summarize", summarize_node) subgraph_builder.add_node("justify", justify_node) subgraph_builder.add_edge(START, "checklist") subgraph_builder.add_edge("checklist", "agent_executor") - subgraph_builder.add_edge("agent_executor", "summarize") + subgraph_builder.add_edge("agent_executor", "generate_cvss") + subgraph_builder.add_edge("generate_cvss", "summarize") subgraph_builder.add_edge("summarize", "justify") subgraph = subgraph_builder.compile() diff --git a/src/vuln_analysis/runtime_context.py b/src/vuln_analysis/runtime_context.py new file mode 100644 index 000000000..4d896f4c5 --- /dev/null +++ b/src/vuln_analysis/runtime_context.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# A shared runtime context for passing workflow state to tools and functions. +# This must be imported by all modules that need to read or set the active state. + +import contextvars + +# Holds the current AgentMorpheusEngineState for the active task +ctx_state = contextvars.ContextVar("ctx_state", default="default_value") \ No newline at end of file diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py new file mode 100644 index 000000000..467a15926 --- /dev/null +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Any + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig + +logger = logging.getLogger(__name__) + + +class ContainerImageAnalysisDataToolConfig(FunctionBaseConfig, name="container_image_analysis_data"): + """ + Tool to retrieve pre-analyzed container image data produced earlier in the workflow. + It does not perform new analysis, it returns structured data gathered previously. + """ + + +@register_function(config_type=ContainerImageAnalysisDataToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def container_image_analysis_data(config: ContainerImageAnalysisDataToolConfig, builder: Builder): + + from vuln_analysis.runtime_context import ctx_state + + async def _arun(query: str) -> list[dict[str, Any]]: + state = ctx_state.get() + vuln_id = getattr(state, "current_vuln_id", None) + + results: list[dict[str, Any]] = [] + + if vuln_id is None: + return results + + try: + for item in state.checklist_results[vuln_id]: + results.append({ + "response": item.get("output"), + "intermediate_steps": item.get("intermediate_steps") + }) + except Exception as e: + logger.exception("Failed to read analysis data from state: %s", e) + return [] + + return results + + yield FunctionInfo.from_fn( + _arun, + description=( + "Useful when you need to retrieve information from the analysis data of a container image. " + "This tool does not perform new analysis, but only returns pre-analyzed data based on a prior scan. " + "This tool is especially useful when your task involves understanding how the container image may be " + "impacted by a reported CVE. The tool provides structured information about the container image's " + "source code. The tool returns a list of analysis results, where each item in the list is an object " + "describing a specific finding. Each object in the list includes a 'response' field containing a " + "concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may " + "contain reasoning or supporting details if not null.")) \ No newline at end of file diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index f570bc002..c5b6da717 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -35,7 +35,7 @@ class LexicalSearchToolConfig(FunctionBaseConfig, name="lexical_code_search"): @register_function(config_type=LexicalSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def lexical_search(config: LexicalSearchToolConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.functions.cve_agent import ctx_state + from vuln_analysis.runtime_context import ctx_state from vuln_analysis.utils.full_text_search import FullTextSearch async def _arun(query: str) -> list: diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 76b889b0c..737104441 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -45,7 +45,7 @@ async def load_vectordb_asretriever(config: LocalVDBRetrieverToolConfig, builder from langchain_community.vectorstores import FAISS from langchain_core.prompts import PromptTemplate - from vuln_analysis.functions.cve_agent import ctx_state + from vuln_analysis.runtime_context import ctx_state embedder = await builder.get_embedder(embedder_name=config.embedder_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 87735f6b2..3160625f9 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -24,6 +24,8 @@ from vuln_analysis.data_models.output import AgentMorpheusOutput from vuln_analysis.data_models.output import ChecklistItemOutput from vuln_analysis.data_models.output import JustificationOutput +from vuln_analysis.data_models.output import CVSSOutput +from vuln_analysis.functions.cve_generate_cvss import CVSS_VECTOR_STRING, CVSS_SCORE from vuln_analysis.data_models.state import AgentMorpheusEngineState from aiq.builder.builder import Builder @@ -92,7 +94,8 @@ def parse_agent_morpheus_engine_output(vuln_id: str, checklist_results: list[dict[str, typing.Any]], summary: str, justification: dict[str, str], - intel_score: int) -> AgentMorpheusEngineOutput: + intel_score: int, + cvss: dict[str, str] | None) -> AgentMorpheusEngineOutput: """ Parse the output fields for a single vulnerability into an AgentMorpheusEngineOutput object. """ @@ -107,11 +110,19 @@ def parse_agent_morpheus_engine_output(vuln_id: str, reason=justification["justification"], status=justification["affected_status"]) + # Combine CVSS model outputs into a single CVSSOutput object + if cvss: + cvss_output = CVSSOutput(vector_string=cvss[CVSS_VECTOR_STRING], + score=cvss[CVSS_SCORE]) + else: + cvss_output=None + return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=checklist_output, summary=summary, justification=justification_output, - intel_score=intel_score) + intel_score=intel_score, + cvss=cvss_output) def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -121,6 +132,8 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: justification = JustificationOutput(label="insufficient_intel", reason="Insufficient intel available for CVE", status="UNKNOWN") + cvss = None + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[ @@ -130,7 +143,8 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: ], summary=summary, justification=justification, - intel_score=0) + intel_score=0, + cvss=cvss) def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -139,6 +153,8 @@ def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: justification = JustificationOutput(label="false_positive", reason="No vulnerable packages or dependencies were detected in the SBOM.", status="FALSE") + cvss = None + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[ @@ -149,7 +165,8 @@ def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: ], summary=summary, justification=justification, - intel_score=0) + intel_score=0, + cvss=cvss) def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -159,6 +176,8 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: justification = JustificationOutput(label="no_sbom_packages", reason="No SBOM packages found for image.", status="UNKNOWN") + cvss = None + return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=[ ChecklistItemOutput( @@ -168,7 +187,8 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: ], summary=summary, justification=justification, - intel_score=0) + intel_score=0, + cvss=cvss) def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheusEngineOutput: @@ -177,12 +197,15 @@ def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheu justification = JustificationOutput(label="poor_quality_intel", reason="Poor quality intel for CVE", status="UNKNOWN") + cvss = None + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[], summary=summary, justification=justification, - intel_score=intel_score + intel_score=intel_score, + cvss=cvss ) @@ -223,7 +246,8 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, checklist_results=result.checklist_results[vuln_id], summary=result.final_summaries[vuln_id], justification=result.justifications[vuln_id], - intel_score=intel_map[vuln_id].intel_score)) + intel_score=intel_map[vuln_id].intel_score, + cvss=result.cvss_results.get(vuln_id, None))) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) elif vuln_id in poor_quality_intel_vul: @@ -234,10 +258,11 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, assert False, "CVE has vulnerable dependencies but there is no workflow output." for out in output: - logger.info("Vulnerability '%s' affected status: %s. Label: %s", + logger.info("Vulnerability '%s' affected status: %s. Label: %s. CVSS score: %s", out.vuln_id, out.justification.status, - out.justification.label) + out.justification.label, + out.cvss.score if out.cvss else "-") return AgentMorpheusOutput(input=message.input, info=message.info, output=output) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index fac1b02e7..d23d9755a 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -34,6 +34,11 @@ "Exposures (CVE) on container images. Information about the container image under investigation is " "stored in vector databases available to you via tools.") +CVSS_SYS_PROMPT = ( + "You are a very powerful and highly knowledgeable cybersecurity assistant specializing in Common Vulnerabilities and Exposures (CVE) analysis. " + "Your task is to determine the correct CVSS (Common Vulnerability Scoring System) metric value to reflect the impact of a reported CVE on a container image, selecting the most appropriate value from a predefined list." +) + # Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. You have access to the following tools: @@ -121,6 +126,46 @@ def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = Fals return f'{sys_prompt} {prompt_template}' +def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> str: + """ + Get the CVSS prompt based on the system prompt and whether to include examples. + """ + + sys_prompt = sys_prompt or CVSS_SYS_PROMPT + + prompt_template = AGENT_PROMPT_TEMPLATE.replace( + "Question: the input question you must answer", + "Metric: the input metric you are evaluating in the following format: metric_name (metric_abbreviation)\n" + "Question: the input question you must answer" + ).replace( + "Question: the input question you must answer", + "Question: the input question you must answer.\n" + "To answer the following question, use the tools available to you to select the most appropriate CVSS metric value.\n" + "The Container Image Analysis Data should be your primary reference for making this selection.\n" + "If the container image analysis data does not contain the necessary information to make a selection, use other tools available to you to gather the required information from the container image.\n" + "Your goal is to make the most accurate selection of the CVSS metric value based on the available evidence. Analyze and reason through the data you have access to — do not guess or default prematurely.\n" + "Only if you have thoroughly evaluated all available tools and resources, and still cannot make a confident selection, may you choose the least impact option as a fallback." + "IMPORTANT: You must not call the same tool with the same input twice. Always try a different input or tool if your first attempt did not get a definitive answer.\n\n", + 1 + ).replace( + "Use the following format (start each response with one of the following prefixes: [Question, Thought, Action, Action Input, Final Answer]):", + "Use the following format (start each response with one of the following prefixes followed by a colon and a line break: " + "[Metric, Question, Thought, Action, Action Input, Final Answer]):", + 1 + ).replace( + "Final Answer: the final answer to the original input question", + "Final Answer: the final answer to the original input question. The final answer must follow this format: :\n\n" + "IMPORTANT: After every Thought, you must provide an Action and Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed." + ) + + if is_openai: + prompt_template = prompt_template.replace("Final Answer: the final answer to the original input question. The final answer must follow this format: :", + "Final Answer: the final answer to the original input question. The final answer must follow this format: : \n" + "Do not include Final Answer together" + " with Action or with Action Input in the same response" + , 1) + + return f'{sys_prompt} {prompt_template}' class PromptBuilder(ABC): diff --git a/uv.lock b/uv.lock index e179dc1f2..b0e93d239 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", @@ -8,9 +8,6 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'darwin'", ] -[options] -prerelease-mode = "allow" - [[package]] name = "aioboto3" version = "15.0.0" From c989c11f66d55adfe528ddcd75f6d111346959e4 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 21 Aug 2025 14:04:42 +0300 Subject: [PATCH 044/286] chore: fine tune prompt and configurations Signed-off-by: Ilona Shishov --- .../configs/config-http-openai.yml | 2 +- src/vuln_analysis/configs/config-tracing.yml | 2 +- src/vuln_analysis/configs/config.yml | 2 +- .../functions/cve_generate_cvss.py | 82 +++++++++++++------ src/vuln_analysis/register.py | 20 ++--- .../tools/container_image_analysis_data.py | 15 ++-- src/vuln_analysis/utils/prompting.py | 55 +++++++------ 7 files changed, 107 insertions(+), 71 deletions(-) diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 50b030859..65776c18b 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -174,7 +174,7 @@ llms: base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 - max_tokens: 2000 + max_tokens: 1024 top_p: 0.01 summarize_llm: _type: openai diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index f261a5ba8..f5a0404da 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -146,7 +146,7 @@ llms: base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 - max_tokens: 2000 + max_tokens: 1024 top_p: 0.01 summarize_llm: _type: nim diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 9e867292a..440fc4480 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -142,7 +142,7 @@ llms: base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 - max_tokens: 2000 + max_tokens: 1024 top_p: 0.01 summarize_llm: _type: nim diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index 3f83e76e2..66f27745f 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -18,6 +18,7 @@ import logging import json import typing +import re from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum @@ -31,6 +32,7 @@ from langchain_core.prompts import PromptTemplate from pydantic import Field from cvss import CVSS3 +from langchain.agents.mrkl.output_parser import MRKLOutputParser from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.prompting import get_cvss_prompt @@ -38,18 +40,21 @@ logger = logging.getLogger(__name__) OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" +OUTPUT_CONTAIN_PARSING_ERROR = "Could not parse" + CVSS_VECTOR_STRING = "vector_string" CVSS_SCORE = "score" + DEFAULT_CVSS_METRICS_LIST = [ { "metric_name": "Attack Vector", "metric_abbreviation": "AV", "question": "Where can the attack be executed from?", "values": [ - {"value": "Network", "abbreviation": "N", "definition": "Remote attack over the internet."}, - {"value": "Adjacent", "abbreviation": "A", "definition": "Requires local network access."}, - {"value": "Local", "abbreviation": "L", "definition": "Requires local access to the system."}, - {"value": "Physical", "abbreviation": "P", "definition": "Requires physical access."} + {"value": "Network", "abbreviation": "N", "definition": "Remote attack over the internet.", "least_impact": False}, + {"value": "Adjacent", "abbreviation": "A", "definition": "Requires local network access.", "least_impact": False}, + {"value": "Local", "abbreviation": "L", "definition": "Requires local access to the system.", "least_impact": False}, + {"value": "Physical", "abbreviation": "P", "definition": "Requires physical access.", "least_impact": True} ] }, { @@ -57,8 +62,8 @@ "metric_abbreviation": "AC", "question": "How difficult is the attack to execute?", "values": [ - {"value": "Low", "abbreviation": "L", "definition": "The attack is straightforward."}, - {"value": "High", "abbreviation": "H", "definition": "The attack requires special conditions."} + {"value": "Low", "abbreviation": "L", "definition": "The attack is straightforward.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "The attack requires special conditions.", "least_impact": True} ] }, { @@ -66,9 +71,9 @@ "metric_abbreviation": "PR", "question": "What level of access does the attacker need?", "values": [ - {"value": "None", "abbreviation": "N", "definition": "No privileges needed."}, - {"value": "Low", "abbreviation": "L", "definition": "Basic user privileges needed."}, - {"value": "High", "abbreviation": "H", "definition": "Administrative privileges needed."} + {"value": "None", "abbreviation": "N", "definition": "No privileges needed.", "least_impact": False}, + {"value": "Low", "abbreviation": "L", "definition": "Basic user privileges needed.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Administrative privileges needed.", "least_impact": True} ] }, { @@ -76,8 +81,8 @@ "metric_abbreviation": "UI", "question": "Does the attack require user action?", "values": [ - {"value": "None", "abbreviation": "N", "definition": "No user interaction needed."}, - {"value": "Required", "abbreviation": "R", "definition": "Requires user action."} + {"value": "None", "abbreviation": "N", "definition": "No user interaction needed.", "least_impact": False}, + {"value": "Required", "abbreviation": "R", "definition": "Requires user action.", "least_impact": True} ] }, { @@ -85,8 +90,10 @@ "metric_abbreviation": "S", "question": "Does the attack affect other components or systems?", "values": [ - {"value": "Unchanged", "abbreviation": "U", "definition": "The impact is limited to the vulnerable system."}, - {"value": "Changed", "abbreviation": "C", "definition": "The impact extends beyond the vulnerable system."} + {"value": "Unchanged", "abbreviation": "U", + "definition": "The impact is limited to the vulnerable system.", "least_impact": True}, + {"value": "Changed", "abbreviation": "C", + "definition": "The impact extends beyond the vulnerable system.", "least_impact": False} ] }, { @@ -94,9 +101,9 @@ "metric_abbreviation": "C", "question": "Does the attack expose sensitive data?", "values": [ - {"value": "None", "abbreviation": "N", "definition": "No impact on data confidentiality."}, - {"value": "Low", "abbreviation": "L", "definition": "Partial exposure of non-sensitive data."}, - {"value": "High", "abbreviation": "H", "definition": "Full exposure of sensitive data."} + {"value": "None", "abbreviation": "N", "definition": "No impact on data confidentiality.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Partial exposure of non-sensitive data.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Full exposure of sensitive data.", "least_impact": False} ] }, { @@ -104,9 +111,9 @@ "metric_abbreviation": "I", "question": "Can the attacker modify critical data?", "values": [ - {"value": "None", "abbreviation": "N", "definition": "No impact on data integrity."}, - {"value": "Low", "abbreviation": "L", "definition": "Limited unauthorized modifications."}, - {"value": "High", "abbreviation": "H", "definition": "Complete control over data integrity."} + {"value": "None", "abbreviation": "N", "definition": "No impact on data integrity.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Limited unauthorized modifications.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Complete control over data integrity.", "least_impact": False} ] }, { @@ -114,9 +121,9 @@ "metric_abbreviation": "A", "question": "Does the attack affect system uptime?", "values": [ - {"value": "None", "abbreviation": "N", "definition": "No impact on system availability."}, - {"value": "Low", "abbreviation": "L", "definition": "Minor service degradation."}, - {"value": "High", "abbreviation": "H", "definition": "Complete system failure."} + {"value": "None", "abbreviation": "N", "definition": "No impact on system availability.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Minor service degradation.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Complete system failure.", "least_impact": False} ] } ] @@ -155,7 +162,13 @@ def handle_parse_error(exception: OutputParserException) -> str: if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: return (f"Check your output and make sure it conforms! Do not output an action " f"and a final answer at the same time") - return "Check your output and make sure it conforms, use the Action/Action Input syntax" + if OUTPUT_CONTAIN_PARSING_ERROR in exception_string: + return ("Check your output and make sure it conforms. " + "Do not execute any tool. " + "Do not repeat Metric or Question. " + "Resume from your last step. " + "Begin your next token with 'Thought:' and re-emit the same content with only formatting fixes.") + return "Check your output and make sure it conforms" return handle_parse_error async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, @@ -177,7 +190,8 @@ async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, agent = create_react_agent(llm=llm, tools=tools, - prompt=prompt) + prompt=prompt, + output_parser=MRKLOutputParser()) agent_executor = AgentExecutor( agent=agent, @@ -202,12 +216,22 @@ async def _run_for_vuln(agent: AgentExecutor, state_copy = state.model_copy(deep=True) state_copy.current_vuln_id = vuln_id token = ctx_state.set(state_copy) + + # Uncomment to enable default to least impact value for non-vulnerable components + # is_vulnerable = state.justifications[vuln_id]['justification_label'] == 'vulnerable' + try: async def _invoke_metric(metric: dict): + # Uncomment to enable default to least impact value for non-vulnerable components + # if (not is_vulnerable) and (metric['metric_abbreviation'] != 'AV'): + # least_impact_value = next((v for v in metric['values'] if v.get('least_impact')), None) + # abbr = least_impact_value['abbreviation'] if least_impact_value else "" + # return {"output": f"{metric['metric_abbreviation']}:{abbr}"} + if semaphore: async with semaphore: - return await agent.ainvoke({"input": metric}) - return await agent.ainvoke({"input": metric}) + return await agent.ainvoke({"input": json.dumps(metric, ensure_ascii=False)}) + return await agent.ainvoke({"input": json.dumps(metric, ensure_ascii=False)}) return await asyncio.gather(*(_invoke_metric(metric) for metric in DEFAULT_CVSS_METRICS_LIST)) finally: @@ -223,6 +247,7 @@ def _postprocess_results(results: list, """ outputs: list[str] = [] + pattern = r'\s*[A-Z\s]+\s*:\s*[A-Z\s]+\s*' for answer in results: # Handle exceptions returned by the agent # OutputParserException is not a subclass of Exception, so we need to check for it separately @@ -232,7 +257,10 @@ def _postprocess_results(results: list, else: return "", f"Failed to generate a CVSS score for this analysis: {str(answer)}" - outputs.append(answer["output"]) + output = answer["output"] + if re.fullmatch(pattern, output): + output = output.replace(" ", "") + outputs.append(output) vector_string = f"CVSS:3.1/" + "/".join(outputs) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 095e6b508..179e70db6 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -66,9 +66,9 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): cve_check_vuln_deps_name: str = Field(description="Function name to check vulnerable dependencies") cve_checklist_name: str = Field(description="Function name to generate checklist") cve_agent_executor_name: str = Field(description="Function name to run CVE agent on checklist") - cve_generate_cvss_name: str = Field(description="Function name to generate CVSS") cve_summarize_name: str = Field(description="Function name to generate summary") cve_justify_name: str = Field(description="Function to generate justifications for each CVE") + cve_generate_cvss_name: str = Field(description="Function name to generate CVSS") cve_output_config_name: str | None = Field(default=None, description="Function to output workflow results " "(e.g. cve_file_output, cve_http_output). " @@ -92,9 +92,9 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): cve_check_vuln_deps_fn = builder.get_function(name=config.cve_check_vuln_deps_name) cve_checklist_fn = builder.get_function(name=config.cve_checklist_name) cve_agent_executor_fn = builder.get_function(name=config.cve_agent_executor_name) - cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_summary_fn = builder.get_function(name=config.cve_summarize_name) cve_justify_fn = builder.get_function(name=config.cve_justify_name) + cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None # Define langgraph node functions @@ -145,11 +145,6 @@ async def agent_executor_node(state: AgentMorpheusEngineState) -> AgentMorpheusE return await cve_agent_executor_fn.ainvoke(state.model_dump()) - async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: - """Generates CVSS for the results of the execution""" - - return await cve_generate_cvss_fn.ainvoke(state.model_dump()) - async def summarize_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: """Summarizes the results of the execution""" @@ -160,6 +155,11 @@ async def justify_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineSt return await cve_justify_fn.ainvoke(state.model_dump()) + async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Generates CVSS for the results of the execution""" + + return await cve_generate_cvss_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def add_completed_time_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Adds the completed time to the output""" @@ -178,15 +178,15 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput subgraph_builder = StateGraph(AgentMorpheusEngineState) subgraph_builder.add_node("checklist", checklist_node) subgraph_builder.add_node("agent_executor", agent_executor_node) - subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_node("summarize", summarize_node) subgraph_builder.add_node("justify", justify_node) + subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_edge(START, "checklist") subgraph_builder.add_edge("checklist", "agent_executor") - subgraph_builder.add_edge("agent_executor", "generate_cvss") - subgraph_builder.add_edge("generate_cvss", "summarize") + subgraph_builder.add_edge("agent_executor", "summarize") subgraph_builder.add_edge("summarize", "justify") + subgraph_builder.add_edge("justify", "generate_cvss") subgraph = subgraph_builder.compile() @catch_pipeline_errors_async diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py index 467a15926..5b138b52c 100644 --- a/src/vuln_analysis/tools/container_image_analysis_data.py +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -62,10 +62,11 @@ async def _arun(query: str) -> list[dict[str, Any]]: _arun, description=( "Useful when you need to retrieve information from the analysis data of a container image. " - "This tool does not perform new analysis, but only returns pre-analyzed data based on a prior scan. " - "This tool is especially useful when your task involves understanding how the container image may be " - "impacted by a reported CVE. The tool provides structured information about the container image's " - "source code. The tool returns a list of analysis results, where each item in the list is an object " - "describing a specific finding. Each object in the list includes a 'response' field containing a " - "concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may " - "contain reasoning or supporting details if not null.")) \ No newline at end of file + "This tool does not require an input; it uses context to retrieve pre-analyzed data based on a prior scan. " + "This tool is especially useful when your task involves understanding how the container image may be impacted by a reported CVE. " + "The tool provides structured information about the container image's source code and context on the reported CVE. " + "The tool returns an object with two fields:\n" + "- vulnerability_context: a list of context statements on the reported CVE (or `null` if none are provided).\n" + "- analysis_data: a list of the container image's source code analysis results, where each item in the list is an object describing a specific finding. " + "Each object in the list includes a 'response' field containing a concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may contain reasoning or supporting details if not null. ")) + \ No newline at end of file diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index d23d9755a..7a46b40ab 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -133,39 +133,46 @@ def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> s sys_prompt = sys_prompt or CVSS_SYS_PROMPT + guidance = [ + "Guidance:\n" + "- At the very beginning, you MUST call the Container Image Analysis Data tool exactly once.\n", + "- After the initial call, you MUST NOT call the Container Image Analysis Data tool again. Reuse the Observation from the first call for all subsequent reasoning and decisions.\n", + "- To answer the question, use the Container Image Analysis Data (retrieved in the first step) as your primary reference for this metric in order to seleect the most appropriate CVSS metric value.\n", + "- If the container image analysis data does not contain sufficient information to make a selection, use other tools available to you to gather the information you require from the container image.\n", + "- Analyze and reason through the data you have access to. Make the most accurate selection based on the available evidence.\n", + "- Only if you have thoroughly evaluated all available tools, and still cannot make a confident selection, may you choose the option where least_impact: true as a fallback.\n", + "- When you reach the final Thought, always include a brief rationale explaining why the selected value follows from prior Observations. Do not call tools in this Thought.\n", + "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your first attempt did not get a definitive answer.\n", + "- REMINDER: In your final Thought, end with exactly: Selected: (). Example: Selected: None (N), Selected: Low (L). Then output Final Answer: :. Do not call tools in this Thought.\n", + "- REMINDER: The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", + "- PROHIBITED: Never output `Action: None`. If you are ready to provide the Final Answer, you may omit Action/Action Input only for the last step.\n", + "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", + *( ["- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n"] if is_openai else [] ), + "- IMPORTANT: After every Thought, you must provide an Action and Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", + "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :\n\n", + "Begin!\n\n" + ] + prompt_template = AGENT_PROMPT_TEMPLATE.replace( "Question: the input question you must answer", - "Metric: the input metric you are evaluating in the following format: metric_name (metric_abbreviation)\n" - "Question: the input question you must answer" - ).replace( - "Question: the input question you must answer", - "Question: the input question you must answer.\n" - "To answer the following question, use the tools available to you to select the most appropriate CVSS metric value.\n" - "The Container Image Analysis Data should be your primary reference for making this selection.\n" - "If the container image analysis data does not contain the necessary information to make a selection, use other tools available to you to gather the required information from the container image.\n" - "Your goal is to make the most accurate selection of the CVSS metric value based on the available evidence. Analyze and reason through the data you have access to — do not guess or default prematurely.\n" - "Only if you have thoroughly evaluated all available tools and resources, and still cannot make a confident selection, may you choose the least impact option as a fallback." - "IMPORTANT: You must not call the same tool with the same input twice. Always try a different input or tool if your first attempt did not get a definitive answer.\n\n", - 1 + "Question: the input question you must answer\n" + "Metric: the input metric you are evaluating in the following format: metric_name (metric_abbreviation)" ).replace( "Use the following format (start each response with one of the following prefixes: [Question, Thought, Action, Action Input, Final Answer]):", "Use the following format (start each response with one of the following prefixes followed by a colon and a line break: " - "[Metric, Question, Thought, Action, Action Input, Final Answer]):", + "[Question, Metric, Thought, Action, Action Input, Final Answer]):", 1 ).replace( - "Final Answer: the final answer to the original input question", - "Final Answer: the final answer to the original input question. The final answer must follow this format: :\n\n" - "IMPORTANT: After every Thought, you must provide an Action and Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed." + "Thought: I now know the final answer", + "Thought: I now know the final answer because " + "I MUST ALWAYS end my final Thought with exactly: Selected: ().\n" + "and my Final Answer must reuse that same verbatim.\n" + ).replace( + "Begin!", + "".join(guidance) ) - - if is_openai: - prompt_template = prompt_template.replace("Final Answer: the final answer to the original input question. The final answer must follow this format: :", - "Final Answer: the final answer to the original input question. The final answer must follow this format: : \n" - "Do not include Final Answer together" - " with Action or with Action Input in the same response" - , 1) - return f'{sys_prompt} {prompt_template}' + return f'{sys_prompt}\n\n{prompt_template}' class PromptBuilder(ABC): From 64e929f1c5d200a9ed7e2b3222340e9224e73a1e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 21 Aug 2025 17:44:57 +0300 Subject: [PATCH 045/286] chore: fine tune cvss prompt and add examples Signed-off-by: Ilona Shishov --- .../functions/cve_generate_cvss.py | 6 +- src/vuln_analysis/utils/prompting.py | 107 +++++++++++++----- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index 66f27745f..384bebdb2 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -167,7 +167,11 @@ def handle_parse_error(exception: OutputParserException) -> str: "Do not execute any tool. " "Do not repeat Metric or Question. " "Resume from your last step. " - "Begin your next token with 'Thought:' and re-emit the same content with only formatting fixes.") + "Begin your next token with 'Thought:'. Fix format only: " + "- NEVER output 'Action: None'.\n." + "- Do NOT call 'Container Image Analysis Data' again; reuse its previous Observation.\n" + "- Final Thought must end with: Selected: ().\n" + "- Final Answer must be: : (reuse the same abbreviation).") return "Check your output and make sure it conforms" return handle_parse_error diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 7a46b40ab..90fe0904c 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -34,11 +34,6 @@ "Exposures (CVE) on container images. Information about the container image under investigation is " "stored in vector databases available to you via tools.") -CVSS_SYS_PROMPT = ( - "You are a very powerful and highly knowledgeable cybersecurity assistant specializing in Common Vulnerabilities and Exposures (CVE) analysis. " - "Your task is to determine the correct CVSS (Common Vulnerability Scoring System) metric value to reflect the impact of a reported CVE on a container image, selecting the most appropriate value from a predefined list." -) - # Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. You have access to the following tools: @@ -112,7 +107,78 @@ """ +CVSS_SYS_PROMPT = ( + "You are a very powerful and highly knowledgeable cybersecurity assistant specializing in Common Vulnerabilities and Exposures (CVE) analysis. " + "Your task is to determine the correct CVSS (Common Vulnerability Scoring System) metric value to reflect the impact of a reported CVE on a container image, selecting the most appropriate value from a predefined list." +) + +# Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py +CVSS_PROMPT_TEMPLATE = """You have access to the following tools: + +{tools} + +Use the following format (start each response with one of the following prefixes followed by a colon and a line break: [Question, Metric, Thought, Action, Action Input, Final Answer]): +Question: the input question you must answer +Metric: the input metric you are evaluating in the following format: () +Thought: you should always think about what to do +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer because . I MUST ALWAYS end my final Thought with exactly: Selected: (). +Final Answer: the final answer to the original input question in the following format: : + +Begin! + +Input: {input} +{agent_scratchpad}""" + +CVSS_EXAMPLES_FOR_PROMPT = """Example 1: + + Question: + Metric: Scope (S) + Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE + Action: Container Image Analysis Data + Action Input: + Observation: + Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, I will use the Lexical Search Container Image Code QA System to search for further information + Action: Lexical Search Container Image Code QA System + Action Input: + Observation: + Thought: I now know the answer because , I can now make a confident selection. + Selected: Changed (C) + Final Answer: S:C + + Example 2: + + Question: + Metric: Attack Vector (AV) + Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE + Action: Container Image Analysis Data + Action Input: + Observation: + Thought: I now know the answer because , I can now make a confident selection. + Selected: Network (N) + Final Answer: AV:N + + Example 3: + + Question: + Metric: Privilege Required (PR) + Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE + Action: Container Image Analysis Data + Action Input: + Observation: + Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, However, it does not provide enough information to make a confident selection. I need to gather more information, I will use the Lexical Search Container Image Code QA System to search for further information. + Action: Lexical Search Container Image Code QA System + Action Input: + Observation: + Thought: I now know the answer because , but due to insuficient information, it is still unclear what the correct selection should be, therefore, I will select the value with `least_impact: true` as a fallback. + Selected: High (H) + Final Answer: PR:H + +""" def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = False) -> str: """ Get the agent prompt based on the system prompt and whether to include examples. @@ -139,38 +205,21 @@ def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> s "- After the initial call, you MUST NOT call the Container Image Analysis Data tool again. Reuse the Observation from the first call for all subsequent reasoning and decisions.\n", "- To answer the question, use the Container Image Analysis Data (retrieved in the first step) as your primary reference for this metric in order to seleect the most appropriate CVSS metric value.\n", "- If the container image analysis data does not contain sufficient information to make a selection, use other tools available to you to gather the information you require from the container image.\n", + "- ABSOLUTE RULE: `Action: None` output is forbidden. Do not output `Action: None` under any circumstances.\n", "- Analyze and reason through the data you have access to. Make the most accurate selection based on the available evidence.\n", "- Only if you have thoroughly evaluated all available tools, and still cannot make a confident selection, may you choose the option where least_impact: true as a fallback.\n", - "- When you reach the final Thought, always include a brief rationale explaining why the selected value follows from prior Observations. Do not call tools in this Thought.\n", + "- REMINDER: When you reach the final Thought, always include a brief rationale explaining why the selected value follows from prior Observations. Do not call tools in this Thought.\n", "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your first attempt did not get a definitive answer.\n", "- REMINDER: In your final Thought, end with exactly: Selected: (). Example: Selected: None (N), Selected: Low (L). Then output Final Answer: :. Do not call tools in this Thought.\n", - "- REMINDER: The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", - "- PROHIBITED: Never output `Action: None`. If you are ready to provide the Final Answer, you may omit Action/Action Input only for the last step.\n", + "- The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", + "- If you are ready to provide the Final Answer, you may omit Action/Action Input only for the last step.\n", "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", *( ["- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n"] if is_openai else [] ), - "- IMPORTANT: After every Thought, you must provide an Action and Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", - "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :\n\n", - "Begin!\n\n" + "- IMPORTANT: After every Thought, you MUST provide an Action AND Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", + "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :" ] - prompt_template = AGENT_PROMPT_TEMPLATE.replace( - "Question: the input question you must answer", - "Question: the input question you must answer\n" - "Metric: the input metric you are evaluating in the following format: metric_name (metric_abbreviation)" - ).replace( - "Use the following format (start each response with one of the following prefixes: [Question, Thought, Action, Action Input, Final Answer]):", - "Use the following format (start each response with one of the following prefixes followed by a colon and a line break: " - "[Question, Metric, Thought, Action, Action Input, Final Answer]):", - 1 - ).replace( - "Thought: I now know the final answer", - "Thought: I now know the final answer because " - "I MUST ALWAYS end my final Thought with exactly: Selected: ().\n" - "and my Final Answer must reuse that same verbatim.\n" - ).replace( - "Begin!", - "".join(guidance) - ) + prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + CVSS_EXAMPLES_FOR_PROMPT + "\n\n" + "Begin!\n\n") return f'{sys_prompt}\n\n{prompt_template}' From 999b5e77fcb801fb7bc0c7352f12dd6a615347f3 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 26 Aug 2025 11:08:33 +0300 Subject: [PATCH 046/286] chore: fine tune cvss prompt Signed-off-by: Ilona Shishov --- .../configs/config-http-openai.yml | 12 ---- .../functions/cve_generate_cvss.py | 4 +- src/vuln_analysis/utils/prompting.py | 57 +++++++++++++------ 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 65776c18b..e2e62bd6d 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -22,18 +22,6 @@ general: endpoint: http://localhost:6006/v1/traces project: cve_agent - telemetry: - tracing: - phoenix: - _type: phoenix - endpoint: http://localhost:6006/v1/traces - project: cve_agent -# tracing: -# langsmith: -# _type: langsmith -# project: default -# api_key: ${LANGSMITH_API_KEY} - functions: cve_generate_vdbs: _type: cve_generate_vdbs diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index 384bebdb2..d746eeca6 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -168,8 +168,8 @@ def handle_parse_error(exception: OutputParserException) -> str: "Do not repeat Metric or Question. " "Resume from your last step. " "Begin your next token with 'Thought:'. Fix format only: " - "- NEVER output 'Action: None'.\n." - "- Do NOT call 'Container Image Analysis Data' again; reuse its previous Observation.\n" + "- NEVER output `Action: None`.\n." + "- Do NOT call `Container Image Analysis Data` again; reuse its previous Observation.\n" "- Final Thought must end with: Selected: ().\n" "- Final Answer must be: : (reuse the same abbreviation).") return "Check your output and make sure it conforms" diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 90fe0904c..5177b05d9 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -134,8 +134,10 @@ Input: {input} {agent_scratchpad}""" -CVSS_EXAMPLES_FOR_PROMPT = """Example 1: +CVSS_EXAMPLES_FOR_PROMPT = """Examples: + Example 1: + Question: Metric: Scope (S) Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE @@ -146,7 +148,7 @@ Action: Lexical Search Container Image Code QA System Action Input: Observation: - Thought: I now know the answer because , I can now make a confident selection. + Thought: I now know the answer because , I can now make a confident selection. Selected: Changed (C) Final Answer: S:C @@ -158,7 +160,7 @@ Action: Container Image Analysis Data Action Input: Observation: - Thought: I now know the answer because , I can now make a confident selection. + Thought: I now know the answer because , I can now make a confident selection. Selected: Network (N) Final Answer: AV:N @@ -174,10 +176,26 @@ Action: Lexical Search Container Image Code QA System Action Input: Observation: - Thought: I now know the answer because , but due to insuficient information, it is still unclear what the correct selection should be, therefore, I will select the value with `least_impact: true` as a fallback. + Thought: I now know the answer because , but due to insuficient information, it is still unclear what the correct selection should be, therefore, I will select the least impact value as a fallback. Selected: High (H) Final Answer: PR:H + Example 4: + + Question: + Metric: User Interaction (UI) + Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE + Action: Container Image Analysis Data + Action Input: + Observation: + Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, However, it does not provide enough information to make a confident selection. I need to gather more information, I will use the Lexical Search Container Image Code QA System to search for further information. + Action: Lexical Search Container Image Code QA System + Action Input: + Observation: + Thought: I will reconsider a prior Observation, . + Thought: I now know the answer because , I can now make a confident selection. + Selected: Required (R) + Final Answer: UI:R """ def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = False) -> str: """ @@ -200,23 +218,26 @@ def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> s sys_prompt = sys_prompt or CVSS_SYS_PROMPT guidance = [ - "Guidance:\n" - "- At the very beginning, you MUST call the Container Image Analysis Data tool exactly once.\n", - "- After the initial call, you MUST NOT call the Container Image Analysis Data tool again. Reuse the Observation from the first call for all subsequent reasoning and decisions.\n", - "- To answer the question, use the Container Image Analysis Data (retrieved in the first step) as your primary reference for this metric in order to seleect the most appropriate CVSS metric value.\n", - "- If the container image analysis data does not contain sufficient information to make a selection, use other tools available to you to gather the information you require from the container image.\n", + "Guidance:\n", + "- At the very beginning, start with `Metric:` and `Question:`.\n", + "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", + "- Exactly one `Thought:` per step is allowed.\n", + "- After every Thought, you MUST provide an Action AND Action Input, unless: (a) you are ready to provide the Final Answer or (b) you need to reconsider a prior Observation. Otherwise, do not stop after a Thought. This rule must always be followed.\n", "- ABSOLUTE RULE: `Action: None` output is forbidden. Do not output `Action: None` under any circumstances.\n", - "- Analyze and reason through the data you have access to. Make the most accurate selection based on the available evidence.\n", - "- Only if you have thoroughly evaluated all available tools, and still cannot make a confident selection, may you choose the option where least_impact: true as a fallback.\n", - "- REMINDER: When you reach the final Thought, always include a brief rationale explaining why the selected value follows from prior Observations. Do not call tools in this Thought.\n", - "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your first attempt did not get a definitive answer.\n", + "- If you need to reconsider a prior Observation, you MUST output `Thought: I will reconsider a prior Observation, `. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", + "- When you reach the final Thought, always include a brief rationale explaining why the selected value's definition matches your findings and what is the supporting evidence to back this up. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", "- REMINDER: In your final Thought, end with exactly: Selected: (). Example: Selected: None (N), Selected: Low (L). Then output Final Answer: :. Do not call tools in this Thought.\n", + "- If you are ready to provide the Final Answer, you may omit Action/Action Input for the last step.\n", + "- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n", "- The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", - "- If you are ready to provide the Final Answer, you may omit Action/Action Input only for the last step.\n", - "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", - *( ["- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n"] if is_openai else [] ), - "- IMPORTANT: After every Thought, you MUST provide an Action AND Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", - "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :" + "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :\n", + "- You MUST start by using the Container Image Analysis Data tool exactly once.\n", + "- After the initial call, you MUST NOT call the Container Image Analysis Data tool again. Reuse the Observation from the first call for all subsequent reasoning and decisions.\n", + "- Use the container image analysis data as your primary reference for this metric to select the most appropriate and accurately descriptive CVSS metric value.\n", + "- If the container image analysis data does not contain sufficient information to make a confident selection, use other tools available to you to gather more information from the container image.\n", + "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your prior attempts did not get a definitive answer.\n", + "- ABSOLUTE RULE: From your Observations, you MUST select the value whose provided definition accurately matches your findings, your Selected and Final Answer must reflect that definition.\n", + "- Only if you have thoroughly evaluated all available tools and prior Observations, and still cannot make a confident selection, may you choose the value with least impact marked by `least_impact: true` as a fallback. NEVER fallback to a value marked by `least_impact: false`." ] prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + CVSS_EXAMPLES_FOR_PROMPT + "\n\n" + "Begin!\n\n") From ddffef3e905555701051c1ec7bd325e4cccd25f7 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 26 Aug 2025 12:25:17 +0300 Subject: [PATCH 047/286] chore: build config and README update Signed-off-by: Ilona Shishov --- README.md | 14 ++++++++-- docker-compose.yml | 2 +- kustomize/base/exploit-iq-config.yml | 28 ++++++++++++++++++- kustomize/config-http-openai-local.yml | 28 ++++++++++++++++++- .../remote-nim-all/kustomization.yaml | 4 ++- .../kustomization.yaml | 4 ++- .../configs/config-http-openai.yml | 7 +++-- src/vuln_analysis/configs/config-tracing.yml | 7 +++-- src/vuln_analysis/configs/config.yml | 7 +++-- .../functions/cve_generate_cvss.py | 10 +++++-- src/vuln_analysis/utils/prompting.py | 7 +++-- 11 files changed, 98 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 26c9a045b..8290810d8 100644 --- a/README.md +++ b/README.md @@ -553,6 +553,16 @@ The configuration defines how the workflow operates, including functions, LLMs, - `replace_exceptions_value`: If `replace_exceptions` is `true`, use this message. Default is `I do not have a definitive answer for this checklist item."` - `return_intermediate_steps`: Controls whether to return intermediate steps taken by the agent, and include them in the output file. Helpful for troubleshooting agent responses. Default is `false`. - `verbose`: Set to true for verbose output. Default is `false`. + - `cve_generate_cvss`: generates and calculates the CVSS (Common Vulnerability Scoring System) vector string and score for the vulnerability analysis. + - `llm_name`: Name of LLM (`generate_cvss_llm`) configured in `llms` section. + - `tool_names`: Container Image Code QA System, Container Image Developer Guide QA System, (Optional) Lexical Search Container Image Code QA System, Container Image Analysis Data + - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency. + - `max_iterations`: The maximum number of iterations for the agent. Default is 10. + - `prompt_examples`: Whether to include examples in agent prompt. Default is `true`. + - `replace_exceptions`: Whether to replace exception message with custom message. Default is `false`. + - `replace_exceptions_value`: If `replace_exceptions` is `true`, use this message. Default is `Failed to generate CVSS for this analysis."` + - `return_intermediate_steps`: Controls whether to return intermediate steps taken by the agent, and include them in the output file. Helpful for troubleshooting agent responses. Default is `false`. + - `verbose`: Set to true for verbose output. Default is `false`. - `cve_summarize`: Generates concise, human-readable summarization paragraph from agent results. - `llm_name`: Name of LLM (`summarize_llm`) configured in `llms` section. - `cve_justify`: Assigns justification label and reason to each CVE based on summary. @@ -563,7 +573,7 @@ The configuration defines how the workflow operates, including functions, LLMs, - `markdown_dir`: Defines the path to the directory where the output will be saved in individual navigable markdown files per CVE-ID. - `overwrite`: Indicates whether the output file should be overwritten when the workflow starts if it already exists. Will throw an error if set to `False` and the file already exists. Note that the overwrite behavior only occurs on workflow initialization. For pipelines started in HTTP mode, each new request will append the existing file until the workflow is restarted. 3. **LLMs** (`llms`): The `llms` section contains the LLMs used by the workflow. Functions can reference LLMs in this section to use. The supported LLM API types in NeMo Agent toolkit are `nim` and `openai` The models in this workflow use `nim`. - - Configured models in this workflow: `checklist_llm`, `code_vdb_retriever_llm`, `doc_vdb_retriever_llm`, `cve_agent_executor_llm`, `summarize_llm`, `justify_llm` + - Configured models in this workflow: `checklist_llm`, `code_vdb_retriever_llm`, `doc_vdb_retriever_llm`, `cve_agent_executor_llm`, `generate_cvss_llm`, `summarize_llm`, `justify_llm` - Each `nim` model is configured with the following attributes defined in the NeMo Agent toolkit's [NimModelConfig](https://docs.nvidia.com/aiqtoolkit/latest/api/aiq/llm/nim_llm/index.html#aiq.llm.nim_llm.NIMModelConfig). Use [OpenAIModelConfig](https://docs.nvidia.com/aiqtoolkit/latest/api/aiq/llm/openai_llm/index.html#aiq.llm.openai_llm.OpenAIModelConfig) for `openai` models. - `base_url`: Optional attribute to override `https://integrate.api.nvidia.com/v1` - `model_name`: The name of the LLM model used by the node. @@ -661,7 +671,7 @@ In any configuration file, locate the `llms` section to see the current settings #### Steps to configure an LLM model 1. Obtain an API key and any other required auth info for the selected service. -2. Update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM APIs](#supported-llm-apis) table. If you choose not to use the default LLM models in your workflow (`meta/llama-3.1-70b-instruct`), you can also add environment variables to override the model names to your `.env` file. In addition to `CHECKLIST_MODEL_NAME`, you can also set `model_name` for the other LLM models using `CODE_VDB_RETRIEVER_MODEL_NAME`, `DOC_VDB_RETRIEVER_MODEL_NAME`, `CVE_AGENT_EXECUTOR_MODEL_NAME`, `SUMMARIZE_MODEL_NAME`, and `JUSTIFY_MODEL_NAME`. +2. Update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM APIs](#supported-llm-apis) table. If you choose not to use the default LLM models in your workflow (`meta/llama-3.1-70b-instruct`), you can also add environment variables to override the model names to your `.env` file. In addition to `CHECKLIST_MODEL_NAME`, you can also set `model_name` for the other LLM models using `CODE_VDB_RETRIEVER_MODEL_NAME`, `DOC_VDB_RETRIEVER_MODEL_NAME`, `CVE_AGENT_EXECUTOR_MODEL_NAME`, `GENERATE_CVSS_MODEL_NAME`, `SUMMARIZE_MODEL_NAME`, and `JUSTIFY_MODEL_NAME`. 3. Update the config file as described above. For example, if you want to use OpenAI's `gpt-4o` model for checklist generation, update `checklist_llm` in the `llms` section to: ``` checklist_llm: diff --git a/docker-compose.yml b/docker-compose.yml index d89b85d6b..42d078f4f 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: - CODE_VDB_RETRIEVER_MODEL_NAME=${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - DOC_VDB_RETRIEVER_MODEL_NAME=${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - CVE_AGENT_EXECUTOR_MODEL_NAME=${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} - - CVE_GENERATE_CVSS_MODEL_NAME=${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + - GENERATE_CVSS_MODEL_NAME=${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} - SUMMARIZE_MODEL_NAME=${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} - JUSTIFY_MODEL_NAME=${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} - EMBEDDER_MODEL_NAME=${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index e93c662df..127b34624 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -77,7 +77,8 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 - + Container Image Analysis Data: + _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -96,6 +97,22 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false cve_summarize: _type: cve_summarize llm_name: summarize_llm @@ -149,6 +166,14 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + generate_cvss_llm: + _type: ${LLM_TYPE_GENERATE_CVSS:-nim} + api_key: ${LLM_API_KEY_GENERATE_CVSS:-"EMPTY"} + base_url: ${GENERATE_CVSS_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 summarize_llm: _type: ${LLM_TYPE_SUMMARIZE:-nim} api_key: ${LLM_API_KEY_SUMMARIZE:-"EMPTY"} @@ -192,6 +217,7 @@ workflow: cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index b6dffde64..9f7aa7c19 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -78,7 +78,8 @@ functions: Internet Search: _type: serp_wrapper max_retries: 5 - + Container Image Analysis Data: + _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -97,6 +98,22 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false cve_summarize: _type: cve_summarize llm_name: summarize_llm @@ -147,6 +164,14 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + generate_cvss_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 summarize_llm: _type: openai api_key: "EMPTY" @@ -190,6 +215,7 @@ workflow: cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output diff --git a/kustomize/overlays/remote-nim-all/kustomization.yaml b/kustomize/overlays/remote-nim-all/kustomization.yaml index ff9cf1d90..994d919e1 100644 --- a/kustomize/overlays/remote-nim-all/kustomization.yaml +++ b/kustomize/overlays/remote-nim-all/kustomization.yaml @@ -71,7 +71,9 @@ patchesStrategicMerge: - name: AGENT_EXECUTOR_LLM_API_BASE value: *nim-llm-baseurl - + - name: GENERATE_CVSS_LLM_API_BASE + value: *nim-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE value: *nim-llm-baseurl diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml index 9446d3e2c..24d8e82de 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -98,7 +98,9 @@ patchesStrategicMerge: - name: AGENT_EXECUTOR_LLM_API_BASE value: *openai-llm-baseurl - + - name: GENERATE_CVSS_LLM_API_BASE + value: *openai-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE value: *openai-llm-baseurl diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index e2e62bd6d..775515a0f 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -94,7 +94,7 @@ functions: cve_generate_cvss: _type: cve_generate_cvss skip: false - llm_name: cve_generate_cvss_llm + llm_name: generate_cvss_llm tool_names: - Container Image Code QA System - Container Image Developer Guide QA System @@ -102,6 +102,7 @@ functions: - Container Image Analysis Data max_concurrency: null max_iterations: 10 + prompt_examples: true replace_exceptions: false replace_exceptions_value: "Failed to generate CVSS for this analysis." return_intermediate_steps: false @@ -156,11 +157,11 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 - cve_generate_cvss_llm: + generate_cvss_llm: _type: openai api_key: "EMPTY" base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 1024 top_p: 0.01 diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index f5a0404da..689f99629 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -82,7 +82,7 @@ functions: cve_generate_cvss: _type: cve_generate_cvss skip: false - llm_name: cve_generate_cvss_llm + llm_name: generate_cvss_llm tool_names: - Container Image Code QA System - Container Image Developer Guide QA System @@ -90,6 +90,7 @@ functions: - Container Image Analysis Data max_concurrency: null max_iterations: 10 + prompt_examples: true replace_exceptions: false replace_exceptions_value: "Failed to generate CVSS for this analysis." return_intermediate_steps: false @@ -141,10 +142,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 - cve_generate_cvss_llm: + generate_cvss_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 1024 top_p: 0.01 diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 440fc4480..2094ab65b 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -78,7 +78,7 @@ functions: cve_generate_cvss: _type: cve_generate_cvss skip: false - llm_name: cve_generate_cvss_llm + llm_name: generate_cvss_llm tool_names: - Container Image Code QA System - Container Image Developer Guide QA System @@ -86,6 +86,7 @@ functions: - Container Image Analysis Data max_concurrency: null max_iterations: 10 + prompt_examples: true replace_exceptions: false replace_exceptions_value: "Failed to generate CVSS for this analysis." return_intermediate_steps: false @@ -137,10 +138,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 - cve_generate_cvss_llm: + generate_cvss_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CVE_GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 1024 top_p: 0.01 diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index d746eeca6..c50514324 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -130,7 +130,12 @@ class CVEGenerateCvssToolConfig(FunctionBaseConfig, name="cve_generate_cvss"): """ - Defines a function that iterates through checklist items using provided tools and gathered intel. + Defines a function that generates and calculates the CVSS (Common Vulnerability Scoring System) + vector string and score for the vulnerability analysis. + + This class evaluates the severity of a given vulnerability by determining its impact and exploitability + based on key CVSS metrics - Attack Vector (AV), Attack Complexity (AC), Privileges Required (PR), + User Interaction (UI), Scope (S), Confidentiality (C), Integrity (I), and Availability (A). """ llm_name: str = Field(description="The LLM model to use with the CVE agent.") tool_names: list[str] = Field(default=[], description="The list of tools to provide to CVE agent.") @@ -143,6 +148,7 @@ class CVEGenerateCvssToolConfig(FunctionBaseConfig, name="cve_generate_cvss"): description= "Manually set the prompt for the specific model in the configuration. The prompt can either be passed in as a " "string of text or as a path to a text file containing the desired prompting.") + prompt_examples: bool = Field(default=True, description="Whether to include examples in agent prompt.") replace_exceptions: bool = Field(default=False, description="Whether to replace exception message with custom message.") replace_exceptions_value: str | None = Field(default=None, description="Message if replace_exceptions is true") @@ -180,7 +186,7 @@ async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) is_openai = "openai" in llm.__class__.__module__.lower() - prompt = PromptTemplate.from_template(get_cvss_prompt(config.prompt, is_openai)) + prompt = PromptTemplate.from_template(get_cvss_prompt(config.prompt, config.prompt_examples, is_openai)) # Filter tools that are not available tools = [ diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 5177b05d9..83a4a2094 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -210,7 +210,7 @@ def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = Fals return f'{sys_prompt} {prompt_template}' -def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> str: +def get_cvss_prompt(sys_prompt: str | None = None, prompt_examples: bool = True, is_openai: bool = False) -> str: """ Get the CVSS prompt based on the system prompt and whether to include examples. """ @@ -240,7 +240,10 @@ def get_cvss_prompt(sys_prompt: str | None = None, is_openai: bool = False) -> s "- Only if you have thoroughly evaluated all available tools and prior Observations, and still cannot make a confident selection, may you choose the value with least impact marked by `least_impact: true` as a fallback. NEVER fallback to a value marked by `least_impact: false`." ] - prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + CVSS_EXAMPLES_FOR_PROMPT + "\n\n" + "Begin!\n\n") + if prompt_examples: + prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + CVSS_EXAMPLES_FOR_PROMPT + "\n\n" + "Begin!\n\n") + else: + prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + "Begin!\n\n") return f'{sys_prompt}\n\n{prompt_template}' From 17a1a83eaff0fbb81757d43199576612211a191e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 26 Aug 2025 14:35:32 +0300 Subject: [PATCH 048/286] fix: updated prompt according to output parser restrictions Signed-off-by: Ilona Shishov --- src/vuln_analysis/utils/prompting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 83a4a2094..cb7390a82 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -192,11 +192,11 @@ Action: Lexical Search Container Image Code QA System Action Input: Observation: - Thought: I will reconsider a prior Observation, . - Thought: I now know the answer because , I can now make a confident selection. + Thought: Reconsidering prior Observations, . I now know the answer because , I can now make a confident selection. Selected: Required (R) Final Answer: UI:R """ + def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = False) -> str: """ Get the agent prompt based on the system prompt and whether to include examples. @@ -221,10 +221,10 @@ def get_cvss_prompt(sys_prompt: str | None = None, prompt_examples: bool = True, "Guidance:\n", "- At the very beginning, start with `Metric:` and `Question:`.\n", "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", - "- Exactly one `Thought:` per step is allowed.\n", - "- After every Thought, you MUST provide an Action AND Action Input, unless: (a) you are ready to provide the Final Answer or (b) you need to reconsider a prior Observation. Otherwise, do not stop after a Thought. This rule must always be followed.\n", + "- After every Thought, you MUST provide an Action AND Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", "- ABSOLUTE RULE: `Action: None` output is forbidden. Do not output `Action: None` under any circumstances.\n", - "- If you need to reconsider a prior Observation, you MUST output `Thought: I will reconsider a prior Observation, `. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", + "- If you need to reconsider a prior Observation, incorporate that reflection inside your Thought immediately before your next Action or Final Answer. Do not emit a separate Thought for reflection.\n", + "- Exactly one `Thought:` per step is allowed.\n", "- When you reach the final Thought, always include a brief rationale explaining why the selected value's definition matches your findings and what is the supporting evidence to back this up. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", "- REMINDER: In your final Thought, end with exactly: Selected: (). Example: Selected: None (N), Selected: Low (L). Then output Final Answer: :. Do not call tools in this Thought.\n", "- If you are ready to provide the Final Answer, you may omit Action/Action Input for the last step.\n", From ef0c9c6d913ce9d44ce402c10ed30656c4cb6797 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 26 Aug 2025 14:36:19 +0300 Subject: [PATCH 049/286] chore: added cvss import to dependency file Signed-off-by: Ilona Shishov --- pyproject.toml | 3 ++- uv.lock | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 462da4d35..3865067dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "tantivy==0.22.2", "tree-sitter==0.21.3", "tree-sitter-languages==1.10.2", - "univers==30.12" + "univers==30.12", + "cvss==3.6" ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/uv.lock b/uv.lock index b0e93d239..334ba636f 100644 --- a/uv.lock +++ b/uv.lock @@ -703,6 +703,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/34/31a1604c9a9ade0fdab61eb48570e09a796f4d9836121266447b0eaf7feb/cryptography-45.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f", size = 3331106, upload-time = "2025-07-02T13:06:18.058Z" }, ] +[[package]] +name = "cvss" +version = "3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/f6/1f7d315de23f39bbc32b94bb6b33a1b6124856037bfaa3f8bdb1a49584fa/cvss-3.6.tar.gz", hash = "sha256:f21d18224efcd3c01b44ff1b37dec2e3208d29a6d0ce6c87a599c73c21ee1a99", size = 30047, upload-time = "2025-08-04T10:50:13.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/6d/fe2c65b94a28ae0481dc254e8cd664b82390069003bea945076d8a445f2b/cvss-3.6-py2.py3-none-any.whl", hash = "sha256:e342c6ad9c7eb69d2aebbbc2768a03cabd57eb947c806e145de5b936219833ea", size = 31154, upload-time = "2025-08-04T10:50:12.328Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -3524,6 +3533,7 @@ dependencies = [ { name = "aioresponses" }, { name = "aiqtoolkit", extra = ["langchain", "profiling", "telemetry"] }, { name = "brotli" }, + { name = "cvss" }, { name = "esprima" }, { name = "faiss-cpu" }, { name = "gitpython" }, @@ -3566,6 +3576,7 @@ requires-dist = [ { name = "aioresponses", specifier = "==0.7.6" }, { name = "aiqtoolkit", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0a2,<1.3.0" }, { name = "brotli" }, + { name = "cvss", specifier = "==3.6" }, { name = "esprima" }, { name = "faiss-cpu", specifier = "==1.9.0" }, { name = "gitpython" }, From 7cd1d195290bb9081b520b129d278ac2010192f2 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 26 Aug 2025 15:21:41 +0300 Subject: [PATCH 050/286] docs: add missing uv command, markdown formatting Signed-off-by: Vladimir Belousov --- kustomize/README.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index d8e4cee2f..b0769a6b2 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -2,14 +2,31 @@ ## Install and Run Locally -One can run ExploitIQ on his local machine ( No GPU dependency is required!), for the purpose of testing, debugging and troubleshooting problems: +One can run ExploitIQ on his local machine ( No GPU dependency is required!), for the purpose of testing, debugging and troubleshooting problems: -1. Install python version 3.12 by any wanted mean ( conda,pyenv, venv, poetry), and switch to that python environment. -2. Install lightweight [uv package manager](https://github.com/astral-sh/uv#installation) -3. inside the new python env, define the following environment variables: +1. Ensure Python 3.12 is installed for your operating system. + +2. Install the lightweight [uv package manager](https://docs.astral.sh/uv/getting-started/installation). + +3. Navigate to the project root, then create and activate a virtual environment using `uv`. + +```shell +# Make sure you're on the repo's root directory. +cd $(git rev-parse --show-toplevel) +# Create and activate the virtual environment +uv venv --python 3.12 +source .venv/bin/activate +``` + +4. **Install Dependencies**: Once the environment is active, install the required packages. + +```shell +uv sync +``` + +5. **Set Environment Variables**: Define the following environment variables. ```shell -PYTHON_VERSION=3.12 export CHECKLIST_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 export CODE_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 export CVE_AGENT_EXECUTOR_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 @@ -20,14 +37,9 @@ export PYTHONUNBUFFERED=1 export SERPAPI_API_KEY=YOUR_SERPAPI_KEY export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 ``` -4. Install all dependencies ( including `aiq` binary): -```shell -# Make sure you're on the repo' root. - cd $(git rev-parse --show-toplevel) - uv venv --python ${PYTHON_VERSION} /workspace/.venv && uv sync -``` -5. After a successfull completed installation, within the new python environment, run: +6. **Run the Application**: After a successful installation, run the application. + ```shell aiq --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 ``` From dd46b4e4572e9f20e288625a158b13904afaae06 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 26 Aug 2025 16:40:12 +0300 Subject: [PATCH 051/286] chore: fine tune cvss prompt to pay special attention to analysis evidence Signed-off-by: Ilona Shishov --- src/vuln_analysis/utils/prompting.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index cb7390a82..ff69a1b46 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -126,7 +126,7 @@ Action Input: the input to the action Observation: the result of the action ... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer because . I MUST ALWAYS end my final Thought with exactly: Selected: (). +Thought: I now know the final answer because . I MUST ALWAYS end my final Thought with exactly: Selected: (); Definition Matched: ; Evidence: <1-2 short quotes>. Final Answer: the final answer to the original input question in the following format: : Begin! @@ -149,7 +149,7 @@ Action Input: Observation: Thought: I now know the answer because , I can now make a confident selection. - Selected: Changed (C) + Selected: Changed (C), Definition Matched: The impact extends beyond the vulnerable system.; Evidence: 'The scope of the impact has changed'. Final Answer: S:C Example 2: @@ -161,7 +161,7 @@ Action Input: Observation: Thought: I now know the answer because , I can now make a confident selection. - Selected: Network (N) + Selected: Network (N); Definition Matched: Remote attack over the internet.; Evidence: 'The vulnerability can be exploited from anywhere with internet access, posing a significant risk.' Final Answer: AV:N Example 3: @@ -177,7 +177,7 @@ Action Input: Observation: Thought: I now know the answer because , but due to insuficient information, it is still unclear what the correct selection should be, therefore, I will select the least impact value as a fallback. - Selected: High (H) + Selected: High (H); Definition Matched: Administrative privileges needed.; Evidence: None. Final Answer: PR:H Example 4: @@ -193,7 +193,7 @@ Action Input: Observation: Thought: Reconsidering prior Observations, . I now know the answer because , I can now make a confident selection. - Selected: Required (R) + Selected: Required (R); Definition Matched: Requires user action.; Evidence: 'User must click a link to trigger the vulnerability'. Final Answer: UI:R """ @@ -226,7 +226,7 @@ def get_cvss_prompt(sys_prompt: str | None = None, prompt_examples: bool = True, "- If you need to reconsider a prior Observation, incorporate that reflection inside your Thought immediately before your next Action or Final Answer. Do not emit a separate Thought for reflection.\n", "- Exactly one `Thought:` per step is allowed.\n", "- When you reach the final Thought, always include a brief rationale explaining why the selected value's definition matches your findings and what is the supporting evidence to back this up. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", - "- REMINDER: In your final Thought, end with exactly: Selected: (). Example: Selected: None (N), Selected: Low (L). Then output Final Answer: :. Do not call tools in this Thought.\n", + "- REMINDER: In your final Thought, end with exactly: Selected: (); Definition Matched: ; Evidence: <1-2 short quotes or 'None' if no evidence>. Example: Selected: None (N); Definition Matched: No privileges needed; Evidence: 'The container image does not require any privileges to be exploited'. Then output Final Answer: :. Do not call tools in this Thought.\n", "- If you are ready to provide the Final Answer, you may omit Action/Action Input for the last step.\n", "- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n", "- The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", @@ -236,8 +236,9 @@ def get_cvss_prompt(sys_prompt: str | None = None, prompt_examples: bool = True, "- Use the container image analysis data as your primary reference for this metric to select the most appropriate and accurately descriptive CVSS metric value.\n", "- If the container image analysis data does not contain sufficient information to make a confident selection, use other tools available to you to gather more information from the container image.\n", "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your prior attempts did not get a definitive answer.\n", - "- ABSOLUTE RULE: From your Observations, you MUST select the value whose provided definition accurately matches your findings, your Selected and Final Answer must reflect that definition.\n", - "- Only if you have thoroughly evaluated all available tools and prior Observations, and still cannot make a confident selection, may you choose the value with least impact marked by `least_impact: true` as a fallback. NEVER fallback to a value marked by `least_impact: false`." + "- ABSOLUTE RULE: From your Observations, you MUST select the value whose provided definition ACCURATELY matches your findings and is supported by the evidence you have collected. Never select a value if its definition contradicts or does not match your evidence.\n", + "- Only if you have thoroughly evaluated all available tools and prior Observations, and still cannot make a confident selection, may you choose the value with least impact marked by `least_impact: true` as a fallback. NEVER fallback to a value marked by `least_impact: false`.\n", + "- IMPORTANT: If using fallback, make ABSOLUTELY sure to pick the value with least_impact: true" ] if prompt_examples: From ee9a733de5ebcd6bcafe8fe18940051e81a4c13d Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Wed, 27 Aug 2025 11:19:31 +0300 Subject: [PATCH 052/286] chore: update container image analysis data tool description Signed-off-by: Ilona Shishov --- src/vuln_analysis/tools/container_image_analysis_data.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py index 5b138b52c..1475e5bad 100644 --- a/src/vuln_analysis/tools/container_image_analysis_data.py +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -64,9 +64,7 @@ async def _arun(query: str) -> list[dict[str, Any]]: "Useful when you need to retrieve information from the analysis data of a container image. " "This tool does not require an input; it uses context to retrieve pre-analyzed data based on a prior scan. " "This tool is especially useful when your task involves understanding how the container image may be impacted by a reported CVE. " - "The tool provides structured information about the container image's source code and context on the reported CVE. " - "The tool returns an object with two fields:\n" - "- vulnerability_context: a list of context statements on the reported CVE (or `null` if none are provided).\n" - "- analysis_data: a list of the container image's source code analysis results, where each item in the list is an object describing a specific finding. " - "Each object in the list includes a 'response' field containing a concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may contain reasoning or supporting details if not null. ")) + "The tool provides structured information about the container image's source code on the reported CVE. " + "The tool returns an object with analysis data, which is a list of the container image's source code analysis results, where each item in the list is an object describing a specific finding. " + "Each item in the list includes a 'response' field containing a concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may contain reasoning or supporting details if not null. ")) \ No newline at end of file From 7cb5212c8853adc88dd82997473ee3c638de2cbb Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 28 Aug 2025 16:08:02 +0300 Subject: [PATCH 053/286] chore: fix not working self hosted deployment variant Signed-off-by: Zvi Grinberg --- .../overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml index 9446d3e2c..c211eb493 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -34,7 +34,7 @@ patchesStrategicMerge: component: exploit-iq spec: containers: - - name: exploit-iq-service + - name: exploit-iq imagePullPolicy: Always workingDir: /workspace/ env: @@ -81,7 +81,7 @@ patchesStrategicMerge: - name: LLM_API_KEY_JUSTIFY value: *api-key - - name: $LLM_API_KEY_INTEL_SOURCE_SCORE + - name: LLM_API_KEY_INTEL_SOURCE_SCORE value: *api-key From 885366cd369fac285f6b7d8fc4b99b904788e208 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 31 Aug 2025 16:08:55 +0300 Subject: [PATCH 054/286] ci: migrate Tekton PipelineRun to NAT Signed-off-by: Vladimir Belousov --- .tekton/on-pull-request.yaml | 92 +++++++++++++++++++++++++++++++++--- Makefile | 85 +++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 Makefile diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index b6e244125..4152e628c 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -10,7 +10,8 @@ metadata: # The branch or tag we are targeting (ie: main, refs/tags/*) pipelinesascode.tekton.dev/on-target-branch: "[main, rh-aiq-main]" - + # Paths that trigger the pipeline on change + pipelinesascode.tekton.dev/on-path-change: "[src/**, metrics_lib/**, pyproject.toml, Dockerfile, .dockerignore]" # Fetch the git-clone task from hub, we are able to reference later on it # with taskRef and it will automatically be embedded into our pipeline. pipelinesascode.tekton.dev/task: "git-clone" @@ -75,9 +76,11 @@ spec: value: $(params.repo_url) - name: REVISION value: $(params.revision) + - name: DEPTH + value: "0" - name: buildah-pvc runAfter: - - fetch-repository + - lint-and-test params: - name: IMAGE value: $(params.output-image) @@ -113,15 +116,92 @@ spec: workspaces: - name: source workspace: source - - name: dockerconfig workspace: dockerconfig-ws - - name: buildah-storage-dir workspace: buildah-storage-dir - - name: buildah-temp-cache workspace: buildah-temp-cache + - name: lint-and-test + runAfter: + - fetch-repository + workspaces: + - name: source + workspace: source + taskSpec: + workspaces: + - name: source + volumes: + - name: env-file-volume + secret: + secretName: integration-tests-env-file + - name: examples-volume + configMap: + name: integration-tests-reports + steps: + - name: lint-test + env: + - name: TARGET_BRANCH_NAME + value: "{{target_branch}}" + image: registry.access.redhat.com/ubi9/python-312:9.6 + workingDir: $(workspaces.source.path) + volumeMounts: + - name: env-file-volume + mountPath: $(workspaces.source.path)/.env + subPath: .env + - name: examples-volume + mountPath: $(workspaces.source.path)/examples + script: | + #!/bin/bash + + set -eou pipefail + # Install uv + curl -LsSf https://astral.sh/uv/install.sh | sh + + print_banner() { + echo + echo "----------- ${1} -----------" + } + + print_banner "CHECKING FOR .ENV FILE" + if [ -f .env ]; then + echo ".env file successfully mounted." + else + echo "ERROR: .env file not found!" + exit 1 + fi + + print_banner "CHECKING FOR EXAMPLES DIR" + if [ -d "examples" ] && [ -n "$(ls -A examples)" ]; then + echo "Examples directory successfully mounted." + else + echo "ERROR: Examples directory not found or is empty!" + exit 1 + fi + + print_banner "CREATING AND ACTIVATING TEST ENV" + uv venv --python 3.12 .venv + source .venv/bin/activate + + print_banner "INSTALLING DEPENDENCIES" + uv sync + + print_banner "RUNNING LINTER" + # Add the current directory to git's safe directories to avoid ownership errors. + git config --global --add safe.directory /workspace/source + echo "Fetching target branch..." + git fetch origin $TARGET_BRANCH_NAME + echo "Git branches:" + git branch -a + # TODO: This step is temporarily configured to pass even if linting errors are found. + # This is handled in the Makefile's lint-pr target and should be reverted after migration. + make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME + + print_banner "RUNNING UNIT TESTS" + # TODO: Uncomment the following line when unit tests are added to the 'src' directory. + # make test-unit + + print_banner "LINT AND TEST COMPLETE" workspaces: - name: source volumeClaimTemplate: @@ -130,7 +210,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 2Gi - name: buildah-storage-dir volumeClaimTemplate: spec: diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..74b2a19c2 --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +# Ensure a consistent shell is used, avoiding potential issues with environment inheritance. +# https://www.gnu.org/software/make/manual/make.html#Makefile-Basics +SHELL = /bin/sh +# Clear the list of suffixes to disable built-in implicit rules. +.SUFFIXES: + +# Directory for the main source code (override via `make lint-all SRC_DIR=./app`) +SRC_DIR ?= src +# Path to the LLM metrics test file +LLM_METRICS_TEST_FILE ?= tests/integration/generate_metrics.py + +# Target branch for PR diffs in CI (override via `make lint-pr TARGET_BRANCH=develop`) +TARGET_BRANCH ?= rh-aiq-main + +# Git remote for PR diffs in CI (override via `make lint-pr GIT_REMOTE=upstream`) +GIT_REMOTE ?= origin + +# The pylint executable (override via `make lint PYLINT=pylint3`) +PYLINT ?= pylint +# The pytest executable (override via `make test PYTEST=pytest-xdist`) +PYTEST ?= pytest +# Extra options for pylint (override via `make lint PYLINT_OPTS="--disable=C0114"`) +PYLINT_OPTS ?= +# Extra options for pytest (override via `make test PYTEST_OPTS="-v -s --maxfail=1"`) +PYTEST_OPTS ?= + +.PHONY: help lint lint-all lint-staged lint-pr test test-unit test-llm-metrics test-integration check + +# Set the default target to "help" +.DEFAULT_GOAL := help + +help: ## Display this help message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ + awk 'BEGIN {FS = ":.*## "}; {printf "\t%-20s %s\n", $$1, $$2}' + +lint: lint-staged ## Lint staged files (default). + +lint-all: ## Lint all source files. + @echo "--- Linting all source files ---" + @$(PYLINT) $(PYLINT_OPTS) $(SRC_DIR) + +lint-staged: ## Lint staged files. + @echo "--- Linting staged files ---" + @{ \ + FILES_TO_LINT=$$(git diff --name-only --cached --diff-filter=ACMRTUXB -- '*.py' ':(exclude)*/__init__.py'); \ + if [ -z "$$FILES_TO_LINT" ]; then \ + echo "No Python files staged for commit. Skipping."; \ + else \ + echo "Linting files:"; \ + echo "$$FILES_TO_LINT" | sed 's/^/ /'; \ + $(PYLINT) $(PYLINT_OPTS) $$FILES_TO_LINT; \ + fi \ + } + +# TODO: During migration, linting errors are ignored to allow the pipeline to pass. +# Remove "|| true" from the pylint command below once the codebase is clean. +lint-pr: ## Lint PR changes (for CI). + @echo "--- Linting changed files in PR against $(GIT_REMOTE)/$(TARGET_BRANCH) ---" + @{ \ + FILES_TO_LINT=$$(git diff --name-only "$(GIT_REMOTE)/$(TARGET_BRANCH)..." -- '*.py' ':(exclude)*/__init__.py'); \ + if [ -z "$$FILES_TO_LINT" ]; then \ + echo "No changed Python files in this PR. Skipping."; \ + else \ + echo "Linting files:"; \ + echo "$$FILES_TO_LINT" | sed 's/^/ /'; \ + $(PYLINT) $(PYLINT_OPTS) $$FILES_TO_LINT || true; \ + fi \ + } + +test: test-unit test-llm-metrics ## Run all tests. + @echo "All tests have been run." + +test-unit: ## Run unit tests. + @echo "Running unit tests in $(SRC_DIR)..." + @python -m pytest $(SRC_DIR) $(PYTEST_OPTS) + +test-llm-metrics: ## Run LLM metrics tests. + @echo "Running LLM metrics tests..." + @python -m pytest $(LLM_METRICS_TEST_FILE) $(PYTEST_OPTS) + +test-integration: ## Run integration tests (placeholder). + @echo "--- SKIPPING: Integration tests are not yet implemented. ---" + +check: lint-all test ## Lint all files and run all tests. + @echo "Lint and all tests have been run." From da916a30732d6a5733311c36c28e5f317a0ac5df Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 2 Sep 2025 11:18:20 +0300 Subject: [PATCH 055/286] fix: import context state from runtime context module Signed-off-by: Ilona Shishov --- src/vuln_analysis/tools/transitive_code_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 43a9a5b3b..c102154d2 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from vuln_analysis.functions.cve_agent import ctx_state +from vuln_analysis.runtime_context import ctx_state from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum From d0f554141d661705817f0f9b449429615524c71c Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 3 Sep 2025 00:04:26 +0300 Subject: [PATCH 056/286] chore: update exploit-iq image tag to 'nat' Signed-off-by: Zvi Grinberg --- kustomize/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index fddd67d64..c9e3cee9c 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -66,4 +66,4 @@ images: newTag: nat - name: quay.io/ecosystem-appeng/agent-morpheus-client - newTag: on-pr-809af874e075a1fb1f2ca5c2d94a08065e675823 + newTag: nat From b5e44dc9c7bcf03a62f2fb025b205acf5b53305c Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 3 Sep 2025 00:06:35 +0300 Subject: [PATCH 057/286] fix: prevent LLM output parsing error in intel score calculation Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/intel_source_score.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index f415ac7d1..18f6c0de6 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -65,6 +65,7 @@ def __get_calculate_score_prompt(self, intel: CveIntel) -> str: 1. The individual scores per criterion. 2. A short justification for each sub-score. 3. The total score out of 100. + 4. Justification of the total score. Ensure the final score does not exceed 100, the weights are respected and the score is returned in total_score tag. @@ -112,4 +113,4 @@ def __render_template(self, template_str: str, intel: CveIntel) -> str: if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): logger.debug("\nrendered_input_list: %s", str(rendered_input_list)) - return rendered_input_list \ No newline at end of file + return rendered_input_list From fe14b8d93787bba38bff69af0cf8497af8300066 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 4 Sep 2025 14:01:04 +0300 Subject: [PATCH 058/286] chore: configure image registry credentials and mount docker config Signed-off-by: Ilona Shishov --- .gitignore | 2 ++ kustomize/README.md | 36 ++++++++++++++----- kustomize/base/exploit_iq_client.yaml | 12 +++++++ kustomize/base/image-registry-credentials.env | 3 ++ kustomize/base/kustomization.yaml | 7 ++++ 5 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 kustomize/base/image-registry-credentials.env diff --git a/.gitignore b/.gitignore index 6e527f541..0f3af34ed 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,8 @@ venv.bak/ # Env files *.env +# but keep this one +!kustomize/base/image-registry-credentials.env # Spyder project settings .spyderproject diff --git a/kustomize/README.md b/kustomize/README.md index b0769a6b2..14ac7488c 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -67,10 +67,28 @@ oc new-project $YOUR_NAMESPACE_NAME 3. Create an image pull secret to authorize pulling the `ExploitIQ` container image: ```shell -oc create secret generic exploit-iq--pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +oc create secret generic exploit-iq-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -4. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. +4. Edit the `image-registry-credentials.env` and provide image registry credentials required by product scanning to access and pull component images. + +```shell +# Example: add your desired registries and tokens to the list +cat > base/image-registry-credentials.env << 'EOF' +{ + "auths": { + "registry.example.io": {"auth": ""}, + "example.io": {"auth": ""}, + "example.docker.io": {"auth": ""} + } +} +EOF +``` + +>[!IMPORTANT] +>This secret is essential for product scanning to authenticate and pull component images. If you skip this step, kustomize will still deploy, but authenticated pulls will not work until you provide real credentials. + +5. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret @@ -86,15 +104,15 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -5. Update `ExploitIQ` configuration file with the correct callback URL for the client service. +6. Update `ExploitIQ` configuration file with the correct callback URL for the client service. ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.json' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` >[!IMPORTANT] -You should only run one of 6,7 steps, depends on if you want to run the service with a self hosted LLM or not. -6. To deploy `EXploitIQ` with a self-hosted LLM , run: +You should only run one of 7,8 steps, depends on if you want to run the service with a self hosted LLM or not. +7. To deploy `EXploitIQ` with a self-hosted LLM , run: ```shell # Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM @@ -102,14 +120,14 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ``` -7. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: +8. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: ```shell # Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` >[!WARNING] -Without Completing the following step with the correct secret from step 4, authentication and logging into the UI App will fail!. -8. If it doesn't already exist, Create the `OAuthClient` Custom Resource using the secret (from step 4) and generated route +Without completing the following step with the correct secret from step 5, authentication and logging into the UI App will fail! +9. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 5) and generated route ```bash oc create -f - < Date: Sun, 7 Sep 2025 10:30:56 +0300 Subject: [PATCH 059/286] fix: agent startup crash chore: update framework name to nat Signed-off-by: Zvi Grinberg --- kustomize/README.md | 12 +- kustomize/base/exploit_iq_service.yaml | 2 +- pyproject.toml | 6 +- uv.lock | 3723 ++++++++++++++++++------ 4 files changed, 2794 insertions(+), 949 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index b0769a6b2..60c652f8d 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -4,16 +4,18 @@ One can run ExploitIQ on his local machine ( No GPU dependency is required!), for the purpose of testing, debugging and troubleshooting problems: -1. Ensure Python 3.12 is installed for your operating system. - -2. Install the lightweight [uv package manager](https://docs.astral.sh/uv/getting-started/installation). - +1. Install the lightweight [uv package manager](https://docs.astral.sh/uv/getting-started/installation). +2. Ensure Python 3.12 is installed for your operating system. +```shell + uv python install 3.12 +``` 3. Navigate to the project root, then create and activate a virtual environment using `uv`. ```shell # Make sure you're on the repo's root directory. cd $(git rev-parse --show-toplevel) # Create and activate the virtual environment +rm -rf .venv || true uv venv --python 3.12 source .venv/bin/activate ``` @@ -41,7 +43,7 @@ export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 6. **Run the Application**: After a successful installation, run the application. ```shell -aiq --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 +nat --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 ``` ## Deploy And Run On OCP diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index b3fb803c9..6480b0ac8 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -47,7 +47,7 @@ spec: imagePullPolicy: Always workingDir: /workspace/ args: - - "aiq" + - "nat" - "--log-level" - "debug" - "serve" diff --git a/pyproject.toml b/pyproject.toml index 3865067dc..3b73bb6ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,11 @@ dynamic = ["version"] dependencies = [ "aiohttp-client-cache==0.11", "aioresponses==0.7.6", - "aiqtoolkit[langchain,profiling,telemetry]>=1.2.0a2,<1.3.0", + "nvidia-nat[langchain,profiling,telemetry]>=1.2.0rc8,<1.3.0", "aiofiles", "brotli", "esprima", + "cvss==3.6", "faiss-cpu==1.9.0", "gitpython", "google-search-results==2.4", @@ -28,8 +29,7 @@ dependencies = [ "tantivy==0.22.2", "tree-sitter==0.21.3", "tree-sitter-languages==1.10.2", - "univers==30.12", - "cvss==3.6" + "univers==30.12" ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/uv.lock b/uv.lock index 334ba636f..1c13b5a16 100644 --- a/uv.lock +++ b/uv.lock @@ -3,27 +3,41 @@ revision = 3 requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'linux'", + "python_full_version >= '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.12' and sys_platform == 'darwin'", - "python_full_version < '3.12' and sys_platform != 'darwin'", + "python_full_version < '3.12' and sys_platform == 'linux'", + "python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", +] + +[[package]] +name = "abnf" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/f2/7b5fac50ee42e8b8d4a098d76743a394546f938c94125adbb93414e5ae7d/abnf-2.2.0.tar.gz", hash = "sha256:433380fd32855bbc60bc7b3d35d40616e21383a32ed1c9b8893d16d9f4a6c2f4", size = 197507, upload-time = "2023-03-17T18:26:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/95/f456ae7928a2f3a913f467d4fd9e662e295dd7349fc58b35f77f6c757a23/abnf-2.2.0-py3-none-any.whl", hash = "sha256:5dc2ae31a84ff454f7de46e08a2a21a442a0e21a092468420587a1590b490d1f", size = 39938, upload-time = "2023-03-17T18:26:22.608Z" }, ] [[package]] name = "aioboto3" -version = "15.0.0" +version = "15.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore", extra = ["boto3"] }, { name = "aiofiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/d0/ed107e16551ba1b93ddcca9a6bf79580450945268a8bc396530687b3189f/aioboto3-15.0.0.tar.gz", hash = "sha256:dce40b701d1f8e0886dc874d27cd9799b8bf6b32d63743f57e7bef7e4a562756", size = 225278, upload-time = "2025-06-26T16:30:48.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b1/b0331786c50f6ef881f9a71c3441ccf7b64c7eed210297d882c37ce31713/aioboto3-15.1.0.tar.gz", hash = "sha256:37763bbc6321ceb479106dc63bc84c8fdb59dd02540034a12941aebef2057c5c", size = 234664, upload-time = "2025-08-14T19:49:15.35Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/95/d69c744f408e5e4592fe53ed98fc244dd13b83d84cf1f83b2499d98bfcc9/aioboto3-15.0.0-py3-none-any.whl", hash = "sha256:9cf54b3627c8b34bb82eaf43ab327e7027e37f92b1e10dd5cfe343cd512568d0", size = 35785, upload-time = "2025-06-26T16:30:47.444Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/28e3ac89e7119b1cb4e6830664060b96a2b5761291e92a10fb3044b5a11d/aioboto3-15.1.0-py3-none-any.whl", hash = "sha256:66006142a2ccc7d6d07aa260ba291c4922b6767d270ba42f95c59e85d8b3e645", size = 35791, upload-time = "2025-08-14T19:49:14.14Z" }, ] [[package]] name = "aiobotocore" -version = "2.23.0" +version = "2.24.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -34,9 +48,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/25/4b06ea1214ddf020a28df27dc7136ac9dfaf87929d51e6f6044dd350ed67/aiobotocore-2.23.0.tar.gz", hash = "sha256:0333931365a6c7053aee292fe6ef50c74690c4ae06bb019afdf706cb6f2f5e32", size = 115825, upload-time = "2025-06-12T23:46:38.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/ca/ac82c0c699815b6d5b4017f3d8fb2c2d49537f4937f4a0bdf58b4c75d321/aiobotocore-2.24.0.tar.gz", hash = "sha256:b32c0c45d38c22a18ce395a0b5448606c5260603296a152895b5bdb40ab3139d", size = 119597, upload-time = "2025-08-08T18:26:50.373Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/43/ccf9b29669cdb09fd4bfc0a8effeb2973b22a0f3c3be4142d0b485975d11/aiobotocore-2.23.0-py3-none-any.whl", hash = "sha256:8202cebbf147804a083a02bc282fbfda873bfdd0065fd34b64784acb7757b66e", size = 84161, upload-time = "2025-06-12T23:46:36.305Z" }, + { url = "https://files.pythonhosted.org/packages/e2/68/b29577197aa2e54b50d6f214524790cc1cb27d289585ad7c7bdfe5125285/aiobotocore-2.24.0-py3-none-any.whl", hash = "sha256:72bb1f8eb1b962779a95e1bcc9cf35bc33196ad763b622a40ae7fa9d2e95c87c", size = 84971, upload-time = "2025-08-08T18:26:48.777Z" }, ] [package.optional-dependencies] @@ -64,7 +78,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.12.14" +version = "3.12.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -75,42 +89,42 @@ dependencies = [ { name = "propcache" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/0b/e39ad954107ebf213a2325038a3e7a506be3d98e1435e1f82086eec4cde2/aiohttp-3.12.14.tar.gz", hash = "sha256:6e06e120e34d93100de448fd941522e11dafa78ef1a893c179901b7d66aa29f2", size = 7822921, upload-time = "2025-07-10T13:05:33.968Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/e1/8029b29316971c5fa89cec170274582619a01b3d82dd1036872acc9bc7e8/aiohttp-3.12.14-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f4552ff7b18bcec18b60a90c6982049cdb9dac1dba48cf00b97934a06ce2e597", size = 709960, upload-time = "2025-07-10T13:03:11.936Z" }, - { url = "https://files.pythonhosted.org/packages/96/bd/4f204cf1e282041f7b7e8155f846583b19149e0872752711d0da5e9cc023/aiohttp-3.12.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8283f42181ff6ccbcf25acaae4e8ab2ff7e92b3ca4a4ced73b2c12d8cd971393", size = 482235, upload-time = "2025-07-10T13:03:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/d6/0f/2a580fcdd113fe2197a3b9df30230c7e85bb10bf56f7915457c60e9addd9/aiohttp-3.12.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:040afa180ea514495aaff7ad34ec3d27826eaa5d19812730fe9e529b04bb2179", size = 470501, upload-time = "2025-07-10T13:03:16.153Z" }, - { url = "https://files.pythonhosted.org/packages/38/78/2c1089f6adca90c3dd74915bafed6d6d8a87df5e3da74200f6b3a8b8906f/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b413c12f14c1149f0ffd890f4141a7471ba4b41234fe4fd4a0ff82b1dc299dbb", size = 1740696, upload-time = "2025-07-10T13:03:18.4Z" }, - { url = "https://files.pythonhosted.org/packages/4a/c8/ce6c7a34d9c589f007cfe064da2d943b3dee5aabc64eaecd21faf927ab11/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d6f607ce2e1a93315414e3d448b831238f1874b9968e1195b06efaa5c87e245", size = 1689365, upload-time = "2025-07-10T13:03:20.629Z" }, - { url = "https://files.pythonhosted.org/packages/18/10/431cd3d089de700756a56aa896faf3ea82bee39d22f89db7ddc957580308/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:565e70d03e924333004ed101599902bba09ebb14843c8ea39d657f037115201b", size = 1788157, upload-time = "2025-07-10T13:03:22.44Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b2/26f4524184e0f7ba46671c512d4b03022633bcf7d32fa0c6f1ef49d55800/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4699979560728b168d5ab63c668a093c9570af2c7a78ea24ca5212c6cdc2b641", size = 1827203, upload-time = "2025-07-10T13:03:24.628Z" }, - { url = "https://files.pythonhosted.org/packages/e0/30/aadcdf71b510a718e3d98a7bfeaea2396ac847f218b7e8edb241b09bd99a/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad5fdf6af93ec6c99bf800eba3af9a43d8bfd66dce920ac905c817ef4a712afe", size = 1729664, upload-time = "2025-07-10T13:03:26.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/7f/7ccf11756ae498fdedc3d689a0c36ace8fc82f9d52d3517da24adf6e9a74/aiohttp-3.12.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac76627c0b7ee0e80e871bde0d376a057916cb008a8f3ffc889570a838f5cc7", size = 1666741, upload-time = "2025-07-10T13:03:28.167Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4d/35ebc170b1856dd020c92376dbfe4297217625ef4004d56587024dc2289c/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:798204af1180885651b77bf03adc903743a86a39c7392c472891649610844635", size = 1715013, upload-time = "2025-07-10T13:03:30.018Z" }, - { url = "https://files.pythonhosted.org/packages/7b/24/46dc0380146f33e2e4aa088b92374b598f5bdcde1718c77e8d1a0094f1a4/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:4f1205f97de92c37dd71cf2d5bcfb65fdaed3c255d246172cce729a8d849b4da", size = 1710172, upload-time = "2025-07-10T13:03:31.821Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0a/46599d7d19b64f4d0fe1b57bdf96a9a40b5c125f0ae0d8899bc22e91fdce/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:76ae6f1dd041f85065d9df77c6bc9c9703da9b5c018479d20262acc3df97d419", size = 1690355, upload-time = "2025-07-10T13:03:34.754Z" }, - { url = "https://files.pythonhosted.org/packages/08/86/b21b682e33d5ca317ef96bd21294984f72379454e689d7da584df1512a19/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a194ace7bc43ce765338ca2dfb5661489317db216ea7ea700b0332878b392cab", size = 1783958, upload-time = "2025-07-10T13:03:36.53Z" }, - { url = "https://files.pythonhosted.org/packages/4f/45/f639482530b1396c365f23c5e3b1ae51c9bc02ba2b2248ca0c855a730059/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:16260e8e03744a6fe3fcb05259eeab8e08342c4c33decf96a9dad9f1187275d0", size = 1804423, upload-time = "2025-07-10T13:03:38.504Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e5/39635a9e06eed1d73671bd4079a3caf9cf09a49df08490686f45a710b80e/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c779e5ebbf0e2e15334ea404fcce54009dc069210164a244d2eac8352a44b28", size = 1717479, upload-time = "2025-07-10T13:03:40.158Z" }, - { url = "https://files.pythonhosted.org/packages/51/e1/7f1c77515d369b7419c5b501196526dad3e72800946c0099594c1f0c20b4/aiohttp-3.12.14-cp311-cp311-win32.whl", hash = "sha256:a289f50bf1bd5be227376c067927f78079a7bdeccf8daa6a9e65c38bae14324b", size = 427907, upload-time = "2025-07-10T13:03:41.801Z" }, - { url = "https://files.pythonhosted.org/packages/06/24/a6bf915c85b7a5b07beba3d42b3282936b51e4578b64a51e8e875643c276/aiohttp-3.12.14-cp311-cp311-win_amd64.whl", hash = "sha256:0b8a69acaf06b17e9c54151a6c956339cf46db4ff72b3ac28516d0f7068f4ced", size = 452334, upload-time = "2025-07-10T13:03:43.485Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0d/29026524e9336e33d9767a1e593ae2b24c2b8b09af7c2bd8193762f76b3e/aiohttp-3.12.14-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a0ecbb32fc3e69bc25efcda7d28d38e987d007096cbbeed04f14a6662d0eee22", size = 701055, upload-time = "2025-07-10T13:03:45.59Z" }, - { url = "https://files.pythonhosted.org/packages/0a/b8/a5e8e583e6c8c1056f4b012b50a03c77a669c2e9bf012b7cf33d6bc4b141/aiohttp-3.12.14-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0400f0ca9bb3e0b02f6466421f253797f6384e9845820c8b05e976398ac1d81a", size = 475670, upload-time = "2025-07-10T13:03:47.249Z" }, - { url = "https://files.pythonhosted.org/packages/29/e8/5202890c9e81a4ec2c2808dd90ffe024952e72c061729e1d49917677952f/aiohttp-3.12.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a56809fed4c8a830b5cae18454b7464e1529dbf66f71c4772e3cfa9cbec0a1ff", size = 468513, upload-time = "2025-07-10T13:03:49.377Z" }, - { url = "https://files.pythonhosted.org/packages/23/e5/d11db8c23d8923d3484a27468a40737d50f05b05eebbb6288bafcb467356/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f2e373276e4755691a963e5d11756d093e346119f0627c2d6518208483fb6d", size = 1715309, upload-time = "2025-07-10T13:03:51.556Z" }, - { url = "https://files.pythonhosted.org/packages/53/44/af6879ca0eff7a16b1b650b7ea4a827301737a350a464239e58aa7c387ef/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ca39e433630e9a16281125ef57ece6817afd1d54c9f1bf32e901f38f16035869", size = 1697961, upload-time = "2025-07-10T13:03:53.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/94/18457f043399e1ec0e59ad8674c0372f925363059c276a45a1459e17f423/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c748b3f8b14c77720132b2510a7d9907a03c20ba80f469e58d5dfd90c079a1c", size = 1753055, upload-time = "2025-07-10T13:03:55.368Z" }, - { url = "https://files.pythonhosted.org/packages/26/d9/1d3744dc588fafb50ff8a6226d58f484a2242b5dd93d8038882f55474d41/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0a568abe1b15ce69d4cc37e23020720423f0728e3cb1f9bcd3f53420ec3bfe7", size = 1799211, upload-time = "2025-07-10T13:03:57.216Z" }, - { url = "https://files.pythonhosted.org/packages/73/12/2530fb2b08773f717ab2d249ca7a982ac66e32187c62d49e2c86c9bba9b4/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9888e60c2c54eaf56704b17feb558c7ed6b7439bca1e07d4818ab878f2083660", size = 1718649, upload-time = "2025-07-10T13:03:59.469Z" }, - { url = "https://files.pythonhosted.org/packages/b9/34/8d6015a729f6571341a311061b578e8b8072ea3656b3d72329fa0faa2c7c/aiohttp-3.12.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3006a1dc579b9156de01e7916d38c63dc1ea0679b14627a37edf6151bc530088", size = 1634452, upload-time = "2025-07-10T13:04:01.698Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4b/08b83ea02595a582447aeb0c1986792d0de35fe7a22fb2125d65091cbaf3/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa8ec5c15ab80e5501a26719eb48a55f3c567da45c6ea5bb78c52c036b2655c7", size = 1695511, upload-time = "2025-07-10T13:04:04.165Z" }, - { url = "https://files.pythonhosted.org/packages/b5/66/9c7c31037a063eec13ecf1976185c65d1394ded4a5120dd5965e3473cb21/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:39b94e50959aa07844c7fe2206b9f75d63cc3ad1c648aaa755aa257f6f2498a9", size = 1716967, upload-time = "2025-07-10T13:04:06.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/02/84406e0ad1acb0fb61fd617651ab6de760b2d6a31700904bc0b33bd0894d/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:04c11907492f416dad9885d503fbfc5dcb6768d90cad8639a771922d584609d3", size = 1657620, upload-time = "2025-07-10T13:04:07.944Z" }, - { url = "https://files.pythonhosted.org/packages/07/53/da018f4013a7a179017b9a274b46b9a12cbeb387570f116964f498a6f211/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:88167bd9ab69bb46cee91bd9761db6dfd45b6e76a0438c7e884c3f8160ff21eb", size = 1737179, upload-time = "2025-07-10T13:04:10.182Z" }, - { url = "https://files.pythonhosted.org/packages/49/e8/ca01c5ccfeaafb026d85fa4f43ceb23eb80ea9c1385688db0ef322c751e9/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:791504763f25e8f9f251e4688195e8b455f8820274320204f7eafc467e609425", size = 1765156, upload-time = "2025-07-10T13:04:12.029Z" }, - { url = "https://files.pythonhosted.org/packages/22/32/5501ab525a47ba23c20613e568174d6c63aa09e2caa22cded5c6ea8e3ada/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2785b112346e435dd3a1a67f67713a3fe692d288542f1347ad255683f066d8e0", size = 1724766, upload-time = "2025-07-10T13:04:13.961Z" }, - { url = "https://files.pythonhosted.org/packages/06/af/28e24574801fcf1657945347ee10df3892311c2829b41232be6089e461e7/aiohttp-3.12.14-cp312-cp312-win32.whl", hash = "sha256:15f5f4792c9c999a31d8decf444e79fcfd98497bf98e94284bf390a7bb8c1729", size = 422641, upload-time = "2025-07-10T13:04:16.018Z" }, - { url = "https://files.pythonhosted.org/packages/98/d5/7ac2464aebd2eecac38dbe96148c9eb487679c512449ba5215d233755582/aiohttp-3.12.14-cp312-cp312-win_amd64.whl", hash = "sha256:3b66e1a182879f579b105a80d5c4bd448b91a57e8933564bf41665064796a338", size = 449316, upload-time = "2025-07-10T13:04:18.289Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9b/e7/d92a237d8802ca88483906c388f7c201bbe96cd80a165ffd0ac2f6a8d59f/aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2", size = 7823716, upload-time = "2025-07-29T05:52:32.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/19/9e86722ec8e835959bd97ce8c1efa78cf361fa4531fca372551abcc9cdd6/aiohttp-3.12.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d3ce17ce0220383a0f9ea07175eeaa6aa13ae5a41f30bc61d84df17f0e9b1117", size = 711246, upload-time = "2025-07-29T05:50:15.937Z" }, + { url = "https://files.pythonhosted.org/packages/71/f9/0a31fcb1a7d4629ac9d8f01f1cb9242e2f9943f47f5d03215af91c3c1a26/aiohttp-3.12.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:010cc9bbd06db80fe234d9003f67e97a10fe003bfbedb40da7d71c1008eda0fe", size = 483515, upload-time = "2025-07-29T05:50:17.442Z" }, + { url = "https://files.pythonhosted.org/packages/62/6c/94846f576f1d11df0c2e41d3001000527c0fdf63fce7e69b3927a731325d/aiohttp-3.12.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f9d7c55b41ed687b9d7165b17672340187f87a773c98236c987f08c858145a9", size = 471776, upload-time = "2025-07-29T05:50:19.568Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6c/f766d0aaafcee0447fad0328da780d344489c042e25cd58fde566bf40aed/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc4fbc61bb3548d3b482f9ac7ddd0f18c67e4225aaa4e8552b9f1ac7e6bda9e5", size = 1741977, upload-time = "2025-07-29T05:50:21.665Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/fb779a05ba6ff44d7bc1e9d24c644e876bfff5abe5454f7b854cace1b9cc/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbc8a7c410bb3ad5d595bb7118147dfbb6449d862cc1125cf8867cb337e8728", size = 1690645, upload-time = "2025-07-29T05:50:23.333Z" }, + { url = "https://files.pythonhosted.org/packages/37/4e/a22e799c2035f5d6a4ad2cf8e7c1d1bd0923192871dd6e367dafb158b14c/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74dad41b3458dbb0511e760fb355bb0b6689e0630de8a22b1b62a98777136e16", size = 1789437, upload-time = "2025-07-29T05:50:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/28/e5/55a33b991f6433569babb56018b2fb8fb9146424f8b3a0c8ecca80556762/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b6f0af863cf17e6222b1735a756d664159e58855da99cfe965134a3ff63b0b0", size = 1828482, upload-time = "2025-07-29T05:50:26.693Z" }, + { url = "https://files.pythonhosted.org/packages/c6/82/1ddf0ea4f2f3afe79dffed5e8a246737cff6cbe781887a6a170299e33204/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5b7fe4972d48a4da367043b8e023fb70a04d1490aa7d68800e465d1b97e493b", size = 1730944, upload-time = "2025-07-29T05:50:28.382Z" }, + { url = "https://files.pythonhosted.org/packages/1b/96/784c785674117b4cb3877522a177ba1b5e4db9ce0fd519430b5de76eec90/aiohttp-3.12.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6443cca89553b7a5485331bc9bedb2342b08d073fa10b8c7d1c60579c4a7b9bd", size = 1668020, upload-time = "2025-07-29T05:50:30.032Z" }, + { url = "https://files.pythonhosted.org/packages/12/8a/8b75f203ea7e5c21c0920d84dd24a5c0e971fe1e9b9ebbf29ae7e8e39790/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c5f40ec615e5264f44b4282ee27628cea221fcad52f27405b80abb346d9f3f8", size = 1716292, upload-time = "2025-07-29T05:50:31.983Z" }, + { url = "https://files.pythonhosted.org/packages/47/0b/a1451543475bb6b86a5cfc27861e52b14085ae232896a2654ff1231c0992/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2abbb216a1d3a2fe86dbd2edce20cdc5e9ad0be6378455b05ec7f77361b3ab50", size = 1711451, upload-time = "2025-07-29T05:50:33.989Z" }, + { url = "https://files.pythonhosted.org/packages/55/fd/793a23a197cc2f0d29188805cfc93aa613407f07e5f9da5cd1366afd9d7c/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:db71ce547012a5420a39c1b744d485cfb823564d01d5d20805977f5ea1345676", size = 1691634, upload-time = "2025-07-29T05:50:35.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/bf/23a335a6670b5f5dfc6d268328e55a22651b440fca341a64fccf1eada0c6/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ced339d7c9b5030abad5854aa5413a77565e5b6e6248ff927d3e174baf3badf7", size = 1785238, upload-time = "2025-07-29T05:50:37.597Z" }, + { url = "https://files.pythonhosted.org/packages/57/4f/ed60a591839a9d85d40694aba5cef86dde9ee51ce6cca0bb30d6eb1581e7/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:7c7dd29c7b5bda137464dc9bfc738d7ceea46ff70309859ffde8c022e9b08ba7", size = 1805701, upload-time = "2025-07-29T05:50:39.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/e0/444747a9455c5de188c0f4a0173ee701e2e325d4b2550e9af84abb20cdba/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:421da6fd326460517873274875c6c5a18ff225b40da2616083c5a34a7570b685", size = 1718758, upload-time = "2025-07-29T05:50:41.292Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/1006278d1ffd13a698e5dd4bfa01e5878f6bddefc296c8b62649753ff249/aiohttp-3.12.15-cp311-cp311-win32.whl", hash = "sha256:4420cf9d179ec8dfe4be10e7d0fe47d6d606485512ea2265b0d8c5113372771b", size = 428868, upload-time = "2025-07-29T05:50:43.063Z" }, + { url = "https://files.pythonhosted.org/packages/10/97/ad2b18700708452400278039272032170246a1bf8ec5d832772372c71f1a/aiohttp-3.12.15-cp311-cp311-win_amd64.whl", hash = "sha256:edd533a07da85baa4b423ee8839e3e91681c7bfa19b04260a469ee94b778bf6d", size = 453273, upload-time = "2025-07-29T05:50:44.613Z" }, + { url = "https://files.pythonhosted.org/packages/63/97/77cb2450d9b35f517d6cf506256bf4f5bda3f93a66b4ad64ba7fc917899c/aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7", size = 702333, upload-time = "2025-07-29T05:50:46.507Z" }, + { url = "https://files.pythonhosted.org/packages/83/6d/0544e6b08b748682c30b9f65640d006e51f90763b41d7c546693bc22900d/aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444", size = 476948, upload-time = "2025-07-29T05:50:48.067Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1d/c8c40e611e5094330284b1aea8a4b02ca0858f8458614fa35754cab42b9c/aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d", size = 469787, upload-time = "2025-07-29T05:50:49.669Z" }, + { url = "https://files.pythonhosted.org/packages/38/7d/b76438e70319796bfff717f325d97ce2e9310f752a267bfdf5192ac6082b/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c", size = 1716590, upload-time = "2025-07-29T05:50:51.368Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/60370d70cdf8b269ee1444b390cbd72ce514f0d1cd1a715821c784d272c9/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0", size = 1699241, upload-time = "2025-07-29T05:50:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2b/4968a7b8792437ebc12186db31523f541943e99bda8f30335c482bea6879/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab", size = 1754335, upload-time = "2025-07-29T05:50:55.394Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/49524ed553f9a0bec1a11fac09e790f49ff669bcd14164f9fab608831c4d/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb", size = 1800491, upload-time = "2025-07-29T05:50:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/3bf5acea47a96a28c121b167f5ef659cf71208b19e52a88cdfa5c37f1fcc/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545", size = 1719929, upload-time = "2025-07-29T05:50:59.192Z" }, + { url = "https://files.pythonhosted.org/packages/39/94/8ae30b806835bcd1cba799ba35347dee6961a11bd507db634516210e91d8/aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c", size = 1635733, upload-time = "2025-07-29T05:51:01.394Z" }, + { url = "https://files.pythonhosted.org/packages/7a/46/06cdef71dd03acd9da7f51ab3a9107318aee12ad38d273f654e4f981583a/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd", size = 1696790, upload-time = "2025-07-29T05:51:03.657Z" }, + { url = "https://files.pythonhosted.org/packages/02/90/6b4cfaaf92ed98d0ec4d173e78b99b4b1a7551250be8937d9d67ecb356b4/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f", size = 1718245, upload-time = "2025-07-29T05:51:05.911Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e6/2593751670fa06f080a846f37f112cbe6f873ba510d070136a6ed46117c6/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d", size = 1658899, upload-time = "2025-07-29T05:51:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/8f/28/c15bacbdb8b8eb5bf39b10680d129ea7410b859e379b03190f02fa104ffd/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519", size = 1738459, upload-time = "2025-07-29T05:51:09.56Z" }, + { url = "https://files.pythonhosted.org/packages/00/de/c269cbc4faa01fb10f143b1670633a8ddd5b2e1ffd0548f7aa49cb5c70e2/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea", size = 1766434, upload-time = "2025-07-29T05:51:11.423Z" }, + { url = "https://files.pythonhosted.org/packages/52/b0/4ff3abd81aa7d929b27d2e1403722a65fc87b763e3a97b3a2a494bfc63bc/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3", size = 1726045, upload-time = "2025-07-29T05:51:13.689Z" }, + { url = "https://files.pythonhosted.org/packages/71/16/949225a6a2dd6efcbd855fbd90cf476052e648fb011aa538e3b15b89a57a/aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1", size = 423591, upload-time = "2025-07-29T05:51:15.452Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266, upload-time = "2025-07-29T05:51:17.239Z" }, ] [[package]] @@ -174,70 +188,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, ] -[[package]] -name = "aiqtoolkit" -version = "1.2.0a20250707" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aioboto3" }, - { name = "click" }, - { name = "colorama" }, - { name = "expandvars" }, - { name = "fastapi" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "jsonpath-ng" }, - { name = "mcp" }, - { name = "networkx" }, - { name = "numpy" }, - { name = "openinference-semantic-conventions" }, - { name = "openpyxl" }, - { name = "pkginfo" }, - { name = "platformdirs" }, - { name = "pydantic" }, - { name = "pymilvus" }, - { name = "pyyaml" }, - { name = "ragas" }, - { name = "rich" }, - { name = "uvicorn", extra = ["standard"] }, - { name = "wikipedia" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/25/e3621910b51233a5e64a7361f620105791092af12e462bde2a37b2dff5f6/aiqtoolkit-1.2.0a20250707-py3-none-any.whl", hash = "sha256:3dfadc2d1eab45511f01c2b7b919ebaf132b1cf4f4cc14a20fc193204de9a892", size = 511867, upload-time = "2025-07-07T09:51:26.332Z" }, -] - -[package.optional-dependencies] -langchain = [ - { name = "aiqtoolkit-langchain" }, -] -profiling = [ - { name = "matplotlib" }, - { name = "prefixspan" }, - { name = "scikit-learn" }, -] -telemetry = [ - { name = "arize-phoenix" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-sdk" }, -] - -[[package]] -name = "aiqtoolkit-langchain" -version = "1.2.0a20250707" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiqtoolkit" }, - { name = "langchain-aws" }, - { name = "langchain-core" }, - { name = "langchain-milvus" }, - { name = "langchain-nvidia-ai-endpoints" }, - { name = "langchain-openai" }, - { name = "langgraph" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/31/5754bd1db8be023b9cfd0a8bd8afb3846e42145445ab2ace7410b0a9188d/aiqtoolkit_langchain-1.2.0a20250707-py3-none-any.whl", hash = "sha256:d8ac542d4e82d971eef991444284d172e36f44f5a83a20f5dac0588f4931566c", size = 13095, upload-time = "2025-07-07T09:52:01.593Z" }, -] - [[package]] name = "alembic" version = "1.16.4" @@ -263,16 +213,16 @@ wheels = [ [[package]] name = "anyio" -version = "4.9.0" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/b4/636b3b65173d3ce9a38ef5f0522789614e590dab6a8d505340a4efe4c567/anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6", size = 213252, upload-time = "2025-08-04T08:54:26.451Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] [[package]] @@ -284,6 +234,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, ] +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, +] + [[package]] name = "arize-phoenix" version = "6.2.0" @@ -333,21 +292,22 @@ wheels = [ [[package]] name = "arize-phoenix-evals" -version = "0.22.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pandas" }, + { name = "pystache" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e0/bb/2bac66f6471f87a765cacda532e5200a94adfc3dbce7ec8c0a6aafdf1f18/arize_phoenix_evals-0.22.0.tar.gz", hash = "sha256:bc84fafc53ffc01f7832c3313ee22418d397543d0b20520b29f175d45968ef23", size = 50110, upload-time = "2025-07-02T18:01:53.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/8f/3096dbb74ea796e9d868aa11269b688162f445afb56d670c13bd9c7d4e6d/arize_phoenix_evals-0.27.0.tar.gz", hash = "sha256:e2838e45b620a316293df46502d9d7b7b7ab4353939e48baa18cfe678be7962a", size = 77293, upload-time = "2025-08-13T15:20:35.738Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/ca/838c91b093bb4f56458eb79c3482d104a5554575bef8a70b97aa11cf0c36/arize_phoenix_evals-0.22.0-py3-none-any.whl", hash = "sha256:30bde46b3c79b15cc8bce42dc2fb213242a30f820f9d7e4717c3192ebb3158a8", size = 64341, upload-time = "2025-07-02T18:01:52.901Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b5/4549cb087cfa0865f30a5680a6a12e4f509f24e1ad739a00a1a47c1783bf/arize_phoenix_evals-0.27.0-py3-none-any.whl", hash = "sha256:b5110a9688c8c9993c8427ecfadf1dead06aeb03b56060a19e42e52c7f5212c5", size = 105443, upload-time = "2025-08-13T15:20:34.269Z" }, ] [[package]] name = "arize-phoenix-otel" -version = "0.12.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openinference-instrumentation" }, @@ -359,9 +319,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a4/b2/e990deaad2eba163c2d6f6af967495263efd4cefdf8545f17ddf3846e221/arize_phoenix_otel-0.12.1.tar.gz", hash = "sha256:0150cdc586abbd3cca49168352f94501d59c2690cab5d283a5e0689bb3dc1234", size = 16402, upload-time = "2025-06-24T15:58:34.914Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/13/bbfaaeac32a304505b7d229b3ef7326f631a1599bce58ce7bf80db9431f0/arize_phoenix_otel-0.13.0.tar.gz", hash = "sha256:1c2061f146528bf39ec7185632a904cf0642b63949b4f6e74ca3833f2ade4f78", size = 18630, upload-time = "2025-08-14T05:07:28.576Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/27/be2ead7273ecf3be0c5b3ebf9e4bccedcd49ab45a7909e82588f56189cd2/arize_phoenix_otel-0.12.1-py3-none-any.whl", hash = "sha256:85167dd061d7d4e14c98edd5733afe151da8027d69f2ecfe29a6aa2484906fa3", size = 13871, upload-time = "2025-06-24T15:58:33.702Z" }, + { url = "https://files.pythonhosted.org/packages/48/72/f17886ca0692854cfd721ea2828dd303bf8361d5b8ac8092f2af7ef94a27/arize_phoenix_otel-0.13.0-py3-none-any.whl", hash = "sha256:7a4d86a9e8bb16f55d3bb3f240891ac663c72c7527bd7094b3dfea58cd1daa7b", size = 16303, upload-time = "2025-08-14T05:07:27.443Z" }, ] [[package]] @@ -379,6 +339,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, ] +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978, upload-time = "2024-11-30T04:30:14.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918, upload-time = "2024-11-30T04:30:10.946Z" }, +] + [[package]] name = "attrs" version = "25.3.0" @@ -390,14 +359,53 @@ wheels = [ [[package]] name = "authlib" -version = "1.6.0" +version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/9d/b1e08d36899c12c8b894a44a5583ee157789f26fc4b176f8e4b6217b56e1/authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210", size = 158371, upload-time = "2025-05-23T00:21:45.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/75/47dbab150ef6f9298e227a40c93c7fed5f3ffb67c9fb62cd49f66285e46e/authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2", size = 147313, upload-time = "2024-08-26T07:10:04.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/4c/9aa0416a403d5cc80292cb030bcd2c918cce2755e314d8c1aa18656e1e12/Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc", size = 225111, upload-time = "2024-08-26T07:10:02.811Z" }, +] + +[[package]] +name = "azure-core" +version = "1.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "six" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/89/f53968635b1b2e53e4aad2dd641488929fef4ca9dfb0b97927fa7697ddf3/azure_core-1.35.0.tar.gz", hash = "sha256:c0be528489485e9ede59b6971eb63c1eaacf83ef53001bfe3904e475e972be5c", size = 339689, upload-time = "2025-07-03T00:55:23.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/78/bf94897361fdd650850f0f2e405b2293e2f12808239046232bdedf554301/azure_core-1.35.0-py3-none-any.whl", hash = "sha256:8db78c72868a58f3de8991eb4d22c4d368fae226dac1002998d6c50437e7dad1", size = 210708, upload-time = "2025-07-03T00:55:25.238Z" }, +] + +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + +[[package]] +name = "banks" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "griffe" }, + { name = "jinja2" }, + { name = "platformdirs" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/f8/25ef24814f77f3fd7f0fd3bd1ef3749e38a9dbd23502fbb53034de49900c/banks-2.2.0.tar.gz", hash = "sha256:d1446280ce6e00301e3e952dd754fd8cee23ff277d29ed160994a84d0d7ffe62", size = 179052, upload-time = "2025-07-18T16:28:26.892Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/29/587c189bbab1ccc8c86a03a5d0e13873df916380ef1be461ebe6acebf48d/authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d", size = 239981, upload-time = "2025-05-23T00:21:43.075Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d6/f9168956276934162ec8d48232f9920f2985ee45aa7602e3c6b4bc203613/banks-2.2.0-py3-none-any.whl", hash = "sha256:963cd5c85a587b122abde4f4064078def35c50c688c1b9d36f43c92503854e7d", size = 29244, upload-time = "2025-07-18T16:28:27.835Z" }, ] [[package]] @@ -413,32 +421,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285, upload-time = "2025-04-15T17:05:12.221Z" }, ] +[[package]] +name = "blis" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/aa/0743c994884de83472c854bb534c9edab8d711e1880d4fa194e6d876bb60/blis-1.2.1.tar.gz", hash = "sha256:1066beedbedc2143c22bd28742658de05694afebacde8d8c2d14dd4b5a96765a", size = 2510297, upload-time = "2025-04-01T12:01:56.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/57/ae6596b1e27859886e0b81fb99497bcfff139895585a9e2284681c8a8846/blis-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:778c4f72b71f97187e3304acfbd30eab98c9ba1a5b03b65128bc3875400ae604", size = 6976808, upload-time = "2025-04-01T12:01:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/ce/35/6225e6ad2bccf23ac124448d59112c098d63a8917462e9f73967bc217168/blis-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5c5f2ffb0ae9c1f5aaa95b9681bcdd9a777d007c501fa220796329b939ca2790", size = 1281913, upload-time = "2025-04-01T12:01:23.202Z" }, + { url = "https://files.pythonhosted.org/packages/7a/84/c6a6d1c0a8a00799d2ec5db05d676bd9a9b0472cac4d3eff2e2fd1953521/blis-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db4dc5d2d57106bb411633603a5c7d178a0845267c3efc7e5ea4fa7a44772976", size = 3104139, upload-time = "2025-04-01T12:01:24.781Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6c/c5fab7ed1fe6e8bdcda732017400d1adc53db5b6dd2c2a6046acab91f4fa/blis-1.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c621271c2843101927407e052b35a67f853da59d5c74e9e070e982c7f82e2e04", size = 3304143, upload-time = "2025-04-01T12:01:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/22/d1/85f03269886253758546fcfdbeddee7e717d843ea134596b60db9c2648c4/blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43f65f882250b817566d7543abd1f6da297f1662e5dd9936e14c04b88285a497", size = 11660080, upload-time = "2025-04-01T12:01:29.478Z" }, + { url = "https://files.pythonhosted.org/packages/78/c8/c81ed3036e8ce0d6ce0d19a032c7f3d69247f221c5357e18548dea9380d3/blis-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78a0613d559ccc426c101c67e8f84e1f93491e29d722c370872c538ee652bd07", size = 3133133, upload-time = "2025-04-01T12:01:31.537Z" }, + { url = "https://files.pythonhosted.org/packages/b8/42/7c296e04b979204777ecae2fe9287ac7b0255d8c4c2111d2a735c439b9d7/blis-1.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f5e32e5e5635fc7087b724b53120dbcd86201f56c0405882ce254bc0e493392", size = 4360695, upload-time = "2025-04-01T12:01:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/aa5c8dfd0068d2cc976830797dd092779259860f964286db05739154e3a7/blis-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d339c97cc83f53e39c1013d0dcd7d5278c853dc102d931132eeb05b226e28429", size = 14828081, upload-time = "2025-04-01T12:01:35.129Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c0/047fef3ac4a531903c52ba7c108fd608556627723bfef7554f040b10e556/blis-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:8d284323cc994e9b818c32046f1aa3e57bcc41c74e02daebdf0d3bc3e14355cb", size = 6232639, upload-time = "2025-04-01T12:01:37.268Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f1/2aecd2447de0eb5deea3a13e471ab43e42e8561afe56a13d830f95c58909/blis-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1cd35e94a1a97b37b31b11f097f998a3a0e75ac06d57e6edf7d9597200f55756", size = 6989811, upload-time = "2025-04-01T12:01:39.013Z" }, + { url = "https://files.pythonhosted.org/packages/cf/39/4c097508f6b9ef7df27dd5ada0a175e8169f58cbe33d40a303a844abdaea/blis-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b6394d27f2259c580df8d13ebe9c0a188a6ace0a689e93d6e49cb15018d4d9c", size = 1282669, upload-time = "2025-04-01T12:01:41.418Z" }, + { url = "https://files.pythonhosted.org/packages/7a/8e/b8a5eafa9824fcc7f3339a283e910f7af110d749fd09f52e83f432124543/blis-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9c127159415dc772f345abc3575e1e2d02bb1ae7cb7f532267d67705be04c66", size = 3063750, upload-time = "2025-04-01T12:01:43.277Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7a/f88e935f2cd3ad52ef363beeddf9a537d5038e519aa7b09dc18c762fbb66/blis-1.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f9fa589aa72448009fd5001afb05e69f3bc953fe778b44580fd7d79ee8201a1", size = 3260903, upload-time = "2025-04-01T12:01:44.815Z" }, + { url = "https://files.pythonhosted.org/packages/4a/26/283f1392974e5c597228f8485f45f89de33f2c85becebc25e846d0485e44/blis-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1aa6150259caf4fa0b527bfc8c1e858542f9ca88a386aa90b93e1ca4c2add6df", size = 11616588, upload-time = "2025-04-01T12:01:46.356Z" }, + { url = "https://files.pythonhosted.org/packages/fa/86/57047b688e42c92e35d0581ef9db15ee3bdf14deff4d9a2481ce331f2dae/blis-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ba67c09883cae52da3d9e9d3f4305464efedd336032c4d5c6c429b27b16f4c1", size = 3072892, upload-time = "2025-04-01T12:01:48.314Z" }, + { url = "https://files.pythonhosted.org/packages/c7/db/85b6f5fa2a2515470cc5a2cbeaedd25aa465fa572801f18d14c24c9e5102/blis-1.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7d9c5fca21b01c4b2f3cb95b71ce7ef95e58b3b62f0d79d1f699178c72c1e03e", size = 4310005, upload-time = "2025-04-01T12:01:49.815Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/6e610e950476ebc9868a0207a827d67433ef65e2b14b837d317e60248e5a/blis-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6952a4a1f15e0d1f73cc1206bd71368b32551f2e94852dae288b50c4ea0daf31", size = 14790198, upload-time = "2025-04-01T12:01:52.601Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/353e29e8dd3d31bba25a3eabbbfb798d82bd19ca2d24fd00583b6d3992f3/blis-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:bd0360427b1669684cd35a8355be126d7a33992ccac6dcb1fbef5e100f4e3026", size = 6260640, upload-time = "2025-04-01T12:01:54.849Z" }, +] + [[package]] name = "boto3" -version = "1.38.27" +version = "1.39.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/fc74d8521d2369dd8c412438401ff12e1350a1cd3eab5c758ed3dd5e5f82/boto3-1.38.27.tar.gz", hash = "sha256:94bd7fdd92d5701b362d4df100d21e28f8307a67ff56b6a8b0398119cf22f859", size = 111875, upload-time = "2025-05-30T19:32:41.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/2e/ed75ea3ee0fd1afacc3379bc2b7457c67a6b0f0e554e1f7ccbdbaed2351b/boto3-1.39.11.tar.gz", hash = "sha256:3027edf20642fe1d5f9dc50a420d0fe2733073ed6a9f0f047b60fe08c3682132", size = 111869, upload-time = "2025-07-22T19:26:50.867Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/8b/b2361188bd1e293eede1bc165e2461d390394f71ec0c8c21211c8dabf62c/boto3-1.38.27-py3-none-any.whl", hash = "sha256:95f5fe688795303a8a15e8b7e7f255cadab35eae459d00cc281a4fd77252ea80", size = 139938, upload-time = "2025-05-30T19:32:38.006Z" }, + { url = "https://files.pythonhosted.org/packages/72/66/88566a6484e746c0b075f7c9bb248e8548eda0a486de4460d150a41e2d57/boto3-1.39.11-py3-none-any.whl", hash = "sha256:af8f1dad35eceff7658fab43b39b0f55892b6e3dd12308733521cc24dd2c9a02", size = 139900, upload-time = "2025-07-22T19:26:48.706Z" }, ] [[package]] name = "botocore" -version = "1.38.27" +version = "1.39.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/5e/67899214ad57f7f26af5bd776ac5eb583dc4ecf5c1e52e2cbfdc200e487a/botocore-1.38.27.tar.gz", hash = "sha256:9788f7efe974328a38cbade64cc0b1e67d27944b899f88cb786ae362973133b6", size = 13919963, upload-time = "2025-05-30T19:32:29.657Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/d0/9d64261186cff650fe63168441edb4f4cd33f085a74c0c54455630a71f91/botocore-1.39.11.tar.gz", hash = "sha256:953b12909d6799350e346ab038e55b6efe622c616f80aef74d7a6683ffdd972c", size = 14217749, upload-time = "2025-07-22T19:26:40.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/83/a753562020b69fa90cebc39e8af2c753b24dcdc74bee8355ee3f6cefdf34/botocore-1.38.27-py3-none-any.whl", hash = "sha256:a785d5e9a5eda88ad6ab9ed8b87d1f2ac409d0226bba6ff801c55359e94d91a8", size = 13580545, upload-time = "2025-05-30T19:32:26.712Z" }, + { url = "https://files.pythonhosted.org/packages/1c/2c/8a0b02d60a1dbbae7faa5af30484b016aa3023f9833dfc0d19b0b770dd6a/botocore-1.39.11-py3-none-any.whl", hash = "sha256:1545352931a8a186f3e977b1e1a4542d7d434796e274c3c62efd0210b5ea76dc", size = 13876276, upload-time = "2025-07-22T19:26:35.164Z" }, ] [[package]] @@ -485,89 +522,114 @@ wheels = [ [[package]] name = "cachetools" -version = "6.1.0" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "catalogue" +version = "2.0.10" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/89/817ad5d0411f136c484d535952aef74af9b25e0d99e90cdffbe121e6d628/cachetools-6.1.0.tar.gz", hash = "sha256:b4c4f404392848db3ce7aac34950d17be4d864da4b8b66911008e430bc544587", size = 30714, upload-time = "2025-06-16T18:51:03.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/244d58127e1cdf04cf2dc7d9566f0d24ef01d5ce21811bab088ecc62b5ea/catalogue-2.0.10.tar.gz", hash = "sha256:4f56daa940913d3f09d589c191c74e5a6d51762b3a9e37dd53b7437afd6cda15", size = 19561, upload-time = "2023-09-25T06:29:24.962Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/f0/2ef431fe4141f5e334759d73e81120492b23b2824336883a91ac04ba710b/cachetools-6.1.0-py3-none-any.whl", hash = "sha256:1c7bb3cf9193deaf3508b7c5f2a79986c13ea38965c5adcff1f84519cf39163e", size = 11189, upload-time = "2025-06-16T18:51:01.514Z" }, + { url = "https://files.pythonhosted.org/packages/9e/96/d32b941a501ab566a16358d68b6eb4e4acc373fab3c3c4d7d9e649f7b4bb/catalogue-2.0.10-py3-none-any.whl", hash = "sha256:58c2de0020aa90f4a2da7dfad161bf7b3b054c86a5f09fcedc0b2b740c109a9f", size = 17325, upload-time = "2023-09-25T06:29:23.337Z" }, ] [[package]] name = "certifi" -version = "2025.7.14" +version = "2025.8.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/76/52c535bcebe74590f296d6c77c86dabf761c41980e1347a2422e4aa2ae41/certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995", size = 163981, upload-time = "2025-07-14T03:29:28.449Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/52/34c6cf5bb9285074dc3531c437b3919e825d976fde097a7a73f79e726d03/certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2", size = 162722, upload-time = "2025-07-14T03:29:26.863Z" }, + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, ] [[package]] name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, +version = "2.0.0b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/84/7930c3586ca7c66a63b2d7a30d9df649ce8c3660f8da241b0661bba4e566/cffi-2.0.0b1.tar.gz", hash = "sha256:4440de58d19c0bebe6a2f3b721253d67b27aabb34e00ab35756d8699876191ea", size = 521625, upload-time = "2025-07-29T01:11:50.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/90/14deaf13603dfff56bb872a4d53e1043486178ae7a2ce8cc17ea5677d97e/cffi-2.0.0b1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:5f373f9bdc3569acd8aaebb6b521080eeb5a298533a58715537caf74e9e27f6b", size = 184383, upload-time = "2025-07-29T01:10:17.675Z" }, + { url = "https://files.pythonhosted.org/packages/f7/36/0a125a1ab354a95aae2165ce4c2b8fcd057706a85380670e3991052dcfcd/cffi-2.0.0b1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a898f76bac81f9a371df6c8664228a85cdea6b283a721f2493f0df6f80afd208", size = 180599, upload-time = "2025-07-29T01:10:18.753Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cb/27237bcd6c4e883104db737929f02838a7405caed422aeeb76ee5ffa14d9/cffi-2.0.0b1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:314afab228f7b45de7bae55059b4e706296e7d3984d53e643cc0389757216221", size = 203212, upload-time = "2025-07-29T01:10:20.057Z" }, + { url = "https://files.pythonhosted.org/packages/12/94/bbeddca63090c5335ad597310bd6f2011f1c8733bc71e88f53c38ac4ff4c/cffi-2.0.0b1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6de033c73dc89f80139c5a7d135fbd6c1d7b28ebb0d2df98cd1f4ef76991b15c", size = 202714, upload-time = "2025-07-29T01:10:21.401Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9b/b7587a1f3f7f52795a7d125d6c6b844f7a8355cbb54ae8fdef2a03488914/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffbbeedd6bac26c0373b71831d3c73181a1c100dc6fc7aadbfcca54cace417db", size = 217093, upload-time = "2025-07-29T01:10:22.481Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b2/af4e0ed2c2aded25ed54107f96d424407839bdfa7e90858f8e0f6fed6ee9/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_i686.manylinux_2_28_i686.whl", hash = "sha256:c5713cac21b2351a53958c765d8e9eda45184bb757c3ccab139608e708788796", size = 209019, upload-time = "2025-07-29T01:10:23.584Z" }, + { url = "https://files.pythonhosted.org/packages/7b/6e/899c5473c3d7cc89815db894abcd81cd976a1f314c142e708aef3c0982a3/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:71ab35c6cc375da1e2c06af65bf0b5049199ad9b264f9ed7c90c0fe9450900e3", size = 215662, upload-time = "2025-07-29T01:10:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8e/953a07806f307bf1089239858013cc81c6d5cc8ca23593704b0530429302/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53c780c2ec8ce0e5db9b74e9b0b55ff5d5f70071202740cef073a2771fa1d2ce", size = 219015, upload-time = "2025-07-29T01:10:27.077Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0a/ffd99099d96a911236decff459cb330a1c046483008456b23554f62c81c6/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:be957dd266facf8e4925643073159b05021a990b46620b06ca27eaf9d900dbc2", size = 212021, upload-time = "2025-07-29T01:10:28.527Z" }, + { url = "https://files.pythonhosted.org/packages/2f/00/c68c1a1665a28dfb8c848668f128d0f1919dc8e843f2e20ce90bce7b60d8/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:16dc303af3630f54186b86aadf1121badf3cba6de17dfeacb84c5091e059a690", size = 217124, upload-time = "2025-07-29T01:10:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/de/a7/194d80668bebc5a6a8d95ec5f3a1f186e8d87c864882c96a9ec2ecbd06a8/cffi-2.0.0b1-cp311-cp311-win32.whl", hash = "sha256:504d264944d0934d7b02164af5c62b175255ef0d39c5142d95968b710c58a8f6", size = 172111, upload-time = "2025-07-29T01:10:30.973Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b6/0002211aab83b6bfbdba09dc8cd354e44c49216e6207999b9f0d1d0053cb/cffi-2.0.0b1-cp311-cp311-win_amd64.whl", hash = "sha256:e2920fa42cf0616c21ea6d3948ad207cf0e420d2d2ef449d86ccad6ef9c13393", size = 182858, upload-time = "2025-07-29T01:10:32.021Z" }, + { url = "https://files.pythonhosted.org/packages/52/9e/c6773b5b91b20c5642166c57503a9c67c6948ae4009aa4d2ce233a6b570f/cffi-2.0.0b1-cp311-cp311-win_arm64.whl", hash = "sha256:142c9c0c75fbc95ce23836e538681bd89e483de37b7cdf251dbdf0975995f8ac", size = 177421, upload-time = "2025-07-29T01:10:33.191Z" }, + { url = "https://files.pythonhosted.org/packages/50/20/432dc366952574ea190bce0a2970f92e676e972c78ef501d58406b459883/cffi-2.0.0b1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9d04b5fc06ba0ce45d7e51dfd8a14dc20708ef301fcf5a215c507f4e084b00c8", size = 185303, upload-time = "2025-07-29T01:10:34.291Z" }, + { url = "https://files.pythonhosted.org/packages/54/2d/e89016a2019212d54be2523756faa5b2c3ab8cb6f520a82e0d6bcacd527d/cffi-2.0.0b1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b17e92900eb61bce62ea07ea8dd0dc33aa476ee8f977918050e52f90f5b645c", size = 181101, upload-time = "2025-07-29T01:10:35.641Z" }, + { url = "https://files.pythonhosted.org/packages/89/4f/6978a38ee0d8976f3087c09e779f9306ed51b9fb68ce5e3606244f6e2469/cffi-2.0.0b1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2155d2a0819c3fdcaa37832fb69e698d455627c23f83bc9c7adbef699fe4be19", size = 208122, upload-time = "2025-07-29T01:10:36.757Z" }, + { url = "https://files.pythonhosted.org/packages/20/2f/568d19b010aa304f6f55aaf160834e0db9677943b0c268462876c4e1c0ef/cffi-2.0.0b1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4210ddc2b41c20739c64dede1304fb81415220ea671885623063fab44066e376", size = 206747, upload-time = "2025-07-29T01:10:37.837Z" }, + { url = "https://files.pythonhosted.org/packages/bf/7b/171907beef5622bc6164ae9db94eaaa8e56bfb986f375742a9669ecc18f7/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31b8e3204cdef043e59a296383e6a43461d17c5c3d73fa9cebf4716a561291b0", size = 220804, upload-time = "2025-07-29T01:10:39.299Z" }, + { url = "https://files.pythonhosted.org/packages/49/2a/539d6021b1570308159745e775d0bd4164e43957e515bffd33cb6e57cf06/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_i686.manylinux_2_28_i686.whl", hash = "sha256:cbde39be02aa7d8fbcd6bf1a9241cb1d84f2e2f0614970c51a707a9a176b85c6", size = 211912, upload-time = "2025-07-29T01:10:40.767Z" }, + { url = "https://files.pythonhosted.org/packages/87/a9/2cddc8eeabd7b32d494de5bb9db95e3816b47ad00e05269b33e2bb8be9f3/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ea57043b545f346b081877737cb0320960012107d0250fa5183a4306f9365d6", size = 219528, upload-time = "2025-07-29T01:10:42.419Z" }, + { url = "https://files.pythonhosted.org/packages/a8/18/49ff9cbe89eae3fff54a7af79474dd897bac44325073a6a7dc9b7ae4b64e/cffi-2.0.0b1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d31ba9f54739dcf98edb87e4881e326fad79e4866137c24afb0da531c1a965ca", size = 223011, upload-time = "2025-07-29T01:10:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/a1/1e/4f10dd0fd9cb8d921620663beb497af0a6175c96cecd87e5baf613d0c947/cffi-2.0.0b1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27309de8cebf48e056550db6607e2fb2c50109b54fc72c02b3b34811233483be", size = 221408, upload-time = "2025-07-29T01:10:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/00/82/cbbb23951d9890475f151c1137d067a712e7f1e59509def619c5d9a645aa/cffi-2.0.0b1-cp312-cp312-win32.whl", hash = "sha256:f4b5acb4cddcaf0ebb82a226f9fa1d5063505e0c206031ee1f4d173750b592fd", size = 172972, upload-time = "2025-07-29T01:10:46.458Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/e52b88ee438acd26fd84963f357a90ce8f4494cc7d94cbde1b26e199bd22/cffi-2.0.0b1-cp312-cp312-win_amd64.whl", hash = "sha256:cf1b2510f1a91c4d7e8f83df6a13404332421e6e4a067059174d455653ae5314", size = 183592, upload-time = "2025-07-29T01:10:47.916Z" }, + { url = "https://files.pythonhosted.org/packages/73/ac/3a5a182637b9a02c16335743b14485cb916ca984dcdc18737851732bff16/cffi-2.0.0b1-cp312-cp312-win_arm64.whl", hash = "sha256:bd7ce5d8224fb5a57bd7f1d9843aa4ecb870ec3f4a2101e1ba8314e91177e184", size = 177583, upload-time = "2025-07-29T01:10:49.091Z" }, +] + +[[package]] +name = "chardet" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, ] [[package]] name = "charset-normalizer" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" }, - { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" }, - { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" }, - { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" }, - { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" }, - { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" }, - { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" }, - { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" }, - { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" }, - { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" }, - { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" }, - { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" }, - { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" }, - { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" }, - { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" }, - { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" }, - { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" }, - { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" }, - { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" }, - { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" }, - { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, + { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, + { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "cint" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/c8/3ae22fa142be0bf9eee856e90c314f4144dfae376cc5e3e55b9a169670fb/cint-1.0.0.tar.gz", hash = "sha256:66f026d28c46ef9ea9635be5cb342506c6a1af80d11cb1c881a8898ca429fc91", size = 4641, upload-time = "2019-03-19T01:07:48.723Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/c2/898e59963084e1e2cbd4aad1dee92c5bd7a79d121dcff1e659c2a0c2174e/cint-1.0.0-py3-none-any.whl", hash = "sha256:8aa33028e04015711c0305f918cb278f1dc8c5c9997acdc45efad2c7cb1abf50", size = 5573, upload-time = "2019-03-19T01:07:46.496Z" }, ] [[package]] @@ -582,6 +644,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] +[[package]] +name = "cloudpathlib" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/15/ae3256348834b92b9594d73eb7230538bae2bf726c2d721b920a668017c5/cloudpathlib-0.21.1.tar.gz", hash = "sha256:f26a855abf34d98f267aafd15efdb2db3c9665913dbabe5fad079df92837a431", size = 45295, upload-time = "2025-05-15T02:32:05.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/e7/6fea57b887f8e367c1e4a496ba03bfaf57824b766f777723ce1faf28834b/cloudpathlib-0.21.1-py3-none-any.whl", hash = "sha256:bfe580ad72ec030472ec233cd7380701b2d3227da7b2898387bd170aa70c803c", size = 52776, upload-time = "2025-05-15T02:32:03.99Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -591,70 +662,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "comm" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, +] + +[[package]] +name = "confection" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "srsly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/d3/57c6631159a1b48d273b40865c315cf51f89df7a9d1101094ef12e3a37c2/confection-0.1.5.tar.gz", hash = "sha256:8e72dd3ca6bd4f48913cd220f10b8275978e740411654b6e8ca6d7008c590f0e", size = 38924, upload-time = "2024-05-31T16:17:01.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/00/3106b1854b45bd0474ced037dfe6b73b90fe68a68968cef47c23de3d43d2/confection-0.1.5-py3-none-any.whl", hash = "sha256:e29d3c3f8eac06b3f77eb9dfb4bf2fc6bcc9622a98ca00a698e3d019c6430b14", size = 35451, upload-time = "2024-05-31T16:16:59.075Z" }, +] + [[package]] name = "contourpy" -version = "1.3.2" +version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/b9/ede788a0b56fc5b071639d06c33cb893f68b1178938f3425debebe2dab78/contourpy-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445", size = 269636, upload-time = "2025-04-15T17:35:54.473Z" }, - { url = "https://files.pythonhosted.org/packages/e6/75/3469f011d64b8bbfa04f709bfc23e1dd71be54d05b1b083be9f5b22750d1/contourpy-1.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773", size = 254636, upload-time = "2025-04-15T17:35:58.283Z" }, - { url = "https://files.pythonhosted.org/packages/8d/2f/95adb8dae08ce0ebca4fd8e7ad653159565d9739128b2d5977806656fcd2/contourpy-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1", size = 313053, upload-time = "2025-04-15T17:36:03.235Z" }, - { url = "https://files.pythonhosted.org/packages/c3/a6/8ccf97a50f31adfa36917707fe39c9a0cbc24b3bbb58185577f119736cc9/contourpy-1.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43", size = 352985, upload-time = "2025-04-15T17:36:08.275Z" }, - { url = "https://files.pythonhosted.org/packages/1d/b6/7925ab9b77386143f39d9c3243fdd101621b4532eb126743201160ffa7e6/contourpy-1.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab", size = 323750, upload-time = "2025-04-15T17:36:13.29Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f3/20c5d1ef4f4748e52d60771b8560cf00b69d5c6368b5c2e9311bcfa2a08b/contourpy-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7", size = 326246, upload-time = "2025-04-15T17:36:18.329Z" }, - { url = "https://files.pythonhosted.org/packages/8c/e5/9dae809e7e0b2d9d70c52b3d24cba134dd3dad979eb3e5e71f5df22ed1f5/contourpy-1.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83", size = 1308728, upload-time = "2025-04-15T17:36:33.878Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/0058ba34aeea35c0b442ae61a4f4d4ca84d6df8f91309bc2d43bb8dd248f/contourpy-1.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd", size = 1375762, upload-time = "2025-04-15T17:36:51.295Z" }, - { url = "https://files.pythonhosted.org/packages/09/33/7174bdfc8b7767ef2c08ed81244762d93d5c579336fc0b51ca57b33d1b80/contourpy-1.3.2-cp311-cp311-win32.whl", hash = "sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f", size = 178196, upload-time = "2025-04-15T17:36:55.002Z" }, - { url = "https://files.pythonhosted.org/packages/5e/fe/4029038b4e1c4485cef18e480b0e2cd2d755448bb071eb9977caac80b77b/contourpy-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878", size = 222017, upload-time = "2025-04-15T17:36:58.576Z" }, - { url = "https://files.pythonhosted.org/packages/34/f7/44785876384eff370c251d58fd65f6ad7f39adce4a093c934d4a67a7c6b6/contourpy-1.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2", size = 271580, upload-time = "2025-04-15T17:37:03.105Z" }, - { url = "https://files.pythonhosted.org/packages/93/3b/0004767622a9826ea3d95f0e9d98cd8729015768075d61f9fea8eeca42a8/contourpy-1.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15", size = 255530, upload-time = "2025-04-15T17:37:07.026Z" }, - { url = "https://files.pythonhosted.org/packages/e7/bb/7bd49e1f4fa805772d9fd130e0d375554ebc771ed7172f48dfcd4ca61549/contourpy-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92", size = 307688, upload-time = "2025-04-15T17:37:11.481Z" }, - { url = "https://files.pythonhosted.org/packages/fc/97/e1d5dbbfa170725ef78357a9a0edc996b09ae4af170927ba8ce977e60a5f/contourpy-1.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87", size = 347331, upload-time = "2025-04-15T17:37:18.212Z" }, - { url = "https://files.pythonhosted.org/packages/6f/66/e69e6e904f5ecf6901be3dd16e7e54d41b6ec6ae3405a535286d4418ffb4/contourpy-1.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415", size = 318963, upload-time = "2025-04-15T17:37:22.76Z" }, - { url = "https://files.pythonhosted.org/packages/a8/32/b8a1c8965e4f72482ff2d1ac2cd670ce0b542f203c8e1d34e7c3e6925da7/contourpy-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe", size = 323681, upload-time = "2025-04-15T17:37:33.001Z" }, - { url = "https://files.pythonhosted.org/packages/30/c6/12a7e6811d08757c7162a541ca4c5c6a34c0f4e98ef2b338791093518e40/contourpy-1.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441", size = 1308674, upload-time = "2025-04-15T17:37:48.64Z" }, - { url = "https://files.pythonhosted.org/packages/2a/8a/bebe5a3f68b484d3a2b8ffaf84704b3e343ef1addea528132ef148e22b3b/contourpy-1.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e", size = 1380480, upload-time = "2025-04-15T17:38:06.7Z" }, - { url = "https://files.pythonhosted.org/packages/34/db/fcd325f19b5978fb509a7d55e06d99f5f856294c1991097534360b307cf1/contourpy-1.3.2-cp312-cp312-win32.whl", hash = "sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912", size = 178489, upload-time = "2025-04-15T17:38:10.338Z" }, - { url = "https://files.pythonhosted.org/packages/01/c8/fadd0b92ffa7b5eb5949bf340a63a4a496a6930a6c37a7ba0f12acb076d6/contourpy-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73", size = 223042, upload-time = "2025-04-15T17:38:14.239Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c0/91f1215d0d9f9f343e4773ba6c9b89e8c0cc7a64a6263f21139da639d848/contourpy-1.3.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0", size = 266807, upload-time = "2025-04-15T17:45:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/d4/79/6be7e90c955c0487e7712660d6cead01fa17bff98e0ea275737cc2bc8e71/contourpy-1.3.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5", size = 318729, upload-time = "2025-04-15T17:45:20.166Z" }, - { url = "https://files.pythonhosted.org/packages/87/68/7f46fb537958e87427d98a4074bcde4b67a70b04900cfc5ce29bc2f556c1/contourpy-1.3.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5", size = 221791, upload-time = "2025-04-15T17:45:24.794Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/2e/c4390a31919d8a78b90e8ecf87cd4b4c4f05a5b48d05ec17db8e5404c6f4/contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1", size = 288773, upload-time = "2025-07-26T12:01:02.277Z" }, + { url = "https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381", size = 270149, upload-time = "2025-07-26T12:01:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/30/2e/dd4ced42fefac8470661d7cb7e264808425e6c5d56d175291e93890cce09/contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7", size = 329222, upload-time = "2025-07-26T12:01:05.688Z" }, + { url = "https://files.pythonhosted.org/packages/f2/74/cc6ec2548e3d276c71389ea4802a774b7aa3558223b7bade3f25787fafc2/contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1", size = 377234, upload-time = "2025-07-26T12:01:07.054Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/64ef723029f917410f75c09da54254c5f9ea90ef89b143ccadb09df14c15/contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a", size = 380555, upload-time = "2025-07-26T12:01:08.801Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db", size = 355238, upload-time = "2025-07-26T12:01:10.319Z" }, + { url = "https://files.pythonhosted.org/packages/98/56/f914f0dd678480708a04cfd2206e7c382533249bc5001eb9f58aa693e200/contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620", size = 1326218, upload-time = "2025-07-26T12:01:12.659Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/4a972334a0c971acd5172389671113ae82aa7527073980c38d5868ff1161/contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f", size = 1392867, upload-time = "2025-07-26T12:01:15.533Z" }, + { url = "https://files.pythonhosted.org/packages/75/3e/f2cc6cd56dc8cff46b1a56232eabc6feea52720083ea71ab15523daab796/contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff", size = 183677, upload-time = "2025-07-26T12:01:17.088Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42", size = 225234, upload-time = "2025-07-26T12:01:18.256Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b6/71771e02c2e004450c12b1120a5f488cad2e4d5b590b1af8bad060360fe4/contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470", size = 193123, upload-time = "2025-07-26T12:01:19.848Z" }, + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/8dcfe16f0107943fa92388c23f6e05cff0ba58058c4c95b00280d4c75a14/contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497", size = 278809, upload-time = "2025-07-26T12:02:52.74Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/8b37ef4f7dafeb335daee3c8254645ef5725be4d9c6aa70b50ec46ef2f7e/contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8", size = 261593, upload-time = "2025-07-26T12:02:54.037Z" }, + { url = "https://files.pythonhosted.org/packages/0a/59/ebfb8c677c75605cc27f7122c90313fd2f375ff3c8d19a1694bda74aaa63/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e", size = 302202, upload-time = "2025-07-26T12:02:55.947Z" }, + { url = "https://files.pythonhosted.org/packages/3c/37/21972a15834d90bfbfb009b9d004779bd5a07a0ec0234e5ba8f64d5736f4/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989", size = 329207, upload-time = "2025-07-26T12:02:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315, upload-time = "2025-07-26T12:02:58.801Z" }, ] [[package]] name = "coverage" -version = "7.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/b7/c0465ca253df10a9e8dae0692a4ae6e9726d245390aaef92360e1d6d3832/coverage-7.9.2.tar.gz", hash = "sha256:997024fa51e3290264ffd7492ec97d0690293ccd2b45a6cd7d82d945a4a80c8b", size = 813556, upload-time = "2025-07-03T10:54:15.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/40/916786453bcfafa4c788abee4ccd6f592b5b5eca0cd61a32a4e5a7ef6e02/coverage-7.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a7a56a2964a9687b6aba5b5ced6971af308ef6f79a91043c05dd4ee3ebc3e9ba", size = 212152, upload-time = "2025-07-03T10:52:53.562Z" }, - { url = "https://files.pythonhosted.org/packages/9f/66/cc13bae303284b546a030762957322bbbff1ee6b6cb8dc70a40f8a78512f/coverage-7.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:123d589f32c11d9be7fe2e66d823a236fe759b0096f5db3fb1b75b2fa414a4fa", size = 212540, upload-time = "2025-07-03T10:52:55.196Z" }, - { url = "https://files.pythonhosted.org/packages/0f/3c/d56a764b2e5a3d43257c36af4a62c379df44636817bb5f89265de4bf8bd7/coverage-7.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:333b2e0ca576a7dbd66e85ab402e35c03b0b22f525eed82681c4b866e2e2653a", size = 245097, upload-time = "2025-07-03T10:52:56.509Z" }, - { url = "https://files.pythonhosted.org/packages/b1/46/bd064ea8b3c94eb4ca5d90e34d15b806cba091ffb2b8e89a0d7066c45791/coverage-7.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:326802760da234baf9f2f85a39e4a4b5861b94f6c8d95251f699e4f73b1835dc", size = 242812, upload-time = "2025-07-03T10:52:57.842Z" }, - { url = "https://files.pythonhosted.org/packages/43/02/d91992c2b29bc7afb729463bc918ebe5f361be7f1daae93375a5759d1e28/coverage-7.9.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19e7be4cfec248df38ce40968c95d3952fbffd57b400d4b9bb580f28179556d2", size = 244617, upload-time = "2025-07-03T10:52:59.239Z" }, - { url = "https://files.pythonhosted.org/packages/b7/4f/8fadff6bf56595a16d2d6e33415841b0163ac660873ed9a4e9046194f779/coverage-7.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0b4a4cb73b9f2b891c1788711408ef9707666501ba23684387277ededab1097c", size = 244263, upload-time = "2025-07-03T10:53:00.601Z" }, - { url = "https://files.pythonhosted.org/packages/9b/d2/e0be7446a2bba11739edb9f9ba4eff30b30d8257370e237418eb44a14d11/coverage-7.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2c8937fa16c8c9fbbd9f118588756e7bcdc7e16a470766a9aef912dd3f117dbd", size = 242314, upload-time = "2025-07-03T10:53:01.932Z" }, - { url = "https://files.pythonhosted.org/packages/9d/7d/dcbac9345000121b8b57a3094c2dfcf1ccc52d8a14a40c1d4bc89f936f80/coverage-7.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:42da2280c4d30c57a9b578bafd1d4494fa6c056d4c419d9689e66d775539be74", size = 242904, upload-time = "2025-07-03T10:53:03.478Z" }, - { url = "https://files.pythonhosted.org/packages/41/58/11e8db0a0c0510cf31bbbdc8caf5d74a358b696302a45948d7c768dfd1cf/coverage-7.9.2-cp311-cp311-win32.whl", hash = "sha256:14fa8d3da147f5fdf9d298cacc18791818f3f1a9f542c8958b80c228320e90c6", size = 214553, upload-time = "2025-07-03T10:53:05.174Z" }, - { url = "https://files.pythonhosted.org/packages/3a/7d/751794ec8907a15e257136e48dc1021b1f671220ecccfd6c4eaf30802714/coverage-7.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:549cab4892fc82004f9739963163fd3aac7a7b0df430669b75b86d293d2df2a7", size = 215441, upload-time = "2025-07-03T10:53:06.472Z" }, - { url = "https://files.pythonhosted.org/packages/62/5b/34abcedf7b946c1c9e15b44f326cb5b0da852885312b30e916f674913428/coverage-7.9.2-cp311-cp311-win_arm64.whl", hash = "sha256:c2667a2b913e307f06aa4e5677f01a9746cd08e4b35e14ebcde6420a9ebb4c62", size = 213873, upload-time = "2025-07-03T10:53:07.699Z" }, - { url = "https://files.pythonhosted.org/packages/53/d7/7deefc6fd4f0f1d4c58051f4004e366afc9e7ab60217ac393f247a1de70a/coverage-7.9.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae9eb07f1cfacd9cfe8eaee6f4ff4b8a289a668c39c165cd0c8548484920ffc0", size = 212344, upload-time = "2025-07-03T10:53:09.3Z" }, - { url = "https://files.pythonhosted.org/packages/95/0c/ee03c95d32be4d519e6a02e601267769ce2e9a91fc8faa1b540e3626c680/coverage-7.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9ce85551f9a1119f02adc46d3014b5ee3f765deac166acf20dbb851ceb79b6f3", size = 212580, upload-time = "2025-07-03T10:53:11.52Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9f/826fa4b544b27620086211b87a52ca67592622e1f3af9e0a62c87aea153a/coverage-7.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8f6389ac977c5fb322e0e38885fbbf901743f79d47f50db706e7644dcdcb6e1", size = 246383, upload-time = "2025-07-03T10:53:13.134Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b3/4477aafe2a546427b58b9c540665feff874f4db651f4d3cb21b308b3a6d2/coverage-7.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff0d9eae8cdfcd58fe7893b88993723583a6ce4dfbfd9f29e001922544f95615", size = 243400, upload-time = "2025-07-03T10:53:14.614Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c2/efffa43778490c226d9d434827702f2dfbc8041d79101a795f11cbb2cf1e/coverage-7.9.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fae939811e14e53ed8a9818dad51d434a41ee09df9305663735f2e2d2d7d959b", size = 245591, upload-time = "2025-07-03T10:53:15.872Z" }, - { url = "https://files.pythonhosted.org/packages/c6/e7/a59888e882c9a5f0192d8627a30ae57910d5d449c80229b55e7643c078c4/coverage-7.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:31991156251ec202c798501e0a42bbdf2169dcb0f137b1f5c0f4267f3fc68ef9", size = 245402, upload-time = "2025-07-03T10:53:17.124Z" }, - { url = "https://files.pythonhosted.org/packages/92/a5/72fcd653ae3d214927edc100ce67440ed8a0a1e3576b8d5e6d066ed239db/coverage-7.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d0d67963f9cbfc7c7f96d4ac74ed60ecbebd2ea6eeb51887af0f8dce205e545f", size = 243583, upload-time = "2025-07-03T10:53:18.781Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f5/84e70e4df28f4a131d580d7d510aa1ffd95037293da66fd20d446090a13b/coverage-7.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49b752a2858b10580969ec6af6f090a9a440a64a301ac1528d7ca5f7ed497f4d", size = 244815, upload-time = "2025-07-03T10:53:20.168Z" }, - { url = "https://files.pythonhosted.org/packages/39/e7/d73d7cbdbd09fdcf4642655ae843ad403d9cbda55d725721965f3580a314/coverage-7.9.2-cp312-cp312-win32.whl", hash = "sha256:88d7598b8ee130f32f8a43198ee02edd16d7f77692fa056cb779616bbea1b355", size = 214719, upload-time = "2025-07-03T10:53:21.521Z" }, - { url = "https://files.pythonhosted.org/packages/9f/d6/7486dcc3474e2e6ad26a2af2db7e7c162ccd889c4c68fa14ea8ec189c9e9/coverage-7.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:9dfb070f830739ee49d7c83e4941cc767e503e4394fdecb3b54bfdac1d7662c0", size = 215509, upload-time = "2025-07-03T10:53:22.853Z" }, - { url = "https://files.pythonhosted.org/packages/b7/34/0439f1ae2593b0346164d907cdf96a529b40b7721a45fdcf8b03c95fcd90/coverage-7.9.2-cp312-cp312-win_arm64.whl", hash = "sha256:4e2c058aef613e79df00e86b6d42a641c877211384ce5bd07585ed7ba71ab31b", size = 213910, upload-time = "2025-07-03T10:53:24.472Z" }, - { url = "https://files.pythonhosted.org/packages/d7/85/f8bbefac27d286386961c25515431482a425967e23d3698b75a250872924/coverage-7.9.2-pp39.pp310.pp311-none-any.whl", hash = "sha256:8a1166db2fb62473285bcb092f586e081e92656c7dfa8e9f62b4d39d7e6b5050", size = 204013, upload-time = "2025-07-03T10:54:12.084Z" }, - { url = "https://files.pythonhosted.org/packages/3c/38/bbe2e63902847cf79036ecc75550d0698af31c91c7575352eb25190d0fb3/coverage-7.9.2-py3-none-any.whl", hash = "sha256:e425cd5b00f6fc0ed7cdbd766c70be8baab4b7839e4d4fe5fac48581dd968ea4", size = 204005, upload-time = "2025-07-03T10:54:13.491Z" }, +version = "7.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/4e/08b493f1f1d8a5182df0044acc970799b58a8d289608e0d891a03e9d269a/coverage-7.10.4.tar.gz", hash = "sha256:25f5130af6c8e7297fd14634955ba9e1697f47143f289e2a23284177c0061d27", size = 823798, upload-time = "2025-08-17T00:26:43.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/ba/2c9817e62018e7d480d14f684c160b3038df9ff69c5af7d80e97d143e4d1/coverage-7.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:05d5f98ec893d4a2abc8bc5f046f2f4367404e7e5d5d18b83de8fde1093ebc4f", size = 216514, upload-time = "2025-08-17T00:24:34.188Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5a/093412a959a6b6261446221ba9fb23bb63f661a5de70b5d130763c87f916/coverage-7.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9267efd28f8994b750d171e58e481e3bbd69e44baed540e4c789f8e368b24b88", size = 216914, upload-time = "2025-08-17T00:24:35.881Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1f/2fdf4a71cfe93b07eae845ebf763267539a7d8b7e16b062f959d56d7e433/coverage-7.10.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4456a039fdc1a89ea60823d0330f1ac6f97b0dbe9e2b6fb4873e889584b085fb", size = 247308, upload-time = "2025-08-17T00:24:37.61Z" }, + { url = "https://files.pythonhosted.org/packages/ba/16/33f6cded458e84f008b9f6bc379609a6a1eda7bffe349153b9960803fc11/coverage-7.10.4-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c2bfbd2a9f7e68a21c5bd191be94bfdb2691ac40d325bac9ef3ae45ff5c753d9", size = 249241, upload-time = "2025-08-17T00:24:38.919Z" }, + { url = "https://files.pythonhosted.org/packages/84/98/9c18e47c889be58339ff2157c63b91a219272503ee32b49d926eea2337f2/coverage-7.10.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab7765f10ae1df7e7fe37de9e64b5a269b812ee22e2da3f84f97b1c7732a0d8", size = 251346, upload-time = "2025-08-17T00:24:40.507Z" }, + { url = "https://files.pythonhosted.org/packages/6d/07/00a6c0d53e9a22d36d8e95ddd049b860eef8f4b9fd299f7ce34d8e323356/coverage-7.10.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a09b13695166236e171ec1627ff8434b9a9bae47528d0ba9d944c912d33b3d2", size = 249037, upload-time = "2025-08-17T00:24:41.904Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0e/1e1b944d6a6483d07bab5ef6ce063fcf3d0cc555a16a8c05ebaab11f5607/coverage-7.10.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5c9e75dfdc0167d5675e9804f04a56b2cf47fb83a524654297000b578b8adcb7", size = 247090, upload-time = "2025-08-17T00:24:43.193Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/2ce5ab8a728b8e25ced077111581290ffaef9efaf860a28e25435ab925cf/coverage-7.10.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c751261bfe6481caba15ec005a194cb60aad06f29235a74c24f18546d8377df0", size = 247732, upload-time = "2025-08-17T00:24:44.906Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f3/706c4a24f42c1c5f3a2ca56637ab1270f84d9e75355160dc34d5e39bb5b7/coverage-7.10.4-cp311-cp311-win32.whl", hash = "sha256:051c7c9e765f003c2ff6e8c81ccea28a70fb5b0142671e4e3ede7cebd45c80af", size = 218961, upload-time = "2025-08-17T00:24:46.241Z" }, + { url = "https://files.pythonhosted.org/packages/e8/aa/6b9ea06e0290bf1cf2a2765bba89d561c5c563b4e9db8298bf83699c8b67/coverage-7.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:1a647b152f10be08fb771ae4a1421dbff66141e3d8ab27d543b5eb9ea5af8e52", size = 219851, upload-time = "2025-08-17T00:24:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/8b/be/f0dc9ad50ee183369e643cd7ed8f2ef5c491bc20b4c3387cbed97dd6e0d1/coverage-7.10.4-cp311-cp311-win_arm64.whl", hash = "sha256:b09b9e4e1de0d406ca9f19a371c2beefe3193b542f64a6dd40cfcf435b7d6aa0", size = 218530, upload-time = "2025-08-17T00:24:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/781c9e4dd57cabda2a28e2ce5b00b6be416015265851060945a5ed4bd85e/coverage-7.10.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a1f0264abcabd4853d4cb9b3d164adbf1565da7dab1da1669e93f3ea60162d79", size = 216706, upload-time = "2025-08-17T00:24:51.528Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8c/51255202ca03d2e7b664770289f80db6f47b05138e06cce112b3957d5dfd/coverage-7.10.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:536cbe6b118a4df231b11af3e0f974a72a095182ff8ec5f4868c931e8043ef3e", size = 216939, upload-time = "2025-08-17T00:24:53.171Z" }, + { url = "https://files.pythonhosted.org/packages/06/7f/df11131483698660f94d3c847dc76461369782d7a7644fcd72ac90da8fd0/coverage-7.10.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9a4c0d84134797b7bf3f080599d0cd501471f6c98b715405166860d79cfaa97e", size = 248429, upload-time = "2025-08-17T00:24:54.934Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fa/13ac5eda7300e160bf98f082e75f5c5b4189bf3a883dd1ee42dbedfdc617/coverage-7.10.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7c155fc0f9cee8c9803ea0ad153ab6a3b956baa5d4cd993405dc0b45b2a0b9e0", size = 251178, upload-time = "2025-08-17T00:24:56.353Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/f63b56a58ad0bec68a840e7be6b7ed9d6f6288d790760647bb88f5fea41e/coverage-7.10.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5f2ab6e451d4b07855d8bcf063adf11e199bff421a4ba57f5bb95b7444ca62", size = 252313, upload-time = "2025-08-17T00:24:57.692Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b6/79338f1ea27b01266f845afb4485976211264ab92407d1c307babe3592a7/coverage-7.10.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:685b67d99b945b0c221be0780c336b303a7753b3e0ec0d618c795aada25d5e7a", size = 250230, upload-time = "2025-08-17T00:24:59.293Z" }, + { url = "https://files.pythonhosted.org/packages/bc/93/3b24f1da3e0286a4dc5832427e1d448d5296f8287464b1ff4a222abeeeb5/coverage-7.10.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0c079027e50c2ae44da51c2e294596cbc9dbb58f7ca45b30651c7e411060fc23", size = 248351, upload-time = "2025-08-17T00:25:00.676Z" }, + { url = "https://files.pythonhosted.org/packages/de/5f/d59412f869e49dcc5b89398ef3146c8bfaec870b179cc344d27932e0554b/coverage-7.10.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3749aa72b93ce516f77cf5034d8e3c0dfd45c6e8a163a602ede2dc5f9a0bb927", size = 249788, upload-time = "2025-08-17T00:25:02.354Z" }, + { url = "https://files.pythonhosted.org/packages/cc/52/04a3b733f40a0cc7c4a5b9b010844111dbf906df3e868b13e1ce7b39ac31/coverage-7.10.4-cp312-cp312-win32.whl", hash = "sha256:fecb97b3a52fa9bcd5a7375e72fae209088faf671d39fae67261f37772d5559a", size = 219131, upload-time = "2025-08-17T00:25:03.79Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/12909fc0b83888197b3ec43a4ac7753589591c08d00d9deda4158df2734e/coverage-7.10.4-cp312-cp312-win_amd64.whl", hash = "sha256:26de58f355626628a21fe6a70e1e1fad95702dafebfb0685280962ae1449f17b", size = 219939, upload-time = "2025-08-17T00:25:05.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/c7/058bb3220fdd6821bada9685eadac2940429ab3c97025ce53549ff423cc1/coverage-7.10.4-cp312-cp312-win_arm64.whl", hash = "sha256:67e8885408f8325198862bc487038a4980c9277d753cb8812510927f2176437a", size = 218572, upload-time = "2025-08-17T00:25:06.897Z" }, + { url = "https://files.pythonhosted.org/packages/bb/78/983efd23200921d9edb6bd40512e1aa04af553d7d5a171e50f9b2b45d109/coverage-7.10.4-py3-none-any.whl", hash = "sha256:065d75447228d05121e5c938ca8f0e91eed60a1eb2d1258d42d5084fecfc3302", size = 208365, upload-time = "2025-08-17T00:26:41.479Z" }, ] [package.optional-dependencies] @@ -664,43 +760,43 @@ toml = [ [[package]] name = "cryptography" -version = "45.0.5" +version = "45.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/1e/49527ac611af559665f71cbb8f92b332b5ec9c6fbc4e88b0f8e92f5e85df/cryptography-45.0.5.tar.gz", hash = "sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a", size = 744903, upload-time = "2025-07-02T13:06:25.941Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/fb/09e28bc0c46d2c547085e60897fea96310574c70fb21cd58a730a45f3403/cryptography-45.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8", size = 7043092, upload-time = "2025-07-02T13:05:01.514Z" }, - { url = "https://files.pythonhosted.org/packages/b1/05/2194432935e29b91fb649f6149c1a4f9e6d3d9fc880919f4ad1bcc22641e/cryptography-45.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d", size = 4205926, upload-time = "2025-07-02T13:05:04.741Z" }, - { url = "https://files.pythonhosted.org/packages/07/8b/9ef5da82350175e32de245646b1884fc01124f53eb31164c77f95a08d682/cryptography-45.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5", size = 4429235, upload-time = "2025-07-02T13:05:07.084Z" }, - { url = "https://files.pythonhosted.org/packages/7c/e1/c809f398adde1994ee53438912192d92a1d0fc0f2d7582659d9ef4c28b0c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57", size = 4209785, upload-time = "2025-07-02T13:05:09.321Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8b/07eb6bd5acff58406c5e806eff34a124936f41a4fb52909ffa4d00815f8c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0", size = 3893050, upload-time = "2025-07-02T13:05:11.069Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ef/3333295ed58d900a13c92806b67e62f27876845a9a908c939f040887cca9/cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d", size = 4457379, upload-time = "2025-07-02T13:05:13.32Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9d/44080674dee514dbb82b21d6fa5d1055368f208304e2ab1828d85c9de8f4/cryptography-45.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9", size = 4209355, upload-time = "2025-07-02T13:05:15.017Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d8/0749f7d39f53f8258e5c18a93131919ac465ee1f9dccaf1b3f420235e0b5/cryptography-45.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27", size = 4456087, upload-time = "2025-07-02T13:05:16.945Z" }, - { url = "https://files.pythonhosted.org/packages/09/d7/92acac187387bf08902b0bf0699816f08553927bdd6ba3654da0010289b4/cryptography-45.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e", size = 4332873, upload-time = "2025-07-02T13:05:18.743Z" }, - { url = "https://files.pythonhosted.org/packages/03/c2/840e0710da5106a7c3d4153c7215b2736151bba60bf4491bdb421df5056d/cryptography-45.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174", size = 4564651, upload-time = "2025-07-02T13:05:21.382Z" }, - { url = "https://files.pythonhosted.org/packages/2e/92/cc723dd6d71e9747a887b94eb3827825c6c24b9e6ce2bb33b847d31d5eaa/cryptography-45.0.5-cp311-abi3-win32.whl", hash = "sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9", size = 2929050, upload-time = "2025-07-02T13:05:23.39Z" }, - { url = "https://files.pythonhosted.org/packages/1f/10/197da38a5911a48dd5389c043de4aec4b3c94cb836299b01253940788d78/cryptography-45.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63", size = 3403224, upload-time = "2025-07-02T13:05:25.202Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2b/160ce8c2765e7a481ce57d55eba1546148583e7b6f85514472b1d151711d/cryptography-45.0.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8", size = 7017143, upload-time = "2025-07-02T13:05:27.229Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e7/2187be2f871c0221a81f55ee3105d3cf3e273c0a0853651d7011eada0d7e/cryptography-45.0.5-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd", size = 4197780, upload-time = "2025-07-02T13:05:29.299Z" }, - { url = "https://files.pythonhosted.org/packages/b9/cf/84210c447c06104e6be9122661159ad4ce7a8190011669afceeaea150524/cryptography-45.0.5-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e", size = 4420091, upload-time = "2025-07-02T13:05:31.221Z" }, - { url = "https://files.pythonhosted.org/packages/3e/6a/cb8b5c8bb82fafffa23aeff8d3a39822593cee6e2f16c5ca5c2ecca344f7/cryptography-45.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0", size = 4198711, upload-time = "2025-07-02T13:05:33.062Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/36d2d69df69c94cbb2473871926daf0f01ad8e00fe3986ac3c1e8c4ca4b3/cryptography-45.0.5-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135", size = 3883299, upload-time = "2025-07-02T13:05:34.94Z" }, - { url = "https://files.pythonhosted.org/packages/82/c7/f0ea40f016de72f81288e9fe8d1f6748036cb5ba6118774317a3ffc6022d/cryptography-45.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7", size = 4450558, upload-time = "2025-07-02T13:05:37.288Z" }, - { url = "https://files.pythonhosted.org/packages/06/ae/94b504dc1a3cdf642d710407c62e86296f7da9e66f27ab12a1ee6fdf005b/cryptography-45.0.5-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42", size = 4198020, upload-time = "2025-07-02T13:05:39.102Z" }, - { url = "https://files.pythonhosted.org/packages/05/2b/aaf0adb845d5dabb43480f18f7ca72e94f92c280aa983ddbd0bcd6ecd037/cryptography-45.0.5-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492", size = 4449759, upload-time = "2025-07-02T13:05:41.398Z" }, - { url = "https://files.pythonhosted.org/packages/91/e4/f17e02066de63e0100a3a01b56f8f1016973a1d67551beaf585157a86b3f/cryptography-45.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0", size = 4319991, upload-time = "2025-07-02T13:05:43.64Z" }, - { url = "https://files.pythonhosted.org/packages/f2/2e/e2dbd629481b499b14516eed933f3276eb3239f7cee2dcfa4ee6b44d4711/cryptography-45.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a", size = 4554189, upload-time = "2025-07-02T13:05:46.045Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ea/a78a0c38f4c8736287b71c2ea3799d173d5ce778c7d6e3c163a95a05ad2a/cryptography-45.0.5-cp37-abi3-win32.whl", hash = "sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f", size = 2911769, upload-time = "2025-07-02T13:05:48.329Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/28ac139109d9005ad3f6b6f8976ffede6706a6478e21c889ce36c840918e/cryptography-45.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97", size = 3390016, upload-time = "2025-07-02T13:05:50.811Z" }, - { url = "https://files.pythonhosted.org/packages/c0/71/9bdbcfd58d6ff5084687fe722c58ac718ebedbc98b9f8f93781354e6d286/cryptography-45.0.5-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e", size = 3587878, upload-time = "2025-07-02T13:06:06.339Z" }, - { url = "https://files.pythonhosted.org/packages/f0/63/83516cfb87f4a8756eaa4203f93b283fda23d210fc14e1e594bd5f20edb6/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6", size = 4152447, upload-time = "2025-07-02T13:06:08.345Z" }, - { url = "https://files.pythonhosted.org/packages/22/11/d2823d2a5a0bd5802b3565437add16f5c8ce1f0778bf3822f89ad2740a38/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18", size = 4386778, upload-time = "2025-07-02T13:06:10.263Z" }, - { url = "https://files.pythonhosted.org/packages/5f/38/6bf177ca6bce4fe14704ab3e93627c5b0ca05242261a2e43ef3168472540/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463", size = 4151627, upload-time = "2025-07-02T13:06:13.097Z" }, - { url = "https://files.pythonhosted.org/packages/38/6a/69fc67e5266bff68a91bcb81dff8fb0aba4d79a78521a08812048913e16f/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1", size = 4385593, upload-time = "2025-07-02T13:06:15.689Z" }, - { url = "https://files.pythonhosted.org/packages/f6/34/31a1604c9a9ade0fdab61eb48570e09a796f4d9836121266447b0eaf7feb/cryptography-45.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f", size = 3331106, upload-time = "2025-07-02T13:06:18.058Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d6/0d/d13399c94234ee8f3df384819dc67e0c5ce215fb751d567a55a1f4b028c7/cryptography-45.0.6.tar.gz", hash = "sha256:5c966c732cf6e4a276ce83b6e4c729edda2df6929083a952cc7da973c539c719", size = 744949, upload-time = "2025-08-05T23:59:27.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/29/2793d178d0eda1ca4a09a7c4e09a5185e75738cc6d526433e8663b460ea6/cryptography-45.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:048e7ad9e08cf4c0ab07ff7f36cc3115924e22e2266e034450a890d9e312dd74", size = 7042702, upload-time = "2025-08-05T23:58:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/b3/b6/cabd07410f222f32c8d55486c464f432808abaa1f12af9afcbe8f2f19030/cryptography-45.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44647c5d796f5fc042bbc6d61307d04bf29bccb74d188f18051b635f20a9c75f", size = 4206483, upload-time = "2025-08-05T23:58:27.132Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/f9c7d36a38b1cfeb1cc74849aabe9bf817990f7603ff6eb485e0d70e0b27/cryptography-45.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e40b80ecf35ec265c452eea0ba94c9587ca763e739b8e559c128d23bff7ebbbf", size = 4429679, upload-time = "2025-08-05T23:58:29.152Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2a/4434c17eb32ef30b254b9e8b9830cee4e516f08b47fdd291c5b1255b8101/cryptography-45.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:00e8724bdad672d75e6f069b27970883179bd472cd24a63f6e620ca7e41cc0c5", size = 4210553, upload-time = "2025-08-05T23:58:30.596Z" }, + { url = "https://files.pythonhosted.org/packages/ef/1d/09a5df8e0c4b7970f5d1f3aff1b640df6d4be28a64cae970d56c6cf1c772/cryptography-45.0.6-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a3085d1b319d35296176af31c90338eeb2ddac8104661df79f80e1d9787b8b2", size = 3894499, upload-time = "2025-08-05T23:58:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/120842ab20d9150a9d3a6bdc07fe2870384e82f5266d41c53b08a3a96b34/cryptography-45.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1b7fa6a1c1188c7ee32e47590d16a5a0646270921f8020efc9a511648e1b2e08", size = 4458484, upload-time = "2025-08-05T23:58:33.526Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/1bc3634d45ddfed0871bfba52cf8f1ad724761662a0c792b97a951fb1b30/cryptography-45.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:275ba5cc0d9e320cd70f8e7b96d9e59903c815ca579ab96c1e37278d231fc402", size = 4210281, upload-time = "2025-08-05T23:58:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/ffb12c2d83d0ee625f124880a1f023b5878f79da92e64c37962bbbe35f3f/cryptography-45.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f4028f29a9f38a2025abedb2e409973709c660d44319c61762202206ed577c42", size = 4456890, upload-time = "2025-08-05T23:58:36.923Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8e/b3f3fe0dc82c77a0deb5f493b23311e09193f2268b77196ec0f7a36e3f3e/cryptography-45.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee411a1b977f40bd075392c80c10b58025ee5c6b47a822a33c1198598a7a5f05", size = 4333247, upload-time = "2025-08-05T23:58:38.781Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a6/c3ef2ab9e334da27a1d7b56af4a2417d77e7806b2e0f90d6267ce120d2e4/cryptography-45.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2a21a8eda2d86bb604934b6b37691585bd095c1f788530c1fcefc53a82b3453", size = 4565045, upload-time = "2025-08-05T23:58:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/77722446b13fa71dddd820a5faab4ce6db49e7e0bf8312ef4192a3f78e2f/cryptography-45.0.6-cp311-abi3-win32.whl", hash = "sha256:d063341378d7ee9c91f9d23b431a3502fc8bfacd54ef0a27baa72a0843b29159", size = 2928923, upload-time = "2025-08-05T23:58:41.919Z" }, + { url = "https://files.pythonhosted.org/packages/38/63/a025c3225188a811b82932a4dcc8457a26c3729d81578ccecbcce2cb784e/cryptography-45.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:833dc32dfc1e39b7376a87b9a6a4288a10aae234631268486558920029b086ec", size = 3403805, upload-time = "2025-08-05T23:58:43.792Z" }, + { url = "https://files.pythonhosted.org/packages/5b/af/bcfbea93a30809f126d51c074ee0fac5bd9d57d068edf56c2a73abedbea4/cryptography-45.0.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:3436128a60a5e5490603ab2adbabc8763613f638513ffa7d311c900a8349a2a0", size = 7020111, upload-time = "2025-08-05T23:58:45.316Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/ea5173689e014f1a8470899cd5beeb358e22bb3cf5a876060f9d1ca78af4/cryptography-45.0.6-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d9ef57b6768d9fa58e92f4947cea96ade1233c0e236db22ba44748ffedca394", size = 4198169, upload-time = "2025-08-05T23:58:47.121Z" }, + { url = "https://files.pythonhosted.org/packages/ba/73/b12995edc0c7e2311ffb57ebd3b351f6b268fed37d93bfc6f9856e01c473/cryptography-45.0.6-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea3c42f2016a5bbf71825537c2ad753f2870191134933196bee408aac397b3d9", size = 4421273, upload-time = "2025-08-05T23:58:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/f7/6e/286894f6f71926bc0da67408c853dd9ba953f662dcb70993a59fd499f111/cryptography-45.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:20ae4906a13716139d6d762ceb3e0e7e110f7955f3bc3876e3a07f5daadec5f3", size = 4199211, upload-time = "2025-08-05T23:58:50.139Z" }, + { url = "https://files.pythonhosted.org/packages/de/34/a7f55e39b9623c5cb571d77a6a90387fe557908ffc44f6872f26ca8ae270/cryptography-45.0.6-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dac5ec199038b8e131365e2324c03d20e97fe214af051d20c49db129844e8b3", size = 3883732, upload-time = "2025-08-05T23:58:52.253Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b9/c6d32edbcba0cd9f5df90f29ed46a65c4631c4fbe11187feb9169c6ff506/cryptography-45.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:18f878a34b90d688982e43f4b700408b478102dd58b3e39de21b5ebf6509c301", size = 4450655, upload-time = "2025-08-05T23:58:53.848Z" }, + { url = "https://files.pythonhosted.org/packages/77/2d/09b097adfdee0227cfd4c699b3375a842080f065bab9014248933497c3f9/cryptography-45.0.6-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5bd6020c80c5b2b2242d6c48487d7b85700f5e0038e67b29d706f98440d66eb5", size = 4198956, upload-time = "2025-08-05T23:58:55.209Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/061ec6689207d54effdff535bbdf85cc380d32dd5377173085812565cf38/cryptography-45.0.6-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:eccddbd986e43014263eda489abbddfbc287af5cddfd690477993dbb31e31016", size = 4449859, upload-time = "2025-08-05T23:58:56.639Z" }, + { url = "https://files.pythonhosted.org/packages/41/ff/e7d5a2ad2d035e5a2af116e1a3adb4d8fcd0be92a18032917a089c6e5028/cryptography-45.0.6-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:550ae02148206beb722cfe4ef0933f9352bab26b087af00e48fdfb9ade35c5b3", size = 4320254, upload-time = "2025-08-05T23:58:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/82/27/092d311af22095d288f4db89fcaebadfb2f28944f3d790a4cf51fe5ddaeb/cryptography-45.0.6-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b64e668fc3528e77efa51ca70fadcd6610e8ab231e3e06ae2bab3b31c2b8ed9", size = 4554815, upload-time = "2025-08-05T23:59:00.283Z" }, + { url = "https://files.pythonhosted.org/packages/7e/01/aa2f4940262d588a8fdf4edabe4cda45854d00ebc6eaac12568b3a491a16/cryptography-45.0.6-cp37-abi3-win32.whl", hash = "sha256:780c40fb751c7d2b0c6786ceee6b6f871e86e8718a8ff4bc35073ac353c7cd02", size = 2912147, upload-time = "2025-08-05T23:59:01.716Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bc/16e0276078c2de3ceef6b5a34b965f4436215efac45313df90d55f0ba2d2/cryptography-45.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:20d15aed3ee522faac1a39fbfdfee25d17b1284bafd808e1640a74846d7c4d1b", size = 3390459, upload-time = "2025-08-05T23:59:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/61/69/c252de4ec047ba2f567ecb53149410219577d408c2aea9c989acae7eafce/cryptography-45.0.6-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fc022c1fa5acff6def2fc6d7819bbbd31ccddfe67d075331a65d9cfb28a20983", size = 3584669, upload-time = "2025-08-05T23:59:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fe/deea71e9f310a31fe0a6bfee670955152128d309ea2d1c79e2a5ae0f0401/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3de77e4df42ac8d4e4d6cdb342d989803ad37707cf8f3fbf7b088c9cbdd46427", size = 4153022, upload-time = "2025-08-05T23:59:16.954Z" }, + { url = "https://files.pythonhosted.org/packages/60/45/a77452f5e49cb580feedba6606d66ae7b82c128947aa754533b3d1bd44b0/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:599c8d7df950aa68baa7e98f7b73f4f414c9f02d0e8104a30c0182a07732638b", size = 4386802, upload-time = "2025-08-05T23:59:18.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b9/a2f747d2acd5e3075fdf5c145c7c3568895daaa38b3b0c960ef830db6cdc/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:31a2b9a10530a1cb04ffd6aa1cd4d3be9ed49f7d77a4dafe198f3b382f41545c", size = 4152706, upload-time = "2025-08-05T23:59:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/81/ec/381b3e8d0685a3f3f304a382aa3dfce36af2d76467da0fd4bb21ddccc7b2/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e5b3dda1b00fb41da3af4c5ef3f922a200e33ee5ba0f0bc9ecf0b0c173958385", size = 4386740, upload-time = "2025-08-05T23:59:21.525Z" }, + { url = "https://files.pythonhosted.org/packages/0a/76/cf8d69da8d0b5ecb0db406f24a63a3f69ba5e791a11b782aeeefef27ccbb/cryptography-45.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:629127cfdcdc6806dfe234734d7cb8ac54edaf572148274fa377a7d3405b0043", size = 3331874, upload-time = "2025-08-05T23:59:23.017Z" }, ] [[package]] @@ -721,6 +817,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] +[[package]] +name = "cymem" +version = "2.0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/4a/1acd761fb6ac4c560e823ce40536a62f886f2d59b2763b5c3fc7e9d92101/cymem-2.0.11.tar.gz", hash = "sha256:efe49a349d4a518be6b6c6b255d4a80f740a341544bde1a807707c058b88d0bd", size = 10346, upload-time = "2025-01-16T21:50:41.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e3/d98e3976f4ffa99cddebc1ce379d4d62e3eb1da22285267f902c99cc3395/cymem-2.0.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ee54039aad3ef65de82d66c40516bf54586287b46d32c91ea0530c34e8a2745", size = 42005, upload-time = "2025-01-16T21:49:34.977Z" }, + { url = "https://files.pythonhosted.org/packages/41/b4/7546faf2ab63e59befc95972316d62276cec153f7d4d60e7b0d5e08f0602/cymem-2.0.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c05ef75b5db217be820604e43a47ccbbafea98ab6659d07cea92fa3c864ea58", size = 41747, upload-time = "2025-01-16T21:49:36.108Z" }, + { url = "https://files.pythonhosted.org/packages/7d/4e/042f372e5b3eb7f5f3dd7677161771d301de2b6fa3f7c74e1cebcd502552/cymem-2.0.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d5381e5793ce531bac0dbc00829c8381f18605bb67e4b61d34f8850463da40", size = 217647, upload-time = "2025-01-16T21:49:37.433Z" }, + { url = "https://files.pythonhosted.org/packages/48/cb/2207679e4b92701f78cf141e1ab4f81f55247dbe154eb426b842a0a993de/cymem-2.0.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b9d3f42d7249ac81802135cad51d707def058001a32f73fc7fbf3de7045ac7", size = 218857, upload-time = "2025-01-16T21:49:40.09Z" }, + { url = "https://files.pythonhosted.org/packages/31/7a/76ae3b7a39ab2531029d281e43fcfcaad728c2341b150a81a3a1f5587cf3/cymem-2.0.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:39b78f2195d20b75c2d465732f6b8e8721c5d4eb012777c2cb89bdb45a043185", size = 206148, upload-time = "2025-01-16T21:49:41.383Z" }, + { url = "https://files.pythonhosted.org/packages/25/f9/d0fc0191ac79f15638ddb59237aa76f234691374d7d7950e10f384bd8a25/cymem-2.0.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2203bd6525a80d8fd0c94654a263af21c0387ae1d5062cceaebb652bf9bad7bc", size = 207112, upload-time = "2025-01-16T21:49:43.986Z" }, + { url = "https://files.pythonhosted.org/packages/56/c8/75f75889401b20f4c3a7c5965dda09df42913e904ddc2ffe7ef3bdf25061/cymem-2.0.11-cp311-cp311-win_amd64.whl", hash = "sha256:aa54af7314de400634448da1f935b61323da80a49484074688d344fb2036681b", size = 39360, upload-time = "2025-01-16T21:49:45.479Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/0d74f7e9d79f934368a78fb1d1466b94bebdbff14f8ae94dd3e4ea8738bb/cymem-2.0.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a0fbe19ce653cd688842d81e5819dc63f911a26e192ef30b0b89f0ab2b192ff2", size = 42621, upload-time = "2025-01-16T21:49:46.585Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d6/f7a19c63b48efc3f00a3ee8d69070ac90202e1e378f6cf81b8671f0cf762/cymem-2.0.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de72101dc0e6326f6a2f73e05a438d1f3c6110d41044236d0fbe62925091267d", size = 42249, upload-time = "2025-01-16T21:49:48.973Z" }, + { url = "https://files.pythonhosted.org/packages/d7/60/cdc434239813eef547fb99b6d0bafe31178501702df9b77c4108c9a216f6/cymem-2.0.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee4395917f6588b8ac1699499128842768b391fe8896e8626950b4da5f9a406", size = 224758, upload-time = "2025-01-16T21:49:51.382Z" }, + { url = "https://files.pythonhosted.org/packages/1d/68/8fa6efae17cd3b2ba9a2f83b824867c5b65b06f7aec3f8a0d0cabdeffb9b/cymem-2.0.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b02f2b17d760dc3fe5812737b1ce4f684641cdd751d67761d333a3b5ea97b83", size = 227995, upload-time = "2025-01-16T21:49:54.538Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f3/ceda70bf6447880140602285b7c6fa171cb7c78b623d35345cc32505cd06/cymem-2.0.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:04ee6b4041ddec24512d6e969ed6445e57917f01e73b9dabbe17b7e6b27fef05", size = 215325, upload-time = "2025-01-16T21:49:57.229Z" }, + { url = "https://files.pythonhosted.org/packages/d3/47/6915eaa521e1ce7a0ba480eecb6870cb4f681bcd64ced88c2f0ed7a744b4/cymem-2.0.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e1048dae7e627ee25f22c87bb670b13e06bc0aecc114b89b959a798d487d1bf4", size = 216447, upload-time = "2025-01-16T21:50:00.432Z" }, + { url = "https://files.pythonhosted.org/packages/7b/be/8e02bdd31e557f642741a06c8e886782ef78f0b00daffd681922dc9bbc88/cymem-2.0.11-cp312-cp312-win_amd64.whl", hash = "sha256:0c269c7a867d74adeb9db65fa1d226342aacf44d64b7931282f0b0eb22eb6275", size = 39283, upload-time = "2025-01-16T21:50:03.384Z" }, +] + +[[package]] +name = "dacite" +version = "1.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/55/a0/7ca79796e799a3e782045d29bf052b5cde7439a2bbb17f15ff44f7aacc63/dacite-1.9.2.tar.gz", hash = "sha256:6ccc3b299727c7aa17582f0021f6ae14d5de47c7227932c47fec4cdfefd26f09", size = 22420, upload-time = "2025-02-05T09:27:29.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/35/386550fd60316d1e37eccdda609b074113298f23cef5bddb2049823fe666/dacite-1.9.2-py3-none-any.whl", hash = "sha256:053f7c3f5128ca2e9aceb66892b1a3c8936d02c686e707bee96e19deef4bc4a0", size = 16600, upload-time = "2025-02-05T09:27:24.345Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -736,11 +863,11 @@ wheels = [ [[package]] name = "datasets" -version = "2.2.1" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, { name = "dill" }, + { name = "filelock" }, { name = "fsspec", extra = ["http"] }, { name = "huggingface-hub" }, { name = "multiprocess" }, @@ -748,23 +875,70 @@ dependencies = [ { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, + { name = "pyyaml" }, { name = "requests" }, - { name = "responses" }, { name = "tqdm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/31/64/1e6fb2a0eb6b0d55117233cf33279ba6d680c0f031ebae81281a47c92760/datasets-2.2.1.tar.gz", hash = "sha256:d362717c4394589b516c8f397ff20a6fe720454aed877ab61d06f3bc05df9544", size = 302132, upload-time = "2022-05-11T17:02:29.543Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566, upload-time = "2025-07-09T14:35:52.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825, upload-time = "2025-07-09T14:35:50.658Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/d4/722d0bcc7986172ac2ef3c979ad56a1030e3afd44ced136d45f8142b1f4a/debugpy-1.8.16.tar.gz", hash = "sha256:31e69a1feb1cf6b51efbed3f6c9b0ef03bc46ff050679c4be7ea6d2e23540870", size = 1643809, upload-time = "2025-08-06T18:00:02.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/d6/ad70ba8b49b23fa286fb21081cf732232cc19374af362051da9c7537ae52/debugpy-1.8.16-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:67371b28b79a6a12bcc027d94a06158f2fde223e35b5c4e0783b6f9d3b39274a", size = 2184063, upload-time = "2025-08-06T18:00:11.885Z" }, + { url = "https://files.pythonhosted.org/packages/aa/49/7b03e88dea9759a4c7910143f87f92beb494daaae25560184ff4ae883f9e/debugpy-1.8.16-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2abae6dd02523bec2dee16bd6b0781cccb53fd4995e5c71cc659b5f45581898", size = 3134837, upload-time = "2025-08-06T18:00:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/5d/52/b348930316921de7565fbe37a487d15409041713004f3d74d03eb077dbd4/debugpy-1.8.16-cp311-cp311-win32.whl", hash = "sha256:f8340a3ac2ed4f5da59e064aa92e39edd52729a88fbde7bbaa54e08249a04493", size = 5159142, upload-time = "2025-08-06T18:00:15.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ef/9aa9549ce1e10cea696d980292e71672a91ee4a6a691ce5f8629e8f48c49/debugpy-1.8.16-cp311-cp311-win_amd64.whl", hash = "sha256:70f5fcd6d4d0c150a878d2aa37391c52de788c3dc680b97bdb5e529cb80df87a", size = 5183117, upload-time = "2025-08-06T18:00:17.251Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/0387c0e108d842c902801bc65ccc53e5b91d8c169702a9bbf4f7efcedf0c/debugpy-1.8.16-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:b202e2843e32e80b3b584bcebfe0e65e0392920dc70df11b2bfe1afcb7a085e4", size = 2511822, upload-time = "2025-08-06T18:00:18.526Z" }, + { url = "https://files.pythonhosted.org/packages/37/44/19e02745cae22bf96440141f94e15a69a1afaa3a64ddfc38004668fcdebf/debugpy-1.8.16-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64473c4a306ba11a99fe0bb14622ba4fbd943eb004847d9b69b107bde45aa9ea", size = 4230135, upload-time = "2025-08-06T18:00:19.997Z" }, + { url = "https://files.pythonhosted.org/packages/f3/0b/19b1ba5ee4412f303475a2c7ad5858efb99c90eae5ec627aa6275c439957/debugpy-1.8.16-cp312-cp312-win32.whl", hash = "sha256:833a61ed446426e38b0dd8be3e9d45ae285d424f5bf6cd5b2b559c8f12305508", size = 5281271, upload-time = "2025-08-06T18:00:21.281Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e0/bc62e2dc141de53bd03e2c7cb9d7011de2e65e8bdcdaa26703e4d28656ba/debugpy-1.8.16-cp312-cp312-win_amd64.whl", hash = "sha256:75f204684581e9ef3dc2f67687c3c8c183fde2d6675ab131d94084baf8084121", size = 5323149, upload-time = "2025-08-06T18:00:23.033Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ecc9ae29fa5b2d90107cd1d9bf8ed19aacb74b2264d986ae9d44fe9bdf87/debugpy-1.8.16-py2.py3-none-any.whl", hash = "sha256:19c9521962475b87da6f673514f7fd610328757ec993bf7ec0d8c96f9a325f9e", size = 5287700, upload-time = "2025-08-06T18:00:42.333Z" }, +] + +[[package]] +name = "decorator" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, +] + +[[package]] +name = "deprecated" +version = "1.2.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/2d/41e8aec8d4bad6f07adfcbc89cf743e0d31c876371d453b2936bcfa7fe34/datasets-2.2.1-py3-none-any.whl", hash = "sha256:1938f3e99599422de50b9b54fe802aca854ed130382dab0b3820c821f7ae6d5e", size = 342193, upload-time = "2022-05-11T17:02:27.047Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, ] [[package]] name = "dill" -version = "0.4.0" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847, upload-time = "2024-01-27T23:42:16.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, +] + +[[package]] +name = "dirtyjson" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, + { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, ] [[package]] @@ -806,6 +980,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "eval-type-backport" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, +] + +[[package]] +name = "executing" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/50/a9d80c47ff289c611ff12e63f7c5d13942c65d68125160cefd768c73e6e4/executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755", size = 978693, upload-time = "2025-01-22T15:41:29.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702, upload-time = "2025-01-22T15:41:25.929Z" }, +] + [[package]] name = "expandvars" version = "1.1.1" @@ -833,7 +1025,6 @@ dependencies = [ { name = "numpy" }, { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/57/f5af8d68f9a8ec943496aa8fa9b01d7c4e654e918b246f5a0e6c85df6e0d/faiss_cpu-1.9.0.tar.gz", hash = "sha256:587fcea9fa478e9307a388754824a032849d317894a607586c3cdd8c8aeb7233", size = 67785, upload-time = "2024-10-08T07:07:00.355Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f5/a2/b346d976c0adcdeab7d23bf6f58eef6d1c5c9c0bf919353923cf91553049/faiss_cpu-1.9.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:b0e9208a36da519dc2eb90e4c44c66a6812a5b68457582d8ed21d04e910e3d1f", size = 7661789, upload-time = "2024-10-08T07:06:26.259Z" }, { url = "https://files.pythonhosted.org/packages/6b/9f/f0a39439a938818f1add48bd7b79d4b8e12e60f2f0c1e4a0b37b295625e0/faiss_cpu-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a4b2871057560020b83ad7bb5aaf3b97b64f980f9af2ca99ba34eeb4fe38bdf", size = 3215612, upload-time = "2024-10-08T07:06:28.359Z" }, @@ -863,20 +1054,32 @@ wheels = [ [[package]] name = "fastjsonschema" -version = "2.21.1" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + +[[package]] +name = "fickling" +version = "0.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/50/4b769ce1ac4071a1ef6d86b1a3fb56cdc3a37615e8c5519e1af96cdac366/fastjsonschema-2.21.1.tar.gz", hash = "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4", size = 373939, upload-time = "2024-12-02T10:55:15.133Z" } +dependencies = [ + { name = "stdlib-list" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/23/0a03d2d01c004ab3f0181bbda3642c7d88226b4a25f47675ef948326504f/fickling-0.1.4.tar.gz", hash = "sha256:cb06bbb7b6a1c443eacf230ab7e212d8b4f3bb2333f307a8c94a144537018888", size = 40956, upload-time = "2025-07-07T13:17:59.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/2b/0817a2b257fe88725c25589d89aec060581aabf668707a8d03b2e9e0cb2a/fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667", size = 23924, upload-time = "2024-12-02T10:55:07.599Z" }, + { url = "https://files.pythonhosted.org/packages/38/40/059cd7c6913cc20b029dd5c8f38578d185f71737c5a62387df4928cd10fe/fickling-0.1.4-py3-none-any.whl", hash = "sha256:110522385a30b7936c50c3860ba42b0605254df9d0ef6cbdaf0ad8fb455a6672", size = 42573, upload-time = "2025-07-07T13:17:58.071Z" }, ] [[package]] name = "filelock" -version = "3.18.0" +version = "3.19.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" }, + { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, ] [[package]] @@ -915,27 +1118,27 @@ wheels = [ [[package]] name = "fonttools" -version = "4.59.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/27/ec3c723bfdf86f34c5c82bf6305df3e0f0d8ea798d2d3a7cb0c0a866d286/fonttools-4.59.0.tar.gz", hash = "sha256:be392ec3529e2f57faa28709d60723a763904f71a2b63aabe14fee6648fe3b14", size = 3532521, upload-time = "2025-07-16T12:04:54.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/96/520733d9602fa1bf6592e5354c6721ac6fc9ea72bc98d112d0c38b967199/fonttools-4.59.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:841b2186adce48903c0fef235421ae21549020eca942c1da773ac380b056ab3c", size = 2782387, upload-time = "2025-07-16T12:03:51.424Z" }, - { url = "https://files.pythonhosted.org/packages/87/6a/170fce30b9bce69077d8eec9bea2cfd9f7995e8911c71be905e2eba6368b/fonttools-4.59.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9bcc1e77fbd1609198966ded6b2a9897bd6c6bcbd2287a2fc7d75f1a254179c5", size = 2342194, upload-time = "2025-07-16T12:03:53.295Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b6/7c8166c0066856f1408092f7968ac744060cf72ca53aec9036106f57eeca/fonttools-4.59.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37c377f7cb2ab2eca8a0b319c68146d34a339792f9420fca6cd49cf28d370705", size = 5032333, upload-time = "2025-07-16T12:03:55.177Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0c/707c5a19598eafcafd489b73c4cb1c142102d6197e872f531512d084aa76/fonttools-4.59.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa39475eaccb98f9199eccfda4298abaf35ae0caec676ffc25b3a5e224044464", size = 4974422, upload-time = "2025-07-16T12:03:57.406Z" }, - { url = "https://files.pythonhosted.org/packages/f6/e7/6d33737d9fe632a0f59289b6f9743a86d2a9d0673de2a0c38c0f54729822/fonttools-4.59.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d3972b13148c1d1fbc092b27678a33b3080d1ac0ca305742b0119b75f9e87e38", size = 5010631, upload-time = "2025-07-16T12:03:59.449Z" }, - { url = "https://files.pythonhosted.org/packages/63/e1/a4c3d089ab034a578820c8f2dff21ef60daf9668034a1e4fb38bb1cc3398/fonttools-4.59.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a408c3c51358c89b29cfa5317cf11518b7ce5de1717abb55c5ae2d2921027de6", size = 5122198, upload-time = "2025-07-16T12:04:01.542Z" }, - { url = "https://files.pythonhosted.org/packages/09/77/ca82b9c12fa4de3c520b7760ee61787640cf3fde55ef1b0bfe1de38c8153/fonttools-4.59.0-cp311-cp311-win32.whl", hash = "sha256:6770d7da00f358183d8fd5c4615436189e4f683bdb6affb02cad3d221d7bb757", size = 2214216, upload-time = "2025-07-16T12:04:03.515Z" }, - { url = "https://files.pythonhosted.org/packages/ab/25/5aa7ca24b560b2f00f260acf32c4cf29d7aaf8656e159a336111c18bc345/fonttools-4.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:84fc186980231a287b28560d3123bd255d3c6b6659828c642b4cf961e2b923d0", size = 2261879, upload-time = "2025-07-16T12:04:05.015Z" }, - { url = "https://files.pythonhosted.org/packages/e2/77/b1c8af22f4265e951cd2e5535dbef8859efcef4fb8dee742d368c967cddb/fonttools-4.59.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f9b3a78f69dcbd803cf2fb3f972779875b244c1115481dfbdd567b2c22b31f6b", size = 2767562, upload-time = "2025-07-16T12:04:06.895Z" }, - { url = "https://files.pythonhosted.org/packages/ff/5a/aeb975699588176bb357e8b398dfd27e5d3a2230d92b81ab8cbb6187358d/fonttools-4.59.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:57bb7e26928573ee7c6504f54c05860d867fd35e675769f3ce01b52af38d48e2", size = 2335168, upload-time = "2025-07-16T12:04:08.695Z" }, - { url = "https://files.pythonhosted.org/packages/54/97/c6101a7e60ae138c4ef75b22434373a0da50a707dad523dd19a4889315bf/fonttools-4.59.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4536f2695fe5c1ffb528d84a35a7d3967e5558d2af58b4775e7ab1449d65767b", size = 4909850, upload-time = "2025-07-16T12:04:10.761Z" }, - { url = "https://files.pythonhosted.org/packages/bd/6c/fa4d18d641054f7bff878cbea14aa9433f292b9057cb1700d8e91a4d5f4f/fonttools-4.59.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:885bde7d26e5b40e15c47bd5def48b38cbd50830a65f98122a8fb90962af7cd1", size = 4955131, upload-time = "2025-07-16T12:04:12.846Z" }, - { url = "https://files.pythonhosted.org/packages/20/5c/331947fc1377deb928a69bde49f9003364f5115e5cbe351eea99e39412a2/fonttools-4.59.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6801aeddb6acb2c42eafa45bc1cb98ba236871ae6f33f31e984670b749a8e58e", size = 4899667, upload-time = "2025-07-16T12:04:14.558Z" }, - { url = "https://files.pythonhosted.org/packages/8a/46/b66469dfa26b8ff0baa7654b2cc7851206c6d57fe3abdabbaab22079a119/fonttools-4.59.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:31003b6a10f70742a63126b80863ab48175fb8272a18ca0846c0482968f0588e", size = 5051349, upload-time = "2025-07-16T12:04:16.388Z" }, - { url = "https://files.pythonhosted.org/packages/2e/05/ebfb6b1f3a4328ab69787d106a7d92ccde77ce66e98659df0f9e3f28d93d/fonttools-4.59.0-cp312-cp312-win32.whl", hash = "sha256:fbce6dae41b692a5973d0f2158f782b9ad05babc2c2019a970a1094a23909b1b", size = 2201315, upload-time = "2025-07-16T12:04:18.557Z" }, - { url = "https://files.pythonhosted.org/packages/09/45/d2bdc9ea20bbadec1016fd0db45696d573d7a26d95ab5174ffcb6d74340b/fonttools-4.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:332bfe685d1ac58ca8d62b8d6c71c2e52a6c64bc218dc8f7825c9ea51385aa01", size = 2249408, upload-time = "2025-07-16T12:04:20.489Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9c/df0ef2c51845a13043e5088f7bb988ca6cd5bb82d5d4203d6a158aa58cf2/fonttools-4.59.0-py3-none-any.whl", hash = "sha256:241313683afd3baacb32a6bd124d0bce7404bc5280e12e291bae1b9bba28711d", size = 1128050, upload-time = "2025-07-16T12:04:52.687Z" }, +version = "4.59.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/7f/29c9c3fe4246f6ad96fee52b88d0dc3a863c7563b0afc959e36d78b965dc/fonttools-4.59.1.tar.gz", hash = "sha256:74995b402ad09822a4c8002438e54940d9f1ecda898d2bb057729d7da983e4cb", size = 3534394, upload-time = "2025-08-14T16:28:14.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/62/9667599561f623d4a523cc9eb4f66f3b94b6155464110fa9aebbf90bbec7/fonttools-4.59.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4909cce2e35706f3d18c54d3dcce0414ba5e0fb436a454dffec459c61653b513", size = 2778815, upload-time = "2025-08-14T16:26:28.484Z" }, + { url = "https://files.pythonhosted.org/packages/8f/78/cc25bcb2ce86033a9df243418d175e58f1956a35047c685ef553acae67d6/fonttools-4.59.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:efbec204fa9f877641747f2d9612b2b656071390d7a7ef07a9dbf0ecf9c7195c", size = 2341631, upload-time = "2025-08-14T16:26:30.396Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cc/fcbb606dd6871f457ac32f281c20bcd6cc77d9fce77b5a4e2b2afab1f500/fonttools-4.59.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39dfd42cc2dc647b2c5469bc7a5b234d9a49e72565b96dd14ae6f11c2c59ef15", size = 5022222, upload-time = "2025-08-14T16:26:32.447Z" }, + { url = "https://files.pythonhosted.org/packages/61/96/c0b1cf2b74d08eb616a80dbf5564351fe4686147291a25f7dce8ace51eb3/fonttools-4.59.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b11bc177a0d428b37890825d7d025040d591aa833f85f8d8878ed183354f47df", size = 4966512, upload-time = "2025-08-14T16:26:34.621Z" }, + { url = "https://files.pythonhosted.org/packages/a4/26/51ce2e3e0835ffc2562b1b11d1fb9dafd0aca89c9041b64a9e903790a761/fonttools-4.59.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b9b4c35b3be45e5bc774d3fc9608bbf4f9a8d371103b858c80edbeed31dd5aa", size = 5001645, upload-time = "2025-08-14T16:26:36.876Z" }, + { url = "https://files.pythonhosted.org/packages/36/11/ef0b23f4266349b6d5ccbd1a07b7adc998d5bce925792aa5d1ec33f593e3/fonttools-4.59.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:01158376b8a418a0bae9625c476cebfcfcb5e6761e9d243b219cd58341e7afbb", size = 5113777, upload-time = "2025-08-14T16:26:39.002Z" }, + { url = "https://files.pythonhosted.org/packages/d0/da/b398fe61ef433da0a0472cdb5d4399124f7581ffe1a31b6242c91477d802/fonttools-4.59.1-cp311-cp311-win32.whl", hash = "sha256:cf7c5089d37787387123f1cb8f1793a47c5e1e3d1e4e7bfbc1cc96e0f925eabe", size = 2215076, upload-time = "2025-08-14T16:26:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/94/bd/e2624d06ab94e41c7c77727b2941f1baed7edb647e63503953e6888020c9/fonttools-4.59.1-cp311-cp311-win_amd64.whl", hash = "sha256:c866eef7a0ba320486ade6c32bfc12813d1a5db8567e6904fb56d3d40acc5116", size = 2262779, upload-time = "2025-08-14T16:26:43.483Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fe/6e069cc4cb8881d164a9bd956e9df555bc62d3eb36f6282e43440200009c/fonttools-4.59.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:43ab814bbba5f02a93a152ee61a04182bb5809bd2bc3609f7822e12c53ae2c91", size = 2769172, upload-time = "2025-08-14T16:26:45.729Z" }, + { url = "https://files.pythonhosted.org/packages/b9/98/ec4e03f748fefa0dd72d9d95235aff6fef16601267f4a2340f0e16b9330f/fonttools-4.59.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4f04c3ffbfa0baafcbc550657cf83657034eb63304d27b05cff1653b448ccff6", size = 2337281, upload-time = "2025-08-14T16:26:47.921Z" }, + { url = "https://files.pythonhosted.org/packages/8b/b1/890360a7e3d04a30ba50b267aca2783f4c1364363797e892e78a4f036076/fonttools-4.59.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d601b153e51a5a6221f0d4ec077b6bfc6ac35bfe6c19aeaa233d8990b2b71726", size = 4909215, upload-time = "2025-08-14T16:26:49.682Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ec/2490599550d6c9c97a44c1e36ef4de52d6acf742359eaa385735e30c05c4/fonttools-4.59.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c735e385e30278c54f43a0d056736942023c9043f84ee1021eff9fd616d17693", size = 4951958, upload-time = "2025-08-14T16:26:51.616Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/bd053f6f7634234a9b9805ff8ae4f32df4f2168bee23cafd1271ba9915a9/fonttools-4.59.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1017413cdc8555dce7ee23720da490282ab7ec1cf022af90a241f33f9a49afc4", size = 4894738, upload-time = "2025-08-14T16:26:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a1/3cd12a010d288325a7cfcf298a84825f0f9c29b01dee1baba64edfe89257/fonttools-4.59.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5c6d8d773470a5107052874341ed3c487c16ecd179976d81afed89dea5cd7406", size = 5045983, upload-time = "2025-08-14T16:26:56.153Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/8a2c3f6619cc43cf87951405337cc8460d08a4e717bb05eaa94b335d11dc/fonttools-4.59.1-cp312-cp312-win32.whl", hash = "sha256:2a2d0d33307f6ad3a2086a95dd607c202ea8852fa9fb52af9b48811154d1428a", size = 2203407, upload-time = "2025-08-14T16:26:58.165Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f2/a19b874ddbd3ebcf11d7e25188ef9ac3f68b9219c62263acb34aca8cde05/fonttools-4.59.1-cp312-cp312-win_amd64.whl", hash = "sha256:0b9e4fa7eaf046ed6ac470f6033d52c052481ff7a6e0a92373d14f556f298dc0", size = 2251561, upload-time = "2025-08-14T16:27:00.646Z" }, + { url = "https://files.pythonhosted.org/packages/0f/64/9d606e66d498917cd7a2ff24f558010d42d6fd4576d9dd57f0bd98333f5a/fonttools-4.59.1-py3-none-any.whl", hash = "sha256:647db657073672a8330608970a984d51573557f328030566521bc03415535042", size = 1130094, upload-time = "2025-08-14T16:28:12.048Z" }, ] [[package]] @@ -983,11 +1186,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2025.7.0" +version = "2025.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432, upload-time = "2025-07-15T16:05:21.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491, upload-time = "2025-03-07T21:47:56.461Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" }, + { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615, upload-time = "2025-03-07T21:47:54.809Z" }, ] [package.optional-dependencies] @@ -1009,14 +1212,47 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.44" +version = "3.1.45" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196, upload-time = "2025-01-02T07:32:43.59Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + +[[package]] +name = "google-auth" +version = "2.40.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/9b/e92ef23b84fa10a64ce4831390b7a4c2e53c0132568d99d4ae61d04c8855/google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77", size = 281029, upload-time = "2025-06-04T18:04:57.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, +] + +[[package]] +name = "google-genai" +version = "1.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "google-auth" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1b/da30fa6e2966942d7028a58eb7aa7d04544dcc3aa66194365b2e0adac570/google_genai-1.31.0.tar.gz", hash = "sha256:8572b47aa684357c3e5e10d290ec772c65414114939e3ad2955203e27cd2fcbc", size = 233482, upload-time = "2025-08-18T23:40:21.733Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599, upload-time = "2025-01-02T07:32:40.731Z" }, + { url = "https://files.pythonhosted.org/packages/41/27/1525bc9cbec58660f0842ebcbfe910a1dde908c2672373804879666e0bb8/google_genai-1.31.0-py3-none-any.whl", hash = "sha256:5c6959bcf862714e8ed0922db3aaf41885bacf6318751b3421bf1e459f78892f", size = 231876, upload-time = "2025-08-18T23:40:20.385Z" }, ] [[package]] @@ -1040,6 +1276,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, ] +[[package]] +name = "gputil" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz", hash = "sha256:099e52c65e512cdfa8c8763fca67f5a5c2afb63469602d5dcb4d296b3661efb9", size = 5545, upload-time = "2018-12-18T09:12:13.63Z" } + +[[package]] +name = "gql" +version = "4.1.0b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "backoff" }, + { name = "graphql-core" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/e7/6a97577e9e1bd679ea31cb375908558497a423f1257f9f6947399069f7b6/gql-4.1.0b0.tar.gz", hash = "sha256:40116b8461345c98c416f516a5a5db9abe5c9d6bde04055e2b4e942e9c76adc3", size = 215609, upload-time = "2025-08-17T14:59:04.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/f9/129acd1856ec69daf1562685d9603c2cb4dda6f7862bd018bc45cdede9ed/gql-4.1.0b0-py3-none-any.whl", hash = "sha256:ca71a6fa3571acbad273df3616acac5f320c69c22f69de736a43a3e9b81b5e44", size = 89933, upload-time = "2025-08-17T14:59:03.316Z" }, +] + +[package.optional-dependencies] +aiohttp = [ + { name = "aiohttp" }, +] +requests = [ + { name = "requests" }, + { name = "requests-toolbelt" }, +] + [[package]] name = "graphql-core" version = "3.3.0a9" @@ -1050,29 +1316,67 @@ wheels = [ ] [[package]] -name = "greenlet" -version = "3.2.3" +name = "graphviz" +version = "0.21" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/92/bb85bd6e80148a4d2e0c59f7c0c2891029f8fd510183afc7d8d2feeed9b6/greenlet-3.2.3.tar.gz", hash = "sha256:8b0dd8ae4c0d6f5e54ee55ba935eeb3d735a9b58a8a1e5b5cbab64e01a39f365", size = 185752, upload-time = "2025-06-05T16:16:09.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/2e/d4fcb2978f826358b673f779f78fa8a32ee37df11920dc2bb5589cbeecef/greenlet-3.2.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:784ae58bba89fa1fa5733d170d42486580cab9decda3484779f4759345b29822", size = 270219, upload-time = "2025-06-05T16:10:10.414Z" }, - { url = "https://files.pythonhosted.org/packages/16/24/929f853e0202130e4fe163bc1d05a671ce8dcd604f790e14896adac43a52/greenlet-3.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0921ac4ea42a5315d3446120ad48f90c3a6b9bb93dd9b3cf4e4d84a66e42de83", size = 630383, upload-time = "2025-06-05T16:38:51.785Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b2/0320715eb61ae70c25ceca2f1d5ae620477d246692d9cc284c13242ec31c/greenlet-3.2.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d2971d93bb99e05f8c2c0c2f4aa9484a18d98c4c3bd3c62b65b7e6ae33dfcfaf", size = 642422, upload-time = "2025-06-05T16:41:35.259Z" }, - { url = "https://files.pythonhosted.org/packages/bd/49/445fd1a210f4747fedf77615d941444349c6a3a4a1135bba9701337cd966/greenlet-3.2.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c667c0bf9d406b77a15c924ef3285e1e05250948001220368e039b6aa5b5034b", size = 638375, upload-time = "2025-06-05T16:48:18.235Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c8/ca19760cf6eae75fa8dc32b487e963d863b3ee04a7637da77b616703bc37/greenlet-3.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:592c12fb1165be74592f5de0d70f82bc5ba552ac44800d632214b76089945147", size = 637627, upload-time = "2025-06-05T16:13:02.858Z" }, - { url = "https://files.pythonhosted.org/packages/65/89/77acf9e3da38e9bcfca881e43b02ed467c1dedc387021fc4d9bd9928afb8/greenlet-3.2.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29e184536ba333003540790ba29829ac14bb645514fbd7e32af331e8202a62a5", size = 585502, upload-time = "2025-06-05T16:12:49.642Z" }, - { url = "https://files.pythonhosted.org/packages/97/c6/ae244d7c95b23b7130136e07a9cc5aadd60d59b5951180dc7dc7e8edaba7/greenlet-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:93c0bb79844a367782ec4f429d07589417052e621aa39a5ac1fb99c5aa308edc", size = 1114498, upload-time = "2025-06-05T16:36:46.598Z" }, - { url = "https://files.pythonhosted.org/packages/89/5f/b16dec0cbfd3070658e0d744487919740c6d45eb90946f6787689a7efbce/greenlet-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:751261fc5ad7b6705f5f76726567375bb2104a059454e0226e1eef6c756748ba", size = 1139977, upload-time = "2025-06-05T16:12:38.262Z" }, - { url = "https://files.pythonhosted.org/packages/66/77/d48fb441b5a71125bcac042fc5b1494c806ccb9a1432ecaa421e72157f77/greenlet-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:83a8761c75312361aa2b5b903b79da97f13f556164a7dd2d5448655425bd4c34", size = 297017, upload-time = "2025-06-05T16:25:05.225Z" }, - { url = "https://files.pythonhosted.org/packages/f3/94/ad0d435f7c48debe960c53b8f60fb41c2026b1d0fa4a99a1cb17c3461e09/greenlet-3.2.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:25ad29caed5783d4bd7a85c9251c651696164622494c00802a139c00d639242d", size = 271992, upload-time = "2025-06-05T16:11:23.467Z" }, - { url = "https://files.pythonhosted.org/packages/93/5d/7c27cf4d003d6e77749d299c7c8f5fd50b4f251647b5c2e97e1f20da0ab5/greenlet-3.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88cd97bf37fe24a6710ec6a3a7799f3f81d9cd33317dcf565ff9950c83f55e0b", size = 638820, upload-time = "2025-06-05T16:38:52.882Z" }, - { url = "https://files.pythonhosted.org/packages/c6/7e/807e1e9be07a125bb4c169144937910bf59b9d2f6d931578e57f0bce0ae2/greenlet-3.2.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:baeedccca94880d2f5666b4fa16fc20ef50ba1ee353ee2d7092b383a243b0b0d", size = 653046, upload-time = "2025-06-05T16:41:36.343Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ab/158c1a4ea1068bdbc78dba5a3de57e4c7aeb4e7fa034320ea94c688bfb61/greenlet-3.2.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:be52af4b6292baecfa0f397f3edb3c6092ce071b499dd6fe292c9ac9f2c8f264", size = 647701, upload-time = "2025-06-05T16:48:19.604Z" }, - { url = "https://files.pythonhosted.org/packages/cc/0d/93729068259b550d6a0288da4ff72b86ed05626eaf1eb7c0d3466a2571de/greenlet-3.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0cc73378150b8b78b0c9fe2ce56e166695e67478550769536a6742dca3651688", size = 649747, upload-time = "2025-06-05T16:13:04.628Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f6/c82ac1851c60851302d8581680573245c8fc300253fc1ff741ae74a6c24d/greenlet-3.2.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:706d016a03e78df129f68c4c9b4c4f963f7d73534e48a24f5f5a7101ed13dbbb", size = 605461, upload-time = "2025-06-05T16:12:50.792Z" }, - { url = "https://files.pythonhosted.org/packages/98/82/d022cf25ca39cf1200650fc58c52af32c90f80479c25d1cbf57980ec3065/greenlet-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:419e60f80709510c343c57b4bb5a339d8767bf9aef9b8ce43f4f143240f88b7c", size = 1121190, upload-time = "2025-06-05T16:36:48.59Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e1/25297f70717abe8104c20ecf7af0a5b82d2f5a980eb1ac79f65654799f9f/greenlet-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:93d48533fade144203816783373f27a97e4193177ebaaf0fc396db19e5d61163", size = 1149055, upload-time = "2025-06-05T16:12:40.457Z" }, - { url = "https://files.pythonhosted.org/packages/1f/8f/8f9e56c5e82eb2c26e8cde787962e66494312dc8cb261c460e1f3a9c88bc/greenlet-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:7454d37c740bb27bdeddfc3f358f26956a07d5220818ceb467a483197d84f849", size = 297817, upload-time = "2025-06-05T16:29:49.244Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, +] + +[[package]] +name = "greenlet" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, + { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, + { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, + { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, + { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, + { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, + { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, + { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, +] + +[[package]] +name = "griffe" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/ca/29f36e00c74844ae50d139cf5a8b1751887b2f4d5023af65d460268ad7aa/griffe-1.12.1.tar.gz", hash = "sha256:29f5a6114c0aeda7d9c86a570f736883f8a2c5b38b57323d56b3d1c000565567", size = 411863, upload-time = "2025-08-14T21:08:15.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/f2/4fab6c3e5bcaf38a44cc8a974d2752eaad4c129e45d6533d926a30edd133/griffe-1.12.1-py3-none-any.whl", hash = "sha256:2d7c12334de00089c31905424a00abcfd931b45b8b516967f224133903d302cc", size = 138940, upload-time = "2025-08-14T21:08:13.382Z" }, +] + +[[package]] +name = "groq" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a2/77fd1460e7d55859219223719aa44ae8902a3a1ad333cd5faf330eb0b894/groq-0.31.0.tar.gz", hash = "sha256:182252e9bf0d696df607c137cbafa851d2c84aaf94bcfe9165c0bc231043490c", size = 136237, upload-time = "2025-08-05T23:14:01.183Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/f8/14672d69a91495f43462c5490067eeafc30346e81bda1a62848e897f9bc3/groq-0.31.0-py3-none-any.whl", hash = "sha256:5e3c7ec9728b7cccf913da982a9b5ebb46dc18a070b35e12a3d6a1e12d6b0f7f", size = 131365, upload-time = "2025-08-05T23:13:59.768Z" }, ] [[package]] @@ -1089,28 +1393,30 @@ wheels = [ [[package]] name = "grpcio" -version = "1.67.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/53/d9282a66a5db45981499190b77790570617a604a38f3d103d0400974aeb5/grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732", size = 12580022, upload-time = "2024-10-29T06:30:07.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/2c/b60d6ea1f63a20a8d09c6db95c4f9a16497913fb3048ce0990ed81aeeca0/grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb", size = 5119075, upload-time = "2024-10-29T06:24:04.696Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9a/e1956f7ca582a22dd1f17b9e26fcb8229051b0ce6d33b47227824772feec/grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e", size = 11009159, upload-time = "2024-10-29T06:24:07.781Z" }, - { url = "https://files.pythonhosted.org/packages/43/a8/35fbbba580c4adb1d40d12e244cf9f7c74a379073c0a0ca9d1b5338675a1/grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f", size = 5629476, upload-time = "2024-10-29T06:24:11.444Z" }, - { url = "https://files.pythonhosted.org/packages/77/c9/864d336e167263d14dfccb4dbfa7fce634d45775609895287189a03f1fc3/grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc", size = 6239901, upload-time = "2024-10-29T06:24:14.2Z" }, - { url = "https://files.pythonhosted.org/packages/f7/1e/0011408ebabf9bd69f4f87cc1515cbfe2094e5a32316f8714a75fd8ddfcb/grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96", size = 5881010, upload-time = "2024-10-29T06:24:17.451Z" }, - { url = "https://files.pythonhosted.org/packages/b4/7d/fbca85ee9123fb296d4eff8df566f458d738186d0067dec6f0aa2fd79d71/grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f", size = 6580706, upload-time = "2024-10-29T06:24:20.038Z" }, - { url = "https://files.pythonhosted.org/packages/75/7a/766149dcfa2dfa81835bf7df623944c1f636a15fcb9b6138ebe29baf0bc6/grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970", size = 6161799, upload-time = "2024-10-29T06:24:22.604Z" }, - { url = "https://files.pythonhosted.org/packages/09/13/5b75ae88810aaea19e846f5380611837de411181df51fd7a7d10cb178dcb/grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744", size = 3616330, upload-time = "2024-10-29T06:24:25.775Z" }, - { url = "https://files.pythonhosted.org/packages/aa/39/38117259613f68f072778c9638a61579c0cfa5678c2558706b10dd1d11d3/grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5", size = 4354535, upload-time = "2024-10-29T06:24:28.614Z" }, - { url = "https://files.pythonhosted.org/packages/6e/25/6f95bd18d5f506364379eabc0d5874873cc7dbdaf0757df8d1e82bc07a88/grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953", size = 5089809, upload-time = "2024-10-29T06:24:31.24Z" }, - { url = "https://files.pythonhosted.org/packages/10/3f/d79e32e5d0354be33a12db2267c66d3cfeff700dd5ccdd09fd44a3ff4fb6/grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb", size = 10981985, upload-time = "2024-10-29T06:24:34.942Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/36fbc14b3542e3a1c20fb98bd60c4732c55a44e374a4eb68f91f28f14aab/grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0", size = 5588770, upload-time = "2024-10-29T06:24:38.145Z" }, - { url = "https://files.pythonhosted.org/packages/0d/af/bbc1305df60c4e65de8c12820a942b5e37f9cf684ef5e49a63fbb1476a73/grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af", size = 6214476, upload-time = "2024-10-29T06:24:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/92/cf/1d4c3e93efa93223e06a5c83ac27e32935f998bc368e276ef858b8883154/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e", size = 5850129, upload-time = "2024-10-29T06:24:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ca/26195b66cb253ac4d5ef59846e354d335c9581dba891624011da0e95d67b/grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75", size = 6568489, upload-time = "2024-10-29T06:24:46.453Z" }, - { url = "https://files.pythonhosted.org/packages/d1/94/16550ad6b3f13b96f0856ee5dfc2554efac28539ee84a51d7b14526da985/grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38", size = 6149369, upload-time = "2024-10-29T06:24:49.112Z" }, - { url = "https://files.pythonhosted.org/packages/33/0d/4c3b2587e8ad7f121b597329e6c2620374fccbc2e4e1aa3c73ccc670fde4/grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78", size = 3599176, upload-time = "2024-10-29T06:24:51.443Z" }, - { url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574, upload-time = "2024-10-29T06:24:54.587Z" }, +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/35feb8f7cab7239c5b94bd2db71abb3d6adb5f335ad8f131abb6060840b6/grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", size = 12756048, upload-time = "2025-07-24T18:54:23.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/77/b2f06db9f240a5abeddd23a0e49eae2b6ac54d85f0e5267784ce02269c3b/grpcio-1.74.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31", size = 5487368, upload-time = "2025-07-24T18:53:03.548Z" }, + { url = "https://files.pythonhosted.org/packages/48/99/0ac8678a819c28d9a370a663007581744a9f2a844e32f0fa95e1ddda5b9e/grpcio-1.74.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4", size = 10999804, upload-time = "2025-07-24T18:53:05.095Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/a2d586300d9e14ad72e8dc211c7aecb45fe9846a51e558c5bca0c9102c7f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce", size = 5987667, upload-time = "2025-07-24T18:53:07.157Z" }, + { url = "https://files.pythonhosted.org/packages/c9/57/5f338bf56a7f22584e68d669632e521f0de460bb3749d54533fc3d0fca4f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3", size = 6655612, upload-time = "2025-07-24T18:53:09.244Z" }, + { url = "https://files.pythonhosted.org/packages/82/ea/a4820c4c44c8b35b1903a6c72a5bdccec92d0840cf5c858c498c66786ba5/grpcio-1.74.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182", size = 6219544, upload-time = "2025-07-24T18:53:11.221Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/0537630a921365928f5abb6d14c79ba4dcb3e662e0dbeede8af4138d9dcf/grpcio-1.74.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d", size = 6334863, upload-time = "2025-07-24T18:53:12.925Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a6/85ca6cb9af3f13e1320d0a806658dca432ff88149d5972df1f7b51e87127/grpcio-1.74.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f", size = 7019320, upload-time = "2025-07-24T18:53:15.002Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/fe2beab970a1e25d2eff108b3cf4f7d9a53c185106377a3d1989216eba45/grpcio-1.74.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4", size = 6514228, upload-time = "2025-07-24T18:53:16.999Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c2/2f9c945c8a248cebc3ccda1b7a1bf1775b9d7d59e444dbb18c0014e23da6/grpcio-1.74.0-cp311-cp311-win32.whl", hash = "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b", size = 3817216, upload-time = "2025-07-24T18:53:20.564Z" }, + { url = "https://files.pythonhosted.org/packages/ff/d1/a9cf9c94b55becda2199299a12b9feef0c79946b0d9d34c989de6d12d05d/grpcio-1.74.0-cp311-cp311-win_amd64.whl", hash = "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11", size = 4495380, upload-time = "2025-07-24T18:53:22.058Z" }, + { url = "https://files.pythonhosted.org/packages/4c/5d/e504d5d5c4469823504f65687d6c8fb97b7f7bf0b34873b7598f1df24630/grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", size = 5445551, upload-time = "2025-07-24T18:53:23.641Z" }, + { url = "https://files.pythonhosted.org/packages/43/01/730e37056f96f2f6ce9f17999af1556df62ee8dab7fa48bceeaab5fd3008/grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", size = 10979810, upload-time = "2025-07-24T18:53:25.349Z" }, + { url = "https://files.pythonhosted.org/packages/79/3d/09fd100473ea5c47083889ca47ffd356576173ec134312f6aa0e13111dee/grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", size = 5941946, upload-time = "2025-07-24T18:53:27.387Z" }, + { url = "https://files.pythonhosted.org/packages/8a/99/12d2cca0a63c874c6d3d195629dcd85cdf5d6f98a30d8db44271f8a97b93/grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", size = 6621763, upload-time = "2025-07-24T18:53:29.193Z" }, + { url = "https://files.pythonhosted.org/packages/9d/2c/930b0e7a2f1029bbc193443c7bc4dc2a46fedb0203c8793dcd97081f1520/grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", size = 6180664, upload-time = "2025-07-24T18:53:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/db/d5/ff8a2442180ad0867717e670f5ec42bfd8d38b92158ad6bcd864e6d4b1ed/grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", size = 6301083, upload-time = "2025-07-24T18:53:32.454Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ba/b361d390451a37ca118e4ec7dccec690422e05bc85fba2ec72b06cefec9f/grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", size = 6994132, upload-time = "2025-07-24T18:53:34.506Z" }, + { url = "https://files.pythonhosted.org/packages/3b/0c/3a5fa47d2437a44ced74141795ac0251bbddeae74bf81df3447edd767d27/grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", size = 6489616, upload-time = "2025-07-24T18:53:36.217Z" }, + { url = "https://files.pythonhosted.org/packages/ae/95/ab64703b436d99dc5217228babc76047d60e9ad14df129e307b5fec81fd0/grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", size = 3807083, upload-time = "2025-07-24T18:53:37.911Z" }, + { url = "https://files.pythonhosted.org/packages/84/59/900aa2445891fc47a33f7d2f76e00ca5d6ae6584b20d19af9c06fa09bf9a/grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", size = 4490123, upload-time = "2025-07-24T18:53:39.528Z" }, ] [[package]] @@ -1124,17 +1430,17 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.1.6rc2" +version = "1.1.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/02/9590927b54d8eca803d5f2c22398c91907bb430fe9082351449ba1b47d21/hf_xet-1.1.6rc2.tar.gz", hash = "sha256:f59e54beb49e3c424d0c8d6a8df74349192e994f7739c18030b32330589d7671", size = 494316, upload-time = "2025-07-14T20:09:24.083Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/49/91010b59debc7c862a5fd426d343134dd9a68778dbe570234b6495a4e204/hf_xet-1.1.8.tar.gz", hash = "sha256:62a0043e441753bbc446dcb5a3fe40a4d03f5fb9f13589ef1df9ab19252beb53", size = 484065, upload-time = "2025-08-18T22:01:03.584Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/55/5e30c2c63122301f184cc442c5aed077bd6f5ae2079a0f4b701a373d81fb/hf_xet-1.1.6rc2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e93026949c596b2b644730d2a1a007a63d1d1c80f293247962c368c777d00c90", size = 2700641, upload-time = "2025-07-14T20:09:19.086Z" }, - { url = "https://files.pythonhosted.org/packages/b3/a0/9b493c6373e8a16a67527449a4719b8dbe3177aa17ed591bc2ec91600bc3/hf_xet-1.1.6rc2-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:cf88c08a4fd8dbe62ee468e9dc1c4f0e37abcfa8cac130189fcffc6ca8d53526", size = 2565067, upload-time = "2025-07-14T20:09:17.852Z" }, - { url = "https://files.pythonhosted.org/packages/56/de/da1c3e840fcb078d20957d69ef150a2492a6b8c8c0272e591770762cffc2/hf_xet-1.1.6rc2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37fff1cc2509f861706af6f5e5d27178bade8a02d84eb069c1b942952cc7b92", size = 3115719, upload-time = "2025-07-14T20:09:16.604Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c9/1719b01de254868cb1e0a6f64dc6f763bfe0e23c56114b392452c07152e4/hf_xet-1.1.6rc2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9b1926694628bffe2589d213596703cce9ac7383712ab15dde638a0b778e9e97", size = 3007969, upload-time = "2025-07-14T20:09:13.963Z" }, - { url = "https://files.pythonhosted.org/packages/66/65/65576be66ecf94f853cdbd5a37babb9693655b8dcf37468f59ed7594f877/hf_xet-1.1.6rc2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:acd8497a218b5db57594aaa9ba39f3fec86440b80d08f84e2595ab2a654f1559", size = 3170371, upload-time = "2025-07-14T20:09:20.981Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3a/86750fd0ec6eb7c4de1394e3921f3357f44ef64c033a5d86129b85cf7dac/hf_xet-1.1.6rc2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38cd088fc7f187ca735c9f47d1a6b79587334b7ffd111494cf867920574194b9", size = 3280665, upload-time = "2025-07-14T20:09:22.566Z" }, - { url = "https://files.pythonhosted.org/packages/f7/72/a45ae75ad1065c05ab79a1f431d14c3e8fb1b64abae95da7e0dcb5265ba5/hf_xet-1.1.6rc2-cp37-abi3-win_amd64.whl", hash = "sha256:3676ed0f45b3f0332e135e99f467a34f1aabe8a49172ca6e8d7e011ddad305fa", size = 2751450, upload-time = "2025-07-14T20:09:25.35Z" }, + { url = "https://files.pythonhosted.org/packages/9c/91/5814db3a0d4a65fb6a87f0931ae28073b87f06307701fe66e7c41513bfb4/hf_xet-1.1.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3d5f82e533fc51c7daad0f9b655d9c7811b5308e5890236828bd1dd3ed8fea74", size = 2752357, upload-time = "2025-08-18T22:00:58.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/72/ce898516e97341a7a9d450609e130e108643389110261eaee6deb1ba8545/hf_xet-1.1.8-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:8e2dba5896bca3ab61d0bef4f01a1647004de59640701b37e37eaa57087bbd9d", size = 2613142, upload-time = "2025-08-18T22:00:57.252Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d6/13af5f916cef795ac2b5e4cc1de31f2e0e375f4475d50799915835f301c2/hf_xet-1.1.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfe5700bc729be3d33d4e9a9b5cc17a951bf8c7ada7ba0c9198a6ab2053b7453", size = 3175859, upload-time = "2025-08-18T22:00:55.978Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ed/34a193c9d1d72b7c3901b3b5153b1be9b2736b832692e1c3f167af537102/hf_xet-1.1.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:09e86514c3c4284ed8a57d6b0f3d089f9836a0af0a1ceb3c9dd664f1f3eaefef", size = 3074178, upload-time = "2025-08-18T22:00:54.147Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1b/de6817b4bf65385280252dff5c9cceeedfbcb27ddb93923639323c1034a4/hf_xet-1.1.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4a9b99ab721d385b83f4fc8ee4e0366b0b59dce03b5888a86029cc0ca634efbf", size = 3238122, upload-time = "2025-08-18T22:01:00.546Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/874c85c7ed519ec101deb654f06703d9e5e68d34416730f64c4755ada36a/hf_xet-1.1.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25b9d43333bbef39aeae1616789ec329c21401a7fe30969d538791076227b591", size = 3344325, upload-time = "2025-08-18T22:01:02.013Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d3/0aaf279f4f3dea58e99401b92c31c0f752924ba0e6c7d7bb07b1dbd7f35e/hf_xet-1.1.8-cp37-abi3-win_amd64.whl", hash = "sha256:4171f31d87b13da4af1ed86c98cf763292e4720c088b4957cf9d564f92904ca9", size = 2801689, upload-time = "2025-08-18T22:01:04.81Z" }, ] [[package]] @@ -1198,7 +1504,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.33.4" +version = "0.35.0rc0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -1210,9 +1516,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/9e/9366b7349fc125dd68b9d384a0fea84d67b7497753fe92c71b67e13f47c4/huggingface_hub-0.33.4.tar.gz", hash = "sha256:6af13478deae120e765bfd92adad0ae1aec1ad8c439b46f23058ad5956cbca0a", size = 426674, upload-time = "2025-07-11T12:32:48.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/6b/d2234ceb0842fb36345e0c36290662f30429cfca3e9726e705f6f079e7ee/huggingface_hub-0.35.0rc0.tar.gz", hash = "sha256:1a8d599a28f8071d45e82f94b5e3437880afda153934198b91ce2d7eea724722", size = 456850, upload-time = "2025-07-24T14:02:10.538Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/7b/98daa50a2db034cab6cd23a3de04fa2358cb691593d28e9130203eb7a805/huggingface_hub-0.33.4-py3-none-any.whl", hash = "sha256:09f9f4e7ca62547c70f8b82767eefadd2667f4e116acba2e3e62a5a81815a7bb", size = 515339, upload-time = "2025-07-11T12:32:46.346Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/4d882ae1577879616e620bc5a4c20c05bd8e4a2367ca630331c48b3193ed/huggingface_hub-0.35.0rc0-py3-none-any.whl", hash = "sha256:812de58926a5ff57b1a1393fff26f26ea8e5aa9de126da62607094018d811296", size = 558670, upload-time = "2025-07-24T14:02:08.744Z" }, ] [[package]] @@ -1245,6 +1551,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "intervaltree" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/fb/396d568039d21344639db96d940d40eb62befe704ef849b27949ded5c3bb/intervaltree-3.1.0.tar.gz", hash = "sha256:902b1b88936918f9b2a19e0e5eb7ccb430ae45cde4f39ea4b36932920d33952d", size = 32861, upload-time = "2020-08-03T08:01:11.392Z" } + +[[package]] +name = "ipykernel" +version = "7.0.0a2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/1a/baaccb506ef660e9e3d420088e3a68f7873b1bcc152c0d069541641fd230/ipykernel-7.0.0a2.tar.gz", hash = "sha256:b3bd996d94ca09012b6df6654a086ddb3a01a40bcd1ad8cc1f5e80b327e0f120", size = 170826, upload-time = "2025-08-13T09:51:57.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/e3/c471afccaedd89576f5591f1622d70aa9f610194b9c4e91f4ff4b6d9d909/ipykernel-7.0.0a2-py3-none-any.whl", hash = "sha256:58336ba0d4c5ad0b9833392d544d882baf520ce8af1b714f89e746197bd7c644", size = 118216, upload-time = "2025-08-13T09:51:55.464Z" }, +] + +[[package]] +name = "ipynbname" +version = "2025.8.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/e791d0ec7125a41c5625dc348ee28c2e335bd79ff8b31e9971333f129337/ipynbname-2025.8.0.0.tar.gz", hash = "sha256:dcc8367c64c4a9f0baa6acea3e509fd6696bf5d81c9abceb386e2ad6efcac935", size = 4374, upload-time = "2025-08-17T16:43:28.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/93/aece384e71b25385bd6a176f09865ee52faa7bda8221acafff8753f3f30a/ipynbname-2025.8.0.0-py3-none-any.whl", hash = "sha256:66f6249ff26d99b0057db2aac5ced4ca8a2ccaad067b9a66d17cfc1725207966", size = 4534, upload-time = "2025-08-17T16:43:27.219Z" }, +] + +[[package]] +name = "ipython" +version = "9.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/80/406f9e3bde1c1fd9bf5a0be9d090f8ae623e401b7670d8f6fdf2ab679891/ipython-9.4.0.tar.gz", hash = "sha256:c033c6d4e7914c3d9768aabe76bbe87ba1dc66a92a05db6bfa1125d81f2ee270", size = 4385338, upload-time = "2025-07-01T11:11:30.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/f8/0031ee2b906a15a33d6bfc12dd09c3dfa966b3cb5b284ecfb7549e6ac3c4/ipython-9.4.0-py3-none-any.whl", hash = "sha256:25850f025a446d9b359e8d296ba175a36aedd32e83ca9b5060430fe16801f066", size = 611021, upload-time = "2025-07-01T11:11:27.85Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + [[package]] name = "isort" version = "5.12.0" @@ -1263,6 +1648,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1327,11 +1724,11 @@ wheels = [ [[package]] name = "json5" -version = "0.12.0" +version = "0.12.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/be/c6c745ec4c4539b25a278b70e29793f10382947df0d9efba2fa09120895d/json5-0.12.0.tar.gz", hash = "sha256:0b4b6ff56801a1c7dc817b0241bca4ce474a0e6a163bfef3fc594d3fd263ff3a", size = 51907, upload-time = "2025-04-03T16:33:13.201Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/ae/929aee9619e9eba9015207a9d2c1c54db18311da7eb4dcf6d41ad6f0eb67/json5-0.12.1.tar.gz", hash = "sha256:b2743e77b3242f8d03c143dd975a6ec7c52e2f2afe76ed934e53503dd4ad4990", size = 52191, upload-time = "2025-08-12T19:47:42.583Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/9f/3500910d5a98549e3098807493851eeef2b89cdd3032227558a104dfe926/json5-0.12.0-py3-none-any.whl", hash = "sha256:6d37aa6c08b0609f16e1ec5ff94697e2cbbfbad5ac112afa05794da9ab7810db", size = 36079, upload-time = "2025-04-03T16:33:11.927Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/05328bd2621be49a6fed9e3030b1e51a2d04537d3f816d211b9cc53c5262/json5-0.12.1-py3-none-any.whl", hash = "sha256:d9c9b3bc34a5f54d43c35e11ef7cb87d8bdd098c6ace87117a7b7e83e705c1d5", size = 36119, upload-time = "2025-08-12T19:47:41.131Z" }, ] [[package]] @@ -1369,7 +1766,7 @@ wheels = [ [[package]] name = "jsonschema" -version = "4.24.0" +version = "4.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -1377,9 +1774,9 @@ dependencies = [ { name = "referencing" }, { name = "rpds-py" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/d3/1cf5326b923a53515d8f3a2cd442e6d7e94fcc444716e879ea70a0ce3177/jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196", size = 353480, upload-time = "2025-05-26T18:48:10.459Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/3d/023389198f69c722d039351050738d6755376c8fd343e91dc493ea485905/jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d", size = 88709, upload-time = "2025-05-26T18:48:08.417Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, ] [[package]] @@ -1394,6 +1791,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/0e/b27cdbaccf30b890c40ed1da9fd4a3593a5cf94dae54fb34f8a4b74fcd3f/jsonschema_specifications-2025.4.1-py3-none-any.whl", hash = "sha256:4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af", size = 18437, upload-time = "2025-04-23T12:34:05.422Z" }, ] +[[package]] +name = "jupyter-client" +version = "8.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, +] + [[package]] name = "jupyter-core" version = "5.8.1" @@ -1408,47 +1821,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/57/6bffd4b20b88da3800c5d691e0337761576ee688eb01299eae865689d2df/jupyter_core-5.8.1-py3-none-any.whl", hash = "sha256:c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0", size = 28880, upload-time = "2025-05-27T07:38:15.137Z" }, ] +[[package]] +name = "kaitaistruct" +version = "0.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/04/dd60b9cb65d580ef6cb6eaee975ad1bdd22d46a3f51b07a1e0606710ea88/kaitaistruct-0.10.tar.gz", hash = "sha256:a044dee29173d6afbacf27bcac39daf89b654dd418cfa009ab82d9178a9ae52a", size = 7061, upload-time = "2022-07-09T00:34:06.729Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/bf/88ad23efc08708bda9a2647169828e3553bb2093a473801db61f75356395/kaitaistruct-0.10-py2.py3-none-any.whl", hash = "sha256:a97350919adbf37fda881f75e9365e2fb88d04832b7a4e57106ec70119efb235", size = 7013, upload-time = "2022-07-09T00:34:03.905Z" }, +] + [[package]] name = "kiwisolver" -version = "1.4.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/59/7c91426a8ac292e1cdd53a63b6d9439abd573c875c3f92c146767dd33faf/kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e", size = 97538, upload-time = "2024-12-24T18:30:51.519Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/ed/c913ee28936c371418cb167b128066ffb20bbf37771eecc2c97edf8a6e4c/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a4d3601908c560bdf880f07d94f31d734afd1bb71e96585cace0e38ef44c6d84", size = 124635, upload-time = "2024-12-24T18:28:51.826Z" }, - { url = "https://files.pythonhosted.org/packages/4c/45/4a7f896f7467aaf5f56ef093d1f329346f3b594e77c6a3c327b2d415f521/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856b269c4d28a5c0d5e6c1955ec36ebfd1651ac00e1ce0afa3e28da95293b561", size = 66717, upload-time = "2024-12-24T18:28:54.256Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b4/c12b3ac0852a3a68f94598d4c8d569f55361beef6159dce4e7b624160da2/kiwisolver-1.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2b9a96e0f326205af81a15718a9073328df1173a2619a68553decb7097fd5d7", size = 65413, upload-time = "2024-12-24T18:28:55.184Z" }, - { url = "https://files.pythonhosted.org/packages/a9/98/1df4089b1ed23d83d410adfdc5947245c753bddfbe06541c4aae330e9e70/kiwisolver-1.4.8-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5020c83e8553f770cb3b5fc13faac40f17e0b205bd237aebd21d53d733adb03", size = 1343994, upload-time = "2024-12-24T18:28:57.493Z" }, - { url = "https://files.pythonhosted.org/packages/8d/bf/b4b169b050c8421a7c53ea1ea74e4ef9c335ee9013216c558a047f162d20/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dace81d28c787956bfbfbbfd72fdcef014f37d9b48830829e488fdb32b49d954", size = 1434804, upload-time = "2024-12-24T18:29:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/66/5a/e13bd341fbcf73325ea60fdc8af752addf75c5079867af2e04cc41f34434/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11e1022b524bd48ae56c9b4f9296bce77e15a2e42a502cceba602f804b32bb79", size = 1450690, upload-time = "2024-12-24T18:29:01.401Z" }, - { url = "https://files.pythonhosted.org/packages/9b/4f/5955dcb376ba4a830384cc6fab7d7547bd6759fe75a09564910e9e3bb8ea/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b9b4d2892fefc886f30301cdd80debd8bb01ecdf165a449eb6e78f79f0fabd6", size = 1376839, upload-time = "2024-12-24T18:29:02.685Z" }, - { url = "https://files.pythonhosted.org/packages/3a/97/5edbed69a9d0caa2e4aa616ae7df8127e10f6586940aa683a496c2c280b9/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a96c0e790ee875d65e340ab383700e2b4891677b7fcd30a699146f9384a2bb0", size = 1435109, upload-time = "2024-12-24T18:29:04.113Z" }, - { url = "https://files.pythonhosted.org/packages/13/fc/e756382cb64e556af6c1809a1bbb22c141bbc2445049f2da06b420fe52bf/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23454ff084b07ac54ca8be535f4174170c1094a4cff78fbae4f73a4bcc0d4dab", size = 2245269, upload-time = "2024-12-24T18:29:05.488Z" }, - { url = "https://files.pythonhosted.org/packages/76/15/e59e45829d7f41c776d138245cabae6515cb4eb44b418f6d4109c478b481/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:87b287251ad6488e95b4f0b4a79a6d04d3ea35fde6340eb38fbd1ca9cd35bbbc", size = 2393468, upload-time = "2024-12-24T18:29:06.79Z" }, - { url = "https://files.pythonhosted.org/packages/e9/39/483558c2a913ab8384d6e4b66a932406f87c95a6080112433da5ed668559/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b21dbe165081142b1232a240fc6383fd32cdd877ca6cc89eab93e5f5883e1c25", size = 2355394, upload-time = "2024-12-24T18:29:08.24Z" }, - { url = "https://files.pythonhosted.org/packages/01/aa/efad1fbca6570a161d29224f14b082960c7e08268a133fe5dc0f6906820e/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:768cade2c2df13db52475bd28d3a3fac8c9eff04b0e9e2fda0f3760f20b3f7fc", size = 2490901, upload-time = "2024-12-24T18:29:09.653Z" }, - { url = "https://files.pythonhosted.org/packages/c9/4f/15988966ba46bcd5ab9d0c8296914436720dd67fca689ae1a75b4ec1c72f/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d47cfb2650f0e103d4bf68b0b5804c68da97272c84bb12850d877a95c056bd67", size = 2312306, upload-time = "2024-12-24T18:29:12.644Z" }, - { url = "https://files.pythonhosted.org/packages/2d/27/bdf1c769c83f74d98cbc34483a972f221440703054894a37d174fba8aa68/kiwisolver-1.4.8-cp311-cp311-win_amd64.whl", hash = "sha256:ed33ca2002a779a2e20eeb06aea7721b6e47f2d4b8a8ece979d8ba9e2a167e34", size = 71966, upload-time = "2024-12-24T18:29:14.089Z" }, - { url = "https://files.pythonhosted.org/packages/4a/c9/9642ea855604aeb2968a8e145fc662edf61db7632ad2e4fb92424be6b6c0/kiwisolver-1.4.8-cp311-cp311-win_arm64.whl", hash = "sha256:16523b40aab60426ffdebe33ac374457cf62863e330a90a0383639ce14bf44b2", size = 65311, upload-time = "2024-12-24T18:29:15.892Z" }, - { url = "https://files.pythonhosted.org/packages/fc/aa/cea685c4ab647f349c3bc92d2daf7ae34c8e8cf405a6dcd3a497f58a2ac3/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502", size = 124152, upload-time = "2024-12-24T18:29:16.85Z" }, - { url = "https://files.pythonhosted.org/packages/c5/0b/8db6d2e2452d60d5ebc4ce4b204feeb16176a851fd42462f66ade6808084/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31", size = 66555, upload-time = "2024-12-24T18:29:19.146Z" }, - { url = "https://files.pythonhosted.org/packages/60/26/d6a0db6785dd35d3ba5bf2b2df0aedc5af089962c6eb2cbf67a15b81369e/kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb", size = 65067, upload-time = "2024-12-24T18:29:20.096Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ed/1d97f7e3561e09757a196231edccc1bcf59d55ddccefa2afc9c615abd8e0/kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f", size = 1378443, upload-time = "2024-12-24T18:29:22.843Z" }, - { url = "https://files.pythonhosted.org/packages/29/61/39d30b99954e6b46f760e6289c12fede2ab96a254c443639052d1b573fbc/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc", size = 1472728, upload-time = "2024-12-24T18:29:24.463Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3e/804163b932f7603ef256e4a715e5843a9600802bb23a68b4e08c8c0ff61d/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a", size = 1478388, upload-time = "2024-12-24T18:29:25.776Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/60eaa75169a154700be74f875a4d9961b11ba048bef315fbe89cb6999056/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a", size = 1413849, upload-time = "2024-12-24T18:29:27.202Z" }, - { url = "https://files.pythonhosted.org/packages/bc/b3/9458adb9472e61a998c8c4d95cfdfec91c73c53a375b30b1428310f923e4/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a", size = 1475533, upload-time = "2024-12-24T18:29:28.638Z" }, - { url = "https://files.pythonhosted.org/packages/e4/7a/0a42d9571e35798de80aef4bb43a9b672aa7f8e58643d7bd1950398ffb0a/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3", size = 2268898, upload-time = "2024-12-24T18:29:30.368Z" }, - { url = "https://files.pythonhosted.org/packages/d9/07/1255dc8d80271400126ed8db35a1795b1a2c098ac3a72645075d06fe5c5d/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b", size = 2425605, upload-time = "2024-12-24T18:29:33.151Z" }, - { url = "https://files.pythonhosted.org/packages/84/df/5a3b4cf13780ef6f6942df67b138b03b7e79e9f1f08f57c49957d5867f6e/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4", size = 2375801, upload-time = "2024-12-24T18:29:34.584Z" }, - { url = "https://files.pythonhosted.org/packages/8f/10/2348d068e8b0f635c8c86892788dac7a6b5c0cb12356620ab575775aad89/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d", size = 2520077, upload-time = "2024-12-24T18:29:36.138Z" }, - { url = "https://files.pythonhosted.org/packages/32/d8/014b89fee5d4dce157d814303b0fce4d31385a2af4c41fed194b173b81ac/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8", size = 2338410, upload-time = "2024-12-24T18:29:39.991Z" }, - { url = "https://files.pythonhosted.org/packages/bd/72/dfff0cc97f2a0776e1c9eb5bef1ddfd45f46246c6533b0191887a427bca5/kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50", size = 71853, upload-time = "2024-12-24T18:29:42.006Z" }, - { url = "https://files.pythonhosted.org/packages/dc/85/220d13d914485c0948a00f0b9eb419efaf6da81b7d72e88ce2391f7aed8d/kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476", size = 65424, upload-time = "2024-12-24T18:29:44.38Z" }, +version = "1.4.10rc0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/de/354c903d772c1cc0a9310344e077b31c6c893cc5a664019b907a04997099/kiwisolver-1.4.10rc0.tar.gz", hash = "sha256:d321718aaa2583577be9836e8cc0ed9fd0863e57a85b1b73b328aac063bc9903", size = 97614, upload-time = "2025-08-10T20:22:27.702Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/21/019c64c58655e6a3b3783197238c92edf8090e30d094d9c2770e50312c85/kiwisolver-1.4.10rc0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7cc57b12496996be722d18b93fb145098c98b7f49cbe06eb92f1a8839fec851f", size = 124257, upload-time = "2025-08-10T20:20:23.798Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/a1459a96995a804ffcb587fa53006b4f598bd2b757e60811cc3538829852/kiwisolver-1.4.10rc0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed726e9bf4ab6d8bf63f214189a080b1ccc2236de8f69a65518ec2e08792d809", size = 66646, upload-time = "2025-08-10T20:20:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0f/6312411778e5d84df581bc67036e5da913013169d3e819ff3fde02766fa9/kiwisolver-1.4.10rc0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:22954717039cdf96df76c2583be41fc211a87375e4d32eb2bbdbd071d2ce0dbb", size = 65386, upload-time = "2025-08-10T20:20:25.845Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/5e917f0bbfcabcab2d46f22da2eda240fb18a15098e1ff75cc4242dc6807/kiwisolver-1.4.10rc0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d1c2b921dcb8dd3c8b8caeb942e8dde2f9efbb4cce5a7cc4ecb119156dc6f63", size = 1435659, upload-time = "2025-08-10T20:20:26.96Z" }, + { url = "https://files.pythonhosted.org/packages/5c/92/e47df418bf28fb7371fd6cc716ac3765b6048010c3f79ca46c97b4f67631/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8655f912cf62c65176ca04e15b937c03c41da59090fb177885a9aa413f3e27c0", size = 1246594, upload-time = "2025-08-10T20:20:28.807Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f8/ff201f9d0a620f830fa624808376a502cd7ab70f3db75e8f0bef8e22854a/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0a6487724ff217c96fcd072289a5b9e48b88ded72d48cfa48105382ac5cae7b6", size = 1263691, upload-time = "2025-08-10T20:20:30.512Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/1fa0c58baa765b93e50809c6d05d451592e23b4b1f3ed7c3aa84e5043543/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fc089ea753b3d23063c5791cfbf781049a885372d8090ed3176f14aa95e8b133", size = 1317497, upload-time = "2025-08-10T20:20:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/11/82/7be342e8384ed7efb91c1a568190d64eab42456619266121047d82da802e/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0ef8610afeec79ae3cbd9cfa36d4520c672fe0615a2a0ddbce235a510736b35d", size = 2195834, upload-time = "2025-08-10T20:20:34.101Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e9/d8a0fd5c4aa341f06258d544ac568f9fcdf4f67ee662b9475e72b42c8808/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:481e143b69de966593116e54e5033ee1fbf6d0c08d557b45fd4319b2011773ba", size = 2290888, upload-time = "2025-08-10T20:20:36.981Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1f/5d4abbceee0facefb7a90ab4fec6966886448323143bd967518c45c92e79/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f339d6f008fe5a175908e2c5d1a7204bfce3da4250f9bf70df5a60a4b16a8760", size = 2461638, upload-time = "2025-08-10T20:20:38.418Z" }, + { url = "https://files.pythonhosted.org/packages/86/d8/9c5f0bc3222ae58674b24ce8262206f59f79aecaf0b1583ff5fe495e9ff0/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f80f88fa20f14817018eab0f3db70e93da4719434f5c16a0058bde6544c26675", size = 2268133, upload-time = "2025-08-10T20:20:40.384Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/82c1e2964bb1b11160afcc692d932be95ebdfe2bc0756cd1f8dee79a2346/kiwisolver-1.4.10rc0-cp311-cp311-win_amd64.whl", hash = "sha256:ef95d952120e64a55d633a1f2973392acd42bced2e7bf3b76816f7dcca5a88a6", size = 73888, upload-time = "2025-08-10T20:20:42.37Z" }, + { url = "https://files.pythonhosted.org/packages/61/a2/d4867a3ae81e2fe28de05f25266d5a6e0f4950e63f050d3cd78cb48bdfb4/kiwisolver-1.4.10rc0-cp311-cp311-win_arm64.whl", hash = "sha256:9453be53a21f813c0472db8a5cea26e8cadc315c5002aa9882fad8bf5212483e", size = 65199, upload-time = "2025-08-10T20:20:43.372Z" }, + { url = "https://files.pythonhosted.org/packages/dc/87/3df31abf12db3ccabfa52a96dc49e6defe233d8ffca1361091a1ea3a109c/kiwisolver-1.4.10rc0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1cb9ae443b2dba2229ac3b8a771420ee76bfce56f610dcb4998676cebed79346", size = 123742, upload-time = "2025-08-10T20:20:44.391Z" }, + { url = "https://files.pythonhosted.org/packages/7b/62/fc9adfd88082b95971969736d777762f7940f3d49f5ffae37c439699156d/kiwisolver-1.4.10rc0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2727d842ff63050315d2a69334d64ed4ada261c04155c09d9534fd4033d38641", size = 66522, upload-time = "2025-08-10T20:20:45.81Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d1/3802735c705ffa861bbb568ed4226936fcfc917a179bf3998fdf97e48e57/kiwisolver-1.4.10rc0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9563b4c98c23f52d98e15be40cbf0c215c824e7268d5222e9ad1803e8c1156d", size = 65016, upload-time = "2025-08-10T20:20:46.911Z" }, + { url = "https://files.pythonhosted.org/packages/72/76/29b4d717f5614fc91cb4542cd67f42b5bcb6c946201a9cf9d4a34231efc2/kiwisolver-1.4.10rc0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75cef7209e3c71dc81d1d5bc8d5eb1b34be7a8d2cd94b83f0eb15533512a45ef", size = 1474820, upload-time = "2025-08-10T20:20:48.447Z" }, + { url = "https://files.pythonhosted.org/packages/f1/9b/828761ad3841b0e7d464514939a980eef7547b5952fb9a33232b17ed1540/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374c11179eaacd3b8bfef677aa28a0a6d703b3474ea399f3b08b8e4d67522016", size = 1276468, upload-time = "2025-08-10T20:20:49.775Z" }, + { url = "https://files.pythonhosted.org/packages/92/6d/cb780c0ebad56e2b7b3c2c0376e9d5c25e90680d2b3b254afaec2507a62f/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9abc647f46a322fd19e0564ce59fcc0f14b9934540ddc7404ed7f3eea54d0d11", size = 1294484, upload-time = "2025-08-10T20:20:51.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/70/44814b447c4e38da6f466b6a3e992d330c3e2c1c9c29731c436997b78f68/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7e3c788da96b50e60f484d9dd790554a80800c22a468cd059b9a7a9c753d273", size = 1343677, upload-time = "2025-08-10T20:20:52.906Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/44eee9dc53be9c4c6ac3b099aedc482f1a1a6b193d0f258ccfa955c291df/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:468814c0e8f41b8f7b537da0c77a05a70f89aa4b7cfff96aa47f7936e57add9b", size = 2225010, upload-time = "2025-08-10T20:20:54.365Z" }, + { url = "https://files.pythonhosted.org/packages/50/5e/e05e24f858352e6985ace1f9ab8ece32b0962f4c5074ddb38fc91617809a/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aa62a0aa711b1f94f80f1407a668362718c64f27e176ac6952d983ec1a5cd745", size = 2321356, upload-time = "2025-08-10T20:20:55.803Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a7/1d21b9aa468bdbd4be190043d73ad07756c078003e43fdc4af5ccb3df75a/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:6e8697067105c536bcc14a33f5c9c0f0c155cf909818d01475f45d012ac441c1", size = 2488062, upload-time = "2025-08-10T20:20:57.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/9b/17512a47070d022499f19078b980531b7be5d50eb9990dfc4ec29aa554ca/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4c168de06cd8a4947a8af1e49d63341face78aca8e9e6b7685625701147ab22d", size = 2291890, upload-time = "2025-08-10T20:20:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/85/c1/084d9b537e33555d8bf5d41ffaee88cf0ee49fa42587fdee181d31a40b61/kiwisolver-1.4.10rc0-cp312-cp312-win_amd64.whl", hash = "sha256:0b8bb7b6b3964d0454f8504e003097f2ae628679a1054ecb63578feeb7671cab", size = 73949, upload-time = "2025-08-10T20:21:00.845Z" }, + { url = "https://files.pythonhosted.org/packages/ff/fa/538442202d639add2f52a814bdfc58207ee6fbb6d1ecd1a6e867f48ec1af/kiwisolver-1.4.10rc0-cp312-cp312-win_arm64.whl", hash = "sha256:44cd6dfea8a6c2becac4f3d60ebdcfe4fed858bbf7fe9cd38ffea7b58df66435", size = 65077, upload-time = "2025-08-10T20:21:01.882Z" }, + { url = "https://files.pythonhosted.org/packages/bb/31/93f123de35b4e62a9debc1d4f35e20e5da42cc8b99824a13b670beaf426e/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81b5d3e873fed82e9ff7b45311c5fa961a191dda6f19898fce6385ecc82ea65e", size = 60156, upload-time = "2025-08-10T20:22:21.821Z" }, + { url = "https://files.pythonhosted.org/packages/bd/92/2777e966406f0ba01e3f7faae07a79d7a0cc530937fdb2883679a5d10eed/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c5645e473af736949870d303d51d6edf2925afcd56ad5779b29bf7e5c620dc86", size = 58657, upload-time = "2025-08-10T20:22:22.903Z" }, + { url = "https://files.pythonhosted.org/packages/86/90/3b73b9a069cf64bd761c12ffd53b93e212a82f7fd36057ad1b8ef0a27399/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81c6204ddf60b409e384a9dfd12f70980bd98556e9c46f681ea40c90201ac236", size = 80335, upload-time = "2025-08-10T20:22:24.371Z" }, + { url = "https://files.pythonhosted.org/packages/42/0e/08307c00313e305336f1faa8c332b8042877670bbf950535643b3310f7d1/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b52feaf694434ba3aea0cfb9515b72d5cbe8c555473f14fc5dba121ced58d73d", size = 78068, upload-time = "2025-08-10T20:22:25.433Z" }, + { url = "https://files.pythonhosted.org/packages/26/05/ed2ad330b22530772f0498431d6f589a18c5eb3bd858da577f1b2ef5980e/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7b4aa27204cb091a4ef96438773c9609f24f217fb3cd53612c41394f39b0d8b6", size = 73983, upload-time = "2025-08-10T20:22:26.498Z" }, ] [[package]] name = "langchain" -version = "0.3.26" +version = "0.3.27" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1459,14 +1882,14 @@ dependencies = [ { name = "requests" }, { name = "sqlalchemy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/13/a9931800ee42bbe0f8850dd540de14e80dda4945e7ee36e20b5d5964286e/langchain-0.3.26.tar.gz", hash = "sha256:8ff034ee0556d3e45eff1f1e96d0d745ced57858414dba7171c8ebdbeb5580c9", size = 10226808, upload-time = "2025-06-20T22:23:01.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f6/f4f7f3a56626fe07e2bb330feb61254dbdf06c506e6b59a536a337da51cf/langchain-0.3.27.tar.gz", hash = "sha256:aa6f1e6274ff055d0fd36254176770f356ed0a8994297d1df47df341953cec62", size = 10233809, upload-time = "2025-07-24T14:42:32.959Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/f2/c09a2e383283e3af1db669ab037ac05a45814f4b9c472c48dc24c0cef039/langchain-0.3.26-py3-none-any.whl", hash = "sha256:361bb2e61371024a8c473da9f9c55f4ee50f269c5ab43afdb2b1309cb7ac36cf", size = 1012336, upload-time = "2025-06-20T22:22:58.874Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d5/4861816a95b2f6993f1360cfb605aacb015506ee2090433a71de9cca8477/langchain-0.3.27-py3-none-any.whl", hash = "sha256:7b20c4f338826acb148d885b20a73a16e410ede9ee4f19bb02011852d5f98798", size = 1018194, upload-time = "2025-07-24T14:42:30.23Z" }, ] [[package]] name = "langchain-aws" -version = "0.2.27" +version = "0.2.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, @@ -1474,9 +1897,9 @@ dependencies = [ { name = "numpy" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/ff/7bfa3e508d7b4f7406c755f93d1db03f5822e91a29a31035bd369d7878fd/langchain_aws-0.2.27.tar.gz", hash = "sha256:3d08c099dafb6536c74cba7de393845a314086ead73a5821fde52d435be34a47", size = 99349, upload-time = "2025-06-24T13:39:46.012Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/58/a3071e27315017fe2320a3d39a1a6fb081a78868034488f3fefec61df971/langchain_aws-0.2.30.tar.gz", hash = "sha256:67c31f0784045a4a73ef78a2c18f392e14c1e9f7b55870e62f18039cadfb925c", size = 109960, upload-time = "2025-07-31T22:02:11.4Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/44/3dec0bfe8666b2b906a3bbec8739a2cb93aee516c032bce8a3ed41957ccd/langchain_aws-0.2.27-py3-none-any.whl", hash = "sha256:d2912c4488ff5ef0361661c1a42476b3586c27c8e86c0c7e3a198e5dc9b8ae6f", size = 121204, upload-time = "2025-06-24T13:39:45.046Z" }, + { url = "https://files.pythonhosted.org/packages/c6/01/368ddf5c488e1a7186a8e72c8687b7c35479ad66f0fe2dcc00f23f35109e/langchain_aws-0.2.30-py3-none-any.whl", hash = "sha256:947fe4ece30bde0a37ea721b87049d7642607bd2ba9d2d93c9890736972b4274", size = 133900, upload-time = "2025-07-31T22:02:10.38Z" }, ] [[package]] @@ -1504,7 +1927,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "0.3.69" +version = "0.3.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -1515,9 +1938,9 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/26/c4770d3933237cde2918d502e3b0a8b6ce100b296840b632658f3e59b341/langchain_core-0.3.69.tar.gz", hash = "sha256:c132961117cc7f0227a4c58dd3e209674a6dd5b7e74abc61a0df93b0d736e283", size = 563824, upload-time = "2025-07-15T21:19:56.626Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/c6/5d755a0f1f4857abbe5ea6f5907ed0e2b5df52bf4dde0a0fd768290e3084/langchain_core-0.3.74.tar.gz", hash = "sha256:ff604441aeade942fbcc0a3860a592daba7671345230c2078ba2eb5f82b6ba76", size = 569553, upload-time = "2025-08-07T20:47:05.094Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/7b/bb7b088440ff9cc55e9e6eba94162cbdcd3b1693c194e1ad4764acba29b9/langchain_core-0.3.69-py3-none-any.whl", hash = "sha256:383e9cb4919f7ef4b24bf8552ef42e4323c064924fea88b28dd5d7ddb740d3b8", size = 441556, upload-time = "2025-07-15T21:19:55.342Z" }, + { url = "https://files.pythonhosted.org/packages/4d/26/545283681ac0379d31c7ad0bac5f195e1982092d76c65ca048db9e3cec0e/langchain_core-0.3.74-py3-none-any.whl", hash = "sha256:088338b5bc2f6a66892f9afc777992c24ee3188f41cbc603d09181e34a228ce7", size = 443453, upload-time = "2025-08-07T20:47:03.853Z" }, ] [[package]] @@ -1535,42 +1958,54 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "0.3.11" +version = "0.3.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d1/69/255f8468d12cf77cca14debc350bbeb3ed42092775fe353b3eb3e6eacbc4/langchain_nvidia_ai_endpoints-0.3.11.tar.gz", hash = "sha256:df06727a7876813e262591c5ca0b1f48052df8dad80de9b2730c799bdb854ed1", size = 38741, upload-time = "2025-07-14T20:52:21.601Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/f0/9be96dbd50faf9d5053fa82beb51c27f19d9da00e07a43755e4fc21c728d/langchain_nvidia_ai_endpoints-0.3.16.tar.gz", hash = "sha256:8c4aafd125284ef12668e5428e18b83864fb44a4677dcf8b456454e45cb1e7b0", size = 40184, upload-time = "2025-08-08T22:18:22.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/5f/47377adf27756b5b469ec1342113edbcfaa86660cca02637a8eddedd853b/langchain_nvidia_ai_endpoints-0.3.11-py3-none-any.whl", hash = "sha256:73253b56e9349dffe5bac633aeb3945375952238bb194623bf043c9f54489e62", size = 42343, upload-time = "2025-07-14T20:52:20.684Z" }, + { url = "https://files.pythonhosted.org/packages/95/68/e40626a8c78895e5fa8cea0f29b80cbdaffba3c9c6eb90532780315ee8e8/langchain_nvidia_ai_endpoints-0.3.16-py3-none-any.whl", hash = "sha256:a8c1c8a316668ff8402b89a97ace5f978ee71e351a487abbc5aa8c47f576e7d0", size = 43572, upload-time = "2025-08-08T22:18:20.873Z" }, ] [[package]] name = "langchain-openai" -version = "0.3.28" +version = "0.3.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/1d/90cd764c62d5eb822113d3debc3abe10c8807d2c0af90917bfe09acd6f86/langchain_openai-0.3.28.tar.gz", hash = "sha256:6c669548dbdea325c034ae5ef699710e2abd054c7354fdb3ef7bf909dc739d9e", size = 753951, upload-time = "2025-07-14T10:50:44.076Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/21/6b2024cdd907812d33d31d42c05baa6a3fc6b341d76f7a982730b6985501/langchain_openai-0.3.30.tar.gz", hash = "sha256:90df37509b2dcf5e057f491326fcbf78cf2a71caff5103a5a7de560320171842", size = 766426, upload-time = "2025-08-12T17:05:55.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/56/75f3d84b69b8bdae521a537697375e1241377627c32b78edcae337093502/langchain_openai-0.3.28-py3-none-any.whl", hash = "sha256:4cd6d80a5b2ae471a168017bc01b2e0f01548328d83532400a001623624ede67", size = 70571, upload-time = "2025-07-14T10:50:42.492Z" }, + { url = "https://files.pythonhosted.org/packages/23/36/cd370071243ae321c22bfafbf75fef1601dd22d0baeeedb71835954ed0ad/langchain_openai-0.3.30-py3-none-any.whl", hash = "sha256:280f1f31004393228e3f75ff8353b1aae86bbc282abc7890a05beb5f43b89923", size = 74362, upload-time = "2025-08-12T17:05:54.415Z" }, ] [[package]] name = "langchain-text-splitters" -version = "0.3.8" +version = "0.3.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/ac/b4a25c5716bb0103b1515f1f52cc69ffb1035a5a225ee5afe3aed28bf57b/langchain_text_splitters-0.3.8.tar.gz", hash = "sha256:116d4b9f2a22dda357d0b79e30acf005c5518177971c66a9f1ab0edfdb0f912e", size = 42128, upload-time = "2025-04-04T14:03:51.521Z" } +sdist = { url = "https://files.pythonhosted.org/packages/91/52/d43ad77acae169210cc476cbc1e4ab37a701017c950211a11ab500fe7d7e/langchain_text_splitters-0.3.9.tar.gz", hash = "sha256:7cd1e5a3aaf609979583eeca2eb34177622570b8fa8f586a605c6b1c34e7ebdb", size = 45260, upload-time = "2025-07-24T14:38:45.14Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/52/7638394b88bc15083fd2c3752a843784d9d2d110d68fed6437c8607fb749/langchain_text_splitters-0.3.9-py3-none-any.whl", hash = "sha256:cee0bb816211584ea79cc79927317c358543f40404bcfdd69e69ba3ccde54401", size = 33314, upload-time = "2025-07-24T14:38:43.953Z" }, +] + +[[package]] +name = "langcodes" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "language-data" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/7a/5a97e327063409a5caa21541e6d08ae4a0f2da328447e9f2c7b39e179226/langcodes-3.5.0.tar.gz", hash = "sha256:1eef8168d07e51e131a2497ffecad4b663f6208e7c3ae3b8dc15c51734a6f801", size = 191030, upload-time = "2024-11-19T10:23:45.546Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/a3/3696ff2444658053c01b6b7443e761f28bb71217d82bb89137a978c5f66f/langchain_text_splitters-0.3.8-py3-none-any.whl", hash = "sha256:e75cc0f4ae58dcf07d9f18776400cf8ade27fadd4ff6d264df6278bb302f6f02", size = 32440, upload-time = "2025-04-04T14:03:50.6Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6b/068c2ea7a712bf805c62445bd9e9c06d7340358ef2824150eceac027444b/langcodes-3.5.0-py3-none-any.whl", hash = "sha256:853c69d1a35e0e13da2f427bb68fb2fa4a8f4fb899e0c62ad8df8d073dcfed33", size = 182974, upload-time = "2024-11-19T10:23:42.824Z" }, ] [[package]] @@ -1589,33 +2024,33 @@ wheels = [ [[package]] name = "langgraph-checkpoint" -version = "2.1.0" +version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "ormsgpack" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/30/c04abcb2ac30f200dbfde5839ca3832552fe2bd852d9e85a68e47418a11c/langgraph_checkpoint-2.1.0.tar.gz", hash = "sha256:cdaa2f0b49aa130ab185c02d82f02b40299a1fbc9ac59ac20cecce09642a1abe", size = 135501, upload-time = "2025-06-16T22:05:01.918Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/3e/d00eb2b56c3846a0cabd2e5aa71c17a95f882d4f799a6ffe96a19b55eba9/langgraph_checkpoint-2.1.1.tar.gz", hash = "sha256:72038c0f9e22260cb9bff1f3ebe5eb06d940b7ee5c1e4765019269d4f21cf92d", size = 136256, upload-time = "2025-07-17T13:07:52.411Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/41/390a97d9d0abe5b71eea2f6fb618d8adadefa674e97f837bae6cda670bc7/langgraph_checkpoint-2.1.0-py3-none-any.whl", hash = "sha256:4cea3e512081da1241396a519cbfe4c5d92836545e2c64e85b6f5c34a1b8bc61", size = 43844, upload-time = "2025-06-16T22:05:00.758Z" }, + { url = "https://files.pythonhosted.org/packages/4c/dd/64686797b0927fb18b290044be12ae9d4df01670dce6bb2498d5ab65cb24/langgraph_checkpoint-2.1.1-py3-none-any.whl", hash = "sha256:5a779134fd28134a9a83d078be4450bbf0e0c79fdf5e992549658899e6fc5ea7", size = 43925, upload-time = "2025-07-17T13:07:51.023Z" }, ] [[package]] name = "langgraph-sdk" -version = "0.1.73" +version = "0.1.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "orjson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/e8/daf0271f91e93b10566533955c00ee16e471066755c2efd1ba9a887a7eab/langgraph_sdk-0.1.73.tar.gz", hash = "sha256:6e6dcdf66bcf8710739899616856527a72a605ce15beb76fbac7f4ce0e2ad080", size = 72157, upload-time = "2025-07-14T23:57:22.765Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/f7/3807b72988f7eef5e0eb41e7e695eca50f3ed31f7cab5602db3b651c85ff/langgraph_sdk-0.1.74.tar.gz", hash = "sha256:7450e0db5b226cc2e5328ca22c5968725873630ef47c4206a30707cb25dc3ad6", size = 72190, upload-time = "2025-07-21T16:36:50.032Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/86/56e01e715e5b0028cdaff1492a89e54fa12e18c21e03b805a10ea36ecd5a/langgraph_sdk-0.1.73-py3-none-any.whl", hash = "sha256:a60ac33f70688ad07051edff1d5ed8089c8f0de1f69dc900be46e095ca20eed8", size = 50222, upload-time = "2025-07-14T23:57:21.42Z" }, + { url = "https://files.pythonhosted.org/packages/1f/1a/3eacc4df8127781ee4b0b1e5cad7dbaf12510f58c42cbcb9d1e2dba2a164/langgraph_sdk-0.1.74-py3-none-any.whl", hash = "sha256:3a265c3757fe0048adad4391d10486db63ef7aa5a2cbd22da22d4503554cb890", size = 50254, upload-time = "2025-07-21T16:36:49.134Z" }, ] [[package]] name = "langsmith" -version = "0.4.6" +version = "0.4.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1626,342 +2061,1055 @@ dependencies = [ { name = "requests-toolbelt" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/9e/11536528c6e351820ad3fca0d2807f0e0f0619ff907529c78f68ba648497/langsmith-0.4.6.tar.gz", hash = "sha256:9189dbc9c60f2086ca3a1f0110cfe3aff6b0b7c2e0e3384f9572e70502e7933c", size = 352364, upload-time = "2025-07-15T19:43:18.541Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/b0/1def3c6d12eb5e412213e39f1ba4ac64a47ec3102cf42a3a1ff86af1402d/langsmith-0.4.14.tar.gz", hash = "sha256:4d29c7a9c85b20ba813ab9c855407bccdf5eb4f397f512ffa89959b2a2cb83ed", size = 921872, upload-time = "2025-08-12T20:39:43.704Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/9b/f2be47db823e89448ea41bfd8fc5ce6a995556bd25be4c23e5b3bb5b6c9b/langsmith-0.4.6-py3-none-any.whl", hash = "sha256:900e83fe59ee672bcf2f75c8bb47cd012bf8154d92a99c0355fc38b6485cbd3e", size = 367901, upload-time = "2025-07-15T19:43:16.508Z" }, + { url = "https://files.pythonhosted.org/packages/9e/08/3f0fb3e2f7cc6fd91c4d06d7abc6607425a66973bee79d04018bac41dd4f/langsmith-0.4.14-py3-none-any.whl", hash = "sha256:b6d070ac425196947d2a98126fb0e35f3b8c001a2e6e5b7049dd1c56f0767d0b", size = 373249, upload-time = "2025-08-12T20:39:41.992Z" }, ] [[package]] -name = "mako" -version = "1.3.10" +name = "language-data" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe" }, + { name = "marisa-trie" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/ce/3f144716a9f2cbf42aa86ebc8b085a184be25c80aa453eea17c294d239c1/language_data-1.3.0.tar.gz", hash = "sha256:7600ef8aa39555145d06c89f0c324bf7dab834ea0b0a439d8243762e3ebad7ec", size = 5129310, upload-time = "2024-11-19T10:21:37.912Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e9/5a5ffd9b286db82be70d677d0a91e4d58f7912bb8dd026ddeeb4abe70679/language_data-1.3.0-py3-none-any.whl", hash = "sha256:e2ee943551b5ae5f89cd0e801d1fc3835bb0ef5b7e9c3a4e8e17b2b214548fbf", size = 5385760, upload-time = "2024-11-19T10:21:36.005Z" }, ] [[package]] -name = "markdown-it-py" -version = "3.0.0" +name = "litellm" +version = "1.75.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl" }, + { name = "aiohttp" }, + { name = "click" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/4e/48e3d6de19afe713223e3bc7009a2003501420de2a5d823c569cefbd9731/litellm-1.75.8.tar.gz", hash = "sha256:92061bd263ff8c33c8fff70ba92cd046adb7ea041a605826a915d108742fe59e", size = 10140384, upload-time = "2025-08-16T21:42:24.23Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, + { url = "https://files.pythonhosted.org/packages/5e/82/c4d00fbeafd93c00dab6ea03f33cadd6a97adeb720ba1d89fc319e5cb10b/litellm-1.75.8-py3-none-any.whl", hash = "sha256:0bf004488df8506381ec6e35e1486e2870e8d578a7c3f2427cd497558ce07a2e", size = 8916305, upload-time = "2025-08-16T21:42:21.387Z" }, ] [[package]] -name = "markupsafe" -version = "3.0.2" +name = "llama-cloud" +version = "0.1.35" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +dependencies = [ + { name = "certifi" }, + { name = "httpx" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/72/816e6e900448e1b4a8137d90e65876b296c5264a23db6ae888bd3e6660ba/llama_cloud-0.1.35.tar.gz", hash = "sha256:200349d5d57424d7461f304cdb1355a58eea3e6ca1e6b0d75c66b2e937216983", size = 106403, upload-time = "2025-07-28T17:22:06.41Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, - { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, - { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, - { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, - { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, - { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, - { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, - { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, - { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, - { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, - { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, - { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, - { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, - { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, - { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, + { url = "https://files.pythonhosted.org/packages/1d/d2/8d18a021ab757cea231428404f21fe3186bf1ebaac3f57a73c379483fd3f/llama_cloud-0.1.35-py3-none-any.whl", hash = "sha256:b7abab4423118e6f638d2f326749e7a07c6426543bea6da99b623c715b22af71", size = 303280, upload-time = "2025-07-28T17:22:04.946Z" }, ] [[package]] -name = "marshmallow" -version = "3.26.1" +name = "llama-cloud-services" +version = "0.6.15" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, + { name = "click" }, + { name = "llama-cloud" }, + { name = "llama-index-core" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "python-dotenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/4e/da311d13340d22705d6ae48732c78a580039f132dfcaa68a7063b066c38c/llama_cloud_services-0.6.15.tar.gz", hash = "sha256:912799d9cdcf48074145c6781f40a6dd7dadb6344ecb30b715407db85a0e675e", size = 31701, upload-time = "2025-04-24T03:39:46.551Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, + { url = "https://files.pythonhosted.org/packages/f5/0d/88805be6a13b368c9e7a2b2cede60fd0298e0e3abc9a6a6923d414c1ab14/llama_cloud_services-0.6.15-py3-none-any.whl", hash = "sha256:c4e24dd41f2cde17eeba7750d41cc70fe26e1179c03ae832122d762572e53de6", size = 36676, upload-time = "2025-04-24T03:39:45.217Z" }, ] [[package]] -name = "matplotlib" -version = "3.10.3" +name = "llama-index" +version = "0.12.42" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "llama-index-agent-openai" }, + { name = "llama-index-cli" }, + { name = "llama-index-core" }, + { name = "llama-index-embeddings-openai" }, + { name = "llama-index-indices-managed-llama-cloud" }, + { name = "llama-index-llms-openai" }, + { name = "llama-index-multi-modal-llms-openai" }, + { name = "llama-index-program-openai" }, + { name = "llama-index-question-gen-openai" }, + { name = "llama-index-readers-file" }, + { name = "llama-index-readers-llama-parse" }, + { name = "nltk" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/91/d49359a21893183ed2a5b6c76bec40e0b1dcbf8ca148f864d134897cfc75/matplotlib-3.10.3.tar.gz", hash = "sha256:2f82d2c5bb7ae93aaaa4cd42aca65d76ce6376f83304fa3a630b569aca274df0", size = 34799811, upload-time = "2025-05-08T19:10:54.39Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/86/7e36d3f7b0606cba5e4862bb62f8e6bef91b7c45229f15262faf2c92f6e3/llama_index-0.12.42.tar.gz", hash = "sha256:d2bd2b9ea06bc42ebe74697aaa0b66c79bd3dee8ff5d06c53294d49ca44512e7", size = 8073, upload-time = "2025-06-12T04:59:38.313Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/bd/af9f655456f60fe1d575f54fb14704ee299b16e999704817a7645dfce6b0/matplotlib-3.10.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0ef061f74cd488586f552d0c336b2f078d43bc00dc473d2c3e7bfee2272f3fa8", size = 8178873, upload-time = "2025-05-08T19:09:53.857Z" }, - { url = "https://files.pythonhosted.org/packages/c2/86/e1c86690610661cd716eda5f9d0b35eaf606ae6c9b6736687cfc8f2d0cd8/matplotlib-3.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d96985d14dc5f4a736bbea4b9de9afaa735f8a0fc2ca75be2fa9e96b2097369d", size = 8052205, upload-time = "2025-05-08T19:09:55.684Z" }, - { url = "https://files.pythonhosted.org/packages/54/51/a9f8e49af3883dacddb2da1af5fca1f7468677f1188936452dd9aaaeb9ed/matplotlib-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5f0283da91e9522bdba4d6583ed9d5521566f63729ffb68334f86d0bb98049", size = 8465823, upload-time = "2025-05-08T19:09:57.442Z" }, - { url = "https://files.pythonhosted.org/packages/e7/e3/c82963a3b86d6e6d5874cbeaa390166458a7f1961bab9feb14d3d1a10f02/matplotlib-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdfa07c0ec58035242bc8b2c8aae37037c9a886370eef6850703d7583e19964b", size = 8606464, upload-time = "2025-05-08T19:09:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/0e/34/24da1027e7fcdd9e82da3194c470143c551852757a4b473a09a012f5b945/matplotlib-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c0b9849a17bce080a16ebcb80a7b714b5677d0ec32161a2cc0a8e5a6030ae220", size = 9413103, upload-time = "2025-05-08T19:10:03.208Z" }, - { url = "https://files.pythonhosted.org/packages/a6/da/948a017c3ea13fd4a97afad5fdebe2f5bbc4d28c0654510ce6fd6b06b7bd/matplotlib-3.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:eef6ed6c03717083bc6d69c2d7ee8624205c29a8e6ea5a31cd3492ecdbaee1e1", size = 8065492, upload-time = "2025-05-08T19:10:05.271Z" }, - { url = "https://files.pythonhosted.org/packages/eb/43/6b80eb47d1071f234ef0c96ca370c2ca621f91c12045f1401b5c9b28a639/matplotlib-3.10.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ab1affc11d1f495ab9e6362b8174a25afc19c081ba5b0775ef00533a4236eea", size = 8179689, upload-time = "2025-05-08T19:10:07.602Z" }, - { url = "https://files.pythonhosted.org/packages/0f/70/d61a591958325c357204870b5e7b164f93f2a8cca1dc6ce940f563909a13/matplotlib-3.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2a818d8bdcafa7ed2eed74487fdb071c09c1ae24152d403952adad11fa3c65b4", size = 8050466, upload-time = "2025-05-08T19:10:09.383Z" }, - { url = "https://files.pythonhosted.org/packages/e7/75/70c9d2306203148cc7902a961240c5927dd8728afedf35e6a77e105a2985/matplotlib-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:748ebc3470c253e770b17d8b0557f0aa85cf8c63fd52f1a61af5b27ec0b7ffee", size = 8456252, upload-time = "2025-05-08T19:10:11.958Z" }, - { url = "https://files.pythonhosted.org/packages/c4/91/ba0ae1ff4b3f30972ad01cd4a8029e70a0ec3b8ea5be04764b128b66f763/matplotlib-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed70453fd99733293ace1aec568255bc51c6361cb0da94fa5ebf0649fdb2150a", size = 8601321, upload-time = "2025-05-08T19:10:14.47Z" }, - { url = "https://files.pythonhosted.org/packages/d2/88/d636041eb54a84b889e11872d91f7cbf036b3b0e194a70fa064eb8b04f7a/matplotlib-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbed9917b44070e55640bd13419de83b4c918e52d97561544814ba463811cbc7", size = 9406972, upload-time = "2025-05-08T19:10:16.569Z" }, - { url = "https://files.pythonhosted.org/packages/b1/79/0d1c165eac44405a86478082e225fce87874f7198300bbebc55faaf6d28d/matplotlib-3.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:cf37d8c6ef1a48829443e8ba5227b44236d7fcaf7647caa3178a4ff9f7a5be05", size = 8067954, upload-time = "2025-05-08T19:10:18.663Z" }, + { url = "https://files.pythonhosted.org/packages/64/9a/3044c50dd7cd7340763ed25e5ee7ee59917281735725eb73f62fe110333e/llama_index-0.12.42-py3-none-any.whl", hash = "sha256:9a304b2bd71d6772fc6c2e9995e119815c00716fc98813b15ea79caee09f1fe2", size = 7085, upload-time = "2025-06-12T04:59:37.071Z" }, ] [[package]] -name = "mccabe" -version = "0.7.0" +name = "llama-index-agent-openai" +version = "0.4.12" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, + { name = "openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/94/69decc46d11e954c6a8c64999cc237af5932d116eeb7a06515856641a6d4/llama_index_agent_openai-0.4.12.tar.gz", hash = "sha256:d2fe53feb69cfe45752edb7328bf0d25f6a9071b3c056787e661b93e5b748a28", size = 12443, upload-time = "2025-06-29T00:52:03.606Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, + { url = "https://files.pythonhosted.org/packages/89/f5/857ea1c136f422234e298e868af74094a71bf98687be40a365ad6551a660/llama_index_agent_openai-0.4.12-py3-none-any.whl", hash = "sha256:6dbb6276b2e5330032a726b28d5eef5140825f36d72d472b231f08ad3af99665", size = 14704, upload-time = "2025-06-29T00:52:02.528Z" }, ] [[package]] -name = "mcp" -version = "1.11.0" +name = "llama-index-cli" +version = "0.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, + { name = "llama-index-core" }, + { name = "llama-index-embeddings-openai" }, + { name = "llama-index-llms-openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/f5/9506eb5578d5bbe9819ee8ba3198d0ad0e2fbe3bab8b257e4131ceb7dfb6/mcp-1.11.0.tar.gz", hash = "sha256:49a213df56bb9472ff83b3132a4825f5c8f5b120a90246f08b0dac6bedac44c8", size = 406907, upload-time = "2025-07-10T16:41:09.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/44/6acba0b8425d15682def89a4dbba68c782fd74ce6e74a4fa48beb08632f6/llama_index_cli-0.4.4.tar.gz", hash = "sha256:c3af0cf1e2a7e5ef44d0bae5aa8e8872b54c5dd6b731afbae9f13ffeb4997be0", size = 25308, upload-time = "2025-07-07T05:17:40.556Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9c/c9ca79f9c512e4113a5d07043013110bb3369fc7770040c61378c7fbcf70/mcp-1.11.0-py3-none-any.whl", hash = "sha256:58deac37f7483e4b338524b98bc949b7c2b7c33d978f5fafab5bde041c5e2595", size = 155880, upload-time = "2025-07-10T16:41:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/cc/21/89989b7fa8ce4b9bc6f0f7326ae7f959a887c1281f007a718eafd6ef614f/llama_index_cli-0.4.4-py3-none-any.whl", hash = "sha256:1070593cf79407054735ab7a23c5a65a26fc18d264661e42ef38fc549b4b7658", size = 28598, upload-time = "2025-07-07T05:17:39.522Z" }, ] [[package]] -name = "mdurl" -version = "0.1.2" +name = "llama-index-core" +version = "0.12.42" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiosqlite" }, + { name = "banks" }, + { name = "dataclasses-json" }, + { name = "deprecated" }, + { name = "dirtyjson" }, + { name = "filetype" }, + { name = "fsspec" }, + { name = "httpx" }, + { name = "nest-asyncio" }, + { name = "networkx" }, + { name = "nltk" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "typing-inspect" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/67/ef1fc8c3b0676b524fd6b67a0e3e082996d768d227ed2b4b39f78119253b/llama_index_core-0.12.42.tar.gz", hash = "sha256:cff21fe15610826997c876a6b1d28d52727932c5f9c2af04b23e041a10f40a24", size = 7292833, upload-time = "2025-06-12T03:07:06.791Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, + { url = "https://files.pythonhosted.org/packages/1f/92/dc47b702da141d0f617ee86d26b5bdb62ed9a52518d1a5f271e91337e4a7/llama_index_core-0.12.42-py3-none-any.whl", hash = "sha256:0534cd9a4f6113175aa406a47ae9a683b5a43fd55532e9dbbffa96838ff18e07", size = 7665930, upload-time = "2025-06-12T03:06:58.387Z" }, ] [[package]] -name = "milvus-lite" -version = "2.5.1" +name = "llama-index-embeddings-openai" +version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tqdm" }, + { name = "llama-index-core" }, + { name = "openai" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/a1/02/a2604ef3a167131fdd701888f45f16c8efa6d523d02efe8c4e640238f4ea/llama_index_embeddings_openai-0.3.1.tar.gz", hash = "sha256:1368aad3ce24cbaed23d5ad251343cef1eb7b4a06d6563d6606d59cb347fef20", size = 5492, upload-time = "2024-11-27T16:04:17.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713, upload-time = "2025-06-30T04:23:37.028Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451, upload-time = "2025-06-30T04:23:51.747Z" }, - { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093, upload-time = "2025-06-30T04:24:06.706Z" }, - { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911, upload-time = "2025-06-30T04:24:19.434Z" }, + { url = "https://files.pythonhosted.org/packages/bb/45/ca55b91c4ac1b6251d4099fa44121a6c012129822906cadcc27b8cfb33a4/llama_index_embeddings_openai-0.3.1-py3-none-any.whl", hash = "sha256:f15a3d13da9b6b21b8bd51d337197879a453d1605e625a1c6d45e741756c0290", size = 6177, upload-time = "2024-11-27T16:04:15.981Z" }, ] [[package]] -name = "multidict" -version = "6.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3d/2c/5dad12e82fbdf7470f29bff2171484bf07cb3b16ada60a6589af8f376440/multidict-6.6.3.tar.gz", hash = "sha256:798a9eb12dab0a6c2e29c1de6f3468af5cb2da6053a20dfa3344907eed0937cc", size = 101006, upload-time = "2025-06-30T15:53:46.929Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/f0/1a39863ced51f639c81a5463fbfa9eb4df59c20d1a8769ab9ef4ca57ae04/multidict-6.6.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:18f4eba0cbac3546b8ae31e0bbc55b02c801ae3cbaf80c247fcdd89b456ff58c", size = 76445, upload-time = "2025-06-30T15:51:24.01Z" }, - { url = "https://files.pythonhosted.org/packages/c9/0e/a7cfa451c7b0365cd844e90b41e21fab32edaa1e42fc0c9f68461ce44ed7/multidict-6.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef43b5dd842382329e4797c46f10748d8c2b6e0614f46b4afe4aee9ac33159df", size = 44610, upload-time = "2025-06-30T15:51:25.158Z" }, - { url = "https://files.pythonhosted.org/packages/c6/bb/a14a4efc5ee748cc1904b0748be278c31b9295ce5f4d2ef66526f410b94d/multidict-6.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bd1fd5eec01494e0f2e8e446a74a85d5e49afb63d75a9934e4a5423dba21d", size = 44267, upload-time = "2025-06-30T15:51:26.326Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f8/410677d563c2d55e063ef74fe578f9d53fe6b0a51649597a5861f83ffa15/multidict-6.6.3-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5bd8d6f793a787153956cd35e24f60485bf0651c238e207b9a54f7458b16d539", size = 230004, upload-time = "2025-06-30T15:51:27.491Z" }, - { url = "https://files.pythonhosted.org/packages/fd/df/2b787f80059314a98e1ec6a4cc7576244986df3e56b3c755e6fc7c99e038/multidict-6.6.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bf99b4daf908c73856bd87ee0a2499c3c9a3d19bb04b9c6025e66af3fd07462", size = 247196, upload-time = "2025-06-30T15:51:28.762Z" }, - { url = "https://files.pythonhosted.org/packages/05/f2/f9117089151b9a8ab39f9019620d10d9718eec2ac89e7ca9d30f3ec78e96/multidict-6.6.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b9e59946b49dafaf990fd9c17ceafa62976e8471a14952163d10a7a630413a9", size = 225337, upload-time = "2025-06-30T15:51:30.025Z" }, - { url = "https://files.pythonhosted.org/packages/93/2d/7115300ec5b699faa152c56799b089a53ed69e399c3c2d528251f0aeda1a/multidict-6.6.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e2db616467070d0533832d204c54eea6836a5e628f2cb1e6dfd8cd6ba7277cb7", size = 257079, upload-time = "2025-06-30T15:51:31.716Z" }, - { url = "https://files.pythonhosted.org/packages/15/ea/ff4bab367623e39c20d3b07637225c7688d79e4f3cc1f3b9f89867677f9a/multidict-6.6.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7394888236621f61dcdd25189b2768ae5cc280f041029a5bcf1122ac63df79f9", size = 255461, upload-time = "2025-06-30T15:51:33.029Z" }, - { url = "https://files.pythonhosted.org/packages/74/07/2c9246cda322dfe08be85f1b8739646f2c4c5113a1422d7a407763422ec4/multidict-6.6.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f114d8478733ca7388e7c7e0ab34b72547476b97009d643644ac33d4d3fe1821", size = 246611, upload-time = "2025-06-30T15:51:34.47Z" }, - { url = "https://files.pythonhosted.org/packages/a8/62/279c13d584207d5697a752a66ffc9bb19355a95f7659140cb1b3cf82180e/multidict-6.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cdf22e4db76d323bcdc733514bf732e9fb349707c98d341d40ebcc6e9318ef3d", size = 243102, upload-time = "2025-06-30T15:51:36.525Z" }, - { url = "https://files.pythonhosted.org/packages/69/cc/e06636f48c6d51e724a8bc8d9e1db5f136fe1df066d7cafe37ef4000f86a/multidict-6.6.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e995a34c3d44ab511bfc11aa26869b9d66c2d8c799fa0e74b28a473a692532d6", size = 238693, upload-time = "2025-06-30T15:51:38.278Z" }, - { url = "https://files.pythonhosted.org/packages/89/a4/66c9d8fb9acf3b226cdd468ed009537ac65b520aebdc1703dd6908b19d33/multidict-6.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:766a4a5996f54361d8d5a9050140aa5362fe48ce51c755a50c0bc3706460c430", size = 246582, upload-time = "2025-06-30T15:51:39.709Z" }, - { url = "https://files.pythonhosted.org/packages/cf/01/c69e0317be556e46257826d5449feb4e6aa0d18573e567a48a2c14156f1f/multidict-6.6.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3893a0d7d28a7fe6ca7a1f760593bc13038d1d35daf52199d431b61d2660602b", size = 253355, upload-time = "2025-06-30T15:51:41.013Z" }, - { url = "https://files.pythonhosted.org/packages/c0/da/9cc1da0299762d20e626fe0042e71b5694f9f72d7d3f9678397cbaa71b2b/multidict-6.6.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:934796c81ea996e61914ba58064920d6cad5d99140ac3167901eb932150e2e56", size = 247774, upload-time = "2025-06-30T15:51:42.291Z" }, - { url = "https://files.pythonhosted.org/packages/e6/91/b22756afec99cc31105ddd4a52f95ab32b1a4a58f4d417979c570c4a922e/multidict-6.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9ed948328aec2072bc00f05d961ceadfd3e9bfc2966c1319aeaf7b7c21219183", size = 242275, upload-time = "2025-06-30T15:51:43.642Z" }, - { url = "https://files.pythonhosted.org/packages/be/f1/adcc185b878036a20399d5be5228f3cbe7f823d78985d101d425af35c800/multidict-6.6.3-cp311-cp311-win32.whl", hash = "sha256:9f5b28c074c76afc3e4c610c488e3493976fe0e596dd3db6c8ddfbb0134dcac5", size = 41290, upload-time = "2025-06-30T15:51:45.264Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d4/27652c1c6526ea6b4f5ddd397e93f4232ff5de42bea71d339bc6a6cc497f/multidict-6.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc7f6fbc61b1c16050a389c630da0b32fc6d4a3d191394ab78972bf5edc568c2", size = 45942, upload-time = "2025-06-30T15:51:46.377Z" }, - { url = "https://files.pythonhosted.org/packages/16/18/23f4932019804e56d3c2413e237f866444b774b0263bcb81df2fdecaf593/multidict-6.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:d4e47d8faffaae822fb5cba20937c048d4f734f43572e7079298a6c39fb172cb", size = 42880, upload-time = "2025-06-30T15:51:47.561Z" }, - { url = "https://files.pythonhosted.org/packages/0e/a0/6b57988ea102da0623ea814160ed78d45a2645e4bbb499c2896d12833a70/multidict-6.6.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:056bebbeda16b2e38642d75e9e5310c484b7c24e3841dc0fb943206a72ec89d6", size = 76514, upload-time = "2025-06-30T15:51:48.728Z" }, - { url = "https://files.pythonhosted.org/packages/07/7a/d1e92665b0850c6c0508f101f9cf0410c1afa24973e1115fe9c6a185ebf7/multidict-6.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e5f481cccb3c5c5e5de5d00b5141dc589c1047e60d07e85bbd7dea3d4580d63f", size = 45394, upload-time = "2025-06-30T15:51:49.986Z" }, - { url = "https://files.pythonhosted.org/packages/52/6f/dd104490e01be6ef8bf9573705d8572f8c2d2c561f06e3826b081d9e6591/multidict-6.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10bea2ee839a759ee368b5a6e47787f399b41e70cf0c20d90dfaf4158dfb4e55", size = 43590, upload-time = "2025-06-30T15:51:51.331Z" }, - { url = "https://files.pythonhosted.org/packages/44/fe/06e0e01b1b0611e6581b7fd5a85b43dacc08b6cea3034f902f383b0873e5/multidict-6.6.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:2334cfb0fa9549d6ce2c21af2bfbcd3ac4ec3646b1b1581c88e3e2b1779ec92b", size = 237292, upload-time = "2025-06-30T15:51:52.584Z" }, - { url = "https://files.pythonhosted.org/packages/ce/71/4f0e558fb77696b89c233c1ee2d92f3e1d5459070a0e89153c9e9e804186/multidict-6.6.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8fee016722550a2276ca2cb5bb624480e0ed2bd49125b2b73b7010b9090e888", size = 258385, upload-time = "2025-06-30T15:51:53.913Z" }, - { url = "https://files.pythonhosted.org/packages/e3/25/cca0e68228addad24903801ed1ab42e21307a1b4b6dd2cf63da5d3ae082a/multidict-6.6.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5511cb35f5c50a2db21047c875eb42f308c5583edf96bd8ebf7d770a9d68f6d", size = 242328, upload-time = "2025-06-30T15:51:55.672Z" }, - { url = "https://files.pythonhosted.org/packages/6e/a3/46f2d420d86bbcb8fe660b26a10a219871a0fbf4d43cb846a4031533f3e0/multidict-6.6.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:712b348f7f449948e0a6c4564a21c7db965af900973a67db432d724619b3c680", size = 268057, upload-time = "2025-06-30T15:51:57.037Z" }, - { url = "https://files.pythonhosted.org/packages/9e/73/1c743542fe00794a2ec7466abd3f312ccb8fad8dff9f36d42e18fb1ec33e/multidict-6.6.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e4e15d2138ee2694e038e33b7c3da70e6b0ad8868b9f8094a72e1414aeda9c1a", size = 269341, upload-time = "2025-06-30T15:51:59.111Z" }, - { url = "https://files.pythonhosted.org/packages/a4/11/6ec9dcbe2264b92778eeb85407d1df18812248bf3506a5a1754bc035db0c/multidict-6.6.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8df25594989aebff8a130f7899fa03cbfcc5d2b5f4a461cf2518236fe6f15961", size = 256081, upload-time = "2025-06-30T15:52:00.533Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2b/631b1e2afeb5f1696846d747d36cda075bfdc0bc7245d6ba5c319278d6c4/multidict-6.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:159ca68bfd284a8860f8d8112cf0521113bffd9c17568579e4d13d1f1dc76b65", size = 253581, upload-time = "2025-06-30T15:52:02.43Z" }, - { url = "https://files.pythonhosted.org/packages/bf/0e/7e3b93f79efeb6111d3bf9a1a69e555ba1d07ad1c11bceb56b7310d0d7ee/multidict-6.6.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e098c17856a8c9ade81b4810888c5ad1914099657226283cab3062c0540b0643", size = 250750, upload-time = "2025-06-30T15:52:04.26Z" }, - { url = "https://files.pythonhosted.org/packages/ad/9e/086846c1d6601948e7de556ee464a2d4c85e33883e749f46b9547d7b0704/multidict-6.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:67c92ed673049dec52d7ed39f8cf9ebbadf5032c774058b4406d18c8f8fe7063", size = 251548, upload-time = "2025-06-30T15:52:06.002Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7b/86ec260118e522f1a31550e87b23542294880c97cfbf6fb18cc67b044c66/multidict-6.6.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:bd0578596e3a835ef451784053cfd327d607fc39ea1a14812139339a18a0dbc3", size = 262718, upload-time = "2025-06-30T15:52:07.707Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bd/22ce8f47abb0be04692c9fc4638508b8340987b18691aa7775d927b73f72/multidict-6.6.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:346055630a2df2115cd23ae271910b4cae40f4e336773550dca4889b12916e75", size = 259603, upload-time = "2025-06-30T15:52:09.58Z" }, - { url = "https://files.pythonhosted.org/packages/07/9c/91b7ac1691be95cd1f4a26e36a74b97cda6aa9820632d31aab4410f46ebd/multidict-6.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:555ff55a359302b79de97e0468e9ee80637b0de1fce77721639f7cd9440b3a10", size = 251351, upload-time = "2025-06-30T15:52:10.947Z" }, - { url = "https://files.pythonhosted.org/packages/6f/5c/4d7adc739884f7a9fbe00d1eac8c034023ef8bad71f2ebe12823ca2e3649/multidict-6.6.3-cp312-cp312-win32.whl", hash = "sha256:73ab034fb8d58ff85c2bcbadc470efc3fafeea8affcf8722855fb94557f14cc5", size = 41860, upload-time = "2025-06-30T15:52:12.334Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a3/0fbc7afdf7cb1aa12a086b02959307848eb6bcc8f66fcb66c0cb57e2a2c1/multidict-6.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:04cbcce84f63b9af41bad04a54d4cc4e60e90c35b9e6ccb130be2d75b71f8c17", size = 45982, upload-time = "2025-06-30T15:52:13.6Z" }, - { url = "https://files.pythonhosted.org/packages/b8/95/8c825bd70ff9b02462dc18d1295dd08d3e9e4eb66856d292ffa62cfe1920/multidict-6.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:0f1130b896ecb52d2a1e615260f3ea2af55fa7dc3d7c3003ba0c3121a759b18b", size = 43210, upload-time = "2025-06-30T15:52:14.893Z" }, - { url = "https://files.pythonhosted.org/packages/d8/30/9aec301e9772b098c1f5c0ca0279237c9766d94b97802e9888010c64b0ed/multidict-6.6.3-py3-none-any.whl", hash = "sha256:8db10f29c7541fc5da4defd8cd697e1ca429db743fa716325f236079b96f775a", size = 12313, upload-time = "2025-06-30T15:53:45.437Z" }, +name = "llama-index-indices-managed-llama-cloud" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-cloud" }, + { name = "llama-index-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/2a/0b42b533e6891150eb9d1189492a08be155e9656889178009bbc4cdf44b9/llama_index_indices_managed_llama_cloud-0.8.0.tar.gz", hash = "sha256:762de10d3949e04997766f6a665ed4503394d82ea4b3339139a365edde6e753e", size = 14722, upload-time = "2025-07-28T19:53:08.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/50/c259cc8b8497ab8f3e245c9bc828e8a269951b222760b5cac072acba3811/llama_index_indices_managed_llama_cloud-0.8.0-py3-none-any.whl", hash = "sha256:817d6bd4715d45522e7165d29208e093d06179cc1bc5f9590382245f73dfd7aa", size = 16451, upload-time = "2025-07-28T19:53:07.826Z" }, ] [[package]] -name = "multiprocess" -version = "0.70.18" +name = "llama-index-llms-openai" +version = "0.4.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dill" }, + { name = "llama-index-core" }, + { name = "openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/39/a7ce514fb500951e9edb713ed918a9ffe49f1a76fccfc531a4ec5c7fe15a/llama_index_llms_openai-0.4.7.tar.gz", hash = "sha256:564af8ab39fb3f3adfeae73a59c0dca46c099ab844a28e725eee0c551d4869f8", size = 24251, upload-time = "2025-06-16T03:38:47.175Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" }, - { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" }, - { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" }, - { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" }, - { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" }, + { url = "https://files.pythonhosted.org/packages/61/e9/391926dad180ced6bb37a62edddb8483fbecde411239bd5e726841bb77b4/llama_index_llms_openai-0.4.7-py3-none-any.whl", hash = "sha256:3b8d9d3c1bcadc2cff09724de70f074f43eafd5b7048a91247c9a41b7cd6216d", size = 25365, upload-time = "2025-06-16T03:38:45.72Z" }, ] [[package]] -name = "mypy-extensions" -version = "1.1.0" +name = "llama-index-multi-modal-llms-openai" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/77/0ece3c7887e9c7242fc7cc2a184374bbeb08a35783d71a8466e1dd18ccfc/llama_index_multi_modal_llms_openai-0.5.1.tar.gz", hash = "sha256:df3aff00c36023c5f8c49f972a325f71823ed0f4dd9cd479955d76afc146575f", size = 3708, upload-time = "2025-05-30T23:08:09.915Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, + { url = "https://files.pythonhosted.org/packages/17/a5/6181a385af1caef88e1d0d3919c399ba7e031190fc0be4fd5242875c5b0e/llama_index_multi_modal_llms_openai-0.5.1-py3-none-any.whl", hash = "sha256:69bb9c310c323ce51038f0d40719f546d0d4e0853835a4f5cfa21f07dbb0b91e", size = 3362, upload-time = "2025-05-30T23:08:08.977Z" }, ] [[package]] -name = "nbformat" -version = "5.10.4" +name = "llama-index-program-openai" +version = "0.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fastjsonschema" }, - { name = "jsonschema" }, - { name = "jupyter-core" }, - { name = "traitlets" }, + { name = "llama-index-agent-openai" }, + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/81/9caa34e80adce1adb715ae083a54ad45c8fc0d9aef0f2d80d61c1b805ab6/llama_index_program_openai-0.3.2.tar.gz", hash = "sha256:04c959a2e616489894bd2eeebb99500d6f1c17d588c3da0ddc75ebd3eb7451ee", size = 6301, upload-time = "2025-05-30T23:00:27.872Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, + { url = "https://files.pythonhosted.org/packages/05/80/d6ac8afafdd38115d61214891c36876e64f429809abff873660fe30862fe/llama_index_program_openai-0.3.2-py3-none-any.whl", hash = "sha256:451829ae53e074e7b47dcc60a9dd155fcf9d1dcbc1754074bdadd6aab4ceb9aa", size = 6129, upload-time = "2025-05-30T23:00:26.64Z" }, ] [[package]] -name = "nemollm" -version = "0.3.5" +name = "llama-index-question-gen-openai" +version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil" }, - { name = "requests-futures" }, - { name = "requests-toolbelt" }, - { name = "tqdm" }, - { name = "typing-extensions" }, - { name = "urllib3" }, + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, + { name = "llama-index-program-openai" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/52/6e/19c5051c81ef5fca597d13c6d41b863535521565b1414ab5ab0e5e8c1297/llama_index_question_gen_openai-0.3.1.tar.gz", hash = "sha256:5e9311b433cc2581ff8a531fa19fb3aa21815baff75aaacdef11760ac9522aa9", size = 4107, upload-time = "2025-05-30T23:00:31.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/c1/0975d3e7b3293fc4480fe6521ddfcba2a02da131f7751c295430c797e5f8/nemollm-0.3.5-py3-none-any.whl", hash = "sha256:de3962d82d557ce5f3d773144361b897a181028b8e1bbdf94804eff865eead88", size = 11573, upload-time = "2024-01-05T19:46:57.009Z" }, + { url = "https://files.pythonhosted.org/packages/15/2a/652593d0bd24f901776db0d1778a42363ea2656530da18215f413ce4f981/llama_index_question_gen_openai-0.3.1-py3-none-any.whl", hash = "sha256:1ce266f6c8373fc8d884ff83a44dfbacecde2301785db7144872db51b8b99429", size = 3733, upload-time = "2025-05-30T23:00:29.965Z" }, ] [[package]] -name = "nest-asyncio" -version = "1.6.0" +name = "llama-index-readers-file" +version = "0.4.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "llama-index-core" }, + { name = "pandas" }, + { name = "pypdf" }, + { name = "striprtf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/55/de3392c59f8f44d20e48b72faa91393061425d68cf7035f167db7732f025/llama_index_readers_file-0.4.8.tar.gz", hash = "sha256:d2821b164c11453b7993456355a6c544631a0ae8656a646df9fc009f16cfb99f", size = 22668, upload-time = "2025-05-27T19:53:18.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/24/62226cbf55e91d97d7b7d659d3dc05a42f15c2413589056ace8a4813fb76/llama_index_readers_file-0.4.8-py3-none-any.whl", hash = "sha256:6cc64c70c3a9f5b146200a68c15b742e6f0d13b308a1c0dc6bb919e3b5fd4275", size = 40972, upload-time = "2025-05-27T19:53:17.047Z" }, +] + +[[package]] +name = "llama-index-readers-llama-parse" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-parse" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/30/4611821286f82ba7b5842295607baa876262db86f88b87d83595eed172bf/llama_index_readers_llama_parse-0.4.0.tar.gz", hash = "sha256:e99ec56f4f8546d7fda1a7c1ae26162fb9acb7ebcac343b5abdb4234b4644e0f", size = 2472, upload-time = "2024-11-18T00:00:08.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/4f/e30d4257fe9e4224f5612b77fe99aaceddae411b2e74ca30534491de3e6f/llama_index_readers_llama_parse-0.4.0-py3-none-any.whl", hash = "sha256:574e48386f28d2c86c3f961ca4a4906910312f3400dd0c53014465bfbc6b32bf", size = 2472, upload-time = "2024-11-18T00:00:07.293Z" }, +] + +[[package]] +name = "llama-parse" +version = "0.6.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-cloud-services" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/27/8014c38cab1e9664153157d3c8693af726c0f7ae0c93adaebace5da688d7/llama_parse-0.6.12.tar.gz", hash = "sha256:c99593fb955c338a69e64a2ec449e09753afe6dcff239ab050989fda74839867", size = 3673, upload-time = "2025-04-11T17:27:49.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/ca/71c9367d3e89d61da2462f535dea1a3a09d4a4085b96f2c9ef5c38864820/llama_parse-0.6.12-py3-none-any.whl", hash = "sha256:2dd1c74b0cba1a2bc300286f6b91a650f6ddc396acfce3497ba3d72d43c53fac", size = 4853, upload-time = "2025-04-11T17:27:48.223Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "marisa-trie" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/df/504ca06cfcc6d67ec034f35b863d6719c26970407c94dc638c1994d78684/marisa_trie-1.3.0.tar.gz", hash = "sha256:39af3060b4ab41a3cce18b1808338db8bf50b6ec4b81be3cc452558aaad95581", size = 212383, upload-time = "2025-08-16T10:05:20.9Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/f2/c98483b63c0cf9f54e283e1689980baca66d348ef214f643688b30a4b28c/marisa_trie-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5d72ffde56fb1515bcb03539803d42d0a119f6782c5812bf2b7313eddc691735", size = 176084, upload-time = "2025-08-16T10:04:04.575Z" }, + { url = "https://files.pythonhosted.org/packages/d5/29/51ff243488d753f62e7fc5111af20cc3437bf85d44bb901a9bf4c89a0e6d/marisa_trie-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a1f0781bccd854184a9c59b095ed09adf16627460eb8df4a91dc3f87e882352", size = 158864, upload-time = "2025-08-16T10:04:06.016Z" }, + { url = "https://files.pythonhosted.org/packages/69/bd/a21c11c15c0e11071e9b4fc983ecc84c9575ae63abc2dd57586aa7bddebf/marisa_trie-1.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:608d965d47f40b8cd402215b95d85db899268d277ae5b8ebe87b7acdd3e2a0bb", size = 1257508, upload-time = "2025-08-16T10:04:07.683Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4b/02e542a8e22779b35bca354cc66a736fe8ca2f20aa93203c93c16cfc4470/marisa_trie-1.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b39a7314f6ad141c9c24acff0a71f4fdae1eab5ea827468c40afafc0662cab3", size = 1275539, upload-time = "2025-08-16T10:04:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/87/a2/308ab72fb6e9a7e40a95f56478ab0dc5bfeb17195ed97e11ed46a7c5790d/marisa_trie-1.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6e8e2f1394eecfb780a25950849d64a799b79f538d17945e42b1652da4e0cae4", size = 2199801, upload-time = "2025-08-16T10:04:10.638Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0a/b8e6627f6fb60d37dabd731d0639816c8c2e2f87ea9ecb34fd2d67863d7e/marisa_trie-1.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a56cc700b1405cc75fde9197f9d2fed66ecbbaee7bdf1f28728494f119dc7f3", size = 2283801, upload-time = "2025-08-16T10:04:11.988Z" }, + { url = "https://files.pythonhosted.org/packages/af/32/fba57bcb72ea8d4331e248fa50ebeb3e9c1889eb7f2b8025974a760c624c/marisa_trie-1.3.0-cp311-cp311-win32.whl", hash = "sha256:58f1b70501c2462583bce5639a65af5516e9785ae6b3158533ddeecde70f0675", size = 117295, upload-time = "2025-08-16T10:04:13.465Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/f8099b17fc1f0301f02556ef7c40253571b0bda2a405d3416fdf0cba2383/marisa_trie-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:938f618d2cece8358899c688591d94db6652d9e1076c15a7efdfcfdc64a96cdb", size = 143966, upload-time = "2025-08-16T10:04:14.873Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/db61a1c950f23b876380928098e69c9a24f2810ef1a68eb7d5bf6732fa47/marisa_trie-1.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28bfd6fada6c87cb31d300bbed5de1bfd338f8c98d1b834cf810a06ce019a020", size = 174778, upload-time = "2025-08-16T10:04:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/e2/de/0b80b66f8fe09b36150e7a1a1e1c9761d136ccb0887186404c3d2c447b25/marisa_trie-1.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:034e483bd35ab6d136d8a91f43088dc78549394cf3787fdeebca144e2e4c82df", size = 155737, upload-time = "2025-08-16T10:04:17.371Z" }, + { url = "https://files.pythonhosted.org/packages/5e/58/5da536b940fa6145744b06714fc25ca0b0b0360d0499e88678d560a0dd68/marisa_trie-1.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b71462677dc6c119589755394086cffbcf4d4d42f906fefb325c982c679406d6", size = 1244807, upload-time = "2025-08-16T10:04:18.58Z" }, + { url = "https://files.pythonhosted.org/packages/45/cd/05bed6d02213da7f2fda63e689300b186a7f16f6b982ea19bb7284ecb1ee/marisa_trie-1.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c891ebce899f35936d4ab9f332b69ab762513d5944b0f43f61427e53671d42", size = 1265122, upload-time = "2025-08-16T10:04:19.901Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/06f718d62f78bfa6d7082533cd187b6425755d77a6ea901228b4085f61b3/marisa_trie-1.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4570850d9b6e6a099797f731652dbe764dfd6dd7eff2934318a7018ba1a82cf1", size = 2172695, upload-time = "2025-08-16T10:04:21.258Z" }, + { url = "https://files.pythonhosted.org/packages/3c/47/a3a50e293f87f3a11082fbb80fdd504e2d8d1a92372476f37b08a0b765dd/marisa_trie-1.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d85a0484f8ecd3a6c843c1b10b42953f14278b35ce30d94bc7cb6305604a6109", size = 2256076, upload-time = "2025-08-16T10:04:23.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/32/b58f7b0c378c05eb3df632ffc2dadb93611266b0f42997e72eb9597dc25c/marisa_trie-1.3.0-cp312-cp312-win32.whl", hash = "sha256:714dabb0ddd4be72841c962d0559d5a80613964dc2a5db72651ae3b2ae3408fc", size = 115589, upload-time = "2025-08-16T10:04:24.243Z" }, + { url = "https://files.pythonhosted.org/packages/1e/90/57830e0403ef2ae9067ec07ccb2fee8297a5c42f518528c8c2e7401cd4b5/marisa_trie-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:bd53e6b99008ff3dab6455791800af405351d98fbf01c4f474642afb1499236d", size = 138547, upload-time = "2025-08-16T10:04:25.637Z" }, +] + +[[package]] +name = "markdown" +version = "3.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071, upload-time = "2025-06-19T17:12:44.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827, upload-time = "2025-06-19T17:12:42.994Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, +] + +[[package]] +name = "marshmallow" +version = "3.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/91/f2939bb60b7ebf12478b030e0d7f340247390f402b3b189616aad790c366/matplotlib-3.10.5.tar.gz", hash = "sha256:352ed6ccfb7998a00881692f38b4ca083c691d3e275b4145423704c34c909076", size = 34804044, upload-time = "2025-07-31T18:09:33.805Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/c7/1f2db90a1d43710478bb1e9b57b162852f79234d28e4f48a28cc415aa583/matplotlib-3.10.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dcfc39c452c6a9f9028d3e44d2d721484f665304857188124b505b2c95e1eecf", size = 8239216, upload-time = "2025-07-31T18:07:51.947Z" }, + { url = "https://files.pythonhosted.org/packages/82/6d/ca6844c77a4f89b1c9e4d481c412e1d1dbabf2aae2cbc5aa2da4a1d6683e/matplotlib-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:903352681b59f3efbf4546985142a9686ea1d616bb054b09a537a06e4b892ccf", size = 8102130, upload-time = "2025-07-31T18:07:53.65Z" }, + { url = "https://files.pythonhosted.org/packages/1d/1e/5e187a30cc673a3e384f3723e5f3c416033c1d8d5da414f82e4e731128ea/matplotlib-3.10.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:080c3676a56b8ee1c762bcf8fca3fe709daa1ee23e6ef06ad9f3fc17332f2d2a", size = 8666471, upload-time = "2025-07-31T18:07:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/03/c0/95540d584d7d645324db99a845ac194e915ef75011a0d5e19e1b5cee7e69/matplotlib-3.10.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b4984d5064a35b6f66d2c11d668565f4389b1119cc64db7a4c1725bc11adffc", size = 9500518, upload-time = "2025-07-31T18:07:57.199Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e019352099ea58b4169adb9c6e1a2ad0c568c6377c2b677ee1f06de2adc7/matplotlib-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3967424121d3a46705c9fa9bdb0931de3228f13f73d7bb03c999c88343a89d89", size = 9552372, upload-time = "2025-07-31T18:07:59.41Z" }, + { url = "https://files.pythonhosted.org/packages/b7/81/3200b792a5e8b354f31f4101ad7834743ad07b6d620259f2059317b25e4d/matplotlib-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:33775bbeb75528555a15ac29396940128ef5613cf9a2d31fb1bfd18b3c0c0903", size = 8100634, upload-time = "2025-07-31T18:08:01.801Z" }, + { url = "https://files.pythonhosted.org/packages/52/46/a944f6f0c1f5476a0adfa501969d229ce5ae60cf9a663be0e70361381f89/matplotlib-3.10.5-cp311-cp311-win_arm64.whl", hash = "sha256:c61333a8e5e6240e73769d5826b9a31d8b22df76c0778f8480baf1b4b01c9420", size = 7978880, upload-time = "2025-07-31T18:08:03.407Z" }, + { url = "https://files.pythonhosted.org/packages/66/1e/c6f6bcd882d589410b475ca1fc22e34e34c82adff519caf18f3e6dd9d682/matplotlib-3.10.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:00b6feadc28a08bd3c65b2894f56cf3c94fc8f7adcbc6ab4516ae1e8ed8f62e2", size = 8253056, upload-time = "2025-07-31T18:08:05.385Z" }, + { url = "https://files.pythonhosted.org/packages/53/e6/d6f7d1b59413f233793dda14419776f5f443bcccb2dfc84b09f09fe05dbe/matplotlib-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee98a5c5344dc7f48dc261b6ba5d9900c008fc12beb3fa6ebda81273602cc389", size = 8110131, upload-time = "2025-07-31T18:08:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/66/2b/bed8a45e74957549197a2ac2e1259671cd80b55ed9e1fe2b5c94d88a9202/matplotlib-3.10.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a17e57e33de901d221a07af32c08870ed4528db0b6059dce7d7e65c1122d4bea", size = 8669603, upload-time = "2025-07-31T18:08:09.064Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a7/315e9435b10d057f5e52dfc603cd353167ae28bb1a4e033d41540c0067a4/matplotlib-3.10.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97b9d6443419085950ee4a5b1ee08c363e5c43d7176e55513479e53669e88468", size = 9508127, upload-time = "2025-07-31T18:08:10.845Z" }, + { url = "https://files.pythonhosted.org/packages/7f/d9/edcbb1f02ca99165365d2768d517898c22c6040187e2ae2ce7294437c413/matplotlib-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ceefe5d40807d29a66ae916c6a3915d60ef9f028ce1927b84e727be91d884369", size = 9566926, upload-time = "2025-07-31T18:08:13.186Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d9/6dd924ad5616c97b7308e6320cf392c466237a82a2040381163b7500510a/matplotlib-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:c04cba0f93d40e45b3c187c6c52c17f24535b27d545f757a2fffebc06c12b98b", size = 8107599, upload-time = "2025-07-31T18:08:15.116Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f3/522dc319a50f7b0279fbe74f86f7a3506ce414bc23172098e8d2bdf21894/matplotlib-3.10.5-cp312-cp312-win_arm64.whl", hash = "sha256:a41bcb6e2c8e79dc99c5511ae6f7787d2fb52efd3d805fff06d5d4f667db16b2", size = 7978173, upload-time = "2025-07-31T18:08:21.518Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/e921be4e1a5f7aca5194e1f016cb67ec294548e530013251f630713e456d/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:160e125da27a749481eaddc0627962990f6029811dbeae23881833a011a0907f", size = 8233224, upload-time = "2025-07-31T18:09:27.512Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/a2b9b04824b9c349c8f1b2d21d5af43fa7010039427f2b133a034cb09e59/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac3d50760394d78a3c9be6b28318fe22b494c4fcf6407e8fd4794b538251899b", size = 8098539, upload-time = "2025-07-31T18:09:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/fc/66/cd29ebc7f6c0d2a15d216fb572573e8fc38bd5d6dec3bd9d7d904c0949f7/matplotlib-3.10.5-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c49465bf689c4d59d174d0c7795fb42a21d4244d11d70e52b8011987367ac61", size = 8672192, upload-time = "2025-07-31T18:09:31.407Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159, upload-time = "2024-04-15T13:44:44.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mcp" +version = "1.12.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/88/f6cb7e7c260cd4b4ce375f2b1614b33ce401f63af0f49f7141a2e9bf0a45/mcp-1.12.4.tar.gz", hash = "sha256:0765585e9a3a5916a3c3ab8659330e493adc7bd8b2ca6120c2d7a0c43e034ca5", size = 431148, upload-time = "2025-08-07T20:31:18.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/68/316cbc54b7163fa22571dcf42c9cc46562aae0a021b974e0a8141e897200/mcp-1.12.4-py3-none-any.whl", hash = "sha256:7aa884648969fab8e78b89399d59a683202972e12e6bc9a1c88ce7eda7743789", size = 160145, upload-time = "2025-08-07T20:31:15.69Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "milvus-lite" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713, upload-time = "2025-06-30T04:23:37.028Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451, upload-time = "2025-06-30T04:23:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093, upload-time = "2025-06-30T04:24:06.706Z" }, + { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911, upload-time = "2025-06-30T04:24:19.434Z" }, +] + +[[package]] +name = "multidict" +version = "6.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" }, + { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" }, + { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" }, + { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" }, + { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" }, + { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" }, + { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" }, + { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" }, + { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" }, + { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" }, + { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" }, + { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" }, + { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, + { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, + { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, + { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, + { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, + { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, + { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, + { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, + { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, + { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, +] + +[[package]] +name = "multiprocess" +version = "0.70.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, +] + +[[package]] +name = "murmurhash" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/e9/02efbc6dfc2dd2085da3daacf9a8c17e8356019eceaedbfa21555e32d2af/murmurhash-1.0.13.tar.gz", hash = "sha256:737246d41ee00ff74b07b0bd1f0888be304d203ce668e642c86aa64ede30f8b7", size = 13258, upload-time = "2025-05-22T12:35:57.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/d1/9d13a02d9c8bfff10b1f68d19df206eaf2a8011defeccf7eb05ea0b8c54e/murmurhash-1.0.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b20d168370bc3ce82920121b78ab35ae244070a9b18798f4a2e8678fa03bd7e0", size = 26410, upload-time = "2025-05-22T12:35:20.786Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/3ee762e98cf9a8c2df9c8b377c326f3dd4495066d4eace9066fca46eba7a/murmurhash-1.0.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cef667d2e83bdceea3bc20c586c491fa442662ace1aea66ff5e3a18bb38268d8", size = 26679, upload-time = "2025-05-22T12:35:21.808Z" }, + { url = "https://files.pythonhosted.org/packages/39/06/24618f79cd5aac48490932e50263bddfd1ea90f7123d49bfe806a5982675/murmurhash-1.0.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:507148e50929ba1fce36898808573b9f81c763d5676f3fc6e4e832ff56b66992", size = 125970, upload-time = "2025-05-22T12:35:23.222Z" }, + { url = "https://files.pythonhosted.org/packages/e8/09/0e7afce0a422692506c85474a26fb3a03c1971b2b5f7e7745276c4b3de7f/murmurhash-1.0.13-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d50f6173d266ad165beb8bca6101d824217fc9279f9e9981f4c0245c1e7ee6", size = 123390, upload-time = "2025-05-22T12:35:24.303Z" }, + { url = "https://files.pythonhosted.org/packages/22/4c/c98f579b1a951b2bcc722a35270a2eec105c1e21585c9b314a02079e3c4d/murmurhash-1.0.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0f272e15a84a8ae5f8b4bc0a68f9f47be38518ddffc72405791178058e9d019a", size = 124007, upload-time = "2025-05-22T12:35:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/df/f8/1b0dcebc8df8e091341617102b5b3b97deb6435f345b84f75382c290ec2c/murmurhash-1.0.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9423e0b0964ed1013a06c970199538c7ef9ca28c0be54798c0f1473a6591761", size = 123705, upload-time = "2025-05-22T12:35:26.709Z" }, + { url = "https://files.pythonhosted.org/packages/79/17/f2a38558e150a0669d843f75e128afb83c1a67af41885ea2acb940e18e2a/murmurhash-1.0.13-cp311-cp311-win_amd64.whl", hash = "sha256:83b81e7084b696df3d853f2c78e0c9bda6b285d643f923f1a6fa9ab145d705c5", size = 24572, upload-time = "2025-05-22T12:35:30.38Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/56ce2d8d4b9ab89557cb1d00ffce346b80a2eb2d8c7944015e5c83eacdec/murmurhash-1.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bbe882e46cb3f86e092d8a1dd7a5a1c992da1ae3b39f7dd4507b6ce33dae7f92", size = 26859, upload-time = "2025-05-22T12:35:31.815Z" }, + { url = "https://files.pythonhosted.org/packages/f8/85/3a0ad54a61257c31496545ae6861515d640316f93681d1dd917e7be06634/murmurhash-1.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:52a33a12ecedc432493692c207c784b06b6427ffaa897fc90b7a76e65846478d", size = 26900, upload-time = "2025-05-22T12:35:34.267Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cd/6651de26744b50ff11c79f0c0d41244db039625de53c0467a7a52876b2d8/murmurhash-1.0.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:950403a7f0dc2d9c8d0710f07c296f2daab66299d9677d6c65d6b6fa2cb30aaa", size = 131367, upload-time = "2025-05-22T12:35:35.258Z" }, + { url = "https://files.pythonhosted.org/packages/50/6c/01ded95ddce33811c9766cae4ce32e0a54288da1d909ee2bcaa6ed13b9f1/murmurhash-1.0.13-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fde9fb5d2c106d86ff3ef2e4a9a69c2a8d23ba46e28c6b30034dc58421bc107b", size = 128943, upload-time = "2025-05-22T12:35:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/ab/27/e539a9622d7bea3ae22706c1eb80d4af80f9dddd93b54d151955c2ae4011/murmurhash-1.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3aa55d62773745616e1ab19345dece122f6e6d09224f7be939cc5b4c513c8473", size = 129108, upload-time = "2025-05-22T12:35:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/7a/84/18af5662e07d06839ad4db18ce026e6f8ef850d7b0ba92817b28dad28ba6/murmurhash-1.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:060dfef1b405cf02c450f182fb629f76ebe7f79657cced2db5054bc29b34938b", size = 129175, upload-time = "2025-05-22T12:35:38.928Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8d/b01d3ee1f1cf3957250223b7c6ce35454f38fbf4abe236bf04a3f769341d/murmurhash-1.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:a8e79627d44a6e20a6487effc30bfe1c74754c13d179106e68cc6d07941b022c", size = 24869, upload-time = "2025-05-22T12:35:40.035Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "nbformat" +version = "5.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, +] + +[[package]] +name = "nemollm" +version = "0.3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "requests-futures" }, + { name = "requests-toolbelt" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/c1/0975d3e7b3293fc4480fe6521ddfcba2a02da131f7751c295430c797e5f8/nemollm-0.3.5-py3-none-any.whl", hash = "sha256:de3962d82d557ce5f3d773144361b897a181028b8e1bbdf94804eff865eead88", size = 11573, upload-time = "2024-01-05T19:46:57.009Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "nltk" +version = "3.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691, upload-time = "2024-08-18T19:48:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442, upload-time = "2024-08-18T19:48:21.909Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, + { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, + { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, + { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, + { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +] + +[[package]] +name = "nvidia-nat" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aioboto3" }, + { name = "authlib" }, + { name = "click" }, + { name = "colorama" }, + { name = "datasets" }, + { name = "expandvars" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "jsonpath-ng" }, + { name = "mcp" }, + { name = "networkx" }, + { name = "numpy" }, + { name = "openinference-semantic-conventions" }, + { name = "openpyxl" }, + { name = "pkce" }, + { name = "pkginfo" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pymilvus" }, + { name = "pyyaml" }, + { name = "ragas" }, + { name = "rich" }, + { name = "tabulate" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "wikipedia" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/95/a487db754505fa0d66fbd13fbd85a5aa3cf0bc81d5fec1a036cd1032be10/nvidia_nat-1.2.0rc8-py3-none-any.whl", hash = "sha256:30c23e798a567b791f2f82a95fd185c0f58597d85c5d9975e78c772b0f063bf4", size = 713049, upload-time = "2025-08-15T01:20:49.358Z" }, +] + +[package.optional-dependencies] +langchain = [ + { name = "nvidia-nat-langchain" }, +] +opentelemetry = [ + { name = "nvidia-nat-opentelemetry" }, +] +profiling = [ + { name = "matplotlib" }, + { name = "prefixspan" }, + { name = "scikit-learn" }, +] +telemetry = [ + { name = "nvidia-nat-opentelemetry" }, + { name = "nvidia-nat-phoenix" }, + { name = "nvidia-nat-ragaai" }, + { name = "nvidia-nat-weave" }, +] + +[[package]] +name = "nvidia-nat-langchain" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-aws" }, + { name = "langchain-core" }, + { name = "langchain-milvus" }, + { name = "langchain-nvidia-ai-endpoints" }, + { name = "langchain-openai" }, + { name = "langgraph" }, + { name = "nvidia-nat" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/ab/90768e1cd4b46d7f19ea6fc27ac1c9b649a8b5566159fe9758764122010a/nvidia_nat_langchain-1.2.0rc8-py3-none-any.whl", hash = "sha256:4d0f142ceb201e3ce041e4b067d6c6fdc217a8a841289542bdc6ea936b3c2958", size = 13669, upload-time = "2025-08-15T01:21:07.928Z" }, +] + +[[package]] +name = "nvidia-nat-opentelemetry" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/91/1c53e70ca026af1422aff99336b2be992d770ab1c1ae930a28b7e5edf75d/nvidia_nat_opentelemetry-1.2.0rc8-py3-none-any.whl", hash = "sha256:cde60971c49683675657ac554bbdd17ee0872da2b8c637b1dec248406cb6af40", size = 18699, upload-time = "2025-08-15T01:19:57.304Z" }, +] + +[[package]] +name = "nvidia-nat-phoenix" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "arize-phoenix" }, + { name = "nvidia-nat", extra = ["opentelemetry"] }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/8b/e1a55c848d66c143865da6f54f29089b01357610a3f13aa87a1500c19c6f/nvidia_nat_phoenix-1.2.0rc8-py3-none-any.whl", hash = "sha256:999b957c4515fda7bb2c2a2ca5ca8bf42297e5a88c52fe1fc32dfa500e1d8ab3", size = 8035, upload-time = "2025-08-15T01:19:22.418Z" }, +] + +[[package]] +name = "nvidia-nat-ragaai" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat", extra = ["opentelemetry"] }, + { name = "ragaai-catalyst" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/b5/1f64bca65c619a0b83b8f745eb6c06ed5b763578e4cc577852db8b23a262/nvidia_nat_ragaai-1.2.0rc8-py3-none-any.whl", hash = "sha256:12fff01606239603477bbcfd424b0559209d27e0d32cdcf3b6f1bf1fa7f63620", size = 10224, upload-time = "2025-08-15T01:20:14.599Z" }, +] + +[[package]] +name = "nvidia-nat-weave" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat" }, + { name = "presidio-analyzer" }, + { name = "presidio-anonymizer" }, + { name = "weave" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/76/b8b74f1d6e39b0462ef70dfacc1cffd9383a0229463c734fe0e7619d3c2f/nvidia_nat_weave-1.2.0rc8-py3-none-any.whl", hash = "sha256:b88e6437f9caa8116ee1331f4a7f90c1434d4f58f3d15715b88d0f102bd85172", size = 7518, upload-time = "2025-08-15T01:21:25.292Z" }, +] + +[[package]] +name = "openai" +version = "1.100.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/71/efa1ddeedcba1a385034b66e6f34c7b0ace4bf99ffbfc2859c025daa147a/openai-1.100.1.tar.gz", hash = "sha256:3e9ae652903e5120514e544af2426334141404657cdcdb6dc6845fc243d66e66", size = 508419, upload-time = "2025-08-18T21:57:44.702Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/b4/686944f0903c65202e86311ec0f42171e697e4f7324caeee0c318046b738/openai-1.100.1-py3-none-any.whl", hash = "sha256:2e8224caaf3136c58e30e6b3984fd7a8e6da0931d2c36fbbb7d668e5c11db914", size = 788286, upload-time = "2025-08-18T21:57:42.754Z" }, +] + +[[package]] +name = "openinference-instrumentation" +version = "0.1.37" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/e0/9d3fe148d27602f794840ba7e2353ba1f25ff6f43ad32fe4390fba393ba4/openinference_instrumentation-0.1.37.tar.gz", hash = "sha256:67fe1c83a864c0cb38a19165b63f28b7287f5c0d7924c47dad599e006a934fd1", size = 23012, upload-time = "2025-08-06T00:29:46.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/68/acb8517ab0b1238114c9470dfb3825d30ab75c871f0f26dc0b28325a8f5c/openinference_instrumentation-0.1.37-py3-none-any.whl", hash = "sha256:4165642efbcad3b59b1fbf22914f8e600af51845aa5290c9b2683022795b4dfa", size = 28829, upload-time = "2025-08-06T00:29:45.037Z" }, +] + +[[package]] +name = "openinference-instrumentation-anthropic" +version = "0.1.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/11/a3f1c8603fc9ed75dc52d6df119e5c59b22ac000cf5d777d4e175dff4875/openinference_instrumentation_anthropic-0.1.19.tar.gz", hash = "sha256:974ea6aaba5bc7441d0f2227f355a781ce860bf9c2996db45a14442c4402ed8e", size = 14262, upload-time = "2025-08-18T18:13:02.274Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/cc/e7d5ee816b7ef287d76168bb61dcef47c39f9b9ebd7cd86487ff1ee9688d/openinference_instrumentation_anthropic-0.1.19-py3-none-any.whl", hash = "sha256:fa513ed575e245953a2124c3d0cf8cc562b263dace00d6a717e002a3dde3ce39", size = 17389, upload-time = "2025-08-18T18:12:58.796Z" }, +] + +[[package]] +name = "openinference-instrumentation-bedrock" +version = "0.1.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dacite" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b6/5c0167e364da21bd0b2aea3b88fbde0a8d77e14fccfe7948d3608c438b22/openinference_instrumentation_bedrock-0.1.26.tar.gz", hash = "sha256:2191f664ec9791f8f42d15fb6af43f3937640249c1aeeb7edbd6a719b3cfd77b", size = 170038, upload-time = "2025-06-19T16:54:19.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/d3/6d52f61360abe0cda5cdd7be71fb64f3953718fe74f03be82cd1087af631/openinference_instrumentation_bedrock-0.1.26-py3-none-any.whl", hash = "sha256:953d1bc8b958d56b03263a5752b2e9a8c6a33069d05e1233527a21a6dd1505ce", size = 50677, upload-time = "2025-06-19T16:54:17.805Z" }, +] + +[[package]] +name = "openinference-instrumentation-crewai" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/30/95276c074a257d8c6a92e7f92637eaf78c1f89fb6281e82f76c17c3d27d3/openinference_instrumentation_crewai-0.1.11.tar.gz", hash = "sha256:61234cce6aead3debd18a2b0eab6b8fb70776a268e14ae71f1f570d707e861aa", size = 10323, upload-time = "2025-07-16T21:17:37.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/7c/7a2ae73f88ea6ab1a6a6242a83aa3070859aafb3a9e5746250b0f6c4f275/openinference_instrumentation_crewai-0.1.11-py3-none-any.whl", hash = "sha256:096975809acc2326c409b5d3fb12cc264f1012f9d207b1b820b172e1b83a147a", size = 11845, upload-time = "2025-07-16T21:17:36.768Z" }, +] + +[[package]] +name = "openinference-instrumentation-google-adk" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/0e/78e1613d97f6d197bf0b59c4e6c975b2360b417e036316736d71d78833e1/openinference_instrumentation_google_adk-0.1.3.tar.gz", hash = "sha256:6cf913d83d08010bd638e02be08a9d74b4d0f76e804b6d9a369cabd18cb3c551", size = 11852, upload-time = "2025-08-04T20:53:47.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3f/561981f1df408a8dfe6131d904f1a6ad65a3fe3d6ad79c01ee060f7e86b5/openinference_instrumentation_google_adk-0.1.3-py3-none-any.whl", hash = "sha256:b4b27c4e3891513f7c056f55ae8bae50ab0f85a2e8cfd918e55fa57e1518fa6e", size = 13660, upload-time = "2025-08-04T20:53:46.086Z" }, ] [[package]] -name = "networkx" -version = "3.5" +name = "openinference-instrumentation-groq" +version = "0.1.11" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/77/4bafc9b2307325e97afc9857e18eaa2d952ebeedb4063f4f4ae82fa9624f/openinference_instrumentation_groq-0.1.11.tar.gz", hash = "sha256:d23d640822dd66682f779c490b7d9cebbf26895cc06ff75d6ae8e1df8210f1e8", size = 11817, upload-time = "2025-04-28T23:14:06.058Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, + { url = "https://files.pythonhosted.org/packages/64/1a/073e8bea94af702bf554f0a1c53a403783ef942fdd07a5ec286a4d642d0a/openinference_instrumentation_groq-0.1.11-py3-none-any.whl", hash = "sha256:b5303469870ba474dc3461558730c18cf5ef787b490386f72d20a250ede9cf36", size = 15717, upload-time = "2025-04-28T23:13:56.832Z" }, ] [[package]] -name = "numpy" -version = "1.26.4" +name = "openinference-instrumentation-haystack" +version = "0.1.24" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/15/d750f65dff58524fb7c1d679bae91948447e1a3547bc32e662e6d3ac7fcc/openinference_instrumentation_haystack-0.1.24.tar.gz", hash = "sha256:14aea1e46d16415e373dd3f65d4e2ae94a1aa705b7b6967e606ab6e5dbb2cc18", size = 13001, upload-time = "2025-05-31T00:10:31.585Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, - { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, - { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, - { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/96/2f/0bad4a4840de04defa9f82fc6ffaa48b02dd19aa085fdf08701ba349cb59/openinference_instrumentation_haystack-0.1.24-py3-none-any.whl", hash = "sha256:55628441fdccb13904c7bc90e3621d18d346acf9bf3884b9fac672a6dba9ad2f", size = 14415, upload-time = "2025-05-31T00:10:29.683Z" }, ] [[package]] -name = "openai" -version = "1.96.1" +name = "openinference-instrumentation-langchain" +version = "0.1.50" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/b5/18fd5e1b6b6c7dca52d60307b3637f9e9e3206a8041a9c8028985dbc6260/openai-1.96.1.tar.gz", hash = "sha256:6d505b5cc550e036bfa3fe99d6cff565b11491d12378d4c353f92ef72b0a408a", size = 489065, upload-time = "2025-07-15T21:39:37.215Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7b/8b0b8ecb147f352363bfbe01202af8fbd69d3cbb4e5c5c40905688ab940f/openinference_instrumentation_langchain-0.1.50.tar.gz", hash = "sha256:9ef7ca19f72d6dd66dabcf84a0d0c5c4ec6fe659a846b6211beb486129a21e4e", size = 66966, upload-time = "2025-07-30T16:50:38.091Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/57/325bbdbdc27b47309be35cb4e0eb8980b0c1bc997194c797c3691d88ae41/openai-1.96.1-py3-none-any.whl", hash = "sha256:0afaab2019bae8e145e7a1baf6953167084f019dd15042c65edd117398c1eb1c", size = 757454, upload-time = "2025-07-15T21:39:34.517Z" }, + { url = "https://files.pythonhosted.org/packages/5a/91/4853bbe5c6df3606f3d660865ca14058cbd65971962f14581be6fcaa5242/openinference_instrumentation_langchain-0.1.50-py3-none-any.whl", hash = "sha256:5fe2f206c6ebd61a6a815042d763ac7dc2576a6033c99e6a54fe0b467192a022", size = 21312, upload-time = "2025-07-30T16:50:36.6Z" }, ] [[package]] -name = "openinference-instrumentation" -version = "0.1.35" +name = "openinference-instrumentation-litellm" +version = "0.1.25" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "openinference-instrumentation" }, { name = "openinference-semantic-conventions" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, + { name = "setuptools" }, + { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/ef/1147c17a23c8a832787fe42a485613097383ec583f3de908b59fbe3ae9e7/openinference_instrumentation-0.1.35.tar.gz", hash = "sha256:cdb302e2007a41a52e2b6a66de3a9fa8c0bc737edefd370703455b50d1ae23e9", size = 22426, upload-time = "2025-07-16T05:12:19.602Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/c3/1b980a8d7142a9087278e8dafce94c80976d5da7ec753d469dacf6ccd722/openinference_instrumentation_litellm-0.1.25.tar.gz", hash = "sha256:d289f670f38c407542220ffdb3f89759134f02926f15ed1b57e975ca12f19ad5", size = 19292, upload-time = "2025-08-18T18:13:05.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/8d/ab5fca6e442ce1a3827aecc3fc00d3773ac662a5c756e8ebc37ee94146fc/openinference_instrumentation-0.1.35-py3-none-any.whl", hash = "sha256:baf5a49a7b017ad3c3e5eb606a7abea66eec8a3a44093d590e3dcd694dd879f8", size = 28194, upload-time = "2025-07-16T05:12:18.735Z" }, + { url = "https://files.pythonhosted.org/packages/11/2a/bb35cbb951edd511fb311c4bd9c934fe07da4748018541b687c968a635b8/openinference_instrumentation_litellm-0.1.25-py3-none-any.whl", hash = "sha256:a09de0fdff8b2141fea128535be2676df452f5faa774694055723b3fbd3fca5c", size = 13137, upload-time = "2025-08-18T18:13:01.256Z" }, ] [[package]] -name = "openinference-instrumentation-langchain" -version = "0.1.46" +name = "openinference-instrumentation-llama-index" +version = "4.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/df/6622fb09eacfad140067c5c6f649c6a3d835f081b5cfba6c8fa5252ab944/openinference_instrumentation_llama_index-4.3.4.tar.gz", hash = "sha256:2e03347bf7d9d6f7ff4b62101239c7408c2a6c69065f7d00f9a3b1e2145ac626", size = 60693, upload-time = "2025-08-01T22:33:33.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/a7/e63f05f49ea66d416c770b92f0670723811c285e2f10c13ce91c0558eb4b/openinference_instrumentation_llama_index-4.3.4-py3-none-any.whl", hash = "sha256:aa94297ede190cd55f62e04dd050fbb7975fec55a63752df8f4faec6b403bac0", size = 28815, upload-time = "2025-08-01T22:33:32.285Z" }, +] + +[[package]] +name = "openinference-instrumentation-mistralai" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/93/40e5f7b951fe309893f0fc4ff615264e021a3ebfc63cbe3815fd6b0f8cb3/openinference_instrumentation_mistralai-1.3.3.tar.gz", hash = "sha256:a55472a52dfa179058b851f2e502346a3cc1acadb023dd5b66b77c542369480a", size = 21307, upload-time = "2025-04-28T23:14:06.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/f0/8e5678d39ff0cac31f2093f194a2510c8cccf443442df01bbab8042aaff9/openinference_instrumentation_mistralai-1.3.3-py3-none-any.whl", hash = "sha256:f7323f4de35dc4aacf80be40d105a2f800f3ff7f9e0404b052d51c657e0d33e0", size = 20386, upload-time = "2025-04-28T23:13:55.994Z" }, +] + +[[package]] +name = "openinference-instrumentation-openai" +version = "0.1.31" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/50/a7c250db10743b10a667d9fd70fee3ac0a56b4934a60a9d7a348f93a8b94/openinference_instrumentation_openai-0.1.31.tar.gz", hash = "sha256:9e52248bdce499fa4eff3a5567cfd55eb916a7b2f533f51b2ce459b03d905bad", size = 21450, upload-time = "2025-08-18T18:12:59.128Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/50/99b1562a6d9093d48548cf8df1820579e4f7239f8ef10f03f01af014355d/openinference_instrumentation_openai-0.1.31-py3-none-any.whl", hash = "sha256:225873a9cebf3a5d1bd258675d55caa88adb91ed057283fbbad904660aec996c", size = 28492, upload-time = "2025-08-18T18:12:57.687Z" }, +] + +[[package]] +name = "openinference-instrumentation-openai-agents" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/d1/965d8b2ef7aa782264ede8c4fc60749f4641004e44aa4e98861253fc1d1d/openinference_instrumentation_openai_agents-1.2.0.tar.gz", hash = "sha256:8c042870fbe4775f8d59d0a72f56bbb4acf555d7b15432042773271645d72e4d", size = 12414, upload-time = "2025-08-18T18:12:58.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/90/746a165fa5878beb6cf2b312c1c100c98de0b3e8dc9a800dae7b179af2b3/openinference_instrumentation_openai_agents-1.2.0-py3-none-any.whl", hash = "sha256:df4de2cb3ceb57a519d7649c5250b44b45cf563da7b758cd207ba16d313ffbeb", size = 14235, upload-time = "2025-08-18T18:12:57.274Z" }, +] + +[[package]] +name = "openinference-instrumentation-smolagents" +version = "0.1.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/3f/7acd78b90ce7a77c7beb32f4af86c1efcf669d45019160ad1ee2af3ccdff/openinference_instrumentation_smolagents-0.1.14.tar.gz", hash = "sha256:9a24a778e5f34c3837576fcf92bf98a4af34fa213249a04d6776e82c51ccdde6", size = 11107, upload-time = "2025-07-10T20:20:23.012Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/b8/43652d8452e8faaed85535b9652412e8393ba1e288a1a4cc267e6c6aff04/openinference_instrumentation_smolagents-0.1.14-py3-none-any.whl", hash = "sha256:8f818ac8df18e57588b267dc033ef76065aa7c2af930864309931b7b1f3a8966", size = 12719, upload-time = "2025-07-10T20:20:22.055Z" }, +] + +[[package]] +name = "openinference-instrumentation-vertexai" +version = "0.1.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openinference-instrumentation" }, @@ -1971,9 +3119,9 @@ dependencies = [ { name = "opentelemetry-semantic-conventions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/d9/2936c480d6f6ccbd2f045ef8a0838a663e17f1513bbf535267c0d28cfffa/openinference_instrumentation_langchain-0.1.46.tar.gz", hash = "sha256:0d3d41723bc985b3804cf916d25890f1ba5245e8e51aba7b4730556d4332de5a", size = 53166, upload-time = "2025-07-03T16:37:56.021Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/3e/a1cd041ed1237e80178c70e5d71d95417225f84065e3aae854b70173f01c/openinference_instrumentation_vertexai-0.1.11.tar.gz", hash = "sha256:600d9be988af6e1371e17187cf6aeed3ebb893d1ca0ef567cc00f67b3242aba6", size = 21061, upload-time = "2025-05-05T16:38:46.18Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/47/46b4b25febbc362b4f4d9406d66ce42421c2e0dcc4ee403d1c1771c0306e/openinference_instrumentation_langchain-0.1.46-py3-none-any.whl", hash = "sha256:e6ee9b6b26e44467bcec846a6b31ee0f0df686a7c887fc7c7f5ad2be64122033", size = 19605, upload-time = "2025-07-03T16:37:54.549Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c9/e2ea586fec1f7650d0f8c6e630417859851995ed5ba693315d1ef282de57/openinference_instrumentation_vertexai-0.1.11-py3-none-any.whl", hash = "sha256:5fceb31a89aea0d8b7c93b7b89f8095888fedf8f8944518b2d803b02f39810dc", size = 17169, upload-time = "2025-05-05T16:38:45.151Z" }, ] [[package]] @@ -1999,45 +3147,45 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/c9/4509bfca6bb43220ce7f863c9f791e0d5001c2ec2b5867d48586008b3d96/opentelemetry_api-1.35.0.tar.gz", hash = "sha256:a111b959bcfa5b4d7dffc2fbd6a241aa72dd78dd8e79b5b1662bda896c5d2ffe", size = 64778, upload-time = "2025-07-11T12:23:28.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/5a/3f8d078dbf55d18442f6a2ecedf6786d81d7245844b2b20ce2b8ad6f0307/opentelemetry_api-1.35.0-py3-none-any.whl", hash = "sha256:c4ea7e258a244858daf18474625e9cc0149b8ee354f37843415771a40c25ee06", size = 65566, upload-time = "2025-07-11T12:23:07.944Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, ] [[package]] name = "opentelemetry-exporter-otlp" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/2e/63718faa67b17f449a7fb7efdc7125a408cbe5d8c0bb35f423f2776d60b5/opentelemetry_exporter_otlp-1.35.0.tar.gz", hash = "sha256:f94feff09b3524df867c7876b79c96cef20068106cb5efe55340e8d08192c8a4", size = 6142, upload-time = "2025-07-11T12:23:30.128Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/7f/d31294ac28d567a14aefd855756bab79fed69c5a75df712f228f10c47e04/opentelemetry_exporter_otlp-1.36.0.tar.gz", hash = "sha256:72f166ea5a8923ac42889337f903e93af57db8893de200369b07401e98e4e06b", size = 6144, upload-time = "2025-07-29T15:12:07.153Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/db/2da28358d3101ca936c1643becbb4ebd69e9e48acf27f153d735a4813c6b/opentelemetry_exporter_otlp-1.35.0-py3-none-any.whl", hash = "sha256:8e6bb9025f6238db7d69bba7ee37c77e4858d0a1ff22a9e126f7c9e017e83afe", size = 7016, upload-time = "2025-07-11T12:23:10.679Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a2/8966111a285124f3d6156a663ddf2aeddd52843c1a3d6b56cbd9b6c3fd0e/opentelemetry_exporter_otlp-1.36.0-py3-none-any.whl", hash = "sha256:de93b7c45bcc78296998775d52add7c63729e83ef2cd6560730a6b336d7f6494", size = 7018, upload-time = "2025-07-29T15:11:50.498Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-proto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/d1/887f860529cba7fc3aba2f6a3597fefec010a17bd1b126810724707d9b51/opentelemetry_exporter_otlp_proto_common-1.35.0.tar.gz", hash = "sha256:6f6d8c39f629b9fa5c79ce19a2829dbd93034f8ac51243cdf40ed2196f00d7eb", size = 20299, upload-time = "2025-07-11T12:23:31.046Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/da/7747e57eb341c59886052d733072bc878424bf20f1d8cf203d508bbece5b/opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf", size = 20302, upload-time = "2025-07-29T15:12:07.71Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/2c/e31dd3c719bff87fa77391eb7f38b1430d22868c52312cba8aad60f280e5/opentelemetry_exporter_otlp_proto_common-1.35.0-py3-none-any.whl", hash = "sha256:863465de697ae81279ede660f3918680b4480ef5f69dcdac04f30722ed7b74cc", size = 18349, upload-time = "2025-07-11T12:23:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/22290dca7db78eb32e0101738366b5bbda00d0407f00feffb9bf8c3fdf87/opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840", size = 18349, upload-time = "2025-07-29T15:11:51.327Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2048,14 +3196,14 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/de/222e4f2f8cd39250991f84d76b661534aef457cafc6a3eb3fcd513627698/opentelemetry_exporter_otlp_proto_grpc-1.35.0.tar.gz", hash = "sha256:ac4c2c3aa5674642db0df0091ab43ec08bbd91a9be469c8d9b18923eb742b9cc", size = 23794, upload-time = "2025-07-11T12:23:31.662Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/6f/6c1b0bdd0446e5532294d1d41bf11fbaea39c8a2423a4cdfe4fe6b708127/opentelemetry_exporter_otlp_proto_grpc-1.36.0.tar.gz", hash = "sha256:b281afbf7036b325b3588b5b6c8bb175069e3978d1bd24071f4a59d04c1e5bbf", size = 23822, upload-time = "2025-07-29T15:12:08.292Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/a6/3f60a77279e6a3dc21fc076dcb51be159a633b0bba5cba9fb804062a9332/opentelemetry_exporter_otlp_proto_grpc-1.35.0-py3-none-any.whl", hash = "sha256:ee31203eb3e50c7967b8fa71db366cc355099aca4e3726e489b248cdb2fd5a62", size = 18846, upload-time = "2025-07-11T12:23:12.957Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/5f6bd188d66d0fd8e81e681bbf5822e53eb150034e2611dd2b935d3ab61a/opentelemetry_exporter_otlp_proto_grpc-1.36.0-py3-none-any.whl", hash = "sha256:734e841fc6a5d6f30e7be4d8053adb703c70ca80c562ae24e8083a28fadef211", size = 18828, upload-time = "2025-07-29T15:11:52.235Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2066,14 +3214,14 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/88/7f/7bdc06e84266a5b4b0fefd9790b3859804bf7682ce2daabcba2e22fdb3b2/opentelemetry_exporter_otlp_proto_http-1.35.0.tar.gz", hash = "sha256:cf940147f91b450ef5f66e9980d40eb187582eed399fa851f4a7a45bb880de79", size = 15908, upload-time = "2025-07-11T12:23:32.335Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/85/6632e7e5700ba1ce5b8a065315f92c1e6d787ccc4fb2bdab15139eaefc82/opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e", size = 16213, upload-time = "2025-07-29T15:12:08.932Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/71/f118cd90dc26797077931dd598bde5e0cc652519db166593f962f8fcd022/opentelemetry_exporter_otlp_proto_http-1.35.0-py3-none-any.whl", hash = "sha256:9a001e3df3c7f160fb31056a28ed7faa2de7df68877ae909516102ae36a54e1d", size = 18589, upload-time = "2025-07-11T12:23:13.906Z" }, + { url = "https://files.pythonhosted.org/packages/7f/41/a680d38b34f8f5ddbd78ed9f0042e1cc712d58ec7531924d71cb1e6c629d/opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902", size = 18752, upload-time = "2025-07-29T15:11:53.164Z" }, ] [[package]] name = "opentelemetry-instrumentation" -version = "0.56b0" +version = "0.57b0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -2081,48 +3229,48 @@ dependencies = [ { name = "packaging" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/14/964e90f524655aed5c699190dad8dd9a05ed0f5fa334b4b33532237c2b51/opentelemetry_instrumentation-0.56b0.tar.gz", hash = "sha256:d2dbb3021188ca0ec8c5606349ee9a2919239627e8341d4d37f1d21ec3291d11", size = 28551, upload-time = "2025-07-11T12:26:19.305Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/37/cf17cf28f945a3aca5a038cfbb45ee01317d4f7f3a0e5209920883fe9b08/opentelemetry_instrumentation-0.57b0.tar.gz", hash = "sha256:f2a30135ba77cdea2b0e1df272f4163c154e978f57214795d72f40befd4fcf05", size = 30807, upload-time = "2025-07-29T15:42:44.746Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/aa/2328f27200b8e51640d4d7ff5343ba6a81ab7d2650a9f574db016aae4adf/opentelemetry_instrumentation-0.56b0-py3-none-any.whl", hash = "sha256:948967f7c8f5bdc6e43512ba74c9ae14acb48eb72a35b61afe8db9909f743be3", size = 31105, upload-time = "2025-07-11T12:25:22.788Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6f/f20cd1542959f43fb26a5bf9bb18cd81a1ea0700e8870c8f369bd07f5c65/opentelemetry_instrumentation-0.57b0-py3-none-any.whl", hash = "sha256:9109280f44882e07cec2850db28210b90600ae9110b42824d196de357cbddf7e", size = 32460, upload-time = "2025-07-29T15:41:40.883Z" }, ] [[package]] name = "opentelemetry-proto" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/a2/7366e32d9a2bccbb8614942dbea2cf93c209610385ea966cb050334f8df7/opentelemetry_proto-1.35.0.tar.gz", hash = "sha256:532497341bd3e1c074def7c5b00172601b28bb83b48afc41a4b779f26eb4ee05", size = 46151, upload-time = "2025-07-11T12:23:38.797Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/a7/3f05de580da7e8a8b8dff041d3d07a20bf3bb62d3bcc027f8fd669a73ff4/opentelemetry_proto-1.35.0-py3-none-any.whl", hash = "sha256:98fffa803164499f562718384e703be8d7dfbe680192279a0429cb150a2f8809", size = 72536, upload-time = "2025-07-11T12:23:23.247Z" }, + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, ] [[package]] name = "opentelemetry-sdk" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/cf/1eb2ed2ce55e0a9aa95b3007f26f55c7943aeef0a783bb006bdd92b3299e/opentelemetry_sdk-1.35.0.tar.gz", hash = "sha256:2a400b415ab68aaa6f04e8a6a9f6552908fb3090ae2ff78d6ae0c597ac581954", size = 160871, upload-time = "2025-07-11T12:23:39.566Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/4f/8e32b757ef3b660511b638ab52d1ed9259b666bdeeceba51a082ce3aea95/opentelemetry_sdk-1.35.0-py3-none-any.whl", hash = "sha256:223d9e5f5678518f4842311bb73966e0b6db5d1e0b74e35074c052cd2487f800", size = 119379, upload-time = "2025-07-11T12:23:24.521Z" }, + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.56b0" +version = "0.57b0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/8e/214fa817f63b9f068519463d8ab46afd5d03b98930c39394a37ae3e741d0/opentelemetry_semantic_conventions-0.56b0.tar.gz", hash = "sha256:c114c2eacc8ff6d3908cb328c811eaf64e6d68623840be9224dc829c4fd6c2ea", size = 124221, upload-time = "2025-07-11T12:23:40.71Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/3f/e80c1b017066a9d999efffe88d1cce66116dcf5cb7f80c41040a83b6e03b/opentelemetry_semantic_conventions-0.56b0-py3-none-any.whl", hash = "sha256:df44492868fd6b482511cc43a942e7194be64e94945f572db24df2e279a001a2", size = 201625, upload-time = "2025-07-11T12:23:25.63Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, ] [[package]] @@ -2136,40 +3284,40 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/87/03ababa86d984952304ac8ce9fbd3a317afb4a225b9a81f9b606ac60c873/orjson-3.11.0.tar.gz", hash = "sha256:2e4c129da624f291bcc607016a99e7f04a353f6874f3bd8d9b47b88597d5f700", size = 5318246, upload-time = "2025-07-15T16:08:29.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/2c/0b71a763f0f5130aa2631ef79e2cd84d361294665acccbb12b7a9813194e/orjson-3.11.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1785df7ada75c18411ff7e20ac822af904a40161ea9dfe8c55b3f6b66939add6", size = 240007, upload-time = "2025-07-15T16:06:45.411Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5a/f79ccd63d378b9c7c771d7a54c203d261b4c618fe3034ae95cd30f934f34/orjson-3.11.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a57899bebbcea146616a2426d20b51b3562b4bc9f8039a3bd14fae361c23053d", size = 129320, upload-time = "2025-07-15T16:06:47.249Z" }, - { url = "https://files.pythonhosted.org/packages/7b/8a/63dafc147fa5ba945ad809c374b8f4ee692bb6b18aa6e161c3e6b69b594e/orjson-3.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b6fbc2fc825aff1456dd358c11a0ad7912a4cb4537d3db92e5334af7463a967", size = 132254, upload-time = "2025-07-15T16:06:48.597Z" }, - { url = "https://files.pythonhosted.org/packages/3c/11/4d1eb230483cc689a2f039c531bb2c980029c40ca5a9b5f64dce9786e955/orjson-3.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4305a638f4cf9bed3746ca3b7c242f14e05177d5baec2527026e0f9ee6c24fb7", size = 127003, upload-time = "2025-07-15T16:06:50.34Z" }, - { url = "https://files.pythonhosted.org/packages/4f/39/b6e96072946d908684e0f4b3de1639062fd5b32016b2929c035bd8e5c847/orjson-3.11.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1235fe7bbc37164f69302199d46f29cfb874018738714dccc5a5a44042c79c77", size = 128674, upload-time = "2025-07-15T16:06:51.659Z" }, - { url = "https://files.pythonhosted.org/packages/1e/dd/c77e3013f35b202ec2cc1f78a95fadf86b8c5a320d56eb1a0bbb965a87bb/orjson-3.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a640e3954e7b4fcb160097551e54cafbde9966be3991932155b71071077881aa", size = 131846, upload-time = "2025-07-15T16:06:53.359Z" }, - { url = "https://files.pythonhosted.org/packages/3f/7d/d83f0f96c2b142f9cdcf12df19052ea3767970989dc757598dc108db208f/orjson-3.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6d750b97d22d5566955e50b02c622f3a1d32744d7a578c878b29a873190ccb7a", size = 134016, upload-time = "2025-07-15T16:06:54.691Z" }, - { url = "https://files.pythonhosted.org/packages/67/4f/d22f79a3c56dde563c4fbc12eebf9224a1b87af5e4ec61beb11f9b3eb499/orjson-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bfcfe498484161e011f8190a400591c52b026de96b3b3cbd3f21e8999b9dc0e", size = 127930, upload-time = "2025-07-15T16:06:56.001Z" }, - { url = "https://files.pythonhosted.org/packages/07/1e/26aede257db2163d974139fd4571f1e80f565216ccbd2c44ee1d43a63dcc/orjson-3.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:feaed3ed43a1d2df75c039798eb5ec92c350c7d86be53369bafc4f3700ce7df2", size = 130569, upload-time = "2025-07-15T16:06:57.275Z" }, - { url = "https://files.pythonhosted.org/packages/b4/bf/2cb57eac8d6054b555cba27203490489a7d3f5dca8c34382f22f2f0f17ba/orjson-3.11.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1120607ec8fc98acf8c54aac6fb0b7b003ba883401fa2d261833111e2fa071", size = 403844, upload-time = "2025-07-15T16:06:59.107Z" }, - { url = "https://files.pythonhosted.org/packages/76/34/36e859ccfc45464df7b35c438c0ecc7751c930b3ebbefb50db7e3a641eb7/orjson-3.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c4b48d9775b0cf1f0aca734f4c6b272cbfacfac38e6a455e6520662f9434afb7", size = 144613, upload-time = "2025-07-15T16:07:00.48Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/5aeb84cdd0b44dc3972668944a1312f7983c2a45fb6b0e5e32b2f9408540/orjson-3.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f018ed1986d79434ac712ff19f951cd00b4dfcb767444410fbb834ebec160abf", size = 132419, upload-time = "2025-07-15T16:07:01.927Z" }, - { url = "https://files.pythonhosted.org/packages/59/0c/95ee1e61a067ad24c4921609156b3beeca8b102f6f36dca62b08e1a7c7a8/orjson-3.11.0-cp311-cp311-win32.whl", hash = "sha256:08e191f8a55ac2c00be48e98a5d10dca004cbe8abe73392c55951bfda60fc123", size = 134620, upload-time = "2025-07-15T16:07:03.304Z" }, - { url = "https://files.pythonhosted.org/packages/94/3e/afd5e284db9387023803553061ea05c785c36fe7845e4fe25912424b343f/orjson-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:b5a4214ea59c8a3b56f8d484b28114af74e9fba0956f9be5c3ce388ae143bf1f", size = 129333, upload-time = "2025-07-15T16:07:04.973Z" }, - { url = "https://files.pythonhosted.org/packages/8b/a4/d29e9995d73f23f2444b4db299a99477a4f7e6f5bf8923b775ef43a4e660/orjson-3.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:57e8e7198a679ab21241ab3f355a7990c7447559e35940595e628c107ef23736", size = 126656, upload-time = "2025-07-15T16:07:06.288Z" }, - { url = "https://files.pythonhosted.org/packages/92/c9/241e304fb1e58ea70b720f1a9e5349c6bb7735ffac401ef1b94f422edd6d/orjson-3.11.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b4089f940c638bb1947d54e46c1cd58f4259072fcc97bc833ea9c78903150ac9", size = 240269, upload-time = "2025-07-15T16:07:08.173Z" }, - { url = "https://files.pythonhosted.org/packages/26/7c/289457cdf40be992b43f1d90ae213ebc03a31a8e2850271ecd79e79a3135/orjson-3.11.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:8335a0ba1c26359fb5c82d643b4c1abbee2bc62875e0f2b5bde6c8e9e25eb68c", size = 129276, upload-time = "2025-07-15T16:07:10.128Z" }, - { url = "https://files.pythonhosted.org/packages/66/de/5c0528d46ded965939b6b7f75b1fe93af42b9906b0039096fc92c9001c12/orjson-3.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63c1c9772dafc811d16d6a7efa3369a739da15d1720d6e58ebe7562f54d6f4a2", size = 131966, upload-time = "2025-07-15T16:07:11.509Z" }, - { url = "https://files.pythonhosted.org/packages/ad/74/39822f267b5935fb6fc961ccc443f4968a74d34fc9270b83caa44e37d907/orjson-3.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9457ccbd8b241fb4ba516417a4c5b95ba0059df4ac801309bcb4ec3870f45ad9", size = 127028, upload-time = "2025-07-15T16:07:13.023Z" }, - { url = "https://files.pythonhosted.org/packages/7c/e3/28f6ed7f03db69bddb3ef48621b2b05b394125188f5909ee0a43fcf4820e/orjson-3.11.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0846e13abe79daece94a00b92574f294acad1d362be766c04245b9b4dd0e47e1", size = 129105, upload-time = "2025-07-15T16:07:14.367Z" }, - { url = "https://files.pythonhosted.org/packages/cb/50/8867fd2fc92c0ab1c3e14673ec5d9d0191202e4ab8ba6256d7a1d6943ad3/orjson-3.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5587c85ae02f608a3f377b6af9eb04829606f518257cbffa8f5081c1aacf2e2f", size = 131902, upload-time = "2025-07-15T16:07:16.176Z" }, - { url = "https://files.pythonhosted.org/packages/13/65/c189deea10342afee08006331082ff67d11b98c2394989998b3ea060354a/orjson-3.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7a1964a71c1567b4570c932a0084ac24ad52c8cf6253d1881400936565ed438", size = 134042, upload-time = "2025-07-15T16:07:17.937Z" }, - { url = "https://files.pythonhosted.org/packages/2b/e4/cf23c3f4231d2a9a043940ab045f799f84a6df1b4fb6c9b4412cdc3ebf8c/orjson-3.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5a8243e73690cc6e9151c9e1dd046a8f21778d775f7d478fa1eb4daa4897c61", size = 128260, upload-time = "2025-07-15T16:07:19.651Z" }, - { url = "https://files.pythonhosted.org/packages/de/b9/2cb94d3a67edb918d19bad4a831af99cd96c3657a23daa239611bcf335d7/orjson-3.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51646f6d995df37b6e1b628f092f41c0feccf1d47e3452c6e95e2474b547d842", size = 130282, upload-time = "2025-07-15T16:07:21.022Z" }, - { url = "https://files.pythonhosted.org/packages/0b/96/df963cc973e689d4c56398647917b4ee95f47e5b6d2779338c09c015b23b/orjson-3.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:2fb8ca8f0b4e31b8aaec674c7540649b64ef02809410506a44dc68d31bd5647b", size = 403765, upload-time = "2025-07-15T16:07:25.469Z" }, - { url = "https://files.pythonhosted.org/packages/fb/92/71429ee1badb69f53281602dbb270fa84fc2e51c83193a814d0208bb63b0/orjson-3.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:64a6a3e94a44856c3f6557e6aa56a6686544fed9816ae0afa8df9077f5759791", size = 144779, upload-time = "2025-07-15T16:07:27.339Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ab/3678b2e5ff0c622a974cb8664ed7cdda5ed26ae2b9d71ba66ec36f32d6cf/orjson-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69f95d484938d8fab5963e09131bcf9fbbb81fa4ec132e316eb2fb9adb8ce78", size = 132797, upload-time = "2025-07-15T16:07:28.717Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/74509f715ff189d2aca90ebb0bd5af6658e0f9aa2512abbe6feca4c78208/orjson-3.11.0-cp312-cp312-win32.whl", hash = "sha256:8514f9f9c667ce7d7ef709ab1a73e7fcab78c297270e90b1963df7126d2b0e23", size = 134695, upload-time = "2025-07-15T16:07:30.034Z" }, - { url = "https://files.pythonhosted.org/packages/82/ba/ef25e3e223f452a01eac6a5b38d05c152d037508dcbf87ad2858cbb7d82e/orjson-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:41b38a894520b8cb5344a35ffafdf6ae8042f56d16771b2c5eb107798cee85ee", size = 129446, upload-time = "2025-07-15T16:07:31.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cd/6f4d93867c5d81bb4ab2d4ac870d3d6e9ba34fa580a03b8d04bf1ce1d8ad/orjson-3.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:5579acd235dd134467340b2f8a670c1c36023b5a69c6a3174c4792af7502bd92", size = 126400, upload-time = "2025-07-15T16:07:34.143Z" }, +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/1d/5e0ae38788bdf0721326695e65fdf41405ed535f633eb0df0f06f57552fa/orjson-3.11.2.tar.gz", hash = "sha256:91bdcf5e69a8fd8e8bdb3de32b31ff01d2bd60c1e8d5fe7d5afabdcf19920309", size = 5470739, upload-time = "2025-08-12T15:12:28.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/7d/e295df1ac9920cbb19fb4c1afa800e86f175cb657143aa422337270a4782/orjson-3.11.2-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:888b64ef7eaeeff63f773881929434a5834a6a140a63ad45183d59287f07fc6a", size = 226502, upload-time = "2025-08-12T15:10:42.284Z" }, + { url = "https://files.pythonhosted.org/packages/65/21/ffb0f10ea04caf418fb4e7ad1fda4b9ab3179df9d7a33b69420f191aadd5/orjson-3.11.2-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:83387cc8b26c9fa0ae34d1ea8861a7ae6cff8fb3e346ab53e987d085315a728e", size = 115999, upload-time = "2025-08-12T15:10:43.738Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/8da1e252ac3353d92e6f754ee0c85027c8a2cda90b6899da2be0df3ef83d/orjson-3.11.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7e35f003692c216d7ee901b6b916b5734d6fc4180fcaa44c52081f974c08e17", size = 111563, upload-time = "2025-08-12T15:10:45.301Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/baabc32e52c570b0e4e1044b1bd2ccbec965e0de3ba2c13082255efa2006/orjson-3.11.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a0a4c29ae90b11d0c00bcc31533854d89f77bde2649ec602f512a7e16e00640", size = 116222, upload-time = "2025-08-12T15:10:46.92Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/da2ad55ad80b49b560dce894c961477d0e76811ee6e614b301de9f2f8728/orjson-3.11.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:585d712b1880f68370108bc5534a257b561672d1592fae54938738fe7f6f1e33", size = 118594, upload-time = "2025-08-12T15:10:48.488Z" }, + { url = "https://files.pythonhosted.org/packages/61/be/014f7eab51449f3c894aa9bbda2707b5340c85650cb7d0db4ec9ae280501/orjson-3.11.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d08e342a7143f8a7c11f1c4033efe81acbd3c98c68ba1b26b96080396019701f", size = 120700, upload-time = "2025-08-12T15:10:49.811Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ae/c217903a30c51341868e2d8c318c59a8413baa35af54d7845071c8ccd6fe/orjson-3.11.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c0f84fc50398773a702732c87cd622737bf11c0721e6db3041ac7802a686fb", size = 123433, upload-time = "2025-08-12T15:10:51.06Z" }, + { url = "https://files.pythonhosted.org/packages/57/c2/b3c346f78b1ff2da310dd300cb0f5d32167f872b4d3bb1ad122c889d97b0/orjson-3.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:140f84e3c8d4c142575898c91e3981000afebf0333df753a90b3435d349a5fe5", size = 121061, upload-time = "2025-08-12T15:10:52.381Z" }, + { url = "https://files.pythonhosted.org/packages/00/c8/c97798f6010327ffc75ad21dd6bca11ea2067d1910777e798c2849f1c68f/orjson-3.11.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96304a2b7235e0f3f2d9363ddccdbfb027d27338722fe469fe656832a017602e", size = 119410, upload-time = "2025-08-12T15:10:53.692Z" }, + { url = "https://files.pythonhosted.org/packages/37/fd/df720f7c0e35694617b7f95598b11a2cb0374661d8389703bea17217da53/orjson-3.11.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d7612bb227d5d9582f1f50a60bd55c64618fc22c4a32825d233a4f2771a428a", size = 392294, upload-time = "2025-08-12T15:10:55.079Z" }, + { url = "https://files.pythonhosted.org/packages/ba/52/0120d18f60ab0fe47531d520372b528a45c9a25dcab500f450374421881c/orjson-3.11.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a134587d18fe493befc2defffef2a8d27cfcada5696cb7234de54a21903ae89a", size = 134134, upload-time = "2025-08-12T15:10:56.568Z" }, + { url = "https://files.pythonhosted.org/packages/ec/10/1f967671966598366de42f07e92b0fc694ffc66eafa4b74131aeca84915f/orjson-3.11.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0b84455e60c4bc12c1e4cbaa5cfc1acdc7775a9da9cec040e17232f4b05458bd", size = 123745, upload-time = "2025-08-12T15:10:57.907Z" }, + { url = "https://files.pythonhosted.org/packages/43/eb/76081238671461cfd0f47e0c24f408ffa66184237d56ef18c33e86abb612/orjson-3.11.2-cp311-cp311-win32.whl", hash = "sha256:f0660efeac223f0731a70884e6914a5f04d613b5ae500744c43f7bf7b78f00f9", size = 124393, upload-time = "2025-08-12T15:10:59.267Z" }, + { url = "https://files.pythonhosted.org/packages/26/76/cc598c1811ba9ba935171267b02e377fc9177489efce525d478a2999d9cc/orjson-3.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:955811c8405251d9e09cbe8606ad8fdef49a451bcf5520095a5ed38c669223d8", size = 119561, upload-time = "2025-08-12T15:11:00.559Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/c48011750f0489006f7617b0a3cebc8230f36d11a34e7e9aca2085f07792/orjson-3.11.2-cp311-cp311-win_arm64.whl", hash = "sha256:2e4d423a6f838552e3a6d9ec734b729f61f88b1124fd697eab82805ea1a2a97d", size = 114186, upload-time = "2025-08-12T15:11:01.931Z" }, + { url = "https://files.pythonhosted.org/packages/40/02/46054ebe7996a8adee9640dcad7d39d76c2000dc0377efa38e55dc5cbf78/orjson-3.11.2-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:901d80d349d8452162b3aa1afb82cec5bee79a10550660bc21311cc61a4c5486", size = 226528, upload-time = "2025-08-12T15:11:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/6b6f0b4d8aea1137436546b990f71be2cd8bd870aa2f5aa14dba0fcc95dc/orjson-3.11.2-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:cf3bd3967a360e87ee14ed82cb258b7f18c710dacf3822fb0042a14313a673a1", size = 115931, upload-time = "2025-08-12T15:11:04.759Z" }, + { url = "https://files.pythonhosted.org/packages/ae/05/4205cc97c30e82a293dd0d149b1a89b138ebe76afeca66fc129fa2aa4e6a/orjson-3.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26693dde66910078229a943e80eeb99fdce6cd2c26277dc80ead9f3ab97d2131", size = 111382, upload-time = "2025-08-12T15:11:06.468Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/b8a951a93caa821f9272a7c917115d825ae2e4e8768f5ddf37968ec9de01/orjson-3.11.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad4c8acb50a28211c33fc7ef85ddf5cb18d4636a5205fd3fa2dce0411a0e30c", size = 116271, upload-time = "2025-08-12T15:11:07.845Z" }, + { url = "https://files.pythonhosted.org/packages/17/03/1006c7f8782d5327439e26d9b0ec66500ea7b679d4bbb6b891d2834ab3ee/orjson-3.11.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:994181e7f1725bb5f2d481d7d228738e0743b16bf319ca85c29369c65913df14", size = 119086, upload-time = "2025-08-12T15:11:09.329Z" }, + { url = "https://files.pythonhosted.org/packages/44/61/57d22bc31f36a93878a6f772aea76b2184102c6993dea897656a66d18c74/orjson-3.11.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dbb79a0476393c07656b69c8e763c3cc925fa8e1d9e9b7d1f626901bb5025448", size = 120724, upload-time = "2025-08-12T15:11:10.674Z" }, + { url = "https://files.pythonhosted.org/packages/78/a9/4550e96b4c490c83aea697d5347b8f7eb188152cd7b5a38001055ca5b379/orjson-3.11.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:191ed27a1dddb305083d8716af413d7219f40ec1d4c9b0e977453b4db0d6fb6c", size = 123577, upload-time = "2025-08-12T15:11:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/3a/86/09b8cb3ebd513d708ef0c92d36ac3eebda814c65c72137b0a82d6d688fc4/orjson-3.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0afb89f16f07220183fd00f5f297328ed0a68d8722ad1b0c8dcd95b12bc82804", size = 121195, upload-time = "2025-08-12T15:11:13.399Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/7b40b39ac2c1c644d4644e706d0de6c9999764341cd85f2a9393cb387661/orjson-3.11.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ab6e6b4e93b1573a026b6ec16fca9541354dd58e514b62c558b58554ae04307", size = 119234, upload-time = "2025-08-12T15:11:15.134Z" }, + { url = "https://files.pythonhosted.org/packages/40/7c/bb6e7267cd80c19023d44d8cbc4ea4ed5429fcd4a7eb9950f50305697a28/orjson-3.11.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:9cb23527efb61fb75527df55d20ee47989c4ee34e01a9c98ee9ede232abf6219", size = 392250, upload-time = "2025-08-12T15:11:16.604Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6730ace05583dbca7c1b406d59f4266e48cd0d360566e71482420fb849fc/orjson-3.11.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a4dd1268e4035af21b8a09e4adf2e61f87ee7bf63b86d7bb0a237ac03fad5b45", size = 134572, upload-time = "2025-08-12T15:11:18.205Z" }, + { url = "https://files.pythonhosted.org/packages/96/0f/7d3e03a30d5aac0432882b539a65b8c02cb6dd4221ddb893babf09c424cc/orjson-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff8b155b145eaf5a9d94d2c476fbe18d6021de93cf36c2ae2c8c5b775763f14e", size = 123869, upload-time = "2025-08-12T15:11:19.554Z" }, + { url = "https://files.pythonhosted.org/packages/45/80/1513265eba6d4a960f078f4b1d2bff94a571ab2d28c6f9835e03dfc65cc6/orjson-3.11.2-cp312-cp312-win32.whl", hash = "sha256:ae3bb10279d57872f9aba68c9931aa71ed3b295fa880f25e68da79e79453f46e", size = 124430, upload-time = "2025-08-12T15:11:20.914Z" }, + { url = "https://files.pythonhosted.org/packages/fb/61/eadf057b68a332351eeb3d89a4cc538d14f31cd8b5ec1b31a280426ccca2/orjson-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:d026e1967239ec11a2559b4146a61d13914504b396f74510a1c4d6b19dfd8732", size = 119598, upload-time = "2025-08-12T15:11:22.372Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/7f4b783402143d965ab7e9a2fc116fdb887fe53bdce7d3523271cd106098/orjson-3.11.2-cp312-cp312-win_arm64.whl", hash = "sha256:59f8d5ad08602711af9589375be98477d70e1d102645430b5a7985fdbf613b36", size = 114052, upload-time = "2025-08-12T15:11:23.762Z" }, ] [[package]] @@ -2233,6 +3381,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/a5/3a92893e7399a691bad7664d977cb5e7c81cf666c81f89ea76ba2bff483d/pandas-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ac942bfd0aca577bef61f2bc8da8147c4ef6879965ef883d8e8d5d2dc3e744b8", size = 10987601, upload-time = "2025-07-07T19:19:09.589Z" }, ] +[[package]] +name = "parso" +version = "0.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609, upload-time = "2024-04-05T09:43:55.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, +] + +[[package]] +name = "pdfminer-six" +version = "20240706" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/37/63cb918ffa21412dd5d54e32e190e69bfc340f3d6aa072ad740bec9386bb/pdfminer.six-20240706.tar.gz", hash = "sha256:c631a46d5da957a9ffe4460c5dce21e8431dabb615fee5f9f4400603a58d95a6", size = 7363505, upload-time = "2024-07-06T13:48:50.795Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/7d/44d6b90e5a293d3a975cefdc4e12a932ebba814995b2a07e37e599dd27c6/pdfminer.six-20240706-py3-none-any.whl", hash = "sha256:f4f70e74174b4b3542fcb8406a210b6e2e27cd0f0b5fd04534a8cc0d8951e38c", size = 5615414, upload-time = "2024-07-06T13:48:48.408Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + [[package]] name = "pgpy13" version = "0.6.1rc1" @@ -2246,6 +3428,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/de/8e43636f1244989fb733513fa7c1252a3f82feda4ee861ec1b6377f2e6e7/pgpy13-0.6.1rc1-py3-none-any.whl", hash = "sha256:c8d6cf3dea0cc23373ff1c4ed79a6a4bf39216bf54970bae7cca656003d1aa4f", size = 84589, upload-time = "2025-03-01T22:27:17.242Z" }, ] +[[package]] +name = "phonenumbers" +version = "9.0.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/07/f76fb718ad78d0cd0926ef750b58928e5b13f0f94cc6c516d310f11228fb/phonenumbers-9.0.12.tar.gz", hash = "sha256:ccadff6b949494bd606836d8c9678bee5b55cb1cbad1e98bf7adae108e6fd0be", size = 2297813, upload-time = "2025-08-15T11:29:03.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/f1/e94b3504078d67d69d024ffb1798c31df9ddfb09ec2ec1680f3e7776414a/phonenumbers-9.0.12-py2.py3-none-any.whl", hash = "sha256:900633afc3e12191458d710262df5efc117838bd1e2e613b64fa254a86bb20a1", size = 2583620, upload-time = "2025-08-15T11:28:59.161Z" }, +] + [[package]] name = "pillow" version = "11.3.0" @@ -2285,11 +3476,20 @@ wheels = [ [[package]] name = "pip" -version = "25.1.1" +version = "25.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/59/de/241caa0ca606f2ec5fe0c1f4261b0465df78d786a38da693864a116c37f4/pip-25.1.1.tar.gz", hash = "sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077", size = 1940155, upload-time = "2025-05-02T15:14:02.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/16/650289cd3f43d5a2fadfd98c68bd1e1e7f2550a1a5326768cddfbcedb2c5/pip-25.2.tar.gz", hash = "sha256:578283f006390f85bb6282dffb876454593d637f5d1be494b5202ce4877e71f2", size = 1840021, upload-time = "2025-07-30T21:50:15.401Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/a2/d40fb2460e883eca5199c62cfc2463fd261f760556ae6290f88488c362c0/pip-25.1.1-py3-none-any.whl", hash = "sha256:2913a38a2abf4ea6b64ab507bd9e967f3b53dc1ede74b01b0931e1ce548751af", size = 1825227, upload-time = "2025-05-02T15:13:59.102Z" }, + { url = "https://files.pythonhosted.org/packages/b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa/pip-25.2-py3-none-any.whl", hash = "sha256:6d67a2b4e7f14d8b31b8b52648866fa717f45a1eb70e83002f4331d07e953717", size = 1752557, upload-time = "2025-07-30T21:50:13.323Z" }, +] + +[[package]] +name = "pkce" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/ea/ddd845c2ec21bf1e8555c782b32dc39b82f0b12764feb9f73ccbb2470f13/pkce-1.0.3.tar.gz", hash = "sha256:9775fd76d8a743d39b87df38af1cd04a58c9b5a5242d5a6350ef343d06814ab6", size = 2757, upload-time = "2021-02-08T18:29:07.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/51/52c22ec0812d25f5bf297a01153604bfa7bfa59ed66f6cd8345beb3c2b2a/pkce-1.0.3-py3-none-any.whl", hash = "sha256:55927e24c7d403b2491ebe182b95d9dcb1807643243d47e3879fbda5aad4471d", size = 3200, upload-time = "2021-02-08T18:29:05.678Z" }, ] [[package]] @@ -2328,6 +3528,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] +[[package]] +name = "polyfile-weave" +version = "0.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "abnf" }, + { name = "chardet" }, + { name = "cint" }, + { name = "fickling" }, + { name = "graphviz" }, + { name = "intervaltree" }, + { name = "jinja2" }, + { name = "kaitaistruct" }, + { name = "networkx" }, + { name = "pdfminer-six" }, + { name = "pillow" }, + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/11/7e0b3908a4f5436197b1fc11713c628cd7f9136dc7c1fb00ac8879991f87/polyfile_weave-0.5.6.tar.gz", hash = "sha256:a9fc41b456272c95a3788a2cab791e052acc24890c512fc5a6f9f4e221d24ed1", size = 5987173, upload-time = "2025-07-28T20:26:32.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/63/04c5c7c2093cf69c9eeea338f4757522a5d048703a35b3ac8a5580ed2369/polyfile_weave-0.5.6-py3-none-any.whl", hash = "sha256:658e5b6ed040a973279a0cd7f54f4566249c85b977dee556788fa6f903c1d30b", size = 1655007, upload-time = "2025-07-28T20:26:30.132Z" }, +] + [[package]] name = "prefixspan" version = "0.5.2" @@ -2338,6 +3563,71 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/c5/b0/e66e9f6e07a0b37aa0f5703c46f54bafbdf65dfba63994247676b19076c4/prefixspan-0.5.2.tar.gz", hash = "sha256:8fbd2f94b3a7f4399d04f9bd6aa214b830fb7828799c472cd43dda10b03f671c", size = 10404, upload-time = "2018-09-29T06:12:27.708Z" } +[[package]] +name = "preshed" +version = "3.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cymem" }, + { name = "murmurhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/3a/db814f67a05b6d7f9c15d38edef5ec9b21415710705b393883de92aee5ef/preshed-3.0.10.tar.gz", hash = "sha256:5a5c8e685e941f4ffec97f1fbf32694b8107858891a4bc34107fac981d8296ff", size = 15039, upload-time = "2025-05-26T15:18:33.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/99/c3709638f687da339504d1daeca48604cadb338bf3556a1484d1f0cd95e6/preshed-3.0.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d96c4fe2b41c1cdcc8c4fc1fdb10f922a6095c0430a3ebe361fe62c78902d068", size = 131486, upload-time = "2025-05-26T15:17:52.231Z" }, + { url = "https://files.pythonhosted.org/packages/e0/27/0fd36b63caa8bbf57b31a121d9565d385bbd7521771d4eb93e17d326873d/preshed-3.0.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb01ea930b96f3301526a2ab26f41347d07555e4378c4144c6b7645074f2ebb0", size = 127938, upload-time = "2025-05-26T15:17:54.19Z" }, + { url = "https://files.pythonhosted.org/packages/90/54/6a876d9cc8d401a9c1fb6bb8ca5a31b3664d0bcb888a9016258a1ae17344/preshed-3.0.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dd1f0a7b7d150e229d073fd4fe94f72610cae992e907cee74687c4695873a98", size = 842263, upload-time = "2025-05-26T15:17:55.398Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7d/ff19f74d15ee587905bafa3582883cfe2f72b574e6d691ee64dc690dc276/preshed-3.0.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fd7b350c280137f324cd447afbf6ba9a849af0e8898850046ac6f34010e08bd", size = 842913, upload-time = "2025-05-26T15:17:56.687Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3a/1c345a26463345557705b61965e1e0a732cc0e9c6dfd4787845dbfa50b4a/preshed-3.0.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cf6a5fdc89ad06079aa6ee63621e417d4f4cf2a3d8b63c72728baad35a9ff641", size = 820548, upload-time = "2025-05-26T15:17:58.057Z" }, + { url = "https://files.pythonhosted.org/packages/7f/6b/71f25e2b7a23dba168f43edfae0bb508552dbef89114ce65c73f2ea7172f/preshed-3.0.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b4c29a7bd66985808ad181c9ad05205a6aa7400cd0f98426acd7bc86588b93f8", size = 840379, upload-time = "2025-05-26T15:17:59.565Z" }, + { url = "https://files.pythonhosted.org/packages/3a/86/d8f32b0b31a36ee8770a9b1a95321430e364cd0ba4bfebb7348aed2f198d/preshed-3.0.10-cp311-cp311-win_amd64.whl", hash = "sha256:1367c1fd6f44296305315d4e1c3fe3171787d4d01c1008a76bc9466bd79c3249", size = 117655, upload-time = "2025-05-26T15:18:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/c3/14/322a4f58bc25991a87f216acb1351800739b0794185d27508ee86c35f382/preshed-3.0.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6e9c46933d55c8898c8f7a6019a8062cd87ef257b075ada2dd5d1e57810189ea", size = 131367, upload-time = "2025-05-26T15:18:02.408Z" }, + { url = "https://files.pythonhosted.org/packages/38/80/67507653c35620cace913f617df6d6f658b87e8da83087b851557d65dd86/preshed-3.0.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c4ebc4f8ef0114d55f2ffdce4965378129c7453d0203664aeeb03055572d9e4", size = 126535, upload-time = "2025-05-26T15:18:03.589Z" }, + { url = "https://files.pythonhosted.org/packages/db/b1/ab4f811aeaf20af0fa47148c1c54b62d7e8120d59025bd0a3f773bb67725/preshed-3.0.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ab5ab4c6dfd3746fb4328e7fbeb2a0544416b872db02903bfac18e6f5cd412f", size = 864907, upload-time = "2025-05-26T15:18:04.794Z" }, + { url = "https://files.pythonhosted.org/packages/fb/db/fe37c1f99cfb26805dd89381ddd54901307feceb267332eaaca228e9f9c1/preshed-3.0.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40586fd96ae3974c552a7cd78781b6844ecb1559ee7556586f487058cf13dd96", size = 869329, upload-time = "2025-05-26T15:18:06.353Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fd/efb6a6233d1cd969966f3f65bdd8e662579c3d83114e5c356cec1927b1f7/preshed-3.0.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a606c24cda931306b98e0edfafed3309bffcf8d6ecfe07804db26024c4f03cd6", size = 846829, upload-time = "2025-05-26T15:18:07.716Z" }, + { url = "https://files.pythonhosted.org/packages/14/49/0e4ce5db3bf86b081abb08a404fb37b7c2dbfd7a73ec6c0bc71b650307eb/preshed-3.0.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:394015566f9354738be903447039e8dbc6d93ba5adf091af694eb03c4e726b1e", size = 874008, upload-time = "2025-05-26T15:18:09.364Z" }, + { url = "https://files.pythonhosted.org/packages/6f/17/76d6593fc2d055d4e413b68a8c87b70aa9b7697d4972cb8062559edcf6e9/preshed-3.0.10-cp312-cp312-win_amd64.whl", hash = "sha256:fd7e38225937e580420c84d1996dde9b4f726aacd9405093455c3a2fa60fede5", size = 116701, upload-time = "2025-05-26T15:18:11.905Z" }, +] + +[[package]] +name = "presidio-analyzer" +version = "2.2.359" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "phonenumbers" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "spacy" }, + { name = "tldextract" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/09/276238e475262a184e4bb9e0c37394442b19d486a2ecd3664d51a91a487b/presidio_analyzer-2.2.359-py3-none-any.whl", hash = "sha256:5f9a71ce5e484b1d9fd10a3f40ba37cb311deeb7cc25c3a87c0ba36b468ee26d", size = 116505, upload-time = "2025-07-13T07:22:35.248Z" }, +] + +[[package]] +name = "presidio-anonymizer" +version = "2.2.357" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "pycryptodome" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/5a/74d4f11e7b111c7570235c2ce4dac923f30c7c85fcc5d8523c8c146b9cdf/presidio_anonymizer-2.2.357-py3-none-any.whl", hash = "sha256:0b3e5e0526f5950bb9b27941e5b1b01b6761295d178a8ba4cedd2771aa2aee52", size = 31209, upload-time = "2025-01-13T13:01:43.477Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.51" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/6e/9d084c929dfe9e3bfe0c6a47e31f78a25c54627d64a66e884a8bf5474f1c/prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed", size = 428940, upload-time = "2025-04-15T09:18:47.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/4f/5249960887b1fbe561d9ff265496d170b55a735b76724f10ef19f9e40716/prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07", size = 387810, upload-time = "2025-04-15T09:18:44.753Z" }, +] + [[package]] name = "propcache" version = "0.3.2" @@ -2395,43 +3685,66 @@ wheels = [ [[package]] name = "psutil" -version = "7.0.0" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/c7/8c6872f7372eb6a6b2e4708b88419fb46b857f7a2e1892966b851cc79fc9/psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2", size = 508067, upload-time = "2024-06-18T21:40:10.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0", size = 250961, upload-time = "2024-06-18T21:41:11.662Z" }, + { url = "https://files.pythonhosted.org/packages/35/56/72f86175e81c656a01c4401cd3b1c923f891b31fbcebe98985894176d7c9/psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0", size = 287478, upload-time = "2024-06-18T21:41:16.18Z" }, + { url = "https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd", size = 290455, upload-time = "2024-06-18T21:41:29.048Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132", size = 292046, upload-time = "2024-06-18T21:41:33.53Z" }, + { url = "https://files.pythonhosted.org/packages/8b/20/2ff69ad9c35c3df1858ac4e094f20bd2374d33c8643cf41da8fd7cdcb78b/psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d", size = 253560, upload-time = "2024-06-18T21:41:46.067Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/561092313ae925f3acfaace6f9ddc4f6a9c748704317bad9c8c8f8a36a79/psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3", size = 257399, upload-time = "2024-06-18T21:41:52.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0", size = 251988, upload-time = "2024-06-18T21:41:57.337Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003, upload-time = "2025-02-13T21:54:07.946Z" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051, upload-time = "2025-02-13T21:54:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535, upload-time = "2025-02-13T21:54:16.07Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004, upload-time = "2025-02-13T21:54:18.662Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986, upload-time = "2025-02-13T21:54:21.811Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544, upload-time = "2025-02-13T21:54:24.68Z" }, - { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053, upload-time = "2025-02-13T21:54:34.31Z" }, - { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, ] [[package]] name = "pyarrow" -version = "20.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/ee/a7810cb9f3d6e9238e61d312076a9859bf3668fd21c69744de9532383912/pyarrow-20.0.0.tar.gz", hash = "sha256:febc4a913592573c8d5805091a6c2b5064c8bd6e002131f01061797d91c783c1", size = 1125187, upload-time = "2025-04-27T12:34:23.264Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/a2/b7930824181ceadd0c63c1042d01fa4ef63eee233934826a7a2a9af6e463/pyarrow-20.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:24ca380585444cb2a31324c546a9a56abbe87e26069189e14bdba19c86c049f0", size = 30856035, upload-time = "2025-04-27T12:28:40.78Z" }, - { url = "https://files.pythonhosted.org/packages/9b/18/c765770227d7f5bdfa8a69f64b49194352325c66a5c3bb5e332dfd5867d9/pyarrow-20.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:95b330059ddfdc591a3225f2d272123be26c8fa76e8c9ee1a77aad507361cfdb", size = 32309552, upload-time = "2025-04-27T12:28:47.051Z" }, - { url = "https://files.pythonhosted.org/packages/44/fb/dfb2dfdd3e488bb14f822d7335653092dde150cffc2da97de6e7500681f9/pyarrow-20.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f0fb1041267e9968c6d0d2ce3ff92e3928b243e2b6d11eeb84d9ac547308232", size = 41334704, upload-time = "2025-04-27T12:28:55.064Z" }, - { url = "https://files.pythonhosted.org/packages/58/0d/08a95878d38808051a953e887332d4a76bc06c6ee04351918ee1155407eb/pyarrow-20.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8ff87cc837601532cc8242d2f7e09b4e02404de1b797aee747dd4ba4bd6313f", size = 42399836, upload-time = "2025-04-27T12:29:02.13Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/efa271234dfe38f0271561086eedcad7bc0f2ddd1efba423916ff0883684/pyarrow-20.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7a3a5dcf54286e6141d5114522cf31dd67a9e7c9133d150799f30ee302a7a1ab", size = 40711789, upload-time = "2025-04-27T12:29:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/46/1f/7f02009bc7fc8955c391defee5348f510e589a020e4b40ca05edcb847854/pyarrow-20.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a6ad3e7758ecf559900261a4df985662df54fb7fdb55e8e3b3aa99b23d526b62", size = 42301124, upload-time = "2025-04-27T12:29:17.187Z" }, - { url = "https://files.pythonhosted.org/packages/4f/92/692c562be4504c262089e86757a9048739fe1acb4024f92d39615e7bab3f/pyarrow-20.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6bb830757103a6cb300a04610e08d9636f0cd223d32f388418ea893a3e655f1c", size = 42916060, upload-time = "2025-04-27T12:29:24.253Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ec/9f5c7e7c828d8e0a3c7ef50ee62eca38a7de2fa6eb1b8fa43685c9414fef/pyarrow-20.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:96e37f0766ecb4514a899d9a3554fadda770fb57ddf42b63d80f14bc20aa7db3", size = 44547640, upload-time = "2025-04-27T12:29:32.782Z" }, - { url = "https://files.pythonhosted.org/packages/54/96/46613131b4727f10fd2ffa6d0d6f02efcc09a0e7374eff3b5771548aa95b/pyarrow-20.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3346babb516f4b6fd790da99b98bed9708e3f02e734c84971faccb20736848dc", size = 25781491, upload-time = "2025-04-27T12:29:38.464Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d6/0c10e0d54f6c13eb464ee9b67a68b8c71bcf2f67760ef5b6fbcddd2ab05f/pyarrow-20.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:75a51a5b0eef32727a247707d4755322cb970be7e935172b6a3a9f9ae98404ba", size = 30815067, upload-time = "2025-04-27T12:29:44.384Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e2/04e9874abe4094a06fd8b0cbb0f1312d8dd7d707f144c2ec1e5e8f452ffa/pyarrow-20.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:211d5e84cecc640c7a3ab900f930aaff5cd2702177e0d562d426fb7c4f737781", size = 32297128, upload-time = "2025-04-27T12:29:52.038Z" }, - { url = "https://files.pythonhosted.org/packages/31/fd/c565e5dcc906a3b471a83273039cb75cb79aad4a2d4a12f76cc5ae90a4b8/pyarrow-20.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ba3cf4182828be7a896cbd232aa8dd6a31bd1f9e32776cc3796c012855e1199", size = 41334890, upload-time = "2025-04-27T12:29:59.452Z" }, - { url = "https://files.pythonhosted.org/packages/af/a9/3bdd799e2c9b20c1ea6dc6fa8e83f29480a97711cf806e823f808c2316ac/pyarrow-20.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c3a01f313ffe27ac4126f4c2e5ea0f36a5fc6ab51f8726cf41fee4b256680bd", size = 42421775, upload-time = "2025-04-27T12:30:06.875Z" }, - { url = "https://files.pythonhosted.org/packages/10/f7/da98ccd86354c332f593218101ae56568d5dcedb460e342000bd89c49cc1/pyarrow-20.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:a2791f69ad72addd33510fec7bb14ee06c2a448e06b649e264c094c5b5f7ce28", size = 40687231, upload-time = "2025-04-27T12:30:13.954Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1b/2168d6050e52ff1e6cefc61d600723870bf569cbf41d13db939c8cf97a16/pyarrow-20.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4250e28a22302ce8692d3a0e8ec9d9dde54ec00d237cff4dfa9c1fbf79e472a8", size = 42295639, upload-time = "2025-04-27T12:30:21.949Z" }, - { url = "https://files.pythonhosted.org/packages/b2/66/2d976c0c7158fd25591c8ca55aee026e6d5745a021915a1835578707feb3/pyarrow-20.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:89e030dc58fc760e4010148e6ff164d2f44441490280ef1e97a542375e41058e", size = 42908549, upload-time = "2025-04-27T12:30:29.551Z" }, - { url = "https://files.pythonhosted.org/packages/31/a9/dfb999c2fc6911201dcbf348247f9cc382a8990f9ab45c12eabfd7243a38/pyarrow-20.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6102b4864d77102dbbb72965618e204e550135a940c2534711d5ffa787df2a5a", size = 44557216, upload-time = "2025-04-27T12:30:36.977Z" }, - { url = "https://files.pythonhosted.org/packages/a0/8e/9adee63dfa3911be2382fb4d92e4b2e7d82610f9d9f668493bebaa2af50f/pyarrow-20.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:96d6a0a37d9c98be08f5ed6a10831d88d52cac7b13f5287f1e0f625a0de8062b", size = 25660496, upload-time = "2025-04-27T12:30:42.809Z" }, +version = "21.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" }, + { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" }, + { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" }, + { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" }, ] [[package]] @@ -2443,6 +3756,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, ] +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycodestyle" version = "2.14.0" @@ -2458,7 +3783,26 @@ version = "2.22" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, ] [[package]] @@ -2564,7 +3908,7 @@ wheels = [ [[package]] name = "pylint" -version = "3.3.7" +version = "3.3.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "astroid" }, @@ -2575,14 +3919,14 @@ dependencies = [ { name = "platformdirs" }, { name = "tomlkit" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/e4/83e487d3ddd64ab27749b66137b26dc0c5b5c161be680e6beffdc99070b3/pylint-3.3.7.tar.gz", hash = "sha256:2b11de8bde49f9c5059452e0c310c079c746a0a8eeaa789e5aa966ecc23e4559", size = 1520709, upload-time = "2025-05-04T17:07:51.089Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/58/1f614a84d3295c542e9f6e2c764533eea3f318f4592dc1ea06c797114767/pylint-3.3.8.tar.gz", hash = "sha256:26698de19941363037e2937d3db9ed94fb3303fdadf7d98847875345a8bb6b05", size = 1523947, upload-time = "2025-08-09T09:12:57.234Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/83/bff755d09e31b5d25cc7fdc4bf3915d1a404e181f1abf0359af376845c24/pylint-3.3.7-py3-none-any.whl", hash = "sha256:43860aafefce92fca4cf6b61fe199cdc5ae54ea28f9bf4cd49de267b5195803d", size = 522565, upload-time = "2025-05-04T17:07:48.714Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1a/711e93a7ab6c392e349428ea56e794a3902bb4e0284c1997cff2d7efdbc1/pylint-3.3.8-py3-none-any.whl", hash = "sha256:7ef94aa692a600e82fabdd17102b73fc226758218c97473c7ad67bd4cb905d83", size = 523153, upload-time = "2025-08-09T09:12:54.836Z" }, ] [[package]] name = "pymilvus" -version = "2.6.0b0" +version = "2.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio" }, @@ -2593,9 +3937,22 @@ dependencies = [ { name = "setuptools" }, { name = "ujson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/fb/5e40ef2c4f5736c8c59084289d065df69510d549761941f7599ab80c1c1e/pymilvus-2.6.0b0.tar.gz", hash = "sha256:af907518417c3698112fdb6fe40db70798457d8c5c1961e56c64bf8fb126a721", size = 1281149, upload-time = "2025-06-18T16:11:15.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/21/5c25a975299415a5a8f26d4759ddf7852aefdf3595f002b5203c4aaf5c8e/pymilvus-2.6.0.tar.gz", hash = "sha256:2b2ca487e098abc34231755e33af2f5294e9f6a64d92d03551532defbac0a3fb", size = 1292994, upload-time = "2025-08-06T09:09:01.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/a2/dfc2a2225aeb90a7dff9443f2d26fe9d04f6f7bcefe537945b5d5220fddd/pymilvus-2.6.0-py3-none-any.whl", hash = "sha256:d743fdd928c9007184d24a52b4f5dfdd18d405a37b4dba66b5ea4bf196fac526", size = 248299, upload-time = "2025-08-06T09:08:58.272Z" }, +] + +[[package]] +name = "pyopenssl" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/8c/cd89ad05804f8e3c17dea8f178c3f40eeab5694c30e0c9f5bcd49f576fc3/pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b", size = 179937, upload-time = "2025-05-17T16:28:31.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/fb/7cf9aaf57c0866ea0df851f052366d4a9b65794dda71f6f140168035c219/pymilvus-2.6.0b0-py3-none-any.whl", hash = "sha256:0a585e38b6492aa298f5a8cb4d59cc332a3fead4068cfe49ffe2998029ae3997", size = 240167, upload-time = "2025-06-18T16:11:11.404Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2659c02301b9500751f8d42f9a6632e1508aa5120de5e43042b8b30f8d5d/pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", size = 56771, upload-time = "2025-05-17T16:28:29.197Z" }, ] [[package]] @@ -2607,6 +3964,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, ] +[[package]] +name = "pypdf" +version = "5.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/3a/584b97a228950ed85aec97c811c68473d9b8d149e6a8c155668287cf1a28/pypdf-5.9.0.tar.gz", hash = "sha256:30f67a614d558e495e1fbb157ba58c1de91ffc1718f5e0dfeb82a029233890a1", size = 5035118, upload-time = "2025-07-27T14:04:52.364Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/d9/6cff57c80a6963e7dd183bf09e9f21604a77716644b1e580e97b259f7612/pypdf-5.9.0-py3-none-any.whl", hash = "sha256:be10a4c54202f46d9daceaa8788be07aa8cd5ea8c25c529c50dd509206382c35", size = 313193, upload-time = "2025-07-27T14:04:50.53Z" }, +] + +[[package]] +name = "pyreadline3" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/49/4cea918a08f02817aabae639e3d0ac046fef9f9180518a3ad394e22da148/pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7", size = 99839, upload-time = "2024-09-19T02:40:10.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, +] + +[[package]] +name = "pystache" +version = "0.6.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/89/0a712ca22930b8c71bced8703e5bb45669c31690ea81afe15f6cb284550c/pystache-0.6.8.tar.gz", hash = "sha256:3707518e6a4d26dd189b07c10c669b1fc17df72684617c327bd3550e7075c72c", size = 101892, upload-time = "2025-03-18T11:54:47.595Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/78/ffd13a516219129cef6a754a11ba2a1c0d69f1e281af4f6bca9ed5327219/pystache-0.6.8-py3-none-any.whl", hash = "sha256:7211e000974a6e06bce2d4d5cad8df03bcfffefd367209117376e4527a1c3cb8", size = 82051, upload-time = "2025-03-18T11:54:45.813Z" }, +] + [[package]] name = "pytest" version = "8.4.1" @@ -2752,6 +4136,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, ] +[[package]] +name = "pyzmq" +version = "27.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/5f/557d2032a2f471edbcc227da724c24a1c05887b5cda1e3ae53af98b9e0a5/pyzmq-27.0.1.tar.gz", hash = "sha256:45c549204bc20e7484ffd2555f6cf02e572440ecf2f3bdd60d4404b20fddf64b", size = 281158, upload-time = "2025-08-03T05:05:40.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/18/a8e0da6ababbe9326116fb1c890bf1920eea880e8da621afb6bc0f39a262/pyzmq-27.0.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:9729190bd770314f5fbba42476abf6abe79a746eeda11d1d68fd56dd70e5c296", size = 1332721, upload-time = "2025-08-03T05:03:15.237Z" }, + { url = "https://files.pythonhosted.org/packages/75/a4/9431ba598651d60ebd50dc25755402b770322cf8432adcc07d2906e53a54/pyzmq-27.0.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:696900ef6bc20bef6a242973943574f96c3f97d2183c1bd3da5eea4f559631b1", size = 908249, upload-time = "2025-08-03T05:03:16.933Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/e624e1793689e4e685d2ee21c40277dd4024d9d730af20446d88f69be838/pyzmq-27.0.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96a63aecec22d3f7fdea3c6c98df9e42973f5856bb6812c3d8d78c262fee808", size = 668649, upload-time = "2025-08-03T05:03:18.49Z" }, + { url = "https://files.pythonhosted.org/packages/6c/29/0652a39d4e876e0d61379047ecf7752685414ad2e253434348246f7a2a39/pyzmq-27.0.1-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c512824360ea7490390566ce00bee880e19b526b312b25cc0bc30a0fe95cb67f", size = 856601, upload-time = "2025-08-03T05:03:20.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/2d/8d5355d7fc55bb6e9c581dd74f58b64fa78c994079e3a0ea09b1b5627cde/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dfb2bb5e0f7198eaacfb6796fb0330afd28f36d985a770745fba554a5903595a", size = 1657750, upload-time = "2025-08-03T05:03:22.055Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f4/cd032352d5d252dc6f5ee272a34b59718ba3af1639a8a4ef4654f9535cf5/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f6886c59ba93ffde09b957d3e857e7950c8fe818bd5494d9b4287bc6d5bc7f1", size = 2034312, upload-time = "2025-08-03T05:03:23.578Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1a/c050d8b6597200e97a4bd29b93c769d002fa0b03083858227e0376ad59bc/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b99ea9d330e86ce1ff7f2456b33f1bf81c43862a5590faf4ef4ed3a63504bdab", size = 1893632, upload-time = "2025-08-03T05:03:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/173ce21d5097e7fcf284a090e8beb64fc683c6582b1f00fa52b1b7e867ce/pyzmq-27.0.1-cp311-cp311-win32.whl", hash = "sha256:571f762aed89025ba8cdcbe355fea56889715ec06d0264fd8b6a3f3fa38154ed", size = 566587, upload-time = "2025-08-03T05:03:26.769Z" }, + { url = "https://files.pythonhosted.org/packages/53/ab/22bd33e7086f0a2cc03a5adabff4bde414288bb62a21a7820951ef86ec20/pyzmq-27.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee16906c8025fa464bea1e48128c048d02359fb40bebe5333103228528506530", size = 632873, upload-time = "2025-08-03T05:03:28.685Z" }, + { url = "https://files.pythonhosted.org/packages/90/14/3e59b4a28194285ceeff725eba9aa5ba8568d1cb78aed381dec1537c705a/pyzmq-27.0.1-cp311-cp311-win_arm64.whl", hash = "sha256:ba068f28028849da725ff9185c24f832ccf9207a40f9b28ac46ab7c04994bd41", size = 558918, upload-time = "2025-08-03T05:03:30.085Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9b/c0957041067c7724b310f22c398be46399297c12ed834c3bc42200a2756f/pyzmq-27.0.1-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:af7ebce2a1e7caf30c0bb64a845f63a69e76a2fadbc1cac47178f7bb6e657bdd", size = 1305432, upload-time = "2025-08-03T05:03:32.177Z" }, + { url = "https://files.pythonhosted.org/packages/8e/55/bd3a312790858f16b7def3897a0c3eb1804e974711bf7b9dcb5f47e7f82c/pyzmq-27.0.1-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8f617f60a8b609a13099b313e7e525e67f84ef4524b6acad396d9ff153f6e4cd", size = 895095, upload-time = "2025-08-03T05:03:33.918Z" }, + { url = "https://files.pythonhosted.org/packages/20/50/fc384631d8282809fb1029a4460d2fe90fa0370a0e866a8318ed75c8d3bb/pyzmq-27.0.1-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59dad4173dc2a111f03e59315c7bd6e73da1a9d20a84a25cf08325b0582b1a", size = 651826, upload-time = "2025-08-03T05:03:35.818Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0a/2356305c423a975000867de56888b79e44ec2192c690ff93c3109fd78081/pyzmq-27.0.1-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5b6133c8d313bde8bd0d123c169d22525300ff164c2189f849de495e1344577", size = 839751, upload-time = "2025-08-03T05:03:37.265Z" }, + { url = "https://files.pythonhosted.org/packages/d7/1b/81e95ad256ca7e7ccd47f5294c1c6da6e2b64fbace65b84fe8a41470342e/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:58cca552567423f04d06a075f4b473e78ab5bdb906febe56bf4797633f54aa4e", size = 1641359, upload-time = "2025-08-03T05:03:38.799Z" }, + { url = "https://files.pythonhosted.org/packages/50/63/9f50ec965285f4e92c265c8f18344e46b12803666d8b73b65d254d441435/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:4b9d8e26fb600d0d69cc9933e20af08552e97cc868a183d38a5c0d661e40dfbb", size = 2020281, upload-time = "2025-08-03T05:03:40.338Z" }, + { url = "https://files.pythonhosted.org/packages/02/4a/19e3398d0dc66ad2b463e4afa1fc541d697d7bc090305f9dfb948d3dfa29/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2329f0c87f0466dce45bba32b63f47018dda5ca40a0085cc5c8558fea7d9fc55", size = 1877112, upload-time = "2025-08-03T05:03:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/bf/42/c562e9151aa90ed1d70aac381ea22a929d6b3a2ce4e1d6e2e135d34fd9c6/pyzmq-27.0.1-cp312-abi3-win32.whl", hash = "sha256:57bb92abdb48467b89c2d21da1ab01a07d0745e536d62afd2e30d5acbd0092eb", size = 558177, upload-time = "2025-08-03T05:03:43.979Z" }, + { url = "https://files.pythonhosted.org/packages/40/96/5c50a7d2d2b05b19994bf7336b97db254299353dd9b49b565bb71b485f03/pyzmq-27.0.1-cp312-abi3-win_amd64.whl", hash = "sha256:ff3f8757570e45da7a5bedaa140489846510014f7a9d5ee9301c61f3f1b8a686", size = 618923, upload-time = "2025-08-03T05:03:45.438Z" }, + { url = "https://files.pythonhosted.org/packages/13/33/1ec89c8f21c89d21a2eaff7def3676e21d8248d2675705e72554fb5a6f3f/pyzmq-27.0.1-cp312-abi3-win_arm64.whl", hash = "sha256:df2c55c958d3766bdb3e9d858b911288acec09a9aab15883f384fc7180df5bed", size = 552358, upload-time = "2025-08-03T05:03:46.887Z" }, + { url = "https://files.pythonhosted.org/packages/b4/1a/49f66fe0bc2b2568dd4280f1f520ac8fafd73f8d762140e278d48aeaf7b9/pyzmq-27.0.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7fb0ee35845bef1e8c4a152d766242164e138c239e3182f558ae15cb4a891f94", size = 835949, upload-time = "2025-08-03T05:05:13.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/94/443c1984b397eab59b14dd7ae8bc2ac7e8f32dbc646474453afcaa6508c4/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f379f11e138dfd56c3f24a04164f871a08281194dd9ddf656a278d7d080c8ad0", size = 799875, upload-time = "2025-08-03T05:05:15.632Z" }, + { url = "https://files.pythonhosted.org/packages/30/f1/fd96138a0f152786a2ba517e9c6a8b1b3516719e412a90bb5d8eea6b660c/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b978c0678cffbe8860ec9edc91200e895c29ae1ac8a7085f947f8e8864c489fb", size = 567403, upload-time = "2025-08-03T05:05:17.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/57/34e53ef2b55b1428dac5aabe3a974a16c8bda3bf20549ba500e3ff6cb426/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ebccf0d760bc92a4a7c751aeb2fef6626144aace76ee8f5a63abeb100cae87f", size = 747032, upload-time = "2025-08-03T05:05:19.074Z" }, + { url = "https://files.pythonhosted.org/packages/81/b7/769598c5ae336fdb657946950465569cf18803140fe89ce466d7f0a57c11/pyzmq-27.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:77fed80e30fa65708546c4119840a46691290efc231f6bfb2ac2a39b52e15811", size = 544566, upload-time = "2025-08-03T05:05:20.798Z" }, +] + +[[package]] +name = "ragaai-catalyst" +version = "2.2.6b7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "google-genai" }, + { name = "gputil" }, + { name = "groq" }, + { name = "ipynbname" }, + { name = "langchain" }, + { name = "langchain-core" }, + { name = "litellm" }, + { name = "llama-index" }, + { name = "markdown" }, + { name = "openai" }, + { name = "openinference-instrumentation-anthropic" }, + { name = "openinference-instrumentation-bedrock" }, + { name = "openinference-instrumentation-crewai" }, + { name = "openinference-instrumentation-google-adk" }, + { name = "openinference-instrumentation-groq" }, + { name = "openinference-instrumentation-haystack" }, + { name = "openinference-instrumentation-langchain" }, + { name = "openinference-instrumentation-litellm" }, + { name = "openinference-instrumentation-llama-index" }, + { name = "openinference-instrumentation-mistralai" }, + { name = "openinference-instrumentation-openai" }, + { name = "openinference-instrumentation-openai-agents" }, + { name = "openinference-instrumentation-smolagents" }, + { name = "openinference-instrumentation-vertexai" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "pandas" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "pyopenssl" }, + { name = "pypdf" }, + { name = "requests" }, + { name = "rich" }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tomli" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/93/92bad2ae2a539e25976a4791d786692af89fa8274c6fafd2e2c4742ab8e1/ragaai_catalyst-2.2.6b7.tar.gz", hash = "sha256:6d28132ed83d16ff9ebdf0b7edf7ac4d941f3f8e669abc0b8b1535050580b513", size = 40941954, upload-time = "2025-07-16T07:56:31.566Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/3f/61cad60f6d617fed4c8cee67285e0173622df417d68a16883f4ebffa39df/ragaai_catalyst-2.2.6b7-py3-none-any.whl", hash = "sha256:50f6a49f5c24845e0eb5676ac2d8b6ea37d2de02d2cef0f608e6ed8606b9a3a9", size = 443209, upload-time = "2025-07-16T07:56:26.31Z" }, +] + [[package]] name = "ragas" version = "0.2.15" @@ -2803,45 +4273,43 @@ wheels = [ [[package]] name = "regex" -version = "2024.11.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494, upload-time = "2024-11-06T20:12:31.635Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/58/7e4d9493a66c88a7da6d205768119f51af0f684fe7be7bac8328e217a52c/regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638", size = 482669, upload-time = "2024-11-06T20:09:31.064Z" }, - { url = "https://files.pythonhosted.org/packages/34/4c/8f8e631fcdc2ff978609eaeef1d6994bf2f028b59d9ac67640ed051f1218/regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7", size = 287684, upload-time = "2024-11-06T20:09:32.915Z" }, - { url = "https://files.pythonhosted.org/packages/c5/1b/f0e4d13e6adf866ce9b069e191f303a30ab1277e037037a365c3aad5cc9c/regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20", size = 284589, upload-time = "2024-11-06T20:09:35.504Z" }, - { url = "https://files.pythonhosted.org/packages/25/4d/ab21047f446693887f25510887e6820b93f791992994f6498b0318904d4a/regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114", size = 792121, upload-time = "2024-11-06T20:09:37.701Z" }, - { url = "https://files.pythonhosted.org/packages/45/ee/c867e15cd894985cb32b731d89576c41a4642a57850c162490ea34b78c3b/regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3", size = 831275, upload-time = "2024-11-06T20:09:40.371Z" }, - { url = "https://files.pythonhosted.org/packages/b3/12/b0f480726cf1c60f6536fa5e1c95275a77624f3ac8fdccf79e6727499e28/regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f", size = 818257, upload-time = "2024-11-06T20:09:43.059Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ce/0d0e61429f603bac433910d99ef1a02ce45a8967ffbe3cbee48599e62d88/regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0", size = 792727, upload-time = "2024-11-06T20:09:48.19Z" }, - { url = "https://files.pythonhosted.org/packages/e4/c1/243c83c53d4a419c1556f43777ccb552bccdf79d08fda3980e4e77dd9137/regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55", size = 780667, upload-time = "2024-11-06T20:09:49.828Z" }, - { url = "https://files.pythonhosted.org/packages/c5/f4/75eb0dd4ce4b37f04928987f1d22547ddaf6c4bae697623c1b05da67a8aa/regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89", size = 776963, upload-time = "2024-11-06T20:09:51.819Z" }, - { url = "https://files.pythonhosted.org/packages/16/5d/95c568574e630e141a69ff8a254c2f188b4398e813c40d49228c9bbd9875/regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d", size = 784700, upload-time = "2024-11-06T20:09:53.982Z" }, - { url = "https://files.pythonhosted.org/packages/8e/b5/f8495c7917f15cc6fee1e7f395e324ec3e00ab3c665a7dc9d27562fd5290/regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34", size = 848592, upload-time = "2024-11-06T20:09:56.222Z" }, - { url = "https://files.pythonhosted.org/packages/1c/80/6dd7118e8cb212c3c60b191b932dc57db93fb2e36fb9e0e92f72a5909af9/regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d", size = 852929, upload-time = "2024-11-06T20:09:58.642Z" }, - { url = "https://files.pythonhosted.org/packages/11/9b/5a05d2040297d2d254baf95eeeb6df83554e5e1df03bc1a6687fc4ba1f66/regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45", size = 781213, upload-time = "2024-11-06T20:10:00.867Z" }, - { url = "https://files.pythonhosted.org/packages/26/b7/b14e2440156ab39e0177506c08c18accaf2b8932e39fb092074de733d868/regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9", size = 261734, upload-time = "2024-11-06T20:10:03.361Z" }, - { url = "https://files.pythonhosted.org/packages/80/32/763a6cc01d21fb3819227a1cc3f60fd251c13c37c27a73b8ff4315433a8e/regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60", size = 274052, upload-time = "2024-11-06T20:10:05.179Z" }, - { url = "https://files.pythonhosted.org/packages/ba/30/9a87ce8336b172cc232a0db89a3af97929d06c11ceaa19d97d84fa90a8f8/regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a", size = 483781, upload-time = "2024-11-06T20:10:07.07Z" }, - { url = "https://files.pythonhosted.org/packages/01/e8/00008ad4ff4be8b1844786ba6636035f7ef926db5686e4c0f98093612add/regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9", size = 288455, upload-time = "2024-11-06T20:10:09.117Z" }, - { url = "https://files.pythonhosted.org/packages/60/85/cebcc0aff603ea0a201667b203f13ba75d9fc8668fab917ac5b2de3967bc/regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2", size = 284759, upload-time = "2024-11-06T20:10:11.155Z" }, - { url = "https://files.pythonhosted.org/packages/94/2b/701a4b0585cb05472a4da28ee28fdfe155f3638f5e1ec92306d924e5faf0/regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4", size = 794976, upload-time = "2024-11-06T20:10:13.24Z" }, - { url = "https://files.pythonhosted.org/packages/4b/bf/fa87e563bf5fee75db8915f7352e1887b1249126a1be4813837f5dbec965/regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577", size = 833077, upload-time = "2024-11-06T20:10:15.37Z" }, - { url = "https://files.pythonhosted.org/packages/a1/56/7295e6bad94b047f4d0834e4779491b81216583c00c288252ef625c01d23/regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3", size = 823160, upload-time = "2024-11-06T20:10:19.027Z" }, - { url = "https://files.pythonhosted.org/packages/fb/13/e3b075031a738c9598c51cfbc4c7879e26729c53aa9cca59211c44235314/regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e", size = 796896, upload-time = "2024-11-06T20:10:21.85Z" }, - { url = "https://files.pythonhosted.org/packages/24/56/0b3f1b66d592be6efec23a795b37732682520b47c53da5a32c33ed7d84e3/regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe", size = 783997, upload-time = "2024-11-06T20:10:24.329Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a1/eb378dada8b91c0e4c5f08ffb56f25fcae47bf52ad18f9b2f33b83e6d498/regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e", size = 781725, upload-time = "2024-11-06T20:10:28.067Z" }, - { url = "https://files.pythonhosted.org/packages/83/f2/033e7dec0cfd6dda93390089864732a3409246ffe8b042e9554afa9bff4e/regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29", size = 789481, upload-time = "2024-11-06T20:10:31.612Z" }, - { url = "https://files.pythonhosted.org/packages/83/23/15d4552ea28990a74e7696780c438aadd73a20318c47e527b47a4a5a596d/regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39", size = 852896, upload-time = "2024-11-06T20:10:34.054Z" }, - { url = "https://files.pythonhosted.org/packages/e3/39/ed4416bc90deedbfdada2568b2cb0bc1fdb98efe11f5378d9892b2a88f8f/regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51", size = 860138, upload-time = "2024-11-06T20:10:36.142Z" }, - { url = "https://files.pythonhosted.org/packages/93/2d/dd56bb76bd8e95bbce684326302f287455b56242a4f9c61f1bc76e28360e/regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad", size = 787692, upload-time = "2024-11-06T20:10:38.394Z" }, - { url = "https://files.pythonhosted.org/packages/0b/55/31877a249ab7a5156758246b9c59539abbeba22461b7d8adc9e8475ff73e/regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54", size = 262135, upload-time = "2024-11-06T20:10:40.367Z" }, - { url = "https://files.pythonhosted.org/packages/38/ec/ad2d7de49a600cdb8dd78434a1aeffe28b9d6fc42eb36afab4a27ad23384/regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b", size = 273567, upload-time = "2024-11-06T20:10:43.467Z" }, +version = "2025.7.34" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/de/e13fa6dc61d78b30ba47481f99933a3b49a57779d625c392d8036770a60d/regex-2025.7.34.tar.gz", hash = "sha256:9ead9765217afd04a86822dfcd4ed2747dfe426e887da413b15ff0ac2457e21a", size = 400714, upload-time = "2025-07-31T00:21:16.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/85/f497b91577169472f7c1dc262a5ecc65e39e146fc3a52c571e5daaae4b7d/regex-2025.7.34-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:da304313761b8500b8e175eb2040c4394a875837d5635f6256d6fa0377ad32c8", size = 484594, upload-time = "2025-07-31T00:19:13.927Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c5/ad2a5c11ce9e6257fcbfd6cd965d07502f6054aaa19d50a3d7fd991ec5d1/regex-2025.7.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:35e43ebf5b18cd751ea81455b19acfdec402e82fe0dc6143edfae4c5c4b3909a", size = 289294, upload-time = "2025-07-31T00:19:15.395Z" }, + { url = "https://files.pythonhosted.org/packages/8e/01/83ffd9641fcf5e018f9b51aa922c3e538ac9439424fda3df540b643ecf4f/regex-2025.7.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96bbae4c616726f4661fe7bcad5952e10d25d3c51ddc388189d8864fbc1b3c68", size = 285933, upload-time = "2025-07-31T00:19:16.704Z" }, + { url = "https://files.pythonhosted.org/packages/77/20/5edab2e5766f0259bc1da7381b07ce6eb4401b17b2254d02f492cd8a81a8/regex-2025.7.34-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9feab78a1ffa4f2b1e27b1bcdaad36f48c2fed4870264ce32f52a393db093c78", size = 792335, upload-time = "2025-07-31T00:19:18.561Z" }, + { url = "https://files.pythonhosted.org/packages/30/bd/744d3ed8777dce8487b2606b94925e207e7c5931d5870f47f5b643a4580a/regex-2025.7.34-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f14b36e6d4d07f1a5060f28ef3b3561c5d95eb0651741474ce4c0a4c56ba8719", size = 858605, upload-time = "2025-07-31T00:19:20.204Z" }, + { url = "https://files.pythonhosted.org/packages/99/3d/93754176289718d7578c31d151047e7b8acc7a8c20e7706716f23c49e45e/regex-2025.7.34-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85c3a958ef8b3d5079c763477e1f09e89d13ad22198a37e9d7b26b4b17438b33", size = 905780, upload-time = "2025-07-31T00:19:21.876Z" }, + { url = "https://files.pythonhosted.org/packages/ee/2e/c689f274a92deffa03999a430505ff2aeace408fd681a90eafa92fdd6930/regex-2025.7.34-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37555e4ae0b93358fa7c2d240a4291d4a4227cc7c607d8f85596cdb08ec0a083", size = 798868, upload-time = "2025-07-31T00:19:23.222Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/39673688805d139b33b4a24851a71b9978d61915c4d72b5ffda324d0668a/regex-2025.7.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee38926f31f1aa61b0232a3a11b83461f7807661c062df9eb88769d86e6195c3", size = 781784, upload-time = "2025-07-31T00:19:24.59Z" }, + { url = "https://files.pythonhosted.org/packages/18/bd/4c1cab12cfabe14beaa076523056b8ab0c882a8feaf0a6f48b0a75dab9ed/regex-2025.7.34-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a664291c31cae9c4a30589bd8bc2ebb56ef880c9c6264cb7643633831e606a4d", size = 852837, upload-time = "2025-07-31T00:19:25.911Z" }, + { url = "https://files.pythonhosted.org/packages/cb/21/663d983cbb3bba537fc213a579abbd0f263fb28271c514123f3c547ab917/regex-2025.7.34-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f3e5c1e0925e77ec46ddc736b756a6da50d4df4ee3f69536ffb2373460e2dafd", size = 844240, upload-time = "2025-07-31T00:19:27.688Z" }, + { url = "https://files.pythonhosted.org/packages/8e/2d/9beeeb913bc5d32faa913cf8c47e968da936af61ec20af5d269d0f84a100/regex-2025.7.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d428fc7731dcbb4e2ffe43aeb8f90775ad155e7db4347a639768bc6cd2df881a", size = 787139, upload-time = "2025-07-31T00:19:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/eb/f5/9b9384415fdc533551be2ba805dd8c4621873e5df69c958f403bfd3b2b6e/regex-2025.7.34-cp311-cp311-win32.whl", hash = "sha256:e154a7ee7fa18333ad90b20e16ef84daaeac61877c8ef942ec8dfa50dc38b7a1", size = 264019, upload-time = "2025-07-31T00:19:31.129Z" }, + { url = "https://files.pythonhosted.org/packages/18/9d/e069ed94debcf4cc9626d652a48040b079ce34c7e4fb174f16874958d485/regex-2025.7.34-cp311-cp311-win_amd64.whl", hash = "sha256:24257953d5c1d6d3c129ab03414c07fc1a47833c9165d49b954190b2b7f21a1a", size = 276047, upload-time = "2025-07-31T00:19:32.497Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cf/3bafbe9d1fd1db77355e7fbbbf0d0cfb34501a8b8e334deca14f94c7b315/regex-2025.7.34-cp311-cp311-win_arm64.whl", hash = "sha256:3157aa512b9e606586900888cd469a444f9b898ecb7f8931996cb715f77477f0", size = 268362, upload-time = "2025-07-31T00:19:34.094Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f0/31d62596c75a33f979317658e8d261574785c6cd8672c06741ce2e2e2070/regex-2025.7.34-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7f7211a746aced993bef487de69307a38c5ddd79257d7be83f7b202cb59ddb50", size = 485492, upload-time = "2025-07-31T00:19:35.57Z" }, + { url = "https://files.pythonhosted.org/packages/d8/16/b818d223f1c9758c3434be89aa1a01aae798e0e0df36c1f143d1963dd1ee/regex-2025.7.34-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fb31080f2bd0681484b275461b202b5ad182f52c9ec606052020fe13eb13a72f", size = 290000, upload-time = "2025-07-31T00:19:37.175Z" }, + { url = "https://files.pythonhosted.org/packages/cd/70/69506d53397b4bd6954061bae75677ad34deb7f6ca3ba199660d6f728ff5/regex-2025.7.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0200a5150c4cf61e407038f4b4d5cdad13e86345dac29ff9dab3d75d905cf130", size = 286072, upload-time = "2025-07-31T00:19:38.612Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/536a216d5f66084fb577bb0543b5cb7de3272eb70a157f0c3a542f1c2551/regex-2025.7.34-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:739a74970e736df0773788377969c9fea3876c2fc13d0563f98e5503e5185f46", size = 797341, upload-time = "2025-07-31T00:19:40.119Z" }, + { url = "https://files.pythonhosted.org/packages/26/af/733f8168449e56e8f404bb807ea7189f59507cbea1b67a7bbcd92f8bf844/regex-2025.7.34-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4fef81b2f7ea6a2029161ed6dea9ae13834c28eb5a95b8771828194a026621e4", size = 862556, upload-time = "2025-07-31T00:19:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/19/dd/59c464d58c06c4f7d87de4ab1f590e430821345a40c5d345d449a636d15f/regex-2025.7.34-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea74cf81fe61a7e9d77989050d0089a927ab758c29dac4e8e1b6c06fccf3ebf0", size = 910762, upload-time = "2025-07-31T00:19:43Z" }, + { url = "https://files.pythonhosted.org/packages/37/a8/b05ccf33ceca0815a1e253693b2c86544932ebcc0049c16b0fbdf18b688b/regex-2025.7.34-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4636a7f3b65a5f340ed9ddf53585c42e3ff37101d383ed321bfe5660481744b", size = 801892, upload-time = "2025-07-31T00:19:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9a/b993cb2e634cc22810afd1652dba0cae156c40d4864285ff486c73cd1996/regex-2025.7.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cef962d7834437fe8d3da6f9bfc6f93f20f218266dcefec0560ed7765f5fe01", size = 786551, upload-time = "2025-07-31T00:19:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/2d/79/7849d67910a0de4e26834b5bb816e028e35473f3d7ae563552ea04f58ca2/regex-2025.7.34-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:cbe1698e5b80298dbce8df4d8d1182279fbdaf1044e864cbc9d53c20e4a2be77", size = 856457, upload-time = "2025-07-31T00:19:47.562Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/de516bc082524b27e45cb4f54e28bd800c01efb26d15646a65b87b13a91e/regex-2025.7.34-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:32b9f9bcf0f605eb094b08e8da72e44badabb63dde6b83bd530580b488d1c6da", size = 848902, upload-time = "2025-07-31T00:19:49.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/22/519ff8ba15f732db099b126f039586bd372da6cd4efb810d5d66a5daeda1/regex-2025.7.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:524c868ba527eab4e8744a9287809579f54ae8c62fbf07d62aacd89f6026b282", size = 788038, upload-time = "2025-07-31T00:19:50.794Z" }, + { url = "https://files.pythonhosted.org/packages/3f/7d/aabb467d8f57d8149895d133c88eb809a1a6a0fe262c1d508eb9dfabb6f9/regex-2025.7.34-cp312-cp312-win32.whl", hash = "sha256:d600e58ee6d036081c89696d2bdd55d507498a7180df2e19945c6642fac59588", size = 264417, upload-time = "2025-07-31T00:19:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/3b/39/bd922b55a4fc5ad5c13753274e5b536f5b06ec8eb9747675668491c7ab7a/regex-2025.7.34-cp312-cp312-win_amd64.whl", hash = "sha256:9a9ab52a466a9b4b91564437b36417b76033e8778e5af8f36be835d8cb370d62", size = 275387, upload-time = "2025-07-31T00:19:53.593Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/c61d2fdcecb754a40475a3d1ef9a000911d3e3fc75c096acf44b0dfb786a/regex-2025.7.34-cp312-cp312-win_arm64.whl", hash = "sha256:c83aec91af9c6fbf7c743274fd952272403ad9a9db05fe9bfc9df8d12b45f176", size = 268482, upload-time = "2025-07-31T00:19:55.183Z" }, ] [[package]] name = "requests" -version = "2.32.4" +version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -2849,46 +4317,45 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] [[package]] -name = "requests-futures" -version = "1.0.2" +name = "requests-file" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/88/f8/175b823241536ba09da033850d66194c372c65c38804847ac9cef0239542/requests_futures-1.0.2.tar.gz", hash = "sha256:6b7eb57940336e800faebc3dab506360edec9478f7b22dc570858ad3aa7458da", size = 10356, upload-time = "2024-11-15T22:14:51.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658", size = 6891, upload-time = "2024-05-21T16:28:00.24Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/23/7c1096731c15c83826cb0dd42078b561a838aed44c36f370aeb815168106/requests_futures-1.0.2-py2.py3-none-any.whl", hash = "sha256:a3534af7c2bf670cd7aa730716e9e7d4386497554f87792be7514063b8912897", size = 7671, upload-time = "2024-11-15T22:14:50.255Z" }, + { url = "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c", size = 4244, upload-time = "2024-05-21T16:27:57.733Z" }, ] [[package]] -name = "requests-toolbelt" -version = "1.0.0" +name = "requests-futures" +version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/f8/175b823241536ba09da033850d66194c372c65c38804847ac9cef0239542/requests_futures-1.0.2.tar.gz", hash = "sha256:6b7eb57940336e800faebc3dab506360edec9478f7b22dc570858ad3aa7458da", size = 10356, upload-time = "2024-11-15T22:14:51.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, + { url = "https://files.pythonhosted.org/packages/91/23/7c1096731c15c83826cb0dd42078b561a838aed44c36f370aeb815168106/requests_futures-1.0.2-py2.py3-none-any.whl", hash = "sha256:a3534af7c2bf670cd7aa730716e9e7d4386497554f87792be7514063b8912897", size = 7671, upload-time = "2024-11-15T22:14:50.255Z" }, ] [[package]] -name = "responses" -version = "0.18.0" +name = "requests-toolbelt" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, - { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/a5/186653e51cb20fe3ac793403334d4d077fbb7bb18a9c5c2fce8304d5a2e2/responses-0.18.0.tar.gz", hash = "sha256:380cad4c1c1dc942e5e8a8eaae0b4d4edf708f4f010db8b7bcfafad1fcd254ff", size = 45885, upload-time = "2022-02-02T19:59:52.834Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/f3/2b3a6dc5986303b3dd1bbbcf482022acb2583c428cd23f0b6d37b1a1a519/responses-0.18.0-py3-none-any.whl", hash = "sha256:15c63ad16de13ee8e7182d99c9334f64fd81f1ee79f90748d527c28f7ca9dd51", size = 38735, upload-time = "2022-02-02T19:59:52.833Z" }, + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] [[package]] @@ -2906,66 +4373,81 @@ wheels = [ [[package]] name = "rpds-py" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/aa/4456d84bbb54adc6a916fb10c9b374f78ac840337644e4a5eda229c81275/rpds_py-0.26.0.tar.gz", hash = "sha256:20dae58a859b0906f0685642e591056f1e787f3a8b39c8e8749a45dc7d26bdb0", size = 27385, upload-time = "2025-07-01T15:57:13.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/4c/4ee8f7e512030ff79fda1df3243c88d70fc874634e2dbe5df13ba4210078/rpds_py-0.26.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9e8cb77286025bdb21be2941d64ac6ca016130bfdcd228739e8ab137eb4406ed", size = 372610, upload-time = "2025-07-01T15:53:58.844Z" }, - { url = "https://files.pythonhosted.org/packages/fa/9d/3dc16be00f14fc1f03c71b1d67c8df98263ab2710a2fbd65a6193214a527/rpds_py-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e09330b21d98adc8ccb2dbb9fc6cb434e8908d4c119aeaa772cb1caab5440a0", size = 358032, upload-time = "2025-07-01T15:53:59.985Z" }, - { url = "https://files.pythonhosted.org/packages/e7/5a/7f1bf8f045da2866324a08ae80af63e64e7bfaf83bd31f865a7b91a58601/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9c1b92b774b2e68d11193dc39620d62fd8ab33f0a3c77ecdabe19c179cdbc1", size = 381525, upload-time = "2025-07-01T15:54:01.162Z" }, - { url = "https://files.pythonhosted.org/packages/45/8a/04479398c755a066ace10e3d158866beb600867cacae194c50ffa783abd0/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:824e6d3503ab990d7090768e4dfd9e840837bae057f212ff9f4f05ec6d1975e7", size = 397089, upload-time = "2025-07-01T15:54:02.319Z" }, - { url = "https://files.pythonhosted.org/packages/72/88/9203f47268db488a1b6d469d69c12201ede776bb728b9d9f29dbfd7df406/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ad7fd2258228bf288f2331f0a6148ad0186b2e3643055ed0db30990e59817a6", size = 514255, upload-time = "2025-07-01T15:54:03.38Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b4/01ce5d1e853ddf81fbbd4311ab1eff0b3cf162d559288d10fd127e2588b5/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0dc23bbb3e06ec1ea72d515fb572c1fea59695aefbffb106501138762e1e915e", size = 402283, upload-time = "2025-07-01T15:54:04.923Z" }, - { url = "https://files.pythonhosted.org/packages/34/a2/004c99936997bfc644d590a9defd9e9c93f8286568f9c16cdaf3e14429a7/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d80bf832ac7b1920ee29a426cdca335f96a2b5caa839811803e999b41ba9030d", size = 383881, upload-time = "2025-07-01T15:54:06.482Z" }, - { url = "https://files.pythonhosted.org/packages/05/1b/ef5fba4a8f81ce04c427bfd96223f92f05e6cd72291ce9d7523db3b03a6c/rpds_py-0.26.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0919f38f5542c0a87e7b4afcafab6fd2c15386632d249e9a087498571250abe3", size = 415822, upload-time = "2025-07-01T15:54:07.605Z" }, - { url = "https://files.pythonhosted.org/packages/16/80/5c54195aec456b292f7bd8aa61741c8232964063fd8a75fdde9c1e982328/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d422b945683e409000c888e384546dbab9009bb92f7c0b456e217988cf316107", size = 558347, upload-time = "2025-07-01T15:54:08.591Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1c/1845c1b1fd6d827187c43afe1841d91678d7241cbdb5420a4c6de180a538/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:77a7711fa562ba2da1aa757e11024ad6d93bad6ad7ede5afb9af144623e5f76a", size = 587956, upload-time = "2025-07-01T15:54:09.963Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ff/9e979329dd131aa73a438c077252ddabd7df6d1a7ad7b9aacf6261f10faa/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:238e8c8610cb7c29460e37184f6799547f7e09e6a9bdbdab4e8edb90986a2318", size = 554363, upload-time = "2025-07-01T15:54:11.073Z" }, - { url = "https://files.pythonhosted.org/packages/00/8b/d78cfe034b71ffbe72873a136e71acc7a831a03e37771cfe59f33f6de8a2/rpds_py-0.26.0-cp311-cp311-win32.whl", hash = "sha256:893b022bfbdf26d7bedb083efeea624e8550ca6eb98bf7fea30211ce95b9201a", size = 220123, upload-time = "2025-07-01T15:54:12.382Z" }, - { url = "https://files.pythonhosted.org/packages/94/c1/3c8c94c7dd3905dbfde768381ce98778500a80db9924731d87ddcdb117e9/rpds_py-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:87a5531de9f71aceb8af041d72fc4cab4943648d91875ed56d2e629bef6d4c03", size = 231732, upload-time = "2025-07-01T15:54:13.434Z" }, - { url = "https://files.pythonhosted.org/packages/67/93/e936fbed1b734eabf36ccb5d93c6a2e9246fbb13c1da011624b7286fae3e/rpds_py-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:de2713f48c1ad57f89ac25b3cb7daed2156d8e822cf0eca9b96a6f990718cc41", size = 221917, upload-time = "2025-07-01T15:54:14.559Z" }, - { url = "https://files.pythonhosted.org/packages/ea/86/90eb87c6f87085868bd077c7a9938006eb1ce19ed4d06944a90d3560fce2/rpds_py-0.26.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:894514d47e012e794f1350f076c427d2347ebf82f9b958d554d12819849a369d", size = 363933, upload-time = "2025-07-01T15:54:15.734Z" }, - { url = "https://files.pythonhosted.org/packages/63/78/4469f24d34636242c924626082b9586f064ada0b5dbb1e9d096ee7a8e0c6/rpds_py-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc921b96fa95a097add244da36a1d9e4f3039160d1d30f1b35837bf108c21136", size = 350447, upload-time = "2025-07-01T15:54:16.922Z" }, - { url = "https://files.pythonhosted.org/packages/ad/91/c448ed45efdfdade82348d5e7995e15612754826ea640afc20915119734f/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e1157659470aa42a75448b6e943c895be8c70531c43cb78b9ba990778955582", size = 384711, upload-time = "2025-07-01T15:54:18.101Z" }, - { url = "https://files.pythonhosted.org/packages/ec/43/e5c86fef4be7f49828bdd4ecc8931f0287b1152c0bb0163049b3218740e7/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:521ccf56f45bb3a791182dc6b88ae5f8fa079dd705ee42138c76deb1238e554e", size = 400865, upload-time = "2025-07-01T15:54:19.295Z" }, - { url = "https://files.pythonhosted.org/packages/55/34/e00f726a4d44f22d5c5fe2e5ddd3ac3d7fd3f74a175607781fbdd06fe375/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9def736773fd56b305c0eef698be5192c77bfa30d55a0e5885f80126c4831a15", size = 517763, upload-time = "2025-07-01T15:54:20.858Z" }, - { url = "https://files.pythonhosted.org/packages/52/1c/52dc20c31b147af724b16104500fba13e60123ea0334beba7b40e33354b4/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdad4ea3b4513b475e027be79e5a0ceac8ee1c113a1a11e5edc3c30c29f964d8", size = 406651, upload-time = "2025-07-01T15:54:22.508Z" }, - { url = "https://files.pythonhosted.org/packages/2e/77/87d7bfabfc4e821caa35481a2ff6ae0b73e6a391bb6b343db2c91c2b9844/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82b165b07f416bdccf5c84546a484cc8f15137ca38325403864bfdf2b5b72f6a", size = 386079, upload-time = "2025-07-01T15:54:23.987Z" }, - { url = "https://files.pythonhosted.org/packages/e3/d4/7f2200c2d3ee145b65b3cddc4310d51f7da6a26634f3ac87125fd789152a/rpds_py-0.26.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d04cab0a54b9dba4d278fe955a1390da3cf71f57feb78ddc7cb67cbe0bd30323", size = 421379, upload-time = "2025-07-01T15:54:25.073Z" }, - { url = "https://files.pythonhosted.org/packages/ae/13/9fdd428b9c820869924ab62236b8688b122baa22d23efdd1c566938a39ba/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:79061ba1a11b6a12743a2b0f72a46aa2758613d454aa6ba4f5a265cc48850158", size = 562033, upload-time = "2025-07-01T15:54:26.225Z" }, - { url = "https://files.pythonhosted.org/packages/f3/e1/b69686c3bcbe775abac3a4c1c30a164a2076d28df7926041f6c0eb5e8d28/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f405c93675d8d4c5ac87364bb38d06c988e11028a64b52a47158a355079661f3", size = 591639, upload-time = "2025-07-01T15:54:27.424Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c9/1e3d8c8863c84a90197ac577bbc3d796a92502124c27092413426f670990/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dafd4c44b74aa4bed4b250f1aed165b8ef5de743bcca3b88fc9619b6087093d2", size = 557105, upload-time = "2025-07-01T15:54:29.93Z" }, - { url = "https://files.pythonhosted.org/packages/9f/c5/90c569649057622959f6dcc40f7b516539608a414dfd54b8d77e3b201ac0/rpds_py-0.26.0-cp312-cp312-win32.whl", hash = "sha256:3da5852aad63fa0c6f836f3359647870e21ea96cf433eb393ffa45263a170d44", size = 223272, upload-time = "2025-07-01T15:54:31.128Z" }, - { url = "https://files.pythonhosted.org/packages/7d/16/19f5d9f2a556cfed454eebe4d354c38d51c20f3db69e7b4ce6cff904905d/rpds_py-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf47cfdabc2194a669dcf7a8dbba62e37a04c5041d2125fae0233b720da6f05c", size = 234995, upload-time = "2025-07-01T15:54:32.195Z" }, - { url = "https://files.pythonhosted.org/packages/83/f0/7935e40b529c0e752dfaa7880224771b51175fce08b41ab4a92eb2fbdc7f/rpds_py-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:20ab1ae4fa534f73647aad289003f1104092890849e0266271351922ed5574f8", size = 223198, upload-time = "2025-07-01T15:54:33.271Z" }, - { url = "https://files.pythonhosted.org/packages/51/f2/b5c85b758a00c513bb0389f8fc8e61eb5423050c91c958cdd21843faa3e6/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f61a9326f80ca59214d1cceb0a09bb2ece5b2563d4e0cd37bfd5515c28510674", size = 373505, upload-time = "2025-07-01T15:56:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/23/e0/25db45e391251118e915e541995bb5f5ac5691a3b98fb233020ba53afc9b/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:183f857a53bcf4b1b42ef0f57ca553ab56bdd170e49d8091e96c51c3d69ca696", size = 359468, upload-time = "2025-07-01T15:56:36.219Z" }, - { url = "https://files.pythonhosted.org/packages/0b/73/dd5ee6075bb6491be3a646b301dfd814f9486d924137a5098e61f0487e16/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:941c1cfdf4799d623cf3aa1d326a6b4fdb7a5799ee2687f3516738216d2262fb", size = 382680, upload-time = "2025-07-01T15:56:37.644Z" }, - { url = "https://files.pythonhosted.org/packages/2f/10/84b522ff58763a5c443f5bcedc1820240e454ce4e620e88520f04589e2ea/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72a8d9564a717ee291f554eeb4bfeafe2309d5ec0aa6c475170bdab0f9ee8e88", size = 397035, upload-time = "2025-07-01T15:56:39.241Z" }, - { url = "https://files.pythonhosted.org/packages/06/ea/8667604229a10a520fcbf78b30ccc278977dcc0627beb7ea2c96b3becef0/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:511d15193cbe013619dd05414c35a7dedf2088fcee93c6bbb7c77859765bd4e8", size = 514922, upload-time = "2025-07-01T15:56:40.645Z" }, - { url = "https://files.pythonhosted.org/packages/24/e6/9ed5b625c0661c4882fc8cdf302bf8e96c73c40de99c31e0b95ed37d508c/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aea1f9741b603a8d8fedb0ed5502c2bc0accbc51f43e2ad1337fe7259c2b77a5", size = 402822, upload-time = "2025-07-01T15:56:42.137Z" }, - { url = "https://files.pythonhosted.org/packages/8a/58/212c7b6fd51946047fb45d3733da27e2fa8f7384a13457c874186af691b1/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4019a9d473c708cf2f16415688ef0b4639e07abaa569d72f74745bbeffafa2c7", size = 384336, upload-time = "2025-07-01T15:56:44.239Z" }, - { url = "https://files.pythonhosted.org/packages/aa/f5/a40ba78748ae8ebf4934d4b88e77b98497378bc2c24ba55ebe87a4e87057/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:093d63b4b0f52d98ebae33b8c50900d3d67e0666094b1be7a12fffd7f65de74b", size = 416871, upload-time = "2025-07-01T15:56:46.284Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a6/33b1fc0c9f7dcfcfc4a4353daa6308b3ece22496ceece348b3e7a7559a09/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2abe21d8ba64cded53a2a677e149ceb76dcf44284202d737178afe7ba540c1eb", size = 559439, upload-time = "2025-07-01T15:56:48.549Z" }, - { url = "https://files.pythonhosted.org/packages/71/2d/ceb3f9c12f8cfa56d34995097f6cd99da1325642c60d1b6680dd9df03ed8/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:4feb7511c29f8442cbbc28149a92093d32e815a28aa2c50d333826ad2a20fdf0", size = 588380, upload-time = "2025-07-01T15:56:50.086Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ed/9de62c2150ca8e2e5858acf3f4f4d0d180a38feef9fdab4078bea63d8dba/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e99685fc95d386da368013e7fb4269dd39c30d99f812a8372d62f244f662709c", size = 555334, upload-time = "2025-07-01T15:56:51.703Z" }, +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/d9/991a0dee12d9fc53ed027e26a26a64b151d77252ac477e22666b9688bc16/rpds_py-0.27.0.tar.gz", hash = "sha256:8b23cf252f180cda89220b378d917180f29d313cd6a07b2431c0d3b776aae86f", size = 27420, upload-time = "2025-08-07T08:26:39.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c1/49d515434c1752e40f5e35b985260cf27af052593378580a2f139a5be6b8/rpds_py-0.27.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dbc2ab5d10544eb485baa76c63c501303b716a5c405ff2469a1d8ceffaabf622", size = 371577, upload-time = "2025-08-07T08:23:25.379Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6d/bf2715b2fee5087fa13b752b5fd573f1a93e4134c74d275f709e38e54fe7/rpds_py-0.27.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7ec85994f96a58cf7ed288caa344b7fe31fd1d503bdf13d7331ead5f70ab60d5", size = 354959, upload-time = "2025-08-07T08:23:26.767Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5c/e7762808c746dd19733a81373c10da43926f6a6adcf4920a21119697a60a/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:190d7285cd3bb6d31d37a0534d7359c1ee191eb194c511c301f32a4afa5a1dd4", size = 381485, upload-time = "2025-08-07T08:23:27.869Z" }, + { url = "https://files.pythonhosted.org/packages/40/51/0d308eb0b558309ca0598bcba4243f52c4cd20e15fe991b5bd75824f2e61/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c10d92fb6d7fd827e44055fcd932ad93dac6a11e832d51534d77b97d1d85400f", size = 396816, upload-time = "2025-08-07T08:23:29.424Z" }, + { url = "https://files.pythonhosted.org/packages/5c/aa/2d585ec911d78f66458b2c91252134ca0c7c70f687a72c87283173dc0c96/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd2c1d27ebfe6a015cfa2005b7fe8c52d5019f7bbdd801bc6f7499aab9ae739e", size = 514950, upload-time = "2025-08-07T08:23:30.576Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ef/aced551cc1148179557aed84343073adadf252c91265263ee6203458a186/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4790c9d5dd565ddb3e9f656092f57268951398cef52e364c405ed3112dc7c7c1", size = 402132, upload-time = "2025-08-07T08:23:32.428Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ac/cf644803d8d417653fe2b3604186861d62ea6afaef1b2284045741baef17/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4300e15e7d03660f04be84a125d1bdd0e6b2f674bc0723bc0fd0122f1a4585dc", size = 383660, upload-time = "2025-08-07T08:23:33.829Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ec/caf47c55ce02b76cbaeeb2d3b36a73da9ca2e14324e3d75cf72b59dcdac5/rpds_py-0.27.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:59195dc244fc183209cf8a93406889cadde47dfd2f0a6b137783aa9c56d67c85", size = 401730, upload-time = "2025-08-07T08:23:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/0b/71/c1f355afdcd5b99ffc253422aa4bdcb04ccf1491dcd1bda3688a0c07fd61/rpds_py-0.27.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fae4a01ef8c4cb2bbe92ef2063149596907dc4a881a8d26743b3f6b304713171", size = 416122, upload-time = "2025-08-07T08:23:36.062Z" }, + { url = "https://files.pythonhosted.org/packages/38/0f/f4b5b1eda724ed0e04d2b26d8911cdc131451a7ee4c4c020a1387e5c6ded/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e3dc8d4ede2dbae6c0fc2b6c958bf51ce9fd7e9b40c0f5b8835c3fde44f5807d", size = 558771, upload-time = "2025-08-07T08:23:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/5f8b834db2289ab48d5cffbecbb75e35410103a77ac0b8da36bf9544ec1c/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3782fb753aa825b4ccabc04292e07897e2fd941448eabf666856c5530277626", size = 587876, upload-time = "2025-08-07T08:23:38.662Z" }, + { url = "https://files.pythonhosted.org/packages/d2/dd/1a1df02ab8eb970115cff2ae31a6f73916609b900dc86961dc382b8c2e5e/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:887ab1f12b0d227e9260558a4a2320024b20102207ada65c43e1ffc4546df72e", size = 554359, upload-time = "2025-08-07T08:23:39.897Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/95a014ab0d51ab6e3bebbdb476a42d992d2bbf9c489d24cff9fda998e925/rpds_py-0.27.0-cp311-cp311-win32.whl", hash = "sha256:5d6790ff400254137b81b8053b34417e2c46921e302d655181d55ea46df58cf7", size = 218084, upload-time = "2025-08-07T08:23:41.086Z" }, + { url = "https://files.pythonhosted.org/packages/49/78/f8d5b71ec65a0376b0de31efcbb5528ce17a9b7fdd19c3763303ccfdedec/rpds_py-0.27.0-cp311-cp311-win_amd64.whl", hash = "sha256:e24d8031a2c62f34853756d9208eeafa6b940a1efcbfe36e8f57d99d52bb7261", size = 230085, upload-time = "2025-08-07T08:23:42.143Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/84429745184091e06b4cc70f8597408e314c2d2f7f5e13249af9ffab9e3d/rpds_py-0.27.0-cp311-cp311-win_arm64.whl", hash = "sha256:08680820d23df1df0a0260f714d12966bc6c42d02e8055a91d61e03f0c47dda0", size = 222112, upload-time = "2025-08-07T08:23:43.233Z" }, + { url = "https://files.pythonhosted.org/packages/cd/17/e67309ca1ac993fa1888a0d9b2f5ccc1f67196ace32e76c9f8e1dbbbd50c/rpds_py-0.27.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:19c990fdf5acecbf0623e906ae2e09ce1c58947197f9bced6bbd7482662231c4", size = 362611, upload-time = "2025-08-07T08:23:44.773Z" }, + { url = "https://files.pythonhosted.org/packages/93/2e/28c2fb84aa7aa5d75933d1862d0f7de6198ea22dfd9a0cca06e8a4e7509e/rpds_py-0.27.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6c27a7054b5224710fcfb1a626ec3ff4f28bcb89b899148c72873b18210e446b", size = 347680, upload-time = "2025-08-07T08:23:46.014Z" }, + { url = "https://files.pythonhosted.org/packages/44/3e/9834b4c8f4f5fe936b479e623832468aa4bd6beb8d014fecaee9eac6cdb1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09965b314091829b378b60607022048953e25f0b396c2b70e7c4c81bcecf932e", size = 384600, upload-time = "2025-08-07T08:23:48Z" }, + { url = "https://files.pythonhosted.org/packages/19/78/744123c7b38865a965cd9e6f691fde7ef989a00a256fa8bf15b75240d12f/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:14f028eb47f59e9169bfdf9f7ceafd29dd64902141840633683d0bad5b04ff34", size = 400697, upload-time = "2025-08-07T08:23:49.407Z" }, + { url = "https://files.pythonhosted.org/packages/32/97/3c3d32fe7daee0a1f1a678b6d4dfb8c4dcf88197fa2441f9da7cb54a8466/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6168af0be75bba990a39f9431cdfae5f0ad501f4af32ae62e8856307200517b8", size = 517781, upload-time = "2025-08-07T08:23:50.557Z" }, + { url = "https://files.pythonhosted.org/packages/b2/be/28f0e3e733680aa13ecec1212fc0f585928a206292f14f89c0b8a684cad1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab47fe727c13c09d0e6f508e3a49e545008e23bf762a245b020391b621f5b726", size = 406449, upload-time = "2025-08-07T08:23:51.732Z" }, + { url = "https://files.pythonhosted.org/packages/95/ae/5d15c83e337c082d0367053baeb40bfba683f42459f6ebff63a2fd7e5518/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa01b3d5e3b7d97efab65bd3d88f164e289ec323a8c033c5c38e53ee25c007e", size = 386150, upload-time = "2025-08-07T08:23:52.822Z" }, + { url = "https://files.pythonhosted.org/packages/bf/65/944e95f95d5931112829e040912b25a77b2e7ed913ea5fe5746aa5c1ce75/rpds_py-0.27.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:6c135708e987f46053e0a1246a206f53717f9fadfba27174a9769ad4befba5c3", size = 406100, upload-time = "2025-08-07T08:23:54.339Z" }, + { url = "https://files.pythonhosted.org/packages/21/a4/1664b83fae02894533cd11dc0b9f91d673797c2185b7be0f7496107ed6c5/rpds_py-0.27.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc327f4497b7087d06204235199daf208fd01c82d80465dc5efa4ec9df1c5b4e", size = 421345, upload-time = "2025-08-07T08:23:55.832Z" }, + { url = "https://files.pythonhosted.org/packages/7c/26/b7303941c2b0823bfb34c71378249f8beedce57301f400acb04bb345d025/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e57906e38583a2cba67046a09c2637e23297618dc1f3caddbc493f2be97c93f", size = 561891, upload-time = "2025-08-07T08:23:56.951Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c8/48623d64d4a5a028fa99576c768a6159db49ab907230edddc0b8468b998b/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f4f69d7a4300fbf91efb1fb4916421bd57804c01ab938ab50ac9c4aa2212f03", size = 591756, upload-time = "2025-08-07T08:23:58.146Z" }, + { url = "https://files.pythonhosted.org/packages/b3/51/18f62617e8e61cc66334c9fb44b1ad7baae3438662098efbc55fb3fda453/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b4c4fbbcff474e1e5f38be1bf04511c03d492d42eec0babda5d03af3b5589374", size = 557088, upload-time = "2025-08-07T08:23:59.6Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4c/e84c3a276e2496a93d245516be6b49e20499aa8ca1c94d59fada0d79addc/rpds_py-0.27.0-cp312-cp312-win32.whl", hash = "sha256:27bac29bbbf39601b2aab474daf99dbc8e7176ca3389237a23944b17f8913d97", size = 221926, upload-time = "2025-08-07T08:24:00.695Z" }, + { url = "https://files.pythonhosted.org/packages/83/89/9d0fbcef64340db0605eb0a0044f258076f3ae0a3b108983b2c614d96212/rpds_py-0.27.0-cp312-cp312-win_amd64.whl", hash = "sha256:8a06aa1197ec0281eb1d7daf6073e199eb832fe591ffa329b88bae28f25f5fe5", size = 233235, upload-time = "2025-08-07T08:24:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b0/e177aa9f39cbab060f96de4a09df77d494f0279604dc2f509263e21b05f9/rpds_py-0.27.0-cp312-cp312-win_arm64.whl", hash = "sha256:e14aab02258cb776a108107bd15f5b5e4a1bbaa61ef33b36693dfab6f89d54f9", size = 223315, upload-time = "2025-08-07T08:24:03.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/64/72ab5b911fdcc48058359b0e786e5363e3fde885156116026f1a2ba9a5b5/rpds_py-0.27.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e6491658dd2569f05860bad645569145c8626ac231877b0fb2d5f9bcb7054089", size = 371658, upload-time = "2025-08-07T08:26:02.369Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4b/90ff04b4da055db53d8fea57640d8d5d55456343a1ec9a866c0ecfe10fd1/rpds_py-0.27.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec77545d188f8bdd29d42bccb9191682a46fb2e655e3d1fb446d47c55ac3b8d", size = 355529, upload-time = "2025-08-07T08:26:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/a4/be/527491fb1afcd86fc5ce5812eb37bc70428ee017d77fee20de18155c3937/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a4aebf8ca02bbb90a9b3e7a463bbf3bee02ab1c446840ca07b1695a68ce424", size = 382822, upload-time = "2025-08-07T08:26:05.52Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a5/dcdb8725ce11e6d0913e6fcf782a13f4b8a517e8acc70946031830b98441/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44524b96481a4c9b8e6c46d6afe43fa1fb485c261e359fbe32b63ff60e3884d8", size = 397233, upload-time = "2025-08-07T08:26:07.179Z" }, + { url = "https://files.pythonhosted.org/packages/33/f9/0947920d1927e9f144660590cc38cadb0795d78fe0d9aae0ef71c1513b7c/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45d04a73c54b6a5fd2bab91a4b5bc8b426949586e61340e212a8484919183859", size = 514892, upload-time = "2025-08-07T08:26:08.622Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ed/d1343398c1417c68f8daa1afce56ef6ce5cc587daaf98e29347b00a80ff2/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:343cf24de9ed6c728abefc5d5c851d5de06497caa7ac37e5e65dd572921ed1b5", size = 402733, upload-time = "2025-08-07T08:26:10.433Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0b/646f55442cd14014fb64d143428f25667a100f82092c90087b9ea7101c74/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aed8118ae20515974650d08eb724150dc2e20c2814bcc307089569995e88a14", size = 384447, upload-time = "2025-08-07T08:26:11.847Z" }, + { url = "https://files.pythonhosted.org/packages/4b/15/0596ef7529828e33a6c81ecf5013d1dd33a511a3e0be0561f83079cda227/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:af9d4fd79ee1cc8e7caf693ee02737daabfc0fcf2773ca0a4735b356c8ad6f7c", size = 402502, upload-time = "2025-08-07T08:26:13.537Z" }, + { url = "https://files.pythonhosted.org/packages/c3/8d/986af3c42f8454a6cafff8729d99fb178ae9b08a9816325ac7a8fa57c0c0/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f0396e894bd1e66c74ecbc08b4f6a03dc331140942c4b1d345dd131b68574a60", size = 416651, upload-time = "2025-08-07T08:26:14.923Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9a/b4ec3629b7b447e896eec574469159b5b60b7781d3711c914748bf32de05/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:59714ab0a5af25d723d8e9816638faf7f4254234decb7d212715c1aa71eee7be", size = 559460, upload-time = "2025-08-07T08:26:16.295Z" }, + { url = "https://files.pythonhosted.org/packages/61/63/d1e127b40c3e4733b3a6f26ae7a063cdf2bc1caa5272c89075425c7d397a/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:88051c3b7d5325409f433c5a40328fcb0685fc04e5db49ff936e910901d10114", size = 588072, upload-time = "2025-08-07T08:26:17.776Z" }, + { url = "https://files.pythonhosted.org/packages/04/7e/8ffc71a8f6833d9c9fb999f5b0ee736b8b159fd66968e05c7afc2dbcd57e/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:181bc29e59e5e5e6e9d63b143ff4d5191224d355e246b5a48c88ce6b35c4e466", size = 555083, upload-time = "2025-08-07T08:26:19.301Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, ] [[package]] name = "s3transfer" -version = "0.13.0" +version = "0.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/5d/9dcc100abc6711e8247af5aa561fc07c4a046f72f659c3adea9a449e191a/s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177", size = 150232, upload-time = "2025-05-22T19:24:50.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/17/22bf8155aa0ea2305eefa3a6402e040df7ebe512d1310165eda1e233c3f8/s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be", size = 85152, upload-time = "2025-05-22T19:24:48.703Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, ] [[package]] name = "scikit-learn" -version = "1.7.0" +version = "1.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "joblib" }, @@ -2973,47 +4455,47 @@ dependencies = [ { name = "scipy" }, { name = "threadpoolctl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/3b/29fa87e76b1d7b3b77cc1fcbe82e6e6b8cd704410705b008822de530277c/scikit_learn-1.7.0.tar.gz", hash = "sha256:c01e869b15aec88e2cdb73d27f15bdbe03bce8e2fb43afbe77c45d399e73a5a3", size = 7178217, upload-time = "2025-06-05T22:02:46.703Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/84/5f4af978fff619706b8961accac84780a6d298d82a8873446f72edb4ead0/scikit_learn-1.7.1.tar.gz", hash = "sha256:24b3f1e976a4665aa74ee0fcaac2b8fccc6ae77c8e07ab25da3ba6d3292b9802", size = 7190445, upload-time = "2025-07-18T08:01:54.5Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/42/c6b41711c2bee01c4800ad8da2862c0b6d2956a399d23ce4d77f2ca7f0c7/scikit_learn-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ef09b1615e1ad04dc0d0054ad50634514818a8eb3ee3dee99af3bffc0ef5007", size = 11719657, upload-time = "2025-06-05T22:01:56.345Z" }, - { url = "https://files.pythonhosted.org/packages/a3/24/44acca76449e391b6b2522e67a63c0454b7c1f060531bdc6d0118fb40851/scikit_learn-1.7.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:7d7240c7b19edf6ed93403f43b0fcb0fe95b53bc0b17821f8fb88edab97085ef", size = 10712636, upload-time = "2025-06-05T22:01:59.093Z" }, - { url = "https://files.pythonhosted.org/packages/9f/1b/fcad1ccb29bdc9b96bcaa2ed8345d56afb77b16c0c47bafe392cc5d1d213/scikit_learn-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80bd3bd4e95381efc47073a720d4cbab485fc483966f1709f1fd559afac57ab8", size = 12242817, upload-time = "2025-06-05T22:02:01.43Z" }, - { url = "https://files.pythonhosted.org/packages/c6/38/48b75c3d8d268a3f19837cb8a89155ead6e97c6892bb64837183ea41db2b/scikit_learn-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dbe48d69aa38ecfc5a6cda6c5df5abef0c0ebdb2468e92437e2053f84abb8bc", size = 12873961, upload-time = "2025-06-05T22:02:03.951Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5a/ba91b8c57aa37dbd80d5ff958576a9a8c14317b04b671ae7f0d09b00993a/scikit_learn-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:8fa979313b2ffdfa049ed07252dc94038def3ecd49ea2a814db5401c07f1ecfa", size = 10717277, upload-time = "2025-06-05T22:02:06.77Z" }, - { url = "https://files.pythonhosted.org/packages/70/3a/bffab14e974a665a3ee2d79766e7389572ffcaad941a246931c824afcdb2/scikit_learn-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c2c7243d34aaede0efca7a5a96d67fddaebb4ad7e14a70991b9abee9dc5c0379", size = 11646758, upload-time = "2025-06-05T22:02:09.51Z" }, - { url = "https://files.pythonhosted.org/packages/58/d8/f3249232fa79a70cb40595282813e61453c1e76da3e1a44b77a63dd8d0cb/scikit_learn-1.7.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f39f6a811bf3f15177b66c82cbe0d7b1ebad9f190737dcdef77cfca1ea3c19c", size = 10673971, upload-time = "2025-06-05T22:02:12.217Z" }, - { url = "https://files.pythonhosted.org/packages/67/93/eb14c50533bea2f77758abe7d60a10057e5f2e2cdcf0a75a14c6bc19c734/scikit_learn-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63017a5f9a74963d24aac7590287149a8d0f1a0799bbe7173c0d8ba1523293c0", size = 11818428, upload-time = "2025-06-05T22:02:14.947Z" }, - { url = "https://files.pythonhosted.org/packages/08/17/804cc13b22a8663564bb0b55fb89e661a577e4e88a61a39740d58b909efe/scikit_learn-1.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b2f8a0b1e73e9a08b7cc498bb2aeab36cdc1f571f8ab2b35c6e5d1c7115d97d", size = 12505887, upload-time = "2025-06-05T22:02:17.824Z" }, - { url = "https://files.pythonhosted.org/packages/68/c7/4e956281a077f4835458c3f9656c666300282d5199039f26d9de1dabd9be/scikit_learn-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:34cc8d9d010d29fb2b7cbcd5ccc24ffdd80515f65fe9f1e4894ace36b267ce19", size = 10668129, upload-time = "2025-06-05T22:02:20.536Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bd/a23177930abd81b96daffa30ef9c54ddbf544d3226b8788ce4c3ef1067b4/scikit_learn-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90c8494ea23e24c0fb371afc474618c1019dc152ce4a10e4607e62196113851b", size = 9334838, upload-time = "2025-07-18T08:01:11.239Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a1/d3a7628630a711e2ac0d1a482910da174b629f44e7dd8cfcd6924a4ef81a/scikit_learn-1.7.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:bb870c0daf3bf3be145ec51df8ac84720d9972170786601039f024bf6d61a518", size = 8651241, upload-time = "2025-07-18T08:01:13.234Z" }, + { url = "https://files.pythonhosted.org/packages/26/92/85ec172418f39474c1cd0221d611345d4f433fc4ee2fc68e01f524ccc4e4/scikit_learn-1.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40daccd1b5623f39e8943ab39735cadf0bdce80e67cdca2adcb5426e987320a8", size = 9718677, upload-time = "2025-07-18T08:01:15.649Z" }, + { url = "https://files.pythonhosted.org/packages/df/ce/abdb1dcbb1d2b66168ec43b23ee0cee356b4cc4100ddee3943934ebf1480/scikit_learn-1.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30d1f413cfc0aa5a99132a554f1d80517563c34a9d3e7c118fde2d273c6fe0f7", size = 9511189, upload-time = "2025-07-18T08:01:18.013Z" }, + { url = "https://files.pythonhosted.org/packages/b2/3b/47b5eaee01ef2b5a80ba3f7f6ecf79587cb458690857d4777bfd77371c6f/scikit_learn-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:c711d652829a1805a95d7fe96654604a8f16eab5a9e9ad87b3e60173415cb650", size = 8914794, upload-time = "2025-07-18T08:01:20.357Z" }, + { url = "https://files.pythonhosted.org/packages/cb/16/57f176585b35ed865f51b04117947fe20f130f78940c6477b6d66279c9c2/scikit_learn-1.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3cee419b49b5bbae8796ecd690f97aa412ef1674410c23fc3257c6b8b85b8087", size = 9260431, upload-time = "2025-07-18T08:01:22.77Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/899317092f5efcab0e9bc929e3391341cec8fb0e816c4789686770024580/scikit_learn-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2fd8b8d35817b0d9ebf0b576f7d5ffbbabdb55536b0655a8aaae629d7ffd2e1f", size = 8637191, upload-time = "2025-07-18T08:01:24.731Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1b/998312db6d361ded1dd56b457ada371a8d8d77ca2195a7d18fd8a1736f21/scikit_learn-1.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:588410fa19a96a69763202f1d6b7b91d5d7a5d73be36e189bc6396bfb355bd87", size = 9486346, upload-time = "2025-07-18T08:01:26.713Z" }, + { url = "https://files.pythonhosted.org/packages/ad/09/a2aa0b4e644e5c4ede7006748f24e72863ba2ae71897fecfd832afea01b4/scikit_learn-1.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3142f0abe1ad1d1c31a2ae987621e41f6b578144a911ff4ac94781a583adad7", size = 9290988, upload-time = "2025-07-18T08:01:28.938Z" }, + { url = "https://files.pythonhosted.org/packages/15/fa/c61a787e35f05f17fc10523f567677ec4eeee5f95aa4798dbbbcd9625617/scikit_learn-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3ddd9092c1bd469acab337d87930067c87eac6bd544f8d5027430983f1e1ae88", size = 8735568, upload-time = "2025-07-18T08:01:30.936Z" }, ] [[package]] name = "scipy" -version = "1.16.0" +version = "1.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/18/b06a83f0c5ee8cddbde5e3f3d0bb9b702abfa5136ef6d4620ff67df7eee5/scipy-1.16.0.tar.gz", hash = "sha256:b5ef54021e832869c8cfb03bc3bf20366cbcd426e02a58e8a58d7584dfbb8f62", size = 30581216, upload-time = "2025-06-22T16:27:55.782Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f8/53fc4884df6b88afd5f5f00240bdc49fee2999c7eff3acf5953eb15bc6f8/scipy-1.16.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:deec06d831b8f6b5fb0b652433be6a09db29e996368ce5911faf673e78d20085", size = 36447362, upload-time = "2025-06-22T16:18:17.817Z" }, - { url = "https://files.pythonhosted.org/packages/c9/25/fad8aa228fa828705142a275fc593d701b1817c98361a2d6b526167d07bc/scipy-1.16.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d30c0fe579bb901c61ab4bb7f3eeb7281f0d4c4a7b52dbf563c89da4fd2949be", size = 28547120, upload-time = "2025-06-22T16:18:24.117Z" }, - { url = "https://files.pythonhosted.org/packages/8d/be/d324ddf6b89fd1c32fecc307f04d095ce84abb52d2e88fab29d0cd8dc7a8/scipy-1.16.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b2243561b45257f7391d0f49972fca90d46b79b8dbcb9b2cb0f9df928d370ad4", size = 20818922, upload-time = "2025-06-22T16:18:28.035Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e0/cf3f39e399ac83fd0f3ba81ccc5438baba7cfe02176be0da55ff3396f126/scipy-1.16.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:e6d7dfc148135e9712d87c5f7e4f2ddc1304d1582cb3a7d698bbadedb61c7afd", size = 23409695, upload-time = "2025-06-22T16:18:32.497Z" }, - { url = "https://files.pythonhosted.org/packages/5b/61/d92714489c511d3ffd6830ac0eb7f74f243679119eed8b9048e56b9525a1/scipy-1.16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90452f6a9f3fe5a2cf3748e7be14f9cc7d9b124dce19667b54f5b429d680d539", size = 33444586, upload-time = "2025-06-22T16:18:37.992Z" }, - { url = "https://files.pythonhosted.org/packages/af/2c/40108915fd340c830aee332bb85a9160f99e90893e58008b659b9f3dddc0/scipy-1.16.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2f0bf2f58031c8701a8b601df41701d2a7be17c7ffac0a4816aeba89c4cdac8", size = 35284126, upload-time = "2025-06-22T16:18:43.605Z" }, - { url = "https://files.pythonhosted.org/packages/d3/30/e9eb0ad3d0858df35d6c703cba0a7e16a18a56a9e6b211d861fc6f261c5f/scipy-1.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c4abb4c11fc0b857474241b812ce69ffa6464b4bd8f4ecb786cf240367a36a7", size = 35608257, upload-time = "2025-06-22T16:18:49.09Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ff/950ee3e0d612b375110d8cda211c1f787764b4c75e418a4b71f4a5b1e07f/scipy-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b370f8f6ac6ef99815b0d5c9f02e7ade77b33007d74802efc8316c8db98fd11e", size = 38040541, upload-time = "2025-06-22T16:18:55.077Z" }, - { url = "https://files.pythonhosted.org/packages/8b/c9/750d34788288d64ffbc94fdb4562f40f609d3f5ef27ab4f3a4ad00c9033e/scipy-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:a16ba90847249bedce8aa404a83fb8334b825ec4a8e742ce6012a7a5e639f95c", size = 38570814, upload-time = "2025-06-22T16:19:00.912Z" }, - { url = "https://files.pythonhosted.org/packages/01/c0/c943bc8d2bbd28123ad0f4f1eef62525fa1723e84d136b32965dcb6bad3a/scipy-1.16.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:7eb6bd33cef4afb9fa5f1fb25df8feeb1e52d94f21a44f1d17805b41b1da3180", size = 36459071, upload-time = "2025-06-22T16:19:06.605Z" }, - { url = "https://files.pythonhosted.org/packages/99/0d/270e2e9f1a4db6ffbf84c9a0b648499842046e4e0d9b2275d150711b3aba/scipy-1.16.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:1dbc8fdba23e4d80394ddfab7a56808e3e6489176d559c6c71935b11a2d59db1", size = 28490500, upload-time = "2025-06-22T16:19:11.775Z" }, - { url = "https://files.pythonhosted.org/packages/1c/22/01d7ddb07cff937d4326198ec8d10831367a708c3da72dfd9b7ceaf13028/scipy-1.16.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7dcf42c380e1e3737b343dec21095c9a9ad3f9cbe06f9c05830b44b1786c9e90", size = 20762345, upload-time = "2025-06-22T16:19:15.813Z" }, - { url = "https://files.pythonhosted.org/packages/34/7f/87fd69856569ccdd2a5873fe5d7b5bbf2ad9289d7311d6a3605ebde3a94b/scipy-1.16.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:26ec28675f4a9d41587266084c626b02899db373717d9312fa96ab17ca1ae94d", size = 23418563, upload-time = "2025-06-22T16:19:20.746Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f1/e4f4324fef7f54160ab749efbab6a4bf43678a9eb2e9817ed71a0a2fd8de/scipy-1.16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:952358b7e58bd3197cfbd2f2f2ba829f258404bdf5db59514b515a8fe7a36c52", size = 33203951, upload-time = "2025-06-22T16:19:25.813Z" }, - { url = "https://files.pythonhosted.org/packages/6d/f0/b6ac354a956384fd8abee2debbb624648125b298f2c4a7b4f0d6248048a5/scipy-1.16.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03931b4e870c6fef5b5c0970d52c9f6ddd8c8d3e934a98f09308377eba6f3824", size = 35070225, upload-time = "2025-06-22T16:19:31.416Z" }, - { url = "https://files.pythonhosted.org/packages/e5/73/5cbe4a3fd4bc3e2d67ffad02c88b83edc88f381b73ab982f48f3df1a7790/scipy-1.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:512c4f4f85912767c351a0306824ccca6fd91307a9f4318efe8fdbd9d30562ef", size = 35389070, upload-time = "2025-06-22T16:19:37.387Z" }, - { url = "https://files.pythonhosted.org/packages/86/e8/a60da80ab9ed68b31ea5a9c6dfd3c2f199347429f229bf7f939a90d96383/scipy-1.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e69f798847e9add03d512eaf5081a9a5c9a98757d12e52e6186ed9681247a1ac", size = 37825287, upload-time = "2025-06-22T16:19:43.375Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b5/29fece1a74c6a94247f8a6fb93f5b28b533338e9c34fdcc9cfe7a939a767/scipy-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:adf9b1999323ba335adc5d1dc7add4781cb5a4b0ef1e98b79768c05c796c4e49", size = 38431929, upload-time = "2025-06-22T16:19:49.385Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f5/4a/b927028464795439faec8eaf0b03b011005c487bb2d07409f28bf30879c4/scipy-1.16.1.tar.gz", hash = "sha256:44c76f9e8b6e8e488a586190ab38016e4ed2f8a038af7cd3defa903c0a2238b3", size = 30580861, upload-time = "2025-07-27T16:33:30.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/91/812adc6f74409b461e3a5fa97f4f74c769016919203138a3bf6fc24ba4c5/scipy-1.16.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c033fa32bab91dc98ca59d0cf23bb876454e2bb02cbe592d5023138778f70030", size = 36552519, upload-time = "2025-07-27T16:26:29.658Z" }, + { url = "https://files.pythonhosted.org/packages/47/18/8e355edcf3b71418d9e9f9acd2708cc3a6c27e8f98fde0ac34b8a0b45407/scipy-1.16.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6e5c2f74e5df33479b5cd4e97a9104c511518fbd979aa9b8f6aec18b2e9ecae7", size = 28638010, upload-time = "2025-07-27T16:26:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/d9/eb/e931853058607bdfbc11b86df19ae7a08686121c203483f62f1ecae5989c/scipy-1.16.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0a55ffe0ba0f59666e90951971a884d1ff6f4ec3275a48f472cfb64175570f77", size = 20909790, upload-time = "2025-07-27T16:26:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/45/0c/be83a271d6e96750cd0be2e000f35ff18880a46f05ce8b5d3465dc0f7a2a/scipy-1.16.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f8a5d6cd147acecc2603fbd382fed6c46f474cccfcf69ea32582e033fb54dcfe", size = 23513352, upload-time = "2025-07-27T16:26:50.017Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bf/fe6eb47e74f762f933cca962db7f2c7183acfdc4483bd1c3813cfe83e538/scipy-1.16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb18899127278058bcc09e7b9966d41a5a43740b5bb8dcba401bd983f82e885b", size = 33534643, upload-time = "2025-07-27T16:26:57.503Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ba/63f402e74875486b87ec6506a4f93f6d8a0d94d10467280f3d9d7837ce3a/scipy-1.16.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adccd93a2fa937a27aae826d33e3bfa5edf9aa672376a4852d23a7cd67a2e5b7", size = 35376776, upload-time = "2025-07-27T16:27:06.639Z" }, + { url = "https://files.pythonhosted.org/packages/c3/b4/04eb9d39ec26a1b939689102da23d505ea16cdae3dbb18ffc53d1f831044/scipy-1.16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:18aca1646a29ee9a0625a1be5637fa798d4d81fdf426481f06d69af828f16958", size = 35698906, upload-time = "2025-07-27T16:27:14.943Z" }, + { url = "https://files.pythonhosted.org/packages/04/d6/bb5468da53321baeb001f6e4e0d9049eadd175a4a497709939128556e3ec/scipy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d85495cef541729a70cdddbbf3e6b903421bc1af3e8e3a9a72a06751f33b7c39", size = 38129275, upload-time = "2025-07-27T16:27:23.873Z" }, + { url = "https://files.pythonhosted.org/packages/c4/94/994369978509f227cba7dfb9e623254d0d5559506fe994aef4bea3ed469c/scipy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:226652fca853008119c03a8ce71ffe1b3f6d2844cc1686e8f9806edafae68596", size = 38644572, upload-time = "2025-07-27T16:27:32.637Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d9/ec4864f5896232133f51382b54a08de91a9d1af7a76dfa372894026dfee2/scipy-1.16.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:81b433bbeaf35728dad619afc002db9b189e45eebe2cd676effe1fb93fef2b9c", size = 36575194, upload-time = "2025-07-27T16:27:41.321Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6d/40e81ecfb688e9d25d34a847dca361982a6addf8e31f0957b1a54fbfa994/scipy-1.16.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:886cc81fdb4c6903a3bb0464047c25a6d1016fef77bb97949817d0c0d79f9e04", size = 28594590, upload-time = "2025-07-27T16:27:49.204Z" }, + { url = "https://files.pythonhosted.org/packages/0e/37/9f65178edfcc629377ce9a64fc09baebea18c80a9e57ae09a52edf84880b/scipy-1.16.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:15240c3aac087a522b4eaedb09f0ad061753c5eebf1ea430859e5bf8640d5919", size = 20866458, upload-time = "2025-07-27T16:27:54.98Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7b/749a66766871ea4cb1d1ea10f27004db63023074c22abed51f22f09770e0/scipy-1.16.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f81a25805f3659b48126b5053d9e823d3215e4a63730b5e1671852a1705921", size = 23539318, upload-time = "2025-07-27T16:28:01.604Z" }, + { url = "https://files.pythonhosted.org/packages/c4/db/8d4afec60eb833a666434d4541a3151eedbf2494ea6d4d468cbe877f00cd/scipy-1.16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c62eea7f607f122069b9bad3f99489ddca1a5173bef8a0c75555d7488b6f725", size = 33292899, upload-time = "2025-07-27T16:28:09.147Z" }, + { url = "https://files.pythonhosted.org/packages/51/1e/79023ca3bbb13a015d7d2757ecca3b81293c663694c35d6541b4dca53e98/scipy-1.16.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f965bbf3235b01c776115ab18f092a95aa74c271a52577bcb0563e85738fd618", size = 35162637, upload-time = "2025-07-27T16:28:17.535Z" }, + { url = "https://files.pythonhosted.org/packages/b6/49/0648665f9c29fdaca4c679182eb972935b3b4f5ace41d323c32352f29816/scipy-1.16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f006e323874ffd0b0b816d8c6a8e7f9a73d55ab3b8c3f72b752b226d0e3ac83d", size = 35490507, upload-time = "2025-07-27T16:28:25.705Z" }, + { url = "https://files.pythonhosted.org/packages/62/8f/66cbb9d6bbb18d8c658f774904f42a92078707a7c71e5347e8bf2f52bb89/scipy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8fd15fc5085ab4cca74cb91fe0a4263b1f32e4420761ddae531ad60934c2119", size = 37923998, upload-time = "2025-07-27T16:28:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/14/c3/61f273ae550fbf1667675701112e380881905e28448c080b23b5a181df7c/scipy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7b8013c6c066609577d910d1a2a077021727af07b6fab0ee22c2f901f22352a", size = 38508060, upload-time = "2025-07-27T16:28:43.242Z" }, ] [[package]] @@ -3034,6 +4516,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, ] +[[package]] +name = "sentry-sdk" +version = "2.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/83/055dc157b719651ef13db569bb8cf2103df11174478649735c1b2bf3f6bc/sentry_sdk-2.35.0.tar.gz", hash = "sha256:5ea58d352779ce45d17bc2fa71ec7185205295b83a9dbb5707273deb64720092", size = 343014, upload-time = "2025-08-14T17:11:20.223Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/3d/742617a7c644deb0c1628dcf6bb2d2165ab7c6aab56fe5222758994007f8/sentry_sdk-2.35.0-py2.py3-none-any.whl", hash = "sha256:6e0c29b9a5d34de8575ffb04d289a987ff3053cf2c98ede445bea995e3830263", size = 363806, upload-time = "2025-08-14T17:11:18.29Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" @@ -3045,15 +4540,24 @@ wheels = [ [[package]] name = "setuptools-scm" -version = "8.3.1" +version = "9.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/19/7ae64b70b2429c48c3a7a4ed36f50f94687d3bfcd0ae2f152367b6410dff/setuptools_scm-8.3.1.tar.gz", hash = "sha256:3d555e92b75dacd037d32bafdf94f97af51ea29ae8c7b234cf94b7a5bd242a63", size = 78088, upload-time = "2025-04-23T11:53:19.739Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/8d/ffdcace33d0480d591057a30285b7c33f8dc431fed3fff7dbadf5f9f128f/setuptools_scm-9.2.0.tar.gz", hash = "sha256:6662c9b9497b6c9bf13bead9d7a9084756f68238302c5ed089fb4dbd29d102d7", size = 201229, upload-time = "2025-08-16T12:56:39.477Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/14/dd3a6053325e882fe191fb4b42289bbdfabf5f44307c302903a8a3236a0a/setuptools_scm-9.2.0-py3-none-any.whl", hash = "sha256:c551ef54e2270727ee17067881c9687ca2aedf179fa5b8f3fab9e8d73bdc421f", size = 62099, upload-time = "2025-08-16T12:56:37.912Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/ac/8f96ba9b4cfe3e4ea201f23f4f97165862395e9331a424ed325ae37024a8/setuptools_scm-8.3.1-py3-none-any.whl", hash = "sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3", size = 43935, upload-time = "2025-04-23T11:53:17.922Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] [[package]] @@ -3065,6 +4569,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smart-open" +version = "7.3.0.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/2b/5e7234c68ed5bc872ad6ae77b8a421c2ed70dcb1190b44dc1abdeed5e347/smart_open-7.3.0.post1.tar.gz", hash = "sha256:ce6a3d9bc1afbf6234ad13c010b77f8cd36d24636811e3c52c3b5160f5214d1e", size = 51557, upload-time = "2025-07-03T10:06:31.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/5b/a2a3d4514c64818925f4e886d39981f1926eeb5288a4549c6b3c17ed66bb/smart_open-7.3.0.post1-py3-none-any.whl", hash = "sha256:c73661a2c24bf045c1e04e08fffc585b59af023fe783d57896f590489db66fb4", size = 61946, upload-time = "2025-07-03T10:06:29.599Z" }, +] + [[package]] name = "smmap" version = "5.0.2" @@ -3101,33 +4617,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/9c/0e6afc12c269578be5c0c1c9f4b49a8d32770a080260c333ac04cc1c832d/soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4", size = 36677, upload-time = "2025-04-20T18:50:07.196Z" }, ] +[[package]] +name = "spacy" +version = "3.8.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, + { name = "cymem" }, + { name = "jinja2" }, + { name = "langcodes" }, + { name = "murmurhash" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "spacy-legacy" }, + { name = "spacy-loggers" }, + { name = "srsly" }, + { name = "thinc" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "wasabi" }, + { name = "weasel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/9e/fb4e1cefe3fbd51ea6a243e5a3d2bc629baa9a28930bf4be6fe5672fa1ca/spacy-3.8.7.tar.gz", hash = "sha256:700fd174c6c552276be142c48e70bb53cae24c4dd86003c4432af9cb93e4c908", size = 1316143, upload-time = "2025-05-23T08:55:39.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/c5/5fbb3a4e694d4855a5bab87af9664377c48b89691f180ad3cde4faeaf35c/spacy-3.8.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bdff8b9b556468a6dd527af17f0ddf9fb0b0bee92ee7703339ddf542361cff98", size = 6746140, upload-time = "2025-05-23T08:54:23.483Z" }, + { url = "https://files.pythonhosted.org/packages/03/2a/43afac516eb82409ca47d7206f982beaf265d2ba06a72ca07cf06b290c20/spacy-3.8.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9194b7cf015ed9b4450ffb162da49c8a9305e76b468de036b0948abdfc748a37", size = 6392440, upload-time = "2025-05-23T08:54:25.12Z" }, + { url = "https://files.pythonhosted.org/packages/6f/83/2ea68c18e2b1b9a6f6b30ef63eb9d07e979626b9595acfdb5394f18923c4/spacy-3.8.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7dc38b78d48b9c2a80a3eea95f776304993f63fc307f07cdd104441442f92f1e", size = 32699126, upload-time = "2025-05-23T08:54:27.385Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0a/bb90e9aa0b3c527876627567d82517aabab08006ccf63796c33b0242254d/spacy-3.8.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e43bd70772751b8fc7a14f338d087a3d297195d43d171832923ef66204b23ab", size = 33008865, upload-time = "2025-05-23T08:54:30.248Z" }, + { url = "https://files.pythonhosted.org/packages/39/dd/8e906ba378457107ab0394976ea9f7b12fdb2cad682ef1a2ccf473d61e5f/spacy-3.8.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c402bf5dcf345fd96d202378c54bc345219681e3531f911d99567d569328c45f", size = 31933169, upload-time = "2025-05-23T08:54:33.199Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b5/42df07eb837a923fbb42509864d5c7c2072d010de933dccdfb3c655b3a76/spacy-3.8.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4234189861e486d86f1269e50542d87e8a6391a1ee190652479cf1a793db115f", size = 32776322, upload-time = "2025-05-23T08:54:36.891Z" }, + { url = "https://files.pythonhosted.org/packages/92/e7/8176484801c67dcd814f141991fe0a3c9b5b4a3583ea30c2062e93d1aa6b/spacy-3.8.7-cp311-cp311-win_amd64.whl", hash = "sha256:e9d12e2eb7f36bc11dd9edae011032fe49ea100d63e83177290d3cbd80eaa650", size = 14938936, upload-time = "2025-05-23T08:54:40.322Z" }, + { url = "https://files.pythonhosted.org/packages/a5/10/89852f40f926e0902c11c34454493ba0d15530b322711e754b89a6d7dfe6/spacy-3.8.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:88b397e37793cea51df298e6c651a763e49877a25bead5ba349761531a456687", size = 6265335, upload-time = "2025-05-23T08:54:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/16/fb/b5d54522969a632c06f4af354763467553b66d5bf0671ac39f3cceb3fd54/spacy-3.8.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f70b676955fa6959347ca86ed6edd8ff0d6eb2ba20561fdfec76924bd3e540f9", size = 5906035, upload-time = "2025-05-23T08:54:44.824Z" }, + { url = "https://files.pythonhosted.org/packages/3a/03/70f06753fd65081404ade30408535eb69f627a36ffce2107116d1aa16239/spacy-3.8.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4b5a624797ade30c25b5b69daa35a93ee24bcc56bd79b0884b2565f76f35d6", size = 33420084, upload-time = "2025-05-23T08:54:46.889Z" }, + { url = "https://files.pythonhosted.org/packages/f9/19/b60e1ebf4985ee2b33d85705b89a5024942b65dad04dbdc3fb46f168b410/spacy-3.8.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9d83e006df66decccefa3872fa958b3756228fb216d83783595444cf42ca10c", size = 33922188, upload-time = "2025-05-23T08:54:49.781Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/1fb1a49dc6d982d96fffc30c3a31bb431526008eea72ac3773f6518720a6/spacy-3.8.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0dca25deba54f3eb5dcfbf63bf16e613e6c601da56f91c4a902d38533c098941", size = 31939285, upload-time = "2025-05-23T08:54:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/2d/55/6cf1aff8e5c01ee683e828f3ccd9282d2aff7ca1143a9349ee3d0c1291ff/spacy-3.8.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5eef3f805a1c118d9b709a23e2d378f5f20da5a0d6258c9cfdc87c4cb234b4fc", size = 32988845, upload-time = "2025-05-23T08:54:57.776Z" }, + { url = "https://files.pythonhosted.org/packages/8c/47/c17ee61b51aa8497d8af0999224b4b62485111a55ec105a06886685b2c68/spacy-3.8.7-cp312-cp312-win_amd64.whl", hash = "sha256:25d7a68e445200c9e9dc0044f8b7278ec0ef01ccc7cb5a95d1de2bd8e3ed6be2", size = 13918682, upload-time = "2025-05-23T08:55:00.387Z" }, +] + +[[package]] +name = "spacy-legacy" +version = "3.0.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/79/91f9d7cc8db5642acad830dcc4b49ba65a7790152832c4eceb305e46d681/spacy-legacy-3.0.12.tar.gz", hash = "sha256:b37d6e0c9b6e1d7ca1cf5bc7152ab64a4c4671f59c85adaf7a3fcb870357a774", size = 23806, upload-time = "2023-01-23T09:04:15.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/55/12e842c70ff8828e34e543a2c7176dac4da006ca6901c9e8b43efab8bc6b/spacy_legacy-3.0.12-py2.py3-none-any.whl", hash = "sha256:476e3bd0d05f8c339ed60f40986c07387c0a71479245d6d0f4298dbd52cda55f", size = 29971, upload-time = "2023-01-23T09:04:13.45Z" }, +] + +[[package]] +name = "spacy-loggers" +version = "1.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/3d/926db774c9c98acf66cb4ed7faf6c377746f3e00b84b700d0868b95d0712/spacy-loggers-1.0.5.tar.gz", hash = "sha256:d60b0bdbf915a60e516cc2e653baeff946f0cfc461b452d11a4d5458c6fe5f24", size = 20811, upload-time = "2023-09-11T12:26:52.323Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/78/d1a1a026ef3af911159398c939b1509d5c36fe524c7b644f34a5146c4e16/spacy_loggers-1.0.5-py3-none-any.whl", hash = "sha256:196284c9c446cc0cdb944005384270d775fdeaf4f494d8e269466cfa497ef645", size = 22343, upload-time = "2023-09-11T12:26:50.586Z" }, +] + [[package]] name = "sqlalchemy" -version = "2.0.41" +version = "2.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/66/45b165c595ec89aa7dcc2c1cd222ab269bc753f1fc7a1e68f8481bd957bf/sqlalchemy-2.0.41.tar.gz", hash = "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9", size = 9689424, upload-time = "2025-05-14T17:10:32.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/4e/b00e3ffae32b74b5180e15d2ab4040531ee1bef4c19755fe7926622dc958/sqlalchemy-2.0.41-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f", size = 2121232, upload-time = "2025-05-14T17:48:20.444Z" }, - { url = "https://files.pythonhosted.org/packages/ef/30/6547ebb10875302074a37e1970a5dce7985240665778cfdee2323709f749/sqlalchemy-2.0.41-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560", size = 2110897, upload-time = "2025-05-14T17:48:21.634Z" }, - { url = "https://files.pythonhosted.org/packages/9e/21/59df2b41b0f6c62da55cd64798232d7349a9378befa7f1bb18cf1dfd510a/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f", size = 3273313, upload-time = "2025-05-14T17:51:56.205Z" }, - { url = "https://files.pythonhosted.org/packages/62/e4/b9a7a0e5c6f79d49bcd6efb6e90d7536dc604dab64582a9dec220dab54b6/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6", size = 3273807, upload-time = "2025-05-14T17:55:26.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/d8/79f2427251b44ddee18676c04eab038d043cff0e764d2d8bb08261d6135d/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04", size = 3209632, upload-time = "2025-05-14T17:51:59.384Z" }, - { url = "https://files.pythonhosted.org/packages/d4/16/730a82dda30765f63e0454918c982fb7193f6b398b31d63c7c3bd3652ae5/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582", size = 3233642, upload-time = "2025-05-14T17:55:29.901Z" }, - { url = "https://files.pythonhosted.org/packages/04/61/c0d4607f7799efa8b8ea3c49b4621e861c8f5c41fd4b5b636c534fcb7d73/sqlalchemy-2.0.41-cp311-cp311-win32.whl", hash = "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8", size = 2086475, upload-time = "2025-05-14T17:56:02.095Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8e/8344f8ae1cb6a479d0741c02cd4f666925b2bf02e2468ddaf5ce44111f30/sqlalchemy-2.0.41-cp311-cp311-win_amd64.whl", hash = "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504", size = 2110903, upload-time = "2025-05-14T17:56:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2a/f1f4e068b371154740dd10fb81afb5240d5af4aa0087b88d8b308b5429c2/sqlalchemy-2.0.41-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9", size = 2119645, upload-time = "2025-05-14T17:55:24.854Z" }, - { url = "https://files.pythonhosted.org/packages/9b/e8/c664a7e73d36fbfc4730f8cf2bf930444ea87270f2825efbe17bf808b998/sqlalchemy-2.0.41-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1", size = 2107399, upload-time = "2025-05-14T17:55:28.097Z" }, - { url = "https://files.pythonhosted.org/packages/5c/78/8a9cf6c5e7135540cb682128d091d6afa1b9e48bd049b0d691bf54114f70/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70", size = 3293269, upload-time = "2025-05-14T17:50:38.227Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/f74add3978c20de6323fb11cb5162702670cc7a9420033befb43d8d5b7a4/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e", size = 3303364, upload-time = "2025-05-14T17:51:49.829Z" }, - { url = "https://files.pythonhosted.org/packages/6a/d4/c990f37f52c3f7748ebe98883e2a0f7d038108c2c5a82468d1ff3eec50b7/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078", size = 3229072, upload-time = "2025-05-14T17:50:39.774Z" }, - { url = "https://files.pythonhosted.org/packages/15/69/cab11fecc7eb64bc561011be2bd03d065b762d87add52a4ca0aca2e12904/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae", size = 3268074, upload-time = "2025-05-14T17:51:51.736Z" }, - { url = "https://files.pythonhosted.org/packages/5c/ca/0c19ec16858585d37767b167fc9602593f98998a68a798450558239fb04a/sqlalchemy-2.0.41-cp312-cp312-win32.whl", hash = "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6", size = 2084514, upload-time = "2025-05-14T17:55:49.915Z" }, - { url = "https://files.pythonhosted.org/packages/7f/23/4c2833d78ff3010a4e17f984c734f52b531a8c9060a50429c9d4b0211be6/sqlalchemy-2.0.41-cp312-cp312-win_amd64.whl", hash = "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0", size = 2111557, upload-time = "2025-05-14T17:55:51.349Z" }, - { url = "https://files.pythonhosted.org/packages/1c/fc/9ba22f01b5cdacc8f5ed0d22304718d2c758fce3fd49a5372b886a86f37c/sqlalchemy-2.0.41-py3-none-any.whl", hash = "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", size = 1911224, upload-time = "2025-05-14T17:39:42.154Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d7/bc/d59b5d97d27229b0e009bd9098cd81af71c2fa5549c580a0a67b9bed0496/sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417", size = 9762949, upload-time = "2025-08-11T14:24:58.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/77/fa7189fe44114658002566c6fe443d3ed0ec1fa782feb72af6ef7fbe98e7/sqlalchemy-2.0.43-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:52d9b73b8fb3e9da34c2b31e6d99d60f5f99fd8c1225c9dad24aeb74a91e1d29", size = 2136472, upload-time = "2025-08-11T15:52:21.789Z" }, + { url = "https://files.pythonhosted.org/packages/99/ea/92ac27f2fbc2e6c1766bb807084ca455265707e041ba027c09c17d697867/sqlalchemy-2.0.43-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f42f23e152e4545157fa367b2435a1ace7571cab016ca26038867eb7df2c3631", size = 2126535, upload-time = "2025-08-11T15:52:23.109Z" }, + { url = "https://files.pythonhosted.org/packages/94/12/536ede80163e295dc57fff69724caf68f91bb40578b6ac6583a293534849/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fb1a8c5438e0c5ea51afe9c6564f951525795cf432bed0c028c1cb081276685", size = 3297521, upload-time = "2025-08-11T15:50:33.536Z" }, + { url = "https://files.pythonhosted.org/packages/03/b5/cacf432e6f1fc9d156eca0560ac61d4355d2181e751ba8c0cd9cb232c8c1/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db691fa174e8f7036afefe3061bc40ac2b770718be2862bfb03aabae09051aca", size = 3297343, upload-time = "2025-08-11T15:57:51.186Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ba/d4c9b526f18457667de4c024ffbc3a0920c34237b9e9dd298e44c7c00ee5/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2b3b4927d0bc03d02ad883f402d5de201dbc8894ac87d2e981e7d87430e60d", size = 3232113, upload-time = "2025-08-11T15:50:34.949Z" }, + { url = "https://files.pythonhosted.org/packages/aa/79/c0121b12b1b114e2c8a10ea297a8a6d5367bc59081b2be896815154b1163/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d3d9b904ad4a6b175a2de0738248822f5ac410f52c2fd389ada0b5262d6a1e3", size = 3258240, upload-time = "2025-08-11T15:57:52.983Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/a2f9be96fb382f3ba027ad42f00dbe30fdb6ba28cda5f11412eee346bec5/sqlalchemy-2.0.43-cp311-cp311-win32.whl", hash = "sha256:5cda6b51faff2639296e276591808c1726c4a77929cfaa0f514f30a5f6156921", size = 2101248, upload-time = "2025-08-11T15:55:01.855Z" }, + { url = "https://files.pythonhosted.org/packages/ee/13/744a32ebe3b4a7a9c7ea4e57babae7aa22070d47acf330d8e5a1359607f1/sqlalchemy-2.0.43-cp311-cp311-win_amd64.whl", hash = "sha256:c5d1730b25d9a07727d20ad74bc1039bbbb0a6ca24e6769861c1aa5bf2c4c4a8", size = 2126109, upload-time = "2025-08-11T15:55:04.092Z" }, + { url = "https://files.pythonhosted.org/packages/61/db/20c78f1081446095450bdc6ee6cc10045fce67a8e003a5876b6eaafc5cc4/sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24", size = 2134891, upload-time = "2025-08-11T15:51:13.019Z" }, + { url = "https://files.pythonhosted.org/packages/45/0a/3d89034ae62b200b4396f0f95319f7d86e9945ee64d2343dcad857150fa2/sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83", size = 2123061, upload-time = "2025-08-11T15:51:14.319Z" }, + { url = "https://files.pythonhosted.org/packages/cb/10/2711f7ff1805919221ad5bee205971254845c069ee2e7036847103ca1e4c/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9", size = 3320384, upload-time = "2025-08-11T15:52:35.088Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0e/3d155e264d2ed2778484006ef04647bc63f55b3e2d12e6a4f787747b5900/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48", size = 3329648, upload-time = "2025-08-11T15:56:34.153Z" }, + { url = "https://files.pythonhosted.org/packages/5b/81/635100fb19725c931622c673900da5efb1595c96ff5b441e07e3dd61f2be/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687", size = 3258030, upload-time = "2025-08-11T15:52:36.933Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ed/a99302716d62b4965fded12520c1cbb189f99b17a6d8cf77611d21442e47/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe", size = 3294469, upload-time = "2025-08-11T15:56:35.553Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a2/3a11b06715149bf3310b55a98b5c1e84a42cfb949a7b800bc75cb4e33abc/sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d", size = 2098906, upload-time = "2025-08-11T15:55:00.645Z" }, + { url = "https://files.pythonhosted.org/packages/bc/09/405c915a974814b90aa591280623adc6ad6b322f61fd5cff80aeaef216c9/sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a", size = 2126260, upload-time = "2025-08-11T15:55:02.965Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d9/13bdde6521f322861fab67473cec4b1cc8999f3871953531cf61945fad92/sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc", size = 1924759, upload-time = "2025-08-11T15:39:53.024Z" }, ] [package.optional-dependencies] @@ -3155,16 +4732,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/d1/8828ec685022e2b86ebd717743359dd5e8eab70a9c1ebff78aa733848216/sqlean_py-3.49.1-cp312-cp312-win_arm64.whl", hash = "sha256:c2537f69879adaed22b16e840d494c440ece5b49d28a5a51094e6c8ca6a2b0a5", size = 739690, upload-time = "2025-05-02T11:58:02.262Z" }, ] +[[package]] +name = "srsly" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/e8/eb51b1349f50bac0222398af0942613fdc9d1453ae67cbe4bf9936a1a54b/srsly-2.5.1.tar.gz", hash = "sha256:ab1b4bf6cf3e29da23dae0493dd1517fb787075206512351421b89b4fc27c77e", size = 466464, upload-time = "2025-01-17T09:26:26.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/9c/a248bb49de499fe0990e3cb0fb341c2373d8863ef9a8b5799353cade5731/srsly-2.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58f0736794ce00a71d62a39cbba1d62ea8d5be4751df956e802d147da20ecad7", size = 635917, upload-time = "2025-01-17T09:25:25.109Z" }, + { url = "https://files.pythonhosted.org/packages/41/47/1bdaad84502df973ecb8ca658117234cf7fb20e1dec60da71dce82de993f/srsly-2.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8269c40859806d71920396d185f4f38dc985cdb6a28d3a326a701e29a5f629", size = 634374, upload-time = "2025-01-17T09:25:26.609Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2a/d73c71989fcf2a6d1fa518d75322aff4db01a8763f167f8c5e00aac11097/srsly-2.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889905900401fefc1032e22b73aecbed8b4251aa363f632b2d1f86fc16f1ad8e", size = 1108390, upload-time = "2025-01-17T09:25:29.32Z" }, + { url = "https://files.pythonhosted.org/packages/35/a3/9eda9997a8bd011caed18fdaa5ce606714eb06d8dab587ed0522b3e92ab1/srsly-2.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf454755f22589df49c25dc799d8af7b47dce3d861dded35baf0f0b6ceab4422", size = 1110712, upload-time = "2025-01-17T09:25:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ef/4b50bc05d06349f905b27f824cc23b652098efd4be19aead3af4981df647/srsly-2.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cc0607c8a59013a51dde5c1b4e465558728e9e0a35dcfa73c7cbefa91a0aad50", size = 1081244, upload-time = "2025-01-17T09:25:32.611Z" }, + { url = "https://files.pythonhosted.org/packages/90/af/d4a2512d9a5048d2b18efead39d4c4404bddd4972935bbc68211292a736c/srsly-2.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d5421ba3ab3c790e8b41939c51a1d0f44326bfc052d7a0508860fb79a47aee7f", size = 1091692, upload-time = "2025-01-17T09:25:34.15Z" }, + { url = "https://files.pythonhosted.org/packages/bb/da/657a685f63028dcb00ccdc4ac125ed347c8bff6fa0dab6a9eb3dc45f3223/srsly-2.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:b96ea5a9a0d0379a79c46d255464a372fb14c30f59a8bc113e4316d131a530ab", size = 632627, upload-time = "2025-01-17T09:25:37.36Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f6/bebc20d75bd02121fc0f65ad8c92a5dd2570e870005e940faa55a263e61a/srsly-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:683b54ed63d7dfee03bc2abc4b4a5f2152f81ec217bbadbac01ef1aaf2a75790", size = 636717, upload-time = "2025-01-17T09:25:40.236Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e8/9372317a4742c70b87b413335adfcdfb2bee4f88f3faba89fabb9e6abf21/srsly-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:459d987130e57e83ce9e160899afbeb871d975f811e6958158763dd9a8a20f23", size = 634697, upload-time = "2025-01-17T09:25:43.605Z" }, + { url = "https://files.pythonhosted.org/packages/d5/00/c6a7b99ab27b051a27bd26fe1a8c1885225bb8980282bf9cb99f70610368/srsly-2.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:184e3c98389aab68ff04aab9095bd5f1a8e5a72cc5edcba9d733bac928f5cf9f", size = 1134655, upload-time = "2025-01-17T09:25:45.238Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/861459e8241ec3b78c111081bd5efa414ef85867e17c45b6882954468d6e/srsly-2.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c2a3e4856e63b7efd47591d049aaee8e5a250e098917f50d93ea68853fab78", size = 1143544, upload-time = "2025-01-17T09:25:47.485Z" }, + { url = "https://files.pythonhosted.org/packages/2d/85/8448fe874dd2042a4eceea5315cfff3af03ac77ff5073812071852c4e7e2/srsly-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:366b4708933cd8d6025c13c2cea3331f079c7bb5c25ec76fca392b6fc09818a0", size = 1098330, upload-time = "2025-01-17T09:25:52.55Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7e/04d0e1417da140b2ac4053a3d4fcfc86cd59bf4829f69d370bb899f74d5d/srsly-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c8a0b03c64eb6e150d772c5149befbadd981cc734ab13184b0561c17c8cef9b1", size = 1110670, upload-time = "2025-01-17T09:25:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/96/1a/a8cd627eaa81a91feb6ceab50155f4ceff3eef6107916cb87ef796958427/srsly-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:7952538f6bba91b9d8bf31a642ac9e8b9ccc0ccbb309feb88518bfb84bb0dc0d", size = 632598, upload-time = "2025-01-17T09:25:55.499Z" }, +] + [[package]] name = "sse-starlette" -version = "2.4.1" +version = "3.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/3e/eae74d8d33e3262bae0a7e023bb43d8bdd27980aa3557333f4632611151f/sse_starlette-2.4.1.tar.gz", hash = "sha256:7c8a800a1ca343e9165fc06bbda45c78e4c6166320707ae30b416c42da070926", size = 18635, upload-time = "2025-07-06T09:41:33.631Z" } +sdist = { url = "https://files.pythonhosted.org/packages/42/6f/22ed6e33f8a9e76ca0a412405f31abb844b779d52c5f96660766edcd737c/sse_starlette-3.0.2.tar.gz", hash = "sha256:ccd60b5765ebb3584d0de2d7a6e4f745672581de4f5005ab31c3a25d10b52b3a", size = 20985, upload-time = "2025-07-27T09:07:44.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/10/c78f463b4ef22eef8491f218f692be838282cd65480f6e423d7730dfd1fb/sse_starlette-3.0.2-py3-none-any.whl", hash = "sha256:16b7cbfddbcd4eaca11f7b586f3b8a080f1afe952c15813455b162edea619e5a", size = 11297, upload-time = "2025-07-27T09:07:43.268Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/f1/6c7eaa8187ba789a6dd6d74430307478d2a91c23a5452ab339b6fbe15a08/sse_starlette-2.4.1-py3-none-any.whl", hash = "sha256:08b77ea898ab1a13a428b2b6f73cfe6d0e607a7b4e15b9bb23e4a37b087fd39a", size = 10824, upload-time = "2025-07-06T09:41:32.321Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] [[package]] @@ -3179,6 +4795,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload-time = "2025-04-13T13:56:16.21Z" }, ] +[[package]] +name = "stdlib-list" +version = "0.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/09/8d5c564931ae23bef17420a6c72618463a59222ca4291a7dd88de8a0d490/stdlib_list-0.11.1.tar.gz", hash = "sha256:95ebd1d73da9333bba03ccc097f5bac05e3aa03e6822a0c0290f87e1047f1857", size = 60442, upload-time = "2025-02-18T15:39:38.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c7/4102536de33c19d090ed2b04e90e7452e2e3dc653cf3323208034eaaca27/stdlib_list-0.11.1-py3-none-any.whl", hash = "sha256:9029ea5e3dfde8cd4294cfd4d1797be56a67fc4693c606181730148c3fd1da29", size = 83620, upload-time = "2025-02-18T15:39:37.02Z" }, +] + [[package]] name = "strawberry-graphql" version = "0.243.1" @@ -3193,6 +4818,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/b4/7c4b5ccce02f111883a56fc1331506bc4a12824a037db7f2904e0d26eb6c/strawberry_graphql-0.243.1-py3-none-any.whl", hash = "sha256:7c4ddb97cd424fa23a540816cb169ef760822c9acfb63901e6717042bcda6cfe", size = 306031, upload-time = "2024-09-26T12:24:22.338Z" }, ] +[[package]] +name = "striprtf" +version = "0.0.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/20/3d419008265346452d09e5dadfd5d045b64b40d8fc31af40588e6c76997a/striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa", size = 6258, upload-time = "2023-07-20T14:30:36.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/cf/0fea4f4ba3fc2772ac2419278aa9f6964124d4302117d61bc055758e000c/striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb", size = 6914, upload-time = "2023-07-20T14:30:35.338Z" }, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, +] + [[package]] name = "tantivy" version = "0.22.2" @@ -3213,11 +4856,43 @@ wheels = [ [[package]] name = "tenacity" -version = "9.1.2" +version = "8.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/32/6c/57df6196ce52c464cf8556e8f697fec5d3469bb8cd319c1685c0a090e0b4/tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2", size = 43608, upload-time = "2024-05-07T08:48:17.099Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/a1/6bb0cbebefb23641f068bb58a2bc56da9beb2b1c550242e3c540b37698f3/tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185", size = 25934, upload-time = "2024-05-07T08:48:14.696Z" }, +] + +[[package]] +name = "thinc" +version = "8.3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +dependencies = [ + { name = "blis" }, + { name = "catalogue" }, + { name = "confection" }, + { name = "cymem" }, + { name = "murmurhash" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "setuptools" }, + { name = "srsly" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ff/60c9bcfe28e56c905aac8e61a838c7afe5dc3073c9beed0b63a26ace0bb7/thinc-8.3.4.tar.gz", hash = "sha256:b5925482498bbb6dca0771e375b35c915818f735891e93d93a662dab15f6ffd8", size = 193903, upload-time = "2025-01-13T12:47:51.698Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, + { url = "https://files.pythonhosted.org/packages/85/47/68187c78a04cdc31cbd3ae393068f994b60476b5ecac6dfe7d04b124aacf/thinc-8.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8bb4b47358a1855803b375f4432cefdf373f46ef249b554418d2e77c7323040", size = 839320, upload-time = "2025-01-13T12:47:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/49/ea/066dd415e61fcef20083bbca41c2c02e640fea71326531f2619708efee1e/thinc-8.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:00ed92f9a34b9794f51fcd48467c863f4eb7c5b41559aef6ef3c980c21378fec", size = 774196, upload-time = "2025-01-13T12:47:15.315Z" }, + { url = "https://files.pythonhosted.org/packages/8c/68/36c1a92a374891e0d496677c59f5f9fdc1e57bbb214c487bb8bb3e9290c2/thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85691fca84a6a1506f7ddbd2c1706a5524d56f65582e76b2e260a06d9e83e86d", size = 3922504, upload-time = "2025-01-13T12:47:22.07Z" }, + { url = "https://files.pythonhosted.org/packages/ec/8a/48e463240a586e91f83c87660986e520aa91fbd839f6631ee9bc0fbb3cbd/thinc-8.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eae1573fc19e514defc1bfd4f93f0b4bfc1dcefdb6d70bad1863825747f24800", size = 4932946, upload-time = "2025-01-13T12:47:24.177Z" }, + { url = "https://files.pythonhosted.org/packages/d9/98/f910b8d8113ab9b955a68e9bbf0d5bd0e828f22dd6d3c226af6ec3970817/thinc-8.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:81e8638f9bdc38e366674acc4b63cf7c6267266a15477963a5db21b3d9f1aa36", size = 1490133, upload-time = "2025-01-13T12:47:26.152Z" }, + { url = "https://files.pythonhosted.org/packages/90/ff/d1b5d7e1a7f95581e9a736f50a5a9aff72327ddbbc629a68070c36acefd9/thinc-8.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c9da6375b106df5186bd2bfd1273bc923c01ab7d482f8942e4ee528a28965c3a", size = 825099, upload-time = "2025-01-13T12:47:27.881Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0b/d207c917886dc40671361de0880ec3ea0443a718aae9dbb0a50ac0849f92/thinc-8.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:07091c6b5faace50857c4cf0982204969d77388d0a6f156dd2442297dceeb838", size = 761024, upload-time = "2025-01-13T12:47:29.739Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a3/3ec5e9d7cbebc3257b8223a3d188216b91ab6ec1e66b6fdd99d22394bc62/thinc-8.3.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd40ad71bcd8b1b9daa0462e1255b1c1e86e901c2fd773966601f44a95878032", size = 3710390, upload-time = "2025-01-13T12:47:33.019Z" }, + { url = "https://files.pythonhosted.org/packages/40/ee/955c74e4e6ff2f694c99dcbbf7be8d478a8868503aeb3474517277c07667/thinc-8.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb10823b3a3f1c6440998b11bf9a3571dd859feaed0fdb510a1c1097d9dc6a86", size = 4731524, upload-time = "2025-01-13T12:47:35.203Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/3786431e5c1eeebed3d7a4c97122896ca6d4a502b03d02c2171c417052fd/thinc-8.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:b5e5e7bf5dae142fd50ed9785971292c4aab4d9ed18e4947653b6a0584d5227c", size = 1455883, upload-time = "2025-01-13T12:47:36.914Z" }, ] [[package]] @@ -3231,26 +4906,66 @@ wheels = [ [[package]] name = "tiktoken" -version = "0.9.0" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "regex" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/cf/756fedf6981e82897f2d570dd25fa597eb3f4459068ae0572d7e888cfd6f/tiktoken-0.9.0.tar.gz", hash = "sha256:d02a5ca6a938e0490e1ff957bc48c8b078c88cb83977be1625b1fd8aac792c5d", size = 35991, upload-time = "2025-02-14T06:03:01.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648, upload-time = "2025-08-08T23:58:08.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/91/912b459799a025d2842566fe1e902f7f50d54a1ce8a0f236ab36b5bd5846/tiktoken-0.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4ae374c46afadad0f501046db3da1b36cd4dfbfa52af23c998773682446097cf", size = 1059743, upload-time = "2025-08-08T23:57:37.516Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/6faa6870489ce64f5f75dcf91512bf35af5864583aee8fcb0dcb593121f5/tiktoken-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25a512ff25dc6c85b58f5dd4f3d8c674dc05f96b02d66cdacf628d26a4e4866b", size = 999334, upload-time = "2025-08-08T23:57:38.595Z" }, + { url = "https://files.pythonhosted.org/packages/a1/3e/a05d1547cf7db9dc75d1461cfa7b556a3b48e0516ec29dfc81d984a145f6/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2130127471e293d385179c1f3f9cd445070c0772be73cdafb7cec9a3684c0458", size = 1129402, upload-time = "2025-08-08T23:57:39.627Z" }, + { url = "https://files.pythonhosted.org/packages/34/9a/db7a86b829e05a01fd4daa492086f708e0a8b53952e1dbc9d380d2b03677/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e43022bf2c33f733ea9b54f6a3f6b4354b909f5a73388fb1b9347ca54a069c", size = 1184046, upload-time = "2025-08-08T23:57:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/bb/52edc8e078cf062ed749248f1454e9e5cfd09979baadb830b3940e522015/tiktoken-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:adb4e308eb64380dc70fa30493e21c93475eaa11669dea313b6bbf8210bfd013", size = 1244691, upload-time = "2025-08-08T23:57:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/884b6cd7ae2570ecdcaffa02b528522b18fef1cbbfdbcaa73799807d0d3b/tiktoken-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:ece6b76bfeeb61a125c44bbefdfccc279b5288e6007fbedc0d32bfec602df2f2", size = 884392, upload-time = "2025-08-08T23:57:43.628Z" }, + { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199, upload-time = "2025-08-08T23:57:45.076Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655, upload-time = "2025-08-08T23:57:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867, upload-time = "2025-08-08T23:57:47.438Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308, upload-time = "2025-08-08T23:57:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301, upload-time = "2025-08-08T23:57:49.642Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282, upload-time = "2025-08-08T23:57:50.759Z" }, +] + +[[package]] +name = "tldextract" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "idna" }, + { name = "requests" }, + { name = "requests-file" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/78/182641ea38e3cfd56e9c7b3c0d48a53d432eea755003aa544af96403d4ac/tldextract-5.3.0.tar.gz", hash = "sha256:b3d2b70a1594a0ecfa6967d57251527d58e00bb5a91a74387baa0d87a0678609", size = 128502, upload-time = "2025-04-22T06:19:37.491Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/ae/4613a59a2a48e761c5161237fc850eb470b4bb93696db89da51b79a871f1/tiktoken-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e", size = 1065987, upload-time = "2025-02-14T06:02:14.174Z" }, - { url = "https://files.pythonhosted.org/packages/3f/86/55d9d1f5b5a7e1164d0f1538a85529b5fcba2b105f92db3622e5d7de6522/tiktoken-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45556bc41241e5294063508caf901bf92ba52d8ef9222023f83d2483a3055348", size = 1009155, upload-time = "2025-02-14T06:02:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/03/58/01fb6240df083b7c1916d1dcb024e2b761213c95d576e9f780dfb5625a76/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33", size = 1142898, upload-time = "2025-02-14T06:02:16.666Z" }, - { url = "https://files.pythonhosted.org/packages/b1/73/41591c525680cd460a6becf56c9b17468d3711b1df242c53d2c7b2183d16/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b3d80aad8d2c6b9238fc1a5524542087c52b860b10cbf952429ffb714bc1136", size = 1197535, upload-time = "2025-02-14T06:02:18.595Z" }, - { url = "https://files.pythonhosted.org/packages/7d/7c/1069f25521c8f01a1a182f362e5c8e0337907fae91b368b7da9c3e39b810/tiktoken-0.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b2a21133be05dc116b1d0372af051cd2c6aa1d2188250c9b553f9fa49301b336", size = 1259548, upload-time = "2025-02-14T06:02:20.729Z" }, - { url = "https://files.pythonhosted.org/packages/6f/07/c67ad1724b8e14e2b4c8cca04b15da158733ac60136879131db05dda7c30/tiktoken-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:11a20e67fdf58b0e2dea7b8654a288e481bb4fc0289d3ad21291f8d0849915fb", size = 893895, upload-time = "2025-02-14T06:02:22.67Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e5/21ff33ecfa2101c1bb0f9b6df750553bd873b7fb532ce2cb276ff40b197f/tiktoken-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e88f121c1c22b726649ce67c089b90ddda8b9662545a8aeb03cfef15967ddd03", size = 1065073, upload-time = "2025-02-14T06:02:24.768Z" }, - { url = "https://files.pythonhosted.org/packages/8e/03/a95e7b4863ee9ceec1c55983e4cc9558bcfd8f4f80e19c4f8a99642f697d/tiktoken-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6600660f2f72369acb13a57fb3e212434ed38b045fd8cc6cdd74947b4b5d210", size = 1008075, upload-time = "2025-02-14T06:02:26.92Z" }, - { url = "https://files.pythonhosted.org/packages/40/10/1305bb02a561595088235a513ec73e50b32e74364fef4de519da69bc8010/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95e811743b5dfa74f4b227927ed86cbc57cad4df859cb3b643be797914e41794", size = 1140754, upload-time = "2025-02-14T06:02:28.124Z" }, - { url = "https://files.pythonhosted.org/packages/1b/40/da42522018ca496432ffd02793c3a72a739ac04c3794a4914570c9bb2925/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99376e1370d59bcf6935c933cb9ba64adc29033b7e73f5f7569f3aad86552b22", size = 1196678, upload-time = "2025-02-14T06:02:29.845Z" }, - { url = "https://files.pythonhosted.org/packages/5c/41/1e59dddaae270ba20187ceb8aa52c75b24ffc09f547233991d5fd822838b/tiktoken-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:badb947c32739fb6ddde173e14885fb3de4d32ab9d8c591cbd013c22b4c31dd2", size = 1259283, upload-time = "2025-02-14T06:02:33.838Z" }, - { url = "https://files.pythonhosted.org/packages/5b/64/b16003419a1d7728d0d8c0d56a4c24325e7b10a21a9dd1fc0f7115c02f0a/tiktoken-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:5a62d7a25225bafed786a524c1b9f0910a1128f4232615bf3f8257a73aaa3b16", size = 894897, upload-time = "2025-02-14T06:02:36.265Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/ea488ef48f2f544566947ced88541bc45fae9e0e422b2edbf165ee07da99/tldextract-5.3.0-py3-none-any.whl", hash = "sha256:f70f31d10b55c83993f55e91ecb7c5d84532a8972f22ec578ecfbe5ea2292db2", size = 107384, upload-time = "2025-04-22T06:19:36.304Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/2f/402986d0823f8d7ca139d969af2917fefaa9b947d1fb32f6168c509f2492/tokenizers-0.21.4.tar.gz", hash = "sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880", size = 351253, upload-time = "2025-07-28T15:48:54.325Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/c6/fdb6f72bf6454f52eb4a2510be7fb0f614e541a2554d6210e370d85efff4/tokenizers-0.21.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133", size = 2863987, upload-time = "2025-07-28T15:48:44.877Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a6/28975479e35ddc751dc1ddc97b9b69bf7fcf074db31548aab37f8116674c/tokenizers-0.21.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60", size = 2732457, upload-time = "2025-07-28T15:48:43.265Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/24f39d7b5c726b7b0be95dca04f344df278a3fe3a4deb15a975d194cbb32/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5", size = 3012624, upload-time = "2025-07-28T13:22:43.895Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/26358925717687a58cb74d7a508de96649544fad5778f0cd9827398dc499/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6", size = 2939681, upload-time = "2025-07-28T13:22:47.499Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/cc300fea5db2ab5ddc2c8aea5757a27b89c84469899710c3aeddc1d39801/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9", size = 3247445, upload-time = "2025-07-28T15:48:39.711Z" }, + { url = "https://files.pythonhosted.org/packages/be/bf/98cb4b9c3c4afd8be89cfa6423704337dc20b73eb4180397a6e0d456c334/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732", size = 3428014, upload-time = "2025-07-28T13:22:49.569Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/96c1cc780e6ca7f01a57c13235dd05b7bc1c0f3588512ebe9d1331b5f5ae/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2", size = 3193197, upload-time = "2025-07-28T13:22:51.471Z" }, + { url = "https://files.pythonhosted.org/packages/f2/90/273b6c7ec78af547694eddeea9e05de771278bd20476525ab930cecaf7d8/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff", size = 3115426, upload-time = "2025-07-28T15:48:41.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/43/c640d5a07e95f1cf9d2c92501f20a25f179ac53a4f71e1489a3dcfcc67ee/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2", size = 9089127, upload-time = "2025-07-28T15:48:46.472Z" }, + { url = "https://files.pythonhosted.org/packages/44/a1/dd23edd6271d4dca788e5200a807b49ec3e6987815cd9d0a07ad9c96c7c2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78", size = 9055243, upload-time = "2025-07-28T15:48:48.539Z" }, + { url = "https://files.pythonhosted.org/packages/21/2b/b410d6e9021c4b7ddb57248304dc817c4d4970b73b6ee343674914701197/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b", size = 9298237, upload-time = "2025-07-28T15:48:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/42348c995c67e2e6e5c89ffb9cfd68507cbaeb84ff39c49ee6e0a6dd0fd2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24", size = 9461980, upload-time = "2025-07-28T15:48:52.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/dacccd834404cd71b5c334882f3ba40331ad2120e69ded32cf5fda9a7436/tokenizers-0.21.4-cp39-abi3-win32.whl", hash = "sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0", size = 2329871, upload-time = "2025-07-28T15:48:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, ] [[package]] @@ -3300,6 +5015,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/98/eb27cc78ad3af8e302c9d8ff4977f5026676e130d28dd7578132a457170c/toolz-1.0.0-py3-none-any.whl", hash = "sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236", size = 56383, upload-time = "2024-10-04T16:17:01.533Z" }, ] +[[package]] +name = "tornado" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/ce/1eb500eae19f4648281bb2186927bb062d2438c2e5093d1360391afd2f90/tornado-6.5.2.tar.gz", hash = "sha256:ab53c8f9a0fa351e2c0741284e06c7a45da86afb544133201c5cc8578eb076a0", size = 510821, upload-time = "2025-08-08T18:27:00.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/48/6a7529df2c9cc12efd2e8f5dd219516184d703b34c06786809670df5b3bd/tornado-6.5.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:2436822940d37cde62771cff8774f4f00b3c8024fe482e16ca8387b8a2724db6", size = 442563, upload-time = "2025-08-08T18:26:42.945Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b5/9b575a0ed3e50b00c40b08cbce82eb618229091d09f6d14bce80fc01cb0b/tornado-6.5.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:583a52c7aa94ee046854ba81d9ebb6c81ec0fd30386d96f7640c96dad45a03ef", size = 440729, upload-time = "2025-08-08T18:26:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4e/619174f52b120efcf23633c817fd3fed867c30bff785e2cd5a53a70e483c/tornado-6.5.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0fe179f28d597deab2842b86ed4060deec7388f1fd9c1b4a41adf8af058907e", size = 444295, upload-time = "2025-08-08T18:26:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/95/fa/87b41709552bbd393c85dd18e4e3499dcd8983f66e7972926db8d96aa065/tornado-6.5.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b186e85d1e3536d69583d2298423744740986018e393d0321df7340e71898882", size = 443644, upload-time = "2025-08-08T18:26:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/fb15f06e33d7430ca89420283a8762a4e6b8025b800ea51796ab5e6d9559/tornado-6.5.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e792706668c87709709c18b353da1f7662317b563ff69f00bab83595940c7108", size = 443878, upload-time = "2025-08-08T18:26:50.599Z" }, + { url = "https://files.pythonhosted.org/packages/11/92/fe6d57da897776ad2e01e279170ea8ae726755b045fe5ac73b75357a5a3f/tornado-6.5.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:06ceb1300fd70cb20e43b1ad8aaee0266e69e7ced38fa910ad2e03285009ce7c", size = 444549, upload-time = "2025-08-08T18:26:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/c8f4f6c9204526daf3d760f4aa555a7a33ad0e60843eac025ccfd6ff4a93/tornado-6.5.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:74db443e0f5251be86cbf37929f84d8c20c27a355dd452a5cfa2aada0d001ec4", size = 443973, upload-time = "2025-08-08T18:26:53.625Z" }, + { url = "https://files.pythonhosted.org/packages/ae/2d/f5f5707b655ce2317190183868cd0f6822a1121b4baeae509ceb9590d0bd/tornado-6.5.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b5e735ab2889d7ed33b32a459cac490eda71a1ba6857b0118de476ab6c366c04", size = 443954, upload-time = "2025-08-08T18:26:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/e8/59/593bd0f40f7355806bf6573b47b8c22f8e1374c9b6fd03114bd6b7a3dcfd/tornado-6.5.2-cp39-abi3-win32.whl", hash = "sha256:c6f29e94d9b37a95013bb669616352ddb82e3bfe8326fccee50583caebc8a5f0", size = 445023, upload-time = "2025-08-08T18:26:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/f609b420c2f564a748a2d80ebfb2ee02a73ca80223af712fca591386cafb/tornado-6.5.2-cp39-abi3-win_amd64.whl", hash = "sha256:e56a5af51cc30dd2cae649429af65ca2f6571da29504a07995175df14c18f35f", size = 445427, upload-time = "2025-08-08T18:26:57.91Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e1f65e8f8c76d73658b33d33b81eed4322fb5085350e4328d5c956f0c8f9/tornado-6.5.2-cp39-abi3-win_arm64.whl", hash = "sha256:d6c33dc3672e3a1f3618eb63b7ef4683a7688e7b9e6e8f0d9aa5726360a004af", size = 444456, upload-time = "2025-08-08T18:26:59.207Z" }, +] + [[package]] name = "tqdm" version = "4.67.1" @@ -3373,13 +5107,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/58/0262e875dd899447476a8ffde7829df3716ffa772990095c65d6de1f053c/tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b", size = 8268983, upload-time = "2024-02-04T10:29:00.987Z" }, ] +[[package]] +name = "typer" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/78/d90f616bf5f88f8710ad067c1f8705bf7618059836ca084e5bb2a0855d75/typer-0.16.1.tar.gz", hash = "sha256:d358c65a464a7a90f338e3bb7ff0c74ac081449e53884b12ba658cbd72990614", size = 102836, upload-time = "2025-08-18T19:18:22.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/76/06dbe78f39b2203d2a47d5facc5df5102d0561e2807396471b5f7c5a30a1/typer-0.16.1-py3-none-any.whl", hash = "sha256:90ee01cb02d9b8395ae21ee3368421faf21fa138cb2a541ed369c08cec5237c9", size = 46397, upload-time = "2025-08-18T19:18:21.663Z" }, +] + [[package]] name = "typing-extensions" -version = "4.14.1" +version = "4.15.0rc1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/d5/1b2b1803ecd99e11a842d054c6b6f3f7938dde30d42b4d0d99611b2ee6fd/typing_extensions-4.15.0rc1.tar.gz", hash = "sha256:49b001798e59fbb7a523f0d36e8cf2d82d8e3f9fc41b04ff958da1ed7cc3b671", size = 109126, upload-time = "2025-08-18T14:31:09.022Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, + { url = "https://files.pythonhosted.org/packages/11/61/1e821439fa89ca57c8a31e285742b7aeb719c8068e06074717b398642fb7/typing_extensions-4.15.0rc1-py3-none-any.whl", hash = "sha256:8fd4191376831cd3503df0cf06a0c0e6c1dae08ea3e6af770a785eeb90934dea", size = 44640, upload-time = "2025-08-18T14:31:07.386Z" }, ] [[package]] @@ -3531,7 +5280,6 @@ dependencies = [ { name = "aiofiles" }, { name = "aiohttp-client-cache" }, { name = "aioresponses" }, - { name = "aiqtoolkit", extra = ["langchain", "profiling", "telemetry"] }, { name = "brotli" }, { name = "cvss" }, { name = "esprima" }, @@ -3541,6 +5289,7 @@ dependencies = [ { name = "json5" }, { name = "nbformat" }, { name = "nemollm" }, + { name = "nvidia-nat", extra = ["langchain", "profiling", "telemetry"] }, { name = "openinference-instrumentation-langchain" }, { name = "ordered-set" }, { name = "pydpkg" }, @@ -3574,9 +5323,8 @@ requires-dist = [ { name = "aiofiles" }, { name = "aiohttp-client-cache", specifier = "==0.11" }, { name = "aioresponses", specifier = "==0.7.6" }, - { name = "aiqtoolkit", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0a2,<1.3.0" }, { name = "brotli" }, - { name = "cvss", specifier = "==3.6" }, + { name = "cvss" }, { name = "esprima" }, { name = "faiss-cpu", specifier = "==1.9.0" }, { name = "gitpython" }, @@ -3584,6 +5332,7 @@ requires-dist = [ { name = "json5" }, { name = "nbformat" }, { name = "nemollm" }, + { name = "nvidia-nat", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0rc8,<1.3.0" }, { name = "openinference-instrumentation-langchain", specifier = "~=0.1.31" }, { name = "ordered-set" }, { name = "pydpkg", specifier = "==1.9.4" }, @@ -3612,6 +5361,47 @@ dev = [ { name = "yapf", specifier = "==0.43.*" }, ] +[[package]] +name = "wandb" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/69/217598886af89350e36bc05c092a67c9c469cff1fd6446edd4c879027e36/wandb-0.21.1.tar.gz", hash = "sha256:753bbdaa3a7703344056e019425b39c17a3d31d8ca0c4d13c4efc046935b08b9", size = 40131395, upload-time = "2025-08-07T18:52:48.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/d0/589f970741f3ead9ad28d4cbb668d1e6a39848df767f004ac9c7bed8f4b5/wandb-0.21.1-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:96f9eedeae428de0d88f9751fb81f1b730ae7902f35c2f5a7a904d7733f124f3", size = 21701698, upload-time = "2025-08-07T18:52:22.399Z" }, + { url = "https://files.pythonhosted.org/packages/41/6c/a6140a0f395a99902aafdfe63088b7aff509e4f14cd7dd084d47eab36f27/wandb-0.21.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41a1ec1b98d9d7e1bcafc483bce82e184b6cbae7531328a0fe8dd0f56d96a92e", size = 21221046, upload-time = "2025-08-07T18:52:26.134Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/dacbb30ed35141d48a387d84f2e792d4b61b5bcdbf5ffdbd3f0b57beb346/wandb-0.21.1-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:f74d4691c38318ed8611e00ca3246b4152a03ff390fdce41816bea5705452a73", size = 21885803, upload-time = "2025-08-07T18:52:28.489Z" }, + { url = "https://files.pythonhosted.org/packages/b0/48/3a7290a33b1f64e29ac8779dab4d4cdef31a9ed3c3d9ea656a4507d64332/wandb-0.21.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c8fbd60b9abf4b9bec201f311602f61394d41a3503c801750b03975a5e36d1b", size = 20825318, upload-time = "2025-08-07T18:52:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/a9/54/c0a087114ff1bb6c32e64aaa58aea4342cebc0ad58b1378c0a5a831d2508/wandb-0.21.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ded9313672630c0630f5b13c598ce9aa0e932e811ebc18823fcc4d73acfb6bb", size = 22362500, upload-time = "2025-08-07T18:52:33.889Z" }, + { url = "https://files.pythonhosted.org/packages/65/68/3aae277ea9fb5d91eec066cf256755bed3a740d92b539888a7ce36cf3f6c/wandb-0.21.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:44f3194d697b409f91708c50c5f9d56e282434a0d60ac380b64f0fb6991cd630", size = 20830372, upload-time = "2025-08-07T18:52:36.76Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bb/58d206e79be1f279ef06cb934ae1e208bcacd2cd73b7a7652236575010d6/wandb-0.21.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0b68bb6dbe94f1910c665c755f438292df40c272feb1a8b42208c1df52cce26", size = 22438521, upload-time = "2025-08-07T18:52:39.672Z" }, + { url = "https://files.pythonhosted.org/packages/e7/b8/dfe01f8e4c40d5dda820fd839c39431608a3453670f79404fa28915972d2/wandb-0.21.1-py3-none-win32.whl", hash = "sha256:98306c3fb369dfafb7194270b938b000ea2bb08dbddff10c19b5a805fd5cab80", size = 21569814, upload-time = "2025-08-07T18:52:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/51/ba/81c77d5d831fcddb89661c85175fcbb91d2ffecf6b0591972829da3eb42f/wandb-0.21.1-py3-none-win_amd64.whl", hash = "sha256:8be92a7e92b5cb5ce00ec0961f9dbaad7757ffdbc5b5a8f2cc7188e23f653f0a", size = 21569817, upload-time = "2025-08-07T18:52:45.559Z" }, +] + +[[package]] +name = "wasabi" +version = "1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/f9/054e6e2f1071e963b5e746b48d1e3727470b2a490834d18ad92364929db3/wasabi-1.1.3.tar.gz", hash = "sha256:4bb3008f003809db0c3e28b4daf20906ea871a2bb43f9914197d540f4f2e0878", size = 30391, upload-time = "2024-05-31T16:56:18.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/7c/34330a89da55610daa5f245ddce5aab81244321101614751e7537f125133/wasabi-1.1.3-py3-none-any.whl", hash = "sha256:f76e16e8f7e79f8c4c8be49b4024ac725713ab10cd7f19350ad18a8e3f71728c", size = 27880, upload-time = "2024-05-31T16:56:16.699Z" }, +] + [[package]] name = "watchfiles" version = "1.1.0" @@ -3653,6 +5443,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/d3/254cea30f918f489db09d6a8435a7de7047f8cb68584477a515f160541d6/watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923fec6e5461c42bd7e3fd5ec37492c6f3468be0499bc0707b4bbbc16ac21792", size = 454009, upload-time = "2025-06-15T19:06:52.896Z" }, ] +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, +] + +[[package]] +name = "weasel" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpathlib" }, + { name = "confection" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "srsly" }, + { name = "typer" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/1a/9c522dd61b52939c217925d3e55c95f9348b73a66a956f52608e1e59a2c0/weasel-0.4.1.tar.gz", hash = "sha256:aabc210f072e13f6744e5c3a28037f93702433405cd35673f7c6279147085aa9", size = 38417, upload-time = "2024-05-15T08:52:54.765Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/87/abd57374044e1f627f0a905ac33c1a7daab35a3a815abfea4e1bafd3fdb1/weasel-0.4.1-py3-none-any.whl", hash = "sha256:24140a090ea1ac512a2b2f479cc64192fd1d527a7f3627671268d08ed5ac418c", size = 50270, upload-time = "2024-05-15T08:52:52.977Z" }, +] + +[[package]] +name = "weave" +version = "0.51.59" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "diskcache" }, + { name = "eval-type-backport" }, + { name = "gql", extra = ["aiohttp", "requests"] }, + { name = "jsonschema" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "polyfile-weave" }, + { name = "pydantic" }, + { name = "rich" }, + { name = "sentry-sdk" }, + { name = "tenacity" }, + { name = "wandb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/53/1b0350a64837df3e29eda6149a542f3a51e706122086f82547153820e982/weave-0.51.59.tar.gz", hash = "sha256:fad34c0478f3470401274cba8fa2bfd45d14a187db0a5724bd507e356761b349", size = 480572, upload-time = "2025-07-25T22:05:07.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/bc/fa5ffb887a1ee28109b29c62416c9e0f41da8e75e6871671208b3d42b392/weave-0.51.59-py3-none-any.whl", hash = "sha256:2238578574ecdf6285efdf028c78987769720242ac75b7b84b1dbc59060468ce", size = 612468, upload-time = "2025-07-25T22:05:05.088Z" }, +] + [[package]] name = "websockets" version = "15.0.1" From 84c3b50681f0f4b39ac37faf56c84856388a95a3 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 7 Sep 2025 11:35:25 +0300 Subject: [PATCH 060/286] chore: add discriptive instructions for auth token Signed-off-by: Ilona Shishov --- kustomize/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 14ac7488c..fe819be53 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -73,13 +73,13 @@ oc create secret generic exploit-iq-pull-secret --from-file=.dockerconfigjson=

:". cat > base/image-registry-credentials.env << 'EOF' { "auths": { - "registry.example.io": {"auth": ""}, - "example.io": {"auth": ""}, - "example.docker.io": {"auth": ""} + "registry.example.io": {"auth": ""}, + "example.io": {"auth": ""}, + "example.docker.io": {"auth": ""} } } EOF From 44103e9e1a9d9933fee5c38e7fa7eb682396275e Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 2 Sep 2025 16:58:30 +0300 Subject: [PATCH 061/286] ci: trigger PipelineRun on non-draft PRs only Signed-off-by: Vladimir Belousov --- .tekton/on-pull-request.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 4152e628c..e23626173 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -4,14 +4,12 @@ kind: PipelineRun metadata: name: vulnerability-analysis-on-pr annotations: - # The event we are targeting as seen from the webhook payload - # this can be an array too, i.e: [pull_request, push] - pipelinesascode.tekton.dev/on-event: "[pull_request]" - - # The branch or tag we are targeting (ie: main, refs/tags/*) - pipelinesascode.tekton.dev/on-target-branch: "[main, rh-aiq-main]" - # Paths that trigger the pipeline on change - pipelinesascode.tekton.dev/on-path-change: "[src/**, metrics_lib/**, pyproject.toml, Dockerfile, .dockerignore]" + # Trigger on non-draft PRs to target branches when specific paths are changed. + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" && + !body.pull_request.draft && + (target_branch == "main" || target_branch == "rh-aiq-main") && + ("src/**".pathChanged() || "metrics_lib/**".pathChanged() || "pyproject.toml".pathChanged() || "uv.lock".pathChanged() || "Dockerfile".pathChanged() || ".dockerignore".pathChanged()) # Fetch the git-clone task from hub, we are able to reference later on it # with taskRef and it will automatically be embedded into our pipeline. pipelinesascode.tekton.dev/task: "git-clone" From bd65b382f878af569f6d710d4c15ae2ea235c9a8 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 7 Sep 2025 18:01:26 +0300 Subject: [PATCH 062/286] chore: add GH PAT env Signed-off-by: Ilona Shishov --- kustomize/base/exploit_iq_client.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index 679015523..95a816fa8 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -70,6 +70,11 @@ spec: value: /config/excludes.json - name: DOCKER_CONFIG value: /tmp/.docker + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: exploit-iq-secret + key: ghsa_api_key volumeMounts: From 0d122de007c6906cbe22b5702960de59b71be2f5 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 7 Sep 2025 19:46:23 +0300 Subject: [PATCH 063/286] chore: fix crash of agent cvss stage in self hosted variant deployment Signed-off-by: Zvi Grinberg --- .../overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml index 83dfeb97e..f44ae4703 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -131,6 +131,10 @@ patchesStrategicMerge: - name: INTEL_SOURCE_SCORE_MODEL_NAME value: *model-name + - name: GENERATE_CVSS_MODEL_NAME + value: *model-name + + From 72e9859164df030176524da3d95dc30b203d4dcd Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf Date: Mon, 1 Sep 2025 14:36:43 +0300 Subject: [PATCH 064/286] feat: Add python support for transitive tool --- .../tools/transitive_code_search.py | 7 +- .../utils/chain_of_calls_retriever.py | 118 +++-- src/vuln_analysis/utils/dep_tree.py | 204 ++++++++- src/vuln_analysis/utils/document_embedding.py | 8 +- .../golang_functions_parsers.py | 43 +- .../lang_functions_parsers.py | 42 +- .../lang_functions_parsers_factory.py | 3 + .../python_functions_parser.py | 405 ++++++++++++++++++ src/vuln_analysis/utils/llm_engine_utils.py | 1 - .../python_segmenters_with_claases_methods.py | 50 +++ .../python_segmenters_with_classes_methods.py | 49 +++ .../utils/transitive_code_searcher_tool.py | 4 +- 12 files changed, 851 insertions(+), 83 deletions(-) create mode 100644 src/vuln_analysis/utils/functions_parsers/python_functions_parser.py create mode 100644 src/vuln_analysis/utils/python_segmenters_with_claases_methods.py create mode 100644 src/vuln_analysis/utils/python_segmenters_with_classes_methods.py diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index c102154d2..ec093470d 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from vuln_analysis.runtime_context import ctx_state from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from aiq.builder.builder import Builder @@ -53,7 +55,10 @@ def get_call_of_chains_retriever(documents_embedder, si): if source_info.type == "code": git_repo = documents_embedder.get_repo_path(source_info) documents = documents_embedder.collect_documents(source_info) - coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=Ecosystem.GO, manifest_path=git_repo) + with open(os.path.join(git_repo, 'ecosystem_data.txt'), 'r') as file: + ecosystem = file.read() + ecosystem = Ecosystem[ecosystem] + coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=ecosystem, manifest_path=git_repo) return coc_retriever diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 44774f413..e290c619a 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -35,7 +35,7 @@ def get_functions_for_package(package_name: str, documents: list[Document], lang if sources_location_packages: for document in documents: doc_extension = get_extension_of_file(document.metadata.get('source')) - if (document.metadata.get('source').startswith(language_parser.dir_name_for_3rd_party_packages()) and + if (not language_parser.is_root_package(document) and document.metadata.get('content_type') == 'functions_classes' and language_parser.is_function(document) and is_function_callable(document, language_parser, callee_function_file_name) and @@ -48,11 +48,12 @@ def get_functions_for_package(package_name: str, documents: list[Document], lang for document in documents: doc_extension = get_extension_of_file(document.metadata.get('source')) if (language_parser.is_root_package(document) - and document.metadata.get('content_type') == 'functions_classes' - and language_parser.is_function(document) and language_parser.is_supported_file_extensions(doc_extension) and function_called_from_caller_body(document, function_to_search, language_parser)): - yield document + if (language_parser.is_script_language() or + (document.metadata.get('content_type') == 'functions_classes' and + language_parser.is_function(document))): + yield document def is_function_callable(document: Document, language_parser, callee_function_file_name: str) -> bool: @@ -87,6 +88,7 @@ class ChainOfCallsRetriever: documents: Optional[List[Document]] documents_of_full_sources: Optional[dict] documents_of_types: Optional[list] + documents_of_functions: list | None language_parser: Optional[LanguageFunctionsParser] dependency_tree: Optional[DependencyTree] tree_dict: Optional[dict] @@ -126,7 +128,8 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat self.tree_dict = dict() # Build a dependency tree using the dependency tree builder logic. - for package, parents in self.dependency_tree.builder.build_tree(manifest_path=manifest_path).items(): + tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) + for package, parents in tree.items(): parents.extend([package]) self.tree_dict[package] = list() self.tree_dict[package].append(parents) @@ -134,20 +137,22 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat logger.debug("Chain of Calls Retriever - populating functions documents") allowed_files_extensions = self.language_parser.supported_files_extensions() # filter out unsupported files extensions. - filtered_documents = [doc for doc in documents + self.documents = [doc for doc in documents if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) .endswith(ext)])] # filter out types and full code documents, retaining only functions/methods documents in this attribute. - self.documents = [doc for doc in filtered_documents + self.documents_of_functions = [doc for doc in self.documents if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] # filter out full code documents and functions/methods docs, retaining only types/classes docs - self.documents_of_types = [doc for doc in filtered_documents + self.documents_of_types = [doc for doc in self.documents if doc.page_content.startswith(self.language_parser.get_type_reserved_word())] # boolean attribute that indicates whether a path was found or not, initially set to False. self.found_path = False # Filter out all documents but full source docs. - self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in filtered_documents + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in self.documents if doc.metadata.get('content_type') == 'simplified_code'} + if not self.language_parser.is_script_language(): + self.documents = self.documents_of_functions logger.debug("Chain of Calls Retriever - after documents_of_full_sources") self.last_visited_parent_package_indexes = dict() # Constructing a map of types and classes to their attributes/members/fields @@ -155,7 +160,7 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat self.documents_of_types) # Create a data structure containing dict of key=(function_name@source_file),value = dict of # local variables names mapped to (types, (values or expressions)) - self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents) + self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) logger.debug("Chain of Calls Retriever - after functions_local_variables_index") def __find_caller_function(self, document_function: Document, function_package: str) -> Document: @@ -170,12 +175,14 @@ def __find_caller_function(self, document_function: Document, function_package: # gets list of all direct parents of function for package_name in package_names: list_of_packages = self.tree_dict.get(package_name) - if list_of_packages is not None: + if list_of_packages: direct_parents.extend(list_of_packages[PARENTS_INDEX]) # Add same package itself to search path. # direct_parents.extend([function_package]) # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) + if function_name_to_search == self.language_parser.get_constructor_method_name(): + function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() last_visited_package_index = (self.last_visited_parent_package_indexes @@ -183,16 +190,18 @@ def __find_caller_function(self, document_function: Document, function_package: function_name_to_search), 0)) package_exclusions = self.tree_dict.get(function_package)[EXCLUSIONS_INDEX] # Search for caller functions only at parents according to dependency tree. - for package_index, package in enumerate(direct_parents[last_visited_package_index:]): + for package in direct_parents[last_visited_package_index:]: sources_location_packages = True if self.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: sources_location_packages = False + possible_docs = self.get_possible_docs(function_name_to_search, package, + package_exclusions, + sources_location_packages) + # Collect all potential caller functions for doc in get_functions_for_package(package_name=package, - documents=self.get_possible_docs(function_name_to_search, package, - package_exclusions, - sources_location_packages), + documents=possible_docs, language_parser=self.language_parser, sources_location_packages=sources_location_packages, function_to_search=function_name_to_search, @@ -201,7 +210,7 @@ def __find_caller_function(self, document_function: Document, function_package: # Perform the search only on the subset of potential caller functions for doc in relevant_docs_to_search_in: function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, - callee_function= + callee_function_name= function_name_to_search, callee_function_package= function_package, @@ -219,8 +228,8 @@ def __find_caller_function(self, document_function: Document, function_package: # If current document is found to be calling document_function in function_package, then return it as a # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. + package_exclusions.append(doc) if function_is_being_called: - package_exclusions.append(doc) # update index of last scanned package for backtracking # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index @@ -229,6 +238,21 @@ def __find_caller_function(self, document_function: Document, function_package: # If didn't find a matching caller function document, returns None. return None + def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: + """ + Checks if a document is in the exclusions list based on its + function name, function body and source metadata. + """ + doc_function_content = doc.page_content.strip() + doc_source = doc.metadata.get('source').strip() + + for exclusion_doc in exclusions: + exclusion_function_content = exclusion_doc.page_content.strip() + exclusion_source = exclusion_doc.metadata.get('source').strip() + + if doc_function_content == exclusion_function_content and doc_source == exclusion_source: + return True + return False # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all # excluded functions, and all function that their body doesn't contain the target function name to search for. @@ -236,14 +260,14 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio sources_location_packages: bool) \ -> (list[ Document], bool): - flatten_docs_functions_names = [self.language_parser.get_function_name(doc) for doc in exclusions] if sources_location_packages: filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') and self.language_parser.is_function(doc) and - not self.language_parser.get_function_name(doc) in flatten_docs_functions_names] + not self._is_doc_excluded(doc, exclusions)] else: - filter_1 = [doc for doc in self.documents if self.language_parser.is_function(doc) and - not self.language_parser.get_function_name(doc) in flatten_docs_functions_names] + filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and + (self.language_parser.is_function(doc) or self.language_parser.is_script_language()) and + not self._is_doc_excluded(doc, exclusions)] return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] @@ -254,12 +278,17 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" self.found_path = False + query = query.splitlines()[0].strip('"\'') (package_name, function) = tuple(query.split(",")) + class_name = None + splitters = [splitter for splitter in ['.'] if splitter in function] + if splitters: + class_name, function = function.split(splitters[0]) found_package = False matching_documents = [] # Check if input package is in dependency tree for package in self.tree_dict: - if package_name.lower() in package.lower(): + if self.language_parser.is_same_package(package_name, package): package_name = package found_package = True break @@ -267,18 +296,30 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: if found_package: target_function_doc = self.__find_initial_function(function, package_name=package_name, documents=self.documents, + language_parser=self.language_parser, + class_name=class_name) + if not target_function_doc and self.language_parser.get_constructor_method_name(): + target_function_doc = self.__find_initial_function(function_name=self.language_parser.get_constructor_method_name(), + package_name=package_name, + documents=self.documents, + language_parser=self.language_parser, + class_name=function) - language_parser=self.language_parser) # If not, there is a chance that the package is some standard library in the ecosystem. else: # Try to create dummy package for ecosystem standard library function, in such case, build a document for # vulnerable function in standard lib package of language. - target_function_doc = Document(page_content=f"func {function + '()' + '{}'}" + page_content = self.language_parser.get_dummy_function(function) + if class_name: + page_content = page_content + f'\n{self.language_parser.get_comment_line_notation()}(class: {class_name})' + target_function_doc = Document(page_content=page_content , metadata={"source": package_name, "ecosystem": self.ecosystem}) + escaped_package_name = re.escape(package_name) + importing_docs = [value for (file, value) in self.documents_of_full_sources.items() if re.search( - rf"(import {package_name}|import\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + rf"(import {escaped_package_name}|(import\s*\(\s*[\w\s\/.\"-]*{escaped_package_name}[\w\s\/.\"-]*\s*\))|from\s+{escaped_package_name}\s+import)" , value.page_content, flags=re.MULTILINE)] root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() @@ -296,7 +337,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: current_package_name = package_name # If an initial document (that represents the vulnerable input function in the input package) was created # then add it to the path and start constructing path that comprise a chain of calls targeting this function. - if target_function_doc is not None: + if target_function_doc: matching_documents.append(target_function_doc) # Otherwise, don't even start the process as the target function is not in dependencies tree or not a standard # library @@ -304,14 +345,12 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: end_loop = True logger.error(f"Cannot find initial function=${function}, in package=${package_name}") # main loop. - while True: - if end_loop: - break + while not end_loop: # Find a caller function and containing package in the dependency tree according to hierarchy found_document = self.__find_caller_function(document_function=target_function_doc, function_package=current_package_name) # If found, then add it to path - if found_document is not None: + if found_document: matching_documents.append(found_document) # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(found_document): @@ -347,12 +386,14 @@ def __determine_doc_package_name(self, target_function_doc): if self.tree_dict.get(package_name, None) is not None][0] def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], - language_parser: LanguageFunctionsParser) -> Document: + language_parser: LanguageFunctionsParser, class_name: str = None) -> Document: relevant_docs = [doc for doc in documents if doc.metadata.get('source').__contains__(package_name) and doc.page_content.__contains__(function_name)] + if class_name: + relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( + f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] - for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): - + for document in get_functions_for_package(package_name, relevant_docs, language_parser): # document_function_calls_input_function = True if function_name.lower() == language_parser.get_function_name(document).lower(): # if language_parser.search_for_called_function(document, callee_function=function_name): @@ -363,7 +404,7 @@ def __find_initial_function(self, function_name: str, package_name: str, documen # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end # (target function). - def print_call_hierarchy(self, call_hierarchy_list: list[Document]) ->list[str]: + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str]: results = [] for i, package_function in enumerate(reversed(call_hierarchy_list)): packages_names = self.language_parser.get_package_names(package_function) @@ -377,10 +418,11 @@ def print_call_hierarchy(self, call_hierarchy_list: list[Document]) ->list[str]: package_name = packages_names[0] else: package_name = package_function.metadata['source'] - function_name = self.language_parser.get_function_name(package_function) - current_level = f"(package={package_name},function={function_name},depth={i})" + try: + function_name = self.language_parser.get_function_name(package_function) + current_level = f"(package={package_name},function={function_name},depth={i})" + except ValueError: + current_level = f"(document={package_name},depth={i})" results.append(current_level) return results - - diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index f8709819a..a947f5539 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -1,15 +1,33 @@ import subprocess from abc import ABC, abstractmethod +from collections import defaultdict from enum import Enum from pathlib import Path +from typing import Any, Optional + +from langchain_core.documents import Document +from packaging.specifiers import SpecifierSet +from tqdm import tqdm + import logging +import os +import ast +import json +import re +from vuln_analysis.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' +TRANSITIVE_ENV_NAME = 'transitive_env' + +PYPROJECT_TOML = 'pyproject.toml' +SETUP_PY = 'setup.py' +README_MD = 'README.md' + class Ecosystem(Enum): GO = 1 @@ -30,6 +48,16 @@ class Ecosystem(Enum): "pom.xml": Ecosystem.JAVA } +def run_command(cmd:str) -> str: + try: + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + logger.debug(f'Successfully run command: {cmd}') + if result.stderr: + logger.debug(f"Stderr: {result.stderr}") + return result.stdout + except Exception as e: + logger.error(f"Failed to run command:{cmd} with error: {e}") + class DependencyTreeBuilder(ABC): """ @@ -42,10 +70,6 @@ class DependencyTreeBuilder(ABC): def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: pass - @abstractmethod - # Return only package name, without version - def extract_package_name(self, package_name: str) -> str: - pass @abstractmethod # A Method that knows how to install the app dependencies based on the manifest path and ecosystem. @@ -148,6 +172,176 @@ def extract_package_name(self, package_name: str) -> str: return package_name +class PythonDependencyTreeBuilder(DependencyTreeBuilder): + + def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: + cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python -m pip install deptree' + run_command(cmd) + cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/deptree', + dependencies = run_command(cmd) + parent_stack = [] + tree = defaultdict(set) + ROOT_PROJECT = 'root_project' + tree[ROOT_PROJECT] = [ROOT_LEVEL_SENTINEL] + for line in dependencies.split(os.linesep): + level = 0 + while line.startswith(' '): + level += 1 + line = line[2:] + package = line.split('==')[0].strip().lower() + package = package.replace('-', '_') + if level < len(parent_stack): + parent_stack = parent_stack[:level] + try: + tree[package].add(parent_stack[-1]) + except IndexError: + pass + parent_stack.append(package) + + installed_dependencies = [] + with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: + for line in manifest: + if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): + installed_dependencies.append(re.split(r"[=>< ]", line.strip())[0]) + for dependency, parents in tree.items(): + if dependency in installed_dependencies: + parents.add(ROOT_PROJECT) + tree[dependency] = list(parents) + return tree + + def extract_version_from_specifier(self, specifier_str: str) -> Optional[str]: + """ + Extracts a most likely specific Python version from a PEP 440 specifier string. + Prioritizes exact matches, then lower bounds. + Examples: + "==3.9" -> "3.9" + ">=3.8,<4.0" -> "3.8" + "~=3.7" -> "3.7" (if ~=3.7 is equivalent to >=3.7,<3.8) + ">3.8" -> "3.9" (returns the next major.minor version) + """ + try: + specifier_set = SpecifierSet(specifier_str) + + for specifier in specifier_set: + if specifier.operator in ('==', '==='): + return specifier.version + + lower_bounds = [] + for specifier in specifier_set: + if specifier.operator in ('>=',): + lower_bounds.append(self.parse_version(specifier.version)) + if lower_bounds: + # Return the highest (most restrictive) lower bound + highest_lower_bound = max(lower_bounds) + return str(highest_lower_bound) + + # Look for a compatible release specifier (~=3.7 implies >=3.7,<3.8) + compatible_match = re.search(r'~=(\d+\.\d+)', specifier_str) + if compatible_match: + return compatible_match.group(1) + + # As a fallback for ">X.Y" which means X.Y+1.0, return the next minor version + greater_than_match = re.search(r'>(\d+\.\d+)', specifier_str) + if greater_than_match: + major, minor = map(int, greater_than_match.group(1).split('.')) + return f"{major}.{minor + 1}" + + except Exception as e: # Catch parsing errors from packaging.specifiers + logger.warning(f"Warning: Could not parse specifier '{specifier_str}': {e}") + return None + + def extract_version_from_readme_hint(self, line: str) -> Optional[str]: + """ + Extracts a Python version (e.g., 3.8, 3.12) from a free-form text line. + Looks for patterns like 'Python 3.9', 'python3.10', 'py3.11'. + """ + match = re.search(r'(?:Python|python|py)\s*(\d+\.\d+)', line, re.IGNORECASE) + return match.group(1) if match else None + + def extract_version_from_pyproject_toml(self, content: str) -> Optional[str]: + #todo: implement more options here, see example: https://github.com/quay/quay/blob/master/pyproject.toml + try: + pyproject_data = json.loads(content) + specifier_from_metadata = pyproject_data.get('project', {}).get('requires-python', None) + if specifier_from_metadata: + logger.debug(f"Found requires-python in pyproject.toml: '{specifier_from_metadata}'") + # Try to extract a specific version from the specifier + specific_version = self.extract_version_from_specifier(specifier_from_metadata) + if specific_version: + return specific_version # Return the most authoritative version immediately + return '' + except json.JSONDecodeError: + logger.warning(f"Warning: Failed to parse pyproject.toml as JSON.") + + def extract_version_from_setup_py(self, content: str): + try: + tree = ast.parse(content) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'setup': + for keyword in node.keywords: + if keyword.arg == 'python_requires': + if isinstance(keyword.value, ast.Constant): + specifier_from_metadata = keyword.value.value + logger.debug(f"Found python_requires in setup.py: '{specifier_from_metadata}'") + specific_version = self.extract_version_from_specifier(specifier_from_metadata) + if specific_version: + return specific_version + break + except SyntaxError: + logger.warning(f"Warning: Failed to parse setup.py as Python code") + return '' + + def extract_version_from_readme_md(self, content: str): + version_from_readme_hint = '' + for line in content.splitlines(): + extracted_version = self.extract_version_from_readme_hint(line) + if extracted_version: + if not version_from_readme_hint or ( + self.extract_version_from_specifier(extracted_version) > self.extract_version_from_specifier( + version_from_readme_hint) + ): + version_from_readme_hint = extracted_version + logger.debug(f"Found Python version hint in README.md: '{extracted_version}'") + return version_from_readme_hint + + def determine_python_version(self, git_repo_path: str) -> Optional[str]: + """ + Determines the most specific Python version from project information documents. + Prioritizes pyproject.toml, then setup.py, then README.md. + """ + + info_docs_mapping = {PYPROJECT_TOML: self.extract_version_from_pyproject_toml, + SETUP_PY: self.extract_version_from_setup_py, + README_MD: self.extract_version_from_readme_md} + + for doc, logic in info_docs_mapping.items(): + for root, _, files in os.walk(git_repo_path): + if doc in files: + doc_full_path = os.path.join(root, doc) + with open(doc_full_path, 'r') as file: + python_version = logic(file.read()) + if python_version: + return python_version + return '3.11' + + def install_dependencies(self, git_repo_path): + cmd = f'cd {git_repo_path} && python -m venv {TRANSITIVE_ENV_NAME}' + run_command(cmd) + with open(git_repo_path / PYTHON_MANIFEST, 'r') as manifest: + for line in tqdm(manifest): + if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): + self.install_dependency(line, git_repo_path) + + def install_dependency(self, dependency, repo_path): + valid_signs = ['==', '>=', '<=', '!='] + if not any([sign in dependency for sign in valid_signs]): + dependency = dependency.replace('=', '==') + cmd = f'{repo_path}/{TRANSITIVE_ENV_NAME}/bin/python -m pip install {dependency}' + res = run_command(cmd) + if not res: + logger.warning(f'Failed to install dependency {dependency}') + + def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTreeBuilder: """ A Factory method function that gets an programming language as argument, and returns a DependencyTreeBuilder' @@ -157,6 +351,8 @@ def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTr """ if programming_language == Ecosystem.GO.value: return GoDependencyTreeBuilder() + elif programming_language == Ecosystem.PYTHON.value: + return PythonDependencyTreeBuilder() else: raise NotImplementedError(f'Unsupported Programming Language for transitive search -> {programming_language}') diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index 8bbeaa276..d153ba4c5 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -38,6 +38,7 @@ from vuln_analysis.data_models.input import SourceDocumentsInfo from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods +from vuln_analysis.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher @@ -141,6 +142,7 @@ class ExtendedLanguageParser(LanguageParser): "js": ExtendedJavaScriptSegmenter, } additional_segmenters["go"] = GoSegmenterWithMethods + additional_segmenters["python"] = PythonSegmenterWithClassesMethods LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, **additional_segmenters, @@ -348,7 +350,11 @@ def get_repo_path(self, source_info: SourceDocumentsInfo): Path Returns the path to the git repository. """ - return self._git_directory / PurePath(source_info.git_repo) + # Sanitize the git repo URL to create a valid filesystem path + # Remove protocol separators and path separators that could cause issues + # Example: 'https://github.com/RHEcosystemAppEng/vulnerability-analysis' -> 'https.github.com.RHEcosystemAppEng.vulnerability-analysis' + sanitized_repo_path = source_info.git_repo.replace('//', '.').replace('/', '.').replace(':', '') + return self._git_directory / PurePath(sanitized_repo_path) def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: """ diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 94258de38..494e74135 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -115,6 +115,14 @@ def handle_imports(code_content: str, identifier: str, callee_package: str) -> b class GoLanguageFunctionsParser(LanguageFunctionsParser): + @staticmethod + def is_same_package(package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() in package_name_from_tree.lower() + + def get_dummy_function(self, function_name): + return f"{self.get_function_reserved_word()} {function_name}() {{}}" + + def __trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], callee_package: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict], @@ -193,7 +201,7 @@ def __lookup_package(self, callee_package, resolved_type, struct_initializer_exp def __get_type_docs_matched_with_callee_package(self, callee_package, checked_type, type_documents) -> list[ Document]: return [a_type for a_type in type_documents if callee_package in a_type.metadata['source'] and - (self.__get_type_name(a_type) == checked_type or self.__get_type_name(a_type) in checked_type)] + (self.get_type_name(a_type) == checked_type or self.get_type_name(a_type) in checked_type)] def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: mappings = dict() @@ -300,7 +308,7 @@ def __get_type_kind(self, the_type: Document) -> str: parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content return parts[2] - def __get_type_name(self, the_type: Document) -> str: + def get_type_name(self, the_type: Document) -> str: if self.__is_struct_or_interface_type(the_type): parts_of_type_header = the_type.page_content.split() return parts_of_type_header[1] if len(parts_of_type_header) > 1 else "" @@ -317,7 +325,7 @@ def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[t types_mapping = dict() for the_type in types: the_kind = self.__get_type_kind(the_type) - type_name = self.__get_type_name(the_type) + type_name = self.get_type_name(the_type) type_key = type_name if the_kind.lower() == "interface": type_key = f"interface;{type_name}" @@ -372,13 +380,11 @@ def is_searchable_file_name(self, function: Document) -> bool: def is_function(self, function: Document) -> bool: return function.page_content.startswith("func") - def is_supported_file_extensions(self, extension: str) -> bool: - return extension in self.supported_files_extensions() - def supported_files_extensions(self) -> list[str]: return [".go"] - def get_comment_line_notation(self) -> str: + @staticmethod + def get_comment_line_notation() -> str: return "//" def dir_name_for_3rd_party_packages(self) -> str: @@ -419,7 +425,7 @@ def get_function_name(self, function: Document) -> str: # TODO Try to extract anonymous function var # else: - def search_for_called_function(self, caller_function: Document, callee_function: str, callee_function_package: str, + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict]) -> bool: @@ -428,7 +434,7 @@ def search_for_called_function(self, caller_function: Document, callee_function: caller_function_body = str( caller_function.page_content[index_of_function_opening + 1: index_of_function_closing]) re.search("", caller_function_body) - regex = fr'[a-zA-Z0-9_\[\]\(\).]*.?{callee_function}\(' + regex = fr'[a-zA-Z0-9_\[\]\(\).]*.?{callee_function_name}\(' matching = re.search(regex, caller_function_body, re.MULTILINE) if matching and matching.group(0): return self.__check_identifier_resolved_to_callee_function_package(function=caller_function, @@ -509,7 +515,7 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum caller_function_body, flags=re.MULTILINE) matches = [match.group(0) for match in match_regex] - if len(matches) > 0: + if len(matches) > 0 or re.search(regex_arguments, function_header): # match_variable = matches[-1] # split = match_variable.split(":=") # if split[0] == match_variable: @@ -522,14 +528,6 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum caller_function_index=caller_function_index) - # Checks if match some argument in function or receiver parameter ( without parenthesis of return - # values) - elif re.search(regex_arguments, function_header): - return self.__trace_down_package(expression=identifier.strip(), code_documents=code_documents, - type_documents=type_documents, callee_package=callee_package, - fields_of_types=fields_of_types, - functions_local_variables_index=functions_local_variables_index, - caller_function_index=caller_function_index) # parameters = [tuple(e.replace(")", "").replace("(", "").split(",")) for e # in re.findall(regex_arguments, function_header)[:2]] # return check_types_from_callee_package(parameter=identifier, params=parameters, @@ -567,15 +565,6 @@ def get_package_names(self, function: Document) -> list[str]: return package_names - def get_package_name(self, function: Document, package_name: str) -> str: - package_names = self.get_package_names(function) - for package in package_names: - if package_name.lower() in package.lower() and package_name.lower() == package.lower(): - return package.lower() - return None def is_root_package(self, function: Document) -> bool: return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) - - def is_comment_line(self, line: str) -> bool: - return line.strip().startswith("//") diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py index 1d28da980..1c6ae986b 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py @@ -27,7 +27,7 @@ def get_function_name(self, function: Document) -> str: # This method get a caller function document, a callee function, and a callee package, and returns True # if caller function @abstractmethod - def search_for_called_function(self, caller_function: Document, callee_function: str, callee_function_package: str, + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict]) -> bool: @@ -41,10 +41,12 @@ def get_package_names(self, function: Document) -> list[str]: # This method gets a function document as an argument, a package name as an argument , and return the containing # package name, when you have multiple options. - @abstractmethod def get_package_name(self, function: Document, package_name: str) -> str: - pass - + package_names = self.get_package_names(function) + for package in package_names: + if package_name.lower() == package.lower(): + return package.lower() + return '' # This method get a function document, and returns true, only if it's a function in the root package ( application) @abstractmethod def is_root_package(self, function: Document) -> bool: @@ -52,13 +54,14 @@ def is_root_package(self, function: Document) -> bool: # This method gets a line from a source code, and returns True if it's a comment line for the ecosystem, # otherwise it returns False. - @abstractmethod - def is_comment_line(self, line: str) -> bool: - pass + @classmethod + def is_comment_line(cls, line: str) -> bool: + return line.strip().startswith(cls.get_comment_line_notation()) # This method returns the ecosystem' comment character(s) + @staticmethod @abstractmethod - def get_comment_line_notation(self) -> str: + def get_comment_line_notation() -> str: pass # This method gets a function document as an argument, and returns True if this contained function is exported to be @@ -85,9 +88,8 @@ def supported_files_extensions(self) -> list[str]: pass # This method gets an extension as an argument, and returns True only if it's supported in the ecosystem. - @abstractmethod def is_supported_file_extensions(self, extension: str) -> bool: - pass + return extension in self.supported_files_extensions() # This method gets a document function as an argument, and returns True only if this function should be taken into # Account as candidate to be included in chains of calls paths ( for example , tests and dev resources files should @@ -106,3 +108,23 @@ def get_function_reserved_word(self) -> str: @abstractmethod def get_type_reserved_word(self) -> str: pass + + @abstractmethod + def get_dummy_function(self, function_name): + pass + + @staticmethod + def is_script_language(): + return False + + @staticmethod + @abstractmethod + def is_same_package(package_name_from_input, package_name_from_tree): + pass + + @staticmethod + def get_constructor_method_name(): + return None + + def get_class_name_from_class_function(self): + pass diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py index 460dd866e..72595e337 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py @@ -1,5 +1,6 @@ from ..dep_tree import Ecosystem from .golang_functions_parsers import GoLanguageFunctionsParser +from .python_functions_parser import PythonLanguageFunctionsParser from .lang_functions_parsers import LanguageFunctionsParser @@ -14,5 +15,7 @@ def get_language_function_parser(ecosystem: Ecosystem) -> LanguageFunctionsParse """ if ecosystem == Ecosystem.GO: return GoLanguageFunctionsParser() + elif ecosystem == Ecosystem.PYTHON: + return PythonLanguageFunctionsParser() else: return LanguageFunctionsParser() diff --git a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py new file mode 100644 index 000000000..c04601e01 --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py @@ -0,0 +1,405 @@ +import ast +import os.path +import re + +from langchain_core.documents import Document + +from vuln_analysis.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +PARAMETER = "parameter" +RETURN_TYPES = "return_types" + +PRIMITIVE_TYPES = [int, + float, + bool, + str, + complex, + type(None), + bytes, + frozenset, + tuple, + list, + dict, + set, + range, + memoryview] + + +class PythonLanguageFunctionsParser(LanguageFunctionsParser): + + @staticmethod + def is_same_package(package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() == package_name_from_tree.lower() + + def get_dummy_function(self, function_name): + return f"{self.get_function_reserved_word()} {function_name}(): pass" + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + for func_method in functions_methods_documents: + func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + all_vars = dict() + for row in func_method.page_content.splitlines(): + if not self.is_comment_line(row): + # Extract arguments and receiver argument of type as parameters + if row.startswith("def"): + match = re.finditer(r"(def|\w+)\s*\([a-zA-Z0-9\s*,.\[\]]+\)" + , func_method.page_content[:func_method.page_content.find(":")] + , flags=re.MULTILINE) + for current_match in match: + current_args = (current_match.group(0).replace("\n\t", "") + .replace("\t", "").replace("\n", "")) + current_params = re.search(r"\(.*\)", current_args) + params = (current_params.group(0).replace("(", "") + .replace(")", "").split(",")) + params_tuple = tuple(params) + param_type: str = "" + for param in reversed(params_tuple): + data = param.strip().split(":") + param_name = data[0] + if ":" in param: + param_type = data[1].split('=')[0] + the_value = PARAMETER + all_vars[param_name] = {"value": the_value, "type": param_type} + # Gets return types from function starting_function_colon - 1].strip() + try: + return_parameters = func_method.page_content.split('->')[1].split(':')[0].strip() + except IndexError: + return_parameters = '' + return_parameters = return_parameters.replace(")", "").replace("(", "") + + if return_parameters.strip() != "": + all_vars[RETURN_TYPES] = return_parameters.split(",") + else: + all_vars[RETURN_TYPES] = [] + + elif re.search(r"^[A-Za-z0-9]*", row.strip()) and re.search(r'(? bool: + for line in code_content.page_content.split(os.linesep): + if not self.is_comment_line(line): + if 'import' in line and callee_package in line: + if identifier: + if identifier == callee_package: + if all(word in line for word in ['import', callee_package]): + return True + else: + if all(word in line for word in ['import', callee_package, 'as', identifier]) \ + or all(word in line for word in ['from', callee_package, 'import', identifier]): + return True + else: + if all(word in line for word in ['from', callee_package, 'import']): + return True + return False + + def get_type_reserved_word(self) -> str: + return "class" + + def get_function_reserved_word(self) -> str: + return "def" + + @staticmethod + def is_searchable_file_name(function: Document) -> bool: + file_path = function.metadata['source'] + file_name = os.path.basename(file_path) + return re.search(r"^test_|.*_test.py", file_name) is None + + def supported_files_extensions(self) -> list[str]: + return [".py"] + + def dir_name_for_3rd_party_packages(self) -> str: + return "site-packages" + + def is_function(self, function: Document) -> bool: + return function.page_content.startswith("def") + + def is_exported_function(self, function: Document) -> bool: + return True + + @staticmethod + def get_comment_line_notation() -> str: + return "#" + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].__contains__(self.dir_name_for_3rd_party_packages()) + + def get_package_names(self, function: Document) -> list[str]: + function_source = function.metadata["source"] + function_names = [] + if self.dir_name_for_3rd_party_packages() in function_source: + function_names.append(function_source.split(self.dir_name_for_3rd_party_packages())[1].split('/')[1]) + else: + function_names.append(function_source.split('/')[0]) + return function_names + + def search_for_called_function(self, caller_function: Document, callee_function_name: str, + callee_function_package: str, code_documents: dict[str, Document], + type_documents: list[Document], callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict]) -> bool: + + calls = self._get_function_calls(caller_function, callee_function_name, code_documents) + if not calls: + return False + for call in calls: + + parts = call.split(".") + # If there is no qualifier identifier , then check whether callee function and caller + # function are in the same package + caller_document = code_documents[caller_function.metadata.get('source')] + if len(parts) == 1: + identifier = parts[0] + identifier = identifier.rstrip('(') + callee_function = code_documents[callee_function_file_name] + callee_function_package = self.get_package_names(callee_function)[0] + + caller_function_package = self.get_package_names(caller_function)[0] + if callee_function_package == caller_function_package: + return True + return (self.is_package_imported(caller_document, callee_function_package, identifier) or + self.is_package_imported(caller_function, caller_function_package, identifier)) + # There is a qualifier identifier. + else: + identifier = parts[-2] + if identifier.startswith("return"): + identifier = identifier.replace("return", "").strip() + if "(" in identifier: + identifier = identifier[identifier.index("(") + 1:] + + # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , + # if it's the same as callee package name + if self.is_package_imported(caller_document, callee_function_package, identifier): + return True + ## otherwise, the identifier is in the caller function body or in signature/receiver function argument. + + else: + callee_class_name = self.get_class_name_from_class_function(callee_function) + if identifier in ['self', 'cls']: + caller_class_name = self.get_class_name_from_class_function(caller_function) + if callee_class_name == caller_class_name: + return True + # todo: test this: + function_header = self.get_function_header_from_document(caller_function) + caller_function_body = self.get_function_body_from_document(caller_function) + escaped_identifier = re.escape(identifier) + match_regex = re.finditer(rf"^(\s*|\n){escaped_identifier}\s*(=)\s*[^=]+\n*$", + caller_function_body, flags=re.MULTILINE) + matches = [match.group(0) for match in match_regex] + + regex_arguments = r"\([a-zA-Z0-9\s*,.]+\)" + if len(matches) > 0 or re.search(regex_arguments, function_header): + caller_function_file = caller_function.metadata.get('source') + caller_function_name = self.get_function_name(caller_function) + caller_function_index = f"{caller_function_name}@{caller_function_file}" + return self._trace_down_package(expression=identifier, code_documents=code_documents, + type_documents=type_documents, + callee_package=callee_function_package, + callee_class=callee_class_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_function_index) + return False + + @staticmethod + def __get_type_docs_matched_with_callee_package(callee_package, checked_type, type_documents) -> list[ + Document]: + return [a_type for a_type in type_documents if callee_package in a_type.metadata['source'] and + f'class {checked_type}' in a_type.page_content] + + def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, + value) -> bool: + result = False + if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + docs = self.__get_type_docs_matched_with_callee_package(callee_package, resolved_type, type_documents) + + if len(docs) > 0: + result = True + elif value == PARAMETER: + result = False + + elif struct_initializer_expression: + struct_type = (struct_initializer_expression.group(0).replace("*", "")) + docs = self.__get_type_docs_matched_with_callee_package(callee_package, struct_type, type_documents) + if len(docs) > 0: + result = True + return result + + def _trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], + callee_package: str, callee_class: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str) -> bool: + variables_mappings = functions_local_variables_index[caller_function_index] + parts = expression.split(".") + result = False + + resolved_type, value, var_properties = self.__prepare_package_lookup(parts[-1], variables_mappings) + + if var_properties and (resolved_type and resolved_type not in PRIMITIVE_TYPES): + if resolved_type.strip() == callee_class.strip(): + return True + + elif var_properties and value: + return self._trace_down_package(value, code_documents, type_documents, callee_package, + callee_class, + fields_of_types, functions_local_variables_index, + caller_function_index) + + # Property/member is not in function, check if it's member/property of a type + elif not var_properties and len(parts) > 1: + identifier_index = -1 + identifier_list = parts[:identifier_index] + property_list = parts[identifier_index:] + + resolved_type, value, var_properties = self.__prepare_package_lookup( + identifier_list[-1], variables_mappings) + if (var_properties and resolved_type and resolved_type not in PRIMITIVE_TYPES): + while property_list: + field_name = property_list[0] + possible_types = {key: value for key, value in fields_of_types.items() + if resolved_type == key + and any([field for field in value if field_name in field])} + for the_type, mappings in possible_types.items(): + result = False + for mapping in mappings: + if field_name in mapping: + matched_types = self.__get_type_docs_matched_with_callee_package(callee_package, + mapping[1], + type_documents) + if matched_types: + result = True + else: + result = False + break + resolved_type = property_list[0] + property_list = property_list[1:] + if result: + return True + identifier_index = identifier_index - 1 + try: + expression[identifier_index] + except IndexError: + return False + identifier_list = expression[:identifier_index] + property_list = expression[identifier_index:] + + return result + + def __prepare_package_lookup(self, expression, variables_mappings): + var_properties = variables_mappings.get(expression, None) + if var_properties: + resolved_type = var_properties.get("type") + value = var_properties.get("value") + return resolved_type, value, var_properties + else: + return None, None, None + + def _get_function_calls(self, caller_function: Document, callee_function: str, code_documets: list[Document] = None): + caller_function_body = self.get_function_body_from_document(caller_function) + regex = fr'\b([a-zA-Z0-9_\[\]\(\).]+\.)?{callee_function}\(' + calls = [matching.group(0) for matching in re.finditer(regex, caller_function_body, re.MULTILINE)] + if code_documets: + code_documet = code_documets[caller_function.metadata['source']] + for line in code_documet.page_content.split(os.linesep): + if all(word in line for word in ['import', callee_function, 'as']): + splitted_identifier = line.split(f'{callee_function} as') + identifier = splitted_identifier[-1].split()[0].rstrip(',') + calls.extend(self._get_function_calls(caller_function, identifier)) + return list(set(calls)) + + def get_function_body_from_document(self, function: Document): + if function.metadata.get('content_type') == 'simplified_code': + return function.page_content + try: + function_tree = ast.parse(function.page_content) + except Exception as e: + # logger.error(e) + raise e + lines = function.page_content.splitlines(keepends=True) + body_start_line = function_tree.body[0].lineno + body_end_line = function_tree.body[-1].end_lineno + body_lines = lines[body_start_line:body_end_line] + body_lines = [line for line in body_lines if not self.is_comment_line(line)] + body_indentation = len(body_lines[0]) - len(body_lines[0].lstrip()) + indent_body_lines = [line[body_indentation:] for line in body_lines] + function_body = ''.join(indent_body_lines) + return function_body + + def get_function_header_from_document(self, function: Document) -> str: + function_tree = ast.parse(function.page_content) + lines = function.page_content.splitlines(keepends=True) + for node in function_tree.body: + if isinstance(node, ast.FunctionDef): + header_start_line = node.lineno - 1 + body_start_line = node.lineno + if node.body: + body_start_line = node.body[0].lineno - 1 + header_lines = lines[header_start_line: body_start_line] + full_header_content = "".join(header_lines) + header_indentation = len(full_header_content) - len(full_header_content.lstrip()) + indent_header_lines = [line[header_indentation:] for line in full_header_content] + return "".join(indent_header_lines).strip() + + def get_function_name(self, function: Document) -> str: + if function.page_content[:3] == self.get_function_reserved_word(): + function_name = function.page_content[4:].split('(')[0].strip() + return function_name + raise ValueError('Only function document is supported') + + @staticmethod + def __get_variable_data(variable_declaration:str) -> tuple[str, str, str]: + """ + Parse a variable declaration string and extract its components. + Args: + variable_declaration (str): A variable declaration string in one of these formats: + Returns: + tuple[str, str, str]: A tuple containing: + - variable_name (str): The name of the variable + - variable_type (str): The type annotation if present, empty string otherwise + - variable_value (str): The assigned value + """ + variable_name = variable_declaration.split()[0].strip() + variable_value = variable_declaration.split('=')[-1] + variable_type = '' + if ':' in variable_declaration: + variable_type = variable_declaration.split(":")[-1].split("=")[0].strip() + + return variable_name, variable_type, variable_value + + def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for the_type in types: + type_key = the_type.page_content.split('class')[1].split('(')[0].strip() + fields_list = [] + for row in the_type.page_content.split(os.linesep): + if re.search(r"^[A-Za-z0-9]*", row.strip()) and '=' in row: + variable_name, variable_type, variable_value = self.__get_variable_data(row) + fields_list.append((variable_name, variable_type)) + types_mapping[(type_key, the_type.metadata['source'])] = fields_list + return types_mapping + + @staticmethod + def is_script_language(): + return True + + @staticmethod + def get_constructor_method_name(): + return '__init__' \ No newline at end of file diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 3160625f9..ef89a65f1 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -208,7 +208,6 @@ def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheu cvss=cvss ) - def postprocess_engine_output(message: AgentMorpheusEngineInput, result: AgentMorpheusEngineState) -> AgentMorpheusOutput: diff --git a/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py b/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py new file mode 100644 index 000000000..7f3fcb226 --- /dev/null +++ b/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py @@ -0,0 +1,50 @@ +import ast +from typing import List +import re + +from langchain_community.document_loaders.parsers.language.python import PythonSegmenter + + +def parse_all_classes_methods(code: str) -> list[str]: + methods = [] + tree = ast.parse(code) + lines = code.splitlines(keepends=True) + for node in tree.body: + if isinstance(node, ast.ClassDef): + class_name = node.name + for item in node.body: + if isinstance(item, ast.FunctionDef): + method_start_line_index = item.lineno - 1 + method_end_line_index = item.end_lineno + method_lines = lines[method_start_line_index:method_end_line_index] + if method_lines: + method_indentation_level = len(method_lines[0]) - len(method_lines[0].lstrip()) + + # Remove the method's specific indentation from all its lines + # This "dedents" the method content to column 0 + dedented_method_lines = [] + for line in method_lines: + if line.startswith(' ' * method_indentation_level): + dedented_method_lines.append(line[method_indentation_level:]) + else: + dedented_method_lines.append(line) + methods.append("".join(dedented_method_lines).strip()) + for i , method in enumerate(methods): + methods[i] = f'{method}\n#(class: {class_name})' + + return methods + + +class PythonSegmenterWithClassesMethods(PythonSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + def extract_functions_classes(self) -> List[str]: + function_classes = super().extract_functions_classes() + classes_methods = [] + for func in function_classes: + if func.startswith('class '): + classes_methods.extend(parse_all_classes_methods(func)) + function_classes.extend(classes_methods) + return function_classes \ No newline at end of file diff --git a/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py b/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py new file mode 100644 index 000000000..78804e396 --- /dev/null +++ b/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py @@ -0,0 +1,49 @@ +import ast +from typing import List + +from langchain_community.document_loaders.parsers.language.python import PythonSegmenter + + +def parse_all_classes_methods(code: str) -> list[str]: + methods = [] + tree = ast.parse(code) + lines = code.splitlines(keepends=True) + for node in tree.body: + if isinstance(node, ast.ClassDef): + class_name = node.name + for item in node.body: + if isinstance(item, ast.FunctionDef): + method_start_line_index = item.lineno - 1 + method_end_line_index = item.end_lineno + method_lines = lines[method_start_line_index:method_end_line_index] + if method_lines: + method_indentation_level = len(method_lines[0]) - len(method_lines[0].lstrip()) + + # Remove the method's specific indentation from all its lines + # This "dedents" the method content to column 0 + dedented_method_lines = [] + for line in method_lines: + if line.startswith(' ' * method_indentation_level): + dedented_method_lines.append(line[method_indentation_level:]) + else: + dedented_method_lines.append(line) + methods.append("".join(dedented_method_lines).strip()) + for i , method in enumerate(methods): + methods[i] = f'{method}\n#(class: {class_name})' + + return methods + + +class PythonSegmenterWithClassesMethods(PythonSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + def extract_functions_classes(self) -> List[str]: + function_classes = super().extract_functions_classes() + classes_methods = [] + for func in function_classes: + if func.startswith('class '): + classes_methods.extend(parse_all_classes_methods(func)) + function_classes.extend(classes_methods) + return function_classes \ No newline at end of file diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py index f5b35b41e..7b1760c5d 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -77,6 +77,8 @@ def download_dependencies(git_repo_path: Path) -> bool: logger.info(f"Started installing packages for {ecosystem}") tree_builder = get_dependency_tree_builder(ecosystem.value) tree_builder.install_dependencies(git_repo_path) + with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: + file.write(ecosystem.name) logger.info(f"Finished installing packages for {ecosystem}") return True except NotImplementedError as err: @@ -89,7 +91,7 @@ def search(self, query: str) -> tuple[bool, list[Document]]: Parameters ---------- query : str - query to search, in the form of "package_name,function_name". + query to search, in the form of "package_name,function_name" or "package_name,class_name.function_name". Returns ------- From 09d25f41f8bd36adaf739f9acd9d39e4808b0bd7 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 9 Sep 2025 07:24:17 +0300 Subject: [PATCH 065/286] refactor: remove redundant duplicate file Signed-off-by: Zvi Grinberg --- .../python_segmenters_with_claases_methods.py | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 src/vuln_analysis/utils/python_segmenters_with_claases_methods.py diff --git a/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py b/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py deleted file mode 100644 index 7f3fcb226..000000000 --- a/src/vuln_analysis/utils/python_segmenters_with_claases_methods.py +++ /dev/null @@ -1,50 +0,0 @@ -import ast -from typing import List -import re - -from langchain_community.document_loaders.parsers.language.python import PythonSegmenter - - -def parse_all_classes_methods(code: str) -> list[str]: - methods = [] - tree = ast.parse(code) - lines = code.splitlines(keepends=True) - for node in tree.body: - if isinstance(node, ast.ClassDef): - class_name = node.name - for item in node.body: - if isinstance(item, ast.FunctionDef): - method_start_line_index = item.lineno - 1 - method_end_line_index = item.end_lineno - method_lines = lines[method_start_line_index:method_end_line_index] - if method_lines: - method_indentation_level = len(method_lines[0]) - len(method_lines[0].lstrip()) - - # Remove the method's specific indentation from all its lines - # This "dedents" the method content to column 0 - dedented_method_lines = [] - for line in method_lines: - if line.startswith(' ' * method_indentation_level): - dedented_method_lines.append(line[method_indentation_level:]) - else: - dedented_method_lines.append(line) - methods.append("".join(dedented_method_lines).strip()) - for i , method in enumerate(methods): - methods[i] = f'{method}\n#(class: {class_name})' - - return methods - - -class PythonSegmenterWithClassesMethods(PythonSegmenter): - - def __init__(self, code: str): - super().__init__(code) - - def extract_functions_classes(self) -> List[str]: - function_classes = super().extract_functions_classes() - classes_methods = [] - for func in function_classes: - if func.startswith('class '): - classes_methods.extend(parse_all_classes_methods(func)) - function_classes.extend(classes_methods) - return function_classes \ No newline at end of file From 901c714e8161b5cc25040bc9fabb6f200fcd61bb Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 9 Sep 2025 15:36:45 +0300 Subject: [PATCH 066/286] fix: configure VDB persistent cache directory Signed-off-by: Vladimir Belousov --- src/vuln_analysis/functions/cve_generate_vdbs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 06c246498..98b57a463 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -132,7 +132,10 @@ def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: if source_infos: # Use DocumentEmbedding class with embedding=None for its utility functions - embedder = DocumentEmbedding(embedding=None) + embedder = DocumentEmbedding(embedding=None, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir) # Determine code index path for either loading from cache or creating new index # Need to add support for configurable base path From 513c5867ab15863ee7ff92c02bb21c097bdea943 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 9 Sep 2025 18:09:05 +0300 Subject: [PATCH 067/286] chore: fix step 6 in OCP Deployment Signed-off-by: Zvi Grinberg --- kustomize/README.md | 2 +- kustomize/base/exploit-iq-config.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 50dbd257f..732db833e 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -110,7 +110,7 @@ EOF ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" -find . -type f -name 'exploit-iq-config.json' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + +find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` >[!IMPORTANT] You should only run one of 7,8 steps, depends on if you want to run the service with a self hosted LLM or not. diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 127b34624..34ded9884 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -41,7 +41,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: https://exploit-iq-client.exploit-iq-nat.svc:8443/vulnerabilities/{vuln_id}/comments + endpoint: CALLBACK_URL_PLACEHOLDER/vulnerabilities/{vuln_id}/comments token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt @@ -121,7 +121,7 @@ functions: llm_name: justify_llm cve_http_output: _type: cve_http_output - url: https://exploit-iq-client.exploit-iq-nat.svc:8443 + url: CALLBACK_URL_PLACEHOLDER endpoint: /reports auth_type: bearer token_path: /var/run/secrets/kubernetes.io/serviceaccount/token From 6d962fdd07ca5d612e182e080a9240619a45daed Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 10 Sep 2025 10:48:28 +0300 Subject: [PATCH 068/286] fix: doc should be excluded from subsequent future searches only if its wrapped function calling the current function Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/chain_of_calls_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index e290c619a..706f539b3 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -228,8 +228,8 @@ def __find_caller_function(self, document_function: Document, function_package: # If current document is found to be calling document_function in function_package, then return it as a # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. - package_exclusions.append(doc) if function_is_being_called: + package_exclusions.append(doc) # update index of last scanned package for backtracking # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index From a946fbb10be76dc1c0e7d12cb2fac34128d0e89c Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf Date: Wed, 10 Sep 2025 12:55:24 +0300 Subject: [PATCH 069/286] fix: add callee_function argument --- src/vuln_analysis/utils/chain_of_calls_retriever.py | 1 + .../utils/functions_parsers/golang_functions_parsers.py | 4 ++-- .../utils/functions_parsers/lang_functions_parsers.py | 4 ++-- .../utils/functions_parsers/python_functions_parser.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 706f539b3..4759df94b 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -212,6 +212,7 @@ def __find_caller_function(self, document_function: Document, function_package: function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, callee_function_name= function_name_to_search, + callee_function=document_function, callee_function_package= function_package, code_documents= diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 494e74135..129fd77ca 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -425,8 +425,8 @@ def get_function_name(self, function: Document) -> str: # TODO Try to extract anonymous function var # else: - def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function_package: str, - code_documents: list[Document], type_documents: list[Document], + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, + callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict]) -> bool: index_of_function_opening = caller_function.page_content.index("{") diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py index 1c6ae986b..bf2503a2e 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py @@ -27,8 +27,8 @@ def get_function_name(self, function: Document) -> str: # This method get a caller function document, a callee function, and a callee package, and returns True # if caller function @abstractmethod - def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function_package: str, - code_documents: list[Document], type_documents: list[Document], + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, + callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict]) -> bool: pass diff --git a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py index c04601e01..49a4573c3 100644 --- a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py +++ b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py @@ -151,7 +151,7 @@ def get_package_names(self, function: Document) -> list[str]: function_names.append(function_source.split('/')[0]) return function_names - def search_for_called_function(self, caller_function: Document, callee_function_name: str, + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, callee_function_package: str, code_documents: dict[str, Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], From 6f986bc36b84e77fd5ebc20e80e091ab411d9e41 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 11 Sep 2025 13:36:36 +0300 Subject: [PATCH 070/286] fix: correct repo path handling and directory traversal Signed-off-by: Vladimir Belousov --- .../functions/cve_generate_vdbs.py | 6 ++-- src/vuln_analysis/utils/document_embedding.py | 3 +- src/vuln_analysis/utils/git_utils.py | 6 ++++ .../utils/source_code_git_loader.py | 30 +++++++++---------- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 98b57a463..7b93b1137 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -205,10 +205,12 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: # Replace ref with specific commit hash for each source info for si in source_infos: try: - repo = get_repo_from_path(config.base_git_dir, si.git_repo) + # Get the sanitized path from the embedder instance + repo_path = embedder.get_repo_path(si) + repo = get_repo_from_path(str(repo_path.parent), repo_path.name) si.ref = repo.commit().hexsha except ValueError as e: - logger.warning("Failed to get commit hash for %s/%s: %s", config.base_git_dir, si.git_repo, e) + logger.warning("Failed to get commit hash for repo defined in %s: %s", si, e) continue except Exception as e: diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index d153ba4c5..f512f6f3a 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -41,6 +41,7 @@ from vuln_analysis.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader +from vuln_analysis.utils.git_utils import sanitize_git_url_for_path from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from vuln_analysis.logging.loggers_factory import LoggingFactory @@ -353,7 +354,7 @@ def get_repo_path(self, source_info: SourceDocumentsInfo): # Sanitize the git repo URL to create a valid filesystem path # Remove protocol separators and path separators that could cause issues # Example: 'https://github.com/RHEcosystemAppEng/vulnerability-analysis' -> 'https.github.com.RHEcosystemAppEng.vulnerability-analysis' - sanitized_repo_path = source_info.git_repo.replace('//', '.').replace('/', '.').replace(':', '') + sanitized_repo_path = sanitize_git_url_for_path(source_info.git_repo) return self._git_directory / PurePath(sanitized_repo_path) def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: diff --git a/src/vuln_analysis/utils/git_utils.py b/src/vuln_analysis/utils/git_utils.py index 87ede34a7..36c4a7ca8 100644 --- a/src/vuln_analysis/utils/git_utils.py +++ b/src/vuln_analysis/utils/git_utils.py @@ -21,6 +21,12 @@ from git import Repo +def sanitize_git_url_for_path(git_url: str) -> str: + """Sanitizes a git repo URL to create a valid filesystem path component.""" + # Example: 'https://github.com/some/repo' -> 'https.github.com.some.repo' + return git_url.replace('//', '.').replace('/', '.').replace(':', '') + + def get_repo_from_path(base_dir: str, git_repo: str = ".git") -> Repo: """ Utility function for getting GitPython `Repo` object representing a Git repository. diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index 766fda168..9ceabd390 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -179,18 +179,18 @@ def yield_blobs(self) -> typing.Iterator[Blob]: logger.info("Processing %d files in the Git repository at path: '%s'", len(final_files), self.repo_path) for f in tqdm(final_files): - - file_path = Path(f) - - abs_file_path = base_path / file_path - - rel_file_path = str(file_path) - - metadata = { - "source": rel_file_path, - "file_path": rel_file_path, - "file_name": file_path.name, - "file_type": file_path.suffix, - } - - yield Blob.from_path(abs_file_path, metadata=metadata) + abs_file_path = base_path / f + if abs_file_path.is_file(): + try: + rel_file_path = str(f) + metadata = { + "source": rel_file_path, + "file_path": rel_file_path, + "file_name": abs_file_path.name, + "file_type": abs_file_path.suffix, + } + yield Blob.from_path(abs_file_path, metadata=metadata) + except Exception as e: + logger.warning("Failed to read blob for '%s'. Ignoring this file. Error: %s", abs_file_path, e) + else: + logger.debug("Skipping path as it is a directory, not a file: '%s'", abs_file_path) From 93dc4a018c1117ce731db69d92e271203896c733 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 14 Sep 2025 13:00:35 +0300 Subject: [PATCH 071/286] fix: remove redundant type casting in GitLoader Signed-off-by: Vladimir Belousov --- src/vuln_analysis/functions/cve_generate_vdbs.py | 2 +- src/vuln_analysis/utils/source_code_git_loader.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 7b93b1137..7772e88d5 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -207,7 +207,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: try: # Get the sanitized path from the embedder instance repo_path = embedder.get_repo_path(si) - repo = get_repo_from_path(str(repo_path.parent), repo_path.name) + repo = get_repo_from_path(repo_path.parent, repo_path.name) si.ref = repo.commit().hexsha except ValueError as e: logger.warning("Failed to get commit hash for repo defined in %s: %s", si, e) diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index 9ceabd390..d596374e6 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -182,10 +182,9 @@ def yield_blobs(self) -> typing.Iterator[Blob]: abs_file_path = base_path / f if abs_file_path.is_file(): try: - rel_file_path = str(f) metadata = { - "source": rel_file_path, - "file_path": rel_file_path, + "source": f, + "file_path": f, "file_name": abs_file_path.name, "file_type": abs_file_path.suffix, } From 42f7abb9e0529e30905cedc2fd01a19e83a0aaf7 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 9 Sep 2025 15:47:46 +0300 Subject: [PATCH 072/286] chore: point cache directories to persistent volume mount path Signed-off-by: Vladimir Belousov --- kustomize/base/exploit-iq-config.yml | 8 ++++---- kustomize/base/exploit_iq_service.yaml | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 127b34624..e076bb136 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -29,10 +29,10 @@ functions: _type: cve_generate_vdbs agent_name: cve_agent_executor # Used to determine which tools are enabled embedder_name: nim_embedder - base_git_dir: .cache/am_cache/git - base_vdb_dir: .cache/am_cache/vdb - base_code_index_dir: .cache/am_cache/code_index - base_pickle_dir: .cache/am_cache/pickle + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_vdb_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}vdb + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 6480b0ac8..96c7a3d4f 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -105,6 +105,8 @@ spec: value: http://nginx-cache:8080/ubuntu - name: ENABLE_EXTENDED_JS_PARSERS value: "True" + - name: EXPLOIT_IQ_DATA_DIR + value: /exploit-iq-data - name: NAMESPACE valueFrom: fieldRef: @@ -113,7 +115,7 @@ spec: - name: config mountPath: /configs - name: cache - mountPath: /morpheus-data + mountPath: /exploit-iq-data - name: ca-bundle mountPath: /app/certs readOnly: true From 063c1bdde8ab06571b4bf278a25a7f87d51a180d Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 17 Sep 2025 13:08:00 +0300 Subject: [PATCH 073/286] fix: pin litellm and enforce lockfile to fix build Signed-off-by: Vladimir Belousov --- Dockerfile | 2 +- pyproject.toml | 3 ++- uv.lock | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f0f899e2..6477049f4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,7 @@ COPY ./ /workspace RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ export SETUPTOOLS_SCM_PRETEND_VERSION=${VULN_ANALYSIS_VERSION} && \ uv venv --python ${PYTHON_VERSION} /workspace/.venv && \ - uv sync --prerelease=allow + uv sync # Activate the environment (make it default for subsequent commands) RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc diff --git a/pyproject.toml b/pyproject.toml index 3b73bb6ce..005bad687 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ dependencies = [ "tantivy==0.22.2", "tree-sitter==0.21.3", "tree-sitter-languages==1.10.2", - "univers==30.12" + "univers==30.12", + "litellm<=1.75.8", ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/uv.lock b/uv.lock index 1c13b5a16..868d8a692 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", @@ -5287,6 +5287,7 @@ dependencies = [ { name = "gitpython" }, { name = "google-search-results" }, { name = "json5" }, + { name = "litellm" }, { name = "nbformat" }, { name = "nemollm" }, { name = "nvidia-nat", extra = ["langchain", "profiling", "telemetry"] }, @@ -5324,12 +5325,13 @@ requires-dist = [ { name = "aiohttp-client-cache", specifier = "==0.11" }, { name = "aioresponses", specifier = "==0.7.6" }, { name = "brotli" }, - { name = "cvss" }, + { name = "cvss", specifier = "==3.6" }, { name = "esprima" }, { name = "faiss-cpu", specifier = "==1.9.0" }, { name = "gitpython" }, { name = "google-search-results", specifier = "==2.4" }, { name = "json5" }, + { name = "litellm", specifier = "<=1.75.8" }, { name = "nbformat" }, { name = "nemollm" }, { name = "nvidia-nat", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0rc8,<1.3.0" }, From fb4dad6b17a09b6d1887036dbb22fd61d82b90e5 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 15 Sep 2025 17:04:02 +0300 Subject: [PATCH 074/286] chore: receive input by image type or source type Signed-off-by: Ilona Shishov --- src/vuln_analysis/data_models/input.py | 9 ++++++++- src/vuln_analysis/functions/cve_check_vuln_deps.py | 11 +++++++++-- src/vuln_analysis/functions/cve_process_sbom.py | 7 ++++++- src/vuln_analysis/utils/llm_engine_utils.py | 13 +++++++------ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/vuln_analysis/data_models/input.py b/src/vuln_analysis/data_models/input.py index c079ca295..5cd242139 100644 --- a/src/vuln_analysis/data_models/input.py +++ b/src/vuln_analysis/data_models/input.py @@ -151,8 +151,15 @@ class ImageInfoInput(HashableModel): platform: str | None = None # i.e. linux/amd64 feed_group: str | None = None # i.e. ubuntu:22.04 + analysis_type: typing.Literal["image", "source"] # i.e. image + """ + The type of analysis to perform: + - "image": Analysis of a container image and tag with SBOM data + - "source": Analysis of source code and commitId without SBOM data + """ + source_info: list[SourceDocumentsInfo] - sbom_info: SBOMInfoInput + sbom_info: SBOMInfoInput | None = None @field_validator('source_info', mode='after') @classmethod diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 56964db8c..ba55f81dd 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -50,13 +50,20 @@ async def cve_check_vuln_deps(config: CVEVulnerableDepsChecksConfig, builder: Bu async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: trace_id.set(message.input.scan.id) - sbom = message.info.sbom.packages - image = f"{message.input.image.name}:{message.input.image.tag}" + if message.input.image.analysis_type == "source": + logger.info("Source code analysis requested. Skipping vulnerable dependency check.") + message.info.vulnerable_dependencies = None + return message + + image = f"{message.input.image.name}:{message.input.image.tag}" + if config.skip: logger.info("`config.skip` is set to True. Skipping vulnerable dependency check for image %s.", image) message.info.vulnerable_dependencies = None return message + + sbom = message.info.sbom.packages if not sbom: logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index ee70572e3..a934970fb 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -70,7 +70,12 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: return packages - if (message.input.image.sbom_info.type == ManualSBOMInfoInput.static_type()): + if message.input.image.analysis_type == "source": + logger.info("Source code analysis requested. No SBOM processing needed.") + return message + + elif (message.input.image.sbom_info.type == ManualSBOMInfoInput.static_type()): + assert isinstance(message.input.image.sbom_info, ManualSBOMInfoInput) # Create the SBOM object diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index ef89a65f1..58f34637e 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -40,11 +40,12 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE assert message.info.intel is not None, "The input message must have intel information" - # Return early if there are no SBOM packages - sbom = message.info.sbom.packages - image = f"{message.input.image.name}:{message.input.image.tag}" - if not sbom: - raise ValueError(f"No SBOM packages found for image {image}. Skipping the LLM Engine.") + # Return early if analysis_type is not source and there are no SBOM packages + if message.input.image.analysis_type != "source": + image = f"{message.input.image.name}:{message.input.image.tag}" + sbom = message.info.sbom.packages + if not sbom: + raise ValueError(f"No SBOM packages found for image {image}. Skipping the LLM Engine.") am_input: AgentMorpheusInput = message.input @@ -229,7 +230,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, # For each vuln_id, get LLM Engine output if it exists # or create placeholder output if it skipped the workflow - if not message.info.sbom.packages: + if not message.input.image.analysis_type == "source" and not message.info.sbom.packages: output = [build_no_sbom_output(vuln_id) for vuln_id in input_vuln_ids] else: output: list[AgentMorpheusEngineOutput] = [] From 10a86d0cdb0df4f1358e6bd248ed8ca5e74bcc31 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 18 Sep 2025 10:59:13 +0300 Subject: [PATCH 075/286] chore: make ecosystem and manifest path optional. update openai schema with new params Signed-off-by: Ilona Shishov --- .../configs/openapi/openapi.json | 62 +++++++++++++++++++ src/vuln_analysis/data_models/input.py | 4 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/configs/openapi/openapi.json b/src/vuln_analysis/configs/openapi/openapi.json index bead3625d..4eeaa5ce8 100644 --- a/src/vuln_analysis/configs/openapi/openapi.json +++ b/src/vuln_analysis/configs/openapi/openapi.json @@ -2144,6 +2144,36 @@ ], "title": "Feed Group" }, + "analysis_type": { + "type": "string", + "enum": [ + "image", + "source" + ], + "title": "Analysis Type" + }, + "ecosystem": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ecosystem" + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, "source_info": { "items": { "$ref": "#/components/schemas/SourceDocumentsInfo" @@ -2168,6 +2198,7 @@ }, "type": "object", "required": [ + "analysis_type", "source_info", "sbom_info" ], @@ -2231,6 +2262,36 @@ ], "title": "Feed Group" }, + "analysis_type": { + "type": "string", + "enum": [ + "image", + "source" + ], + "title": "Analysis Type" + }, + "ecosystem": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ecosystem" + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, "source_info": { "items": { "$ref": "#/components/schemas/SourceDocumentsInfo" @@ -2255,6 +2316,7 @@ }, "type": "object", "required": [ + "analysis_type", "source_info", "sbom_info" ], diff --git a/src/vuln_analysis/data_models/input.py b/src/vuln_analysis/data_models/input.py index 5cd242139..f5941f0c7 100644 --- a/src/vuln_analysis/data_models/input.py +++ b/src/vuln_analysis/data_models/input.py @@ -150,7 +150,9 @@ class ImageInfoInput(HashableModel): digest: str | None = None # i.e. sha256:... platform: str | None = None # i.e. linux/amd64 feed_group: str | None = None # i.e. ubuntu:22.04 - + ecosystem: str | None = None # i.e. python + manifest_path: str | None = None # i.e. path/to/requirements.txt + analysis_type: typing.Literal["image", "source"] # i.e. image """ The type of analysis to perform: From 83c4af588b72326a7f4b7a42a02b404fae1a4ce3 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 18 Sep 2025 13:07:16 +0300 Subject: [PATCH 076/286] chore: use enumeration to define analysis type and ecosystem params Signed-off-by: Ilona Shishov --- .../configs/openapi/openapi.json | 28 +++++++++---------- src/vuln_analysis/data_models/common.py | 5 ++++ src/vuln_analysis/data_models/input.py | 6 ++-- .../functions/cve_check_vuln_deps.py | 3 +- .../functions/cve_process_sbom.py | 3 +- src/vuln_analysis/utils/dep_tree.py | 15 +++++----- src/vuln_analysis/utils/llm_engine_utils.py | 5 ++-- 7 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/vuln_analysis/configs/openapi/openapi.json b/src/vuln_analysis/configs/openapi/openapi.json index 4eeaa5ce8..2f2a49c54 100644 --- a/src/vuln_analysis/configs/openapi/openapi.json +++ b/src/vuln_analysis/configs/openapi/openapi.json @@ -2153,13 +2153,13 @@ "title": "Analysis Type" }, "ecosystem": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": "string", + "enum": [ + "go", + "python", + "javascript", + "java", + "c", ], "title": "Ecosystem" }, @@ -2271,13 +2271,13 @@ "title": "Analysis Type" }, "ecosystem": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": "string", + "enum": [ + "go", + "python", + "javascript", + "java", + "c", ], "title": "Ecosystem" }, diff --git a/src/vuln_analysis/data_models/common.py b/src/vuln_analysis/data_models/common.py index 39455cf77..077a98fa9 100644 --- a/src/vuln_analysis/data_models/common.py +++ b/src/vuln_analysis/data_models/common.py @@ -16,6 +16,7 @@ import sys import typing from hashlib import sha512 +from enum import Enum from pydantic import BaseModel from pydantic import Field @@ -23,6 +24,10 @@ _LT = typing.TypeVar("_LT") +class AnalysisType(str, Enum): + IMAGE = "image" + SOURCE = "source" + class HashableModel(BaseModel): """ Subclass of a Pydantic BaseModel that is hashable. Use in objects that need to be hashed for caching purposes. diff --git a/src/vuln_analysis/data_models/input.py b/src/vuln_analysis/data_models/input.py index f5941f0c7..5ee3c1d11 100644 --- a/src/vuln_analysis/data_models/input.py +++ b/src/vuln_analysis/data_models/input.py @@ -28,6 +28,8 @@ from ..utils.string_utils import is_valid_cve_id from ..utils.string_utils import is_valid_ghsa_id +from ..utils.dep_tree import Ecosystem +from .common import AnalysisType from .common import HashableModel from .common import TypedBaseModel from .info import AgentMorpheusInfo @@ -150,10 +152,10 @@ class ImageInfoInput(HashableModel): digest: str | None = None # i.e. sha256:... platform: str | None = None # i.e. linux/amd64 feed_group: str | None = None # i.e. ubuntu:22.04 - ecosystem: str | None = None # i.e. python + ecosystem: Ecosystem | None = None # i.e. python manifest_path: str | None = None # i.e. path/to/requirements.txt - analysis_type: typing.Literal["image", "source"] # i.e. image + analysis_type: AnalysisType # i.e. image """ The type of analysis to perform: - "image": Analysis of a container image and tag with SBOM data diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index ba55f81dd..2da2185a7 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -24,6 +24,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field +from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) @@ -51,7 +52,7 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: trace_id.set(message.input.scan.id) - if message.input.image.analysis_type == "source": + if message.input.image.analysis_type == AnalysisType.SOURCE: logger.info("Source code analysis requested. Skipping vulnerable dependency check.") message.info.vulnerable_dependencies = None return message diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index a934970fb..ada78bb6c 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -23,6 +23,7 @@ from pydantic import Field from pydantic import NonNegativeInt +from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -70,7 +71,7 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: return packages - if message.input.image.analysis_type == "source": + if message.input.image.analysis_type == AnalysisType.SOURCE: logger.info("Source code analysis requested. No SBOM processing needed.") return message diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index a947f5539..7267e034e 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -17,8 +17,8 @@ from vuln_analysis.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from vuln_analysis.logging.loggers_factory import LoggingFactory -logger = LoggingFactory.get_agent_logger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' @@ -29,11 +29,12 @@ README_MD = 'README.md' -class Ecosystem(Enum): - GO = 1 - PYTHON = 2 - JAVASCRIPT = 3 - JAVA = 4 +class Ecosystem(str, Enum): + GO = "go" + PYTHON = "python" + JAVASCRIPT = "javascript" + JAVA = "java" + C = "c" GOLANG_MANIFEST = "go.mod" @@ -48,7 +49,7 @@ class Ecosystem(Enum): "pom.xml": Ecosystem.JAVA } -def run_command(cmd:str) -> str: +def run_command(cmd: str) -> str: try: result = subprocess.run(cmd, shell=True, capture_output=True, text=True) logger.debug(f'Successfully run command: {cmd}') diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 58f34637e..78043e704 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -17,6 +17,7 @@ from ordered_set import OrderedSet +from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.data_models.dependencies import VulnerableDependencies from vuln_analysis.data_models.input import AgentMorpheusEngineInput from vuln_analysis.data_models.input import AgentMorpheusInput @@ -41,7 +42,7 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE assert message.info.intel is not None, "The input message must have intel information" # Return early if analysis_type is not source and there are no SBOM packages - if message.input.image.analysis_type != "source": + if message.input.image.analysis_type != AnalysisType.SOURCE: image = f"{message.input.image.name}:{message.input.image.tag}" sbom = message.info.sbom.packages if not sbom: @@ -230,7 +231,7 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, # For each vuln_id, get LLM Engine output if it exists # or create placeholder output if it skipped the workflow - if not message.input.image.analysis_type == "source" and not message.info.sbom.packages: + if not message.input.image.analysis_type == AnalysisType.SOURCE and not message.info.sbom.packages: output = [build_no_sbom_output(vuln_id) for vuln_id in input_vuln_ids] else: output: list[AgentMorpheusEngineOutput] = [] From 5eb26090a89ad29568f20a73c44e763eac898f2c Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 17 Sep 2025 15:30:21 +0300 Subject: [PATCH 077/286] chore: APPENG-3770 - enable Swagger UI OpenAPI in deployment Signed-off-by: Zvi Grinberg --- .tekton/on-tag.yaml | 1 + kustomize/base/exploit_iq_client.yaml | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml index b0a3ba181..00d0d0458 100644 --- a/.tekton/on-tag.yaml +++ b/.tekton/on-tag.yaml @@ -122,6 +122,7 @@ spec: --label=exploit_iq_vcs_branch={{source_branch}} --label=exploit_iq_vcs_build_commit_id={{revision}} --label=exploit_iq_vcs_repo={{repo_url}} + --env=EXPLOIT_IQ_VERSION={{git_tag}} --format=docker taskRef: resolver: cluster diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index 95a816fa8..80e635068 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -143,6 +143,22 @@ spec: kind: Service name: exploit-iq-client --- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq-client-swagger-ui + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: reencrypt + port: + targetPort: metrics + to: + kind: Service + name: exploit-iq-client +--- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: From f72fe1498715075303a5ef053db3e960eac69963 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 18 Sep 2025 15:49:09 +0300 Subject: [PATCH 078/286] docs: add step to install ExploitIQ API route as server in swagger-ui Signed-off-by: Zvi Grinberg --- kustomize/README.md | 7 ++++++- kustomize/base/exploit_iq_client.yaml | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 732db833e..df9d5bd29 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -158,8 +158,13 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` +10. On Non Production environments, the Swagger-UI endpoint is enabled, and can be configured for testing on environment +that way: +```shell +oc set env deployment -l component=exploit-iq-client QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') +``` -9. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: +11. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: ```shell DEPLOYMENT_VARIANT_NAME=remote-nim-all diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index 80e635068..4d1a4beeb 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -152,7 +152,6 @@ metadata: spec: tls: insecureEdgeTerminationPolicy: Redirect - termination: reencrypt port: targetPort: metrics to: From 0ec02b1500acd45c5e1c7b98ea4db5215073852e Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 17 Sep 2025 15:39:13 +0300 Subject: [PATCH 079/286] test: add transitive code search tool test and utilities functions to ease writing transitive code search tools unit tests in all ecosystems Signed-off-by: Zvi Grinberg --- pyproject.toml | 1 + .../tools/test_transitive_code_search.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/vuln_analysis/tools/test_transitive_code_search.py diff --git a/pyproject.toml b/pyproject.toml index 005bad687..20b33fb4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "pytest-asyncio==0.24.*", "pytest-cov~=6.1", "pytest-pretty~=1.2.0", + "pytest-asyncio~=0.24.0", "pytest~=8.3", "setuptools >= 64", "setuptools_scm>=8", diff --git a/src/vuln_analysis/tools/test_transitive_code_search.py b/src/vuln_analysis/tools/test_transitive_code_search.py new file mode 100644 index 000000000..5dcefed44 --- /dev/null +++ b/src/vuln_analysis/tools/test_transitive_code_search.py @@ -0,0 +1,45 @@ +import pytest +import asyncio +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig +from vuln_analysis.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, + ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput +,SBOMPackage,ScanInfoInput,VulnInfo,AgentMorpheusInfo) +from vuln_analysis.runtime_context import ctx_state + + +@pytest.mark.asyncio +async def test_transitive_search_golang_1(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/kuadrant/authorino", + git_ref="f792cd138891dc1ead99fd089aa757fbca3aace9", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("crypto/x509,ParsePKCS1PrivateKey") + (path_found, list_path) = result + assert path_found is True + assert len(list_path) == 3 + + +def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], + excluded_extensions: list[str]): + source_code_info = [SourceDocumentsInfo(type='code', git_repo=git_repository, + ref=git_ref, include=included_extensions, + exclude=excluded_extensions)] + sbom_info_input = ManualSBOMInfoInput(packages=[SBOMPackage(name="a", version="1.0", system="blabla")]) + morpheus_input = AgentMorpheusInput(image=ImageInfoInput(source_info=source_code_info, + sbom_info=sbom_info_input), + scan=ScanInfoInput(vulns=[VulnInfo(vuln_id="CVE-2025-1234")])) + engine_input = AgentMorpheusEngineInput(input=morpheus_input,info=AgentMorpheusInfo()) + state: AgentMorpheusEngineState = AgentMorpheusEngineState(original_input=engine_input, + code_vdb_path="", + doc_vdb_path="", + code_index_path="", + cve_intel=[]) + ctx_state.set(state) + + +async def get_transitive_code_runner_function(): + transitive_code_search = transitive_search(config=TransitiveCodeSearchToolConfig(), builder=None) + async for function in transitive_code_search.gen: + return function.single_fn From 8c133da037e96bcb6ade955d6b8152ce73cce6a6 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 23 Sep 2025 19:20:28 +0300 Subject: [PATCH 080/286] Complete all golang UT tests Signed-off-by: Zvi Grinberg --- .../tools/test_transitive_code_search.py | 71 +++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/tools/test_transitive_code_search.py b/src/vuln_analysis/tools/test_transitive_code_search.py index 5dcefed44..d2e2e0af9 100644 --- a/src/vuln_analysis/tools/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/test_transitive_code_search.py @@ -1,16 +1,22 @@ +import logging + import pytest -import asyncio + +from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig from vuln_analysis.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput -,SBOMPackage,ScanInfoInput,VulnInfo,AgentMorpheusInfo) +, SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) from vuln_analysis.runtime_context import ctx_state +transitive_code_search_runner_coroutine = None + @pytest.mark.asyncio async def test_transitive_search_golang_1(): transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) set_input_for_next_run(git_repository="https://github.com/kuadrant/authorino", git_ref="f792cd138891dc1ead99fd089aa757fbca3aace9", included_extensions=["**/*.go"], @@ -21,6 +27,62 @@ async def test_transitive_search_golang_1(): assert len(list_path) == 3 +@pytest.mark.asyncio +async def test_transitive_search_golang_2(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/oc", + git_ref="0000b3ef257d07f423dfdf9b6d274214d1b0c846", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("crypto/rsa,Verify") + (path_found, list_path) = result + + assert path_found is False + assert len(list_path) == 1 + + +@pytest.mark.asyncio +async def test_transitive_search_golang_3(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/zvigrinberg/router", + git_ref="b49f382f59d6479af9ea26f067ee5cb4e1dd13d9", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/beorn7/perks,NewTargeted") + (path_found, list_path) = result + + assert path_found is True + assert len(list_path) == 5 + + +@pytest.mark.asyncio +async def test_transitive_search_golang_4(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/oc-mirror", + git_ref="b137a53a5360a41a70432ea2bfc98a6cee6f7a4a", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/mholt/archiver,Unarchive") + (path_found, list_path) = result + + assert path_found is True + assert len(list_path) == 2 + + +@pytest.mark.asyncio +async def test_transitive_search_golang_5(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/metallb", + git_ref="3be9d86e5752c6974a2fea99d9373af4f2225e6b", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/jpillora/backoff,ForAttempt") + (path_found, list_path) = result + + assert path_found is False + assert len(list_path) is 1 + + def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], excluded_extensions: list[str]): source_code_info = [SourceDocumentsInfo(type='code', git_repo=git_repository, @@ -28,9 +90,10 @@ def set_input_for_next_run(git_repository: str, git_ref: str, included_extension exclude=excluded_extensions)] sbom_info_input = ManualSBOMInfoInput(packages=[SBOMPackage(name="a", version="1.0", system="blabla")]) morpheus_input = AgentMorpheusInput(image=ImageInfoInput(source_info=source_code_info, - sbom_info=sbom_info_input), + sbom_info=sbom_info_input, + analysis_type=AnalysisType.IMAGE), scan=ScanInfoInput(vulns=[VulnInfo(vuln_id="CVE-2025-1234")])) - engine_input = AgentMorpheusEngineInput(input=morpheus_input,info=AgentMorpheusInfo()) + engine_input = AgentMorpheusEngineInput(input=morpheus_input, info=AgentMorpheusInfo()) state: AgentMorpheusEngineState = AgentMorpheusEngineState(original_input=engine_input, code_vdb_path="", doc_vdb_path="", From cd6c8ddaa907c8bf220190d82958eeaf8e276199 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 23 Sep 2025 20:32:21 +0300 Subject: [PATCH 081/286] fix: infine recursive loop APPENG-2957 https://issues.redhat.com/browse/APPENG-2957 Signed-off-by: Zvi Grinberg --- .../utils/functions_parsers/golang_functions_parsers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 129fd77ca..beadfa125 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -129,6 +129,7 @@ def __trace_down_package(self, expression: str, code_documents: dict[str, Docume caller_function_index: str) -> bool: variables_mappings = functions_local_variables_index[caller_function_index] parts = expression.split(".") + result = False (resolved_type, struct_initializer_expression, @@ -161,7 +162,7 @@ def __trace_down_package(self, expression: str, code_documents: dict[str, Docume elif var_properties is not None: value = var_properties.get("value", None) - if value is not None and value.strip() != "": + if value is not None and value.strip() != "" and len(parts) > 1: return self.__trace_down_package(value, code_documents, type_documents, callee_package, fields_of_types, functions_local_variables_index, caller_function_index) @@ -487,6 +488,9 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum identifier = identifier.replace("return", " ").strip() if "(" in identifier: identifier = identifier[identifier.index("(") + 1:] + if ")" in identifier: + identifier = identifier[:identifier.index(")")] + # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , # if it's the same as callee package name From 86db99daa73d917b09d770b1a5d669453654725f Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 28 Sep 2025 13:42:34 +0300 Subject: [PATCH 082/286] fix: prevent agent answer from containing extensive guidance/instructions Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/prompting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index ff69a1b46..46988fdf8 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -35,7 +35,7 @@ "stored in vector databases available to you via tools.") # Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py -AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. You have access to the following tools: +AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. Avoid giving instructions or guidance on how to achieve a task on final answer, rather focus merely on answering the question. You have access to the following tools: {tools} From 5ce1cd434f1a67fa2afb5e7f6ca8938293a9dd77 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf Date: Tue, 30 Sep 2025 13:22:20 +0300 Subject: [PATCH 083/286] fix: Move ecosystem saving to earlier stage --- src/vuln_analysis/utils/transitive_code_searcher_tool.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py index 7b1760c5d..e43af8d6a 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -72,13 +72,12 @@ def download_dependencies(git_repo_path: Path) -> bool: else: logger.info("Didn't find manifest to install, skipping.. ") return False - + with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: + file.write(ecosystem.name) try: logger.info(f"Started installing packages for {ecosystem}") tree_builder = get_dependency_tree_builder(ecosystem.value) tree_builder.install_dependencies(git_repo_path) - with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: - file.write(ecosystem.name) logger.info(f"Finished installing packages for {ecosystem}") return True except NotImplementedError as err: From 0f3a346c05842922efa069dcf9ba94f0dba083dc Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 30 Sep 2025 13:24:18 +0300 Subject: [PATCH 084/286] fix: Failing to parse C Type aliases in go Fixes: https://issues.redhat.com/browse/APPENG-3435 Signed-off-by: Zvi Grinberg --- .../tools/test_transitive_code_search.py | 14 ++ .../golang_functions_parsers.py | 150 +++++++++++------- 2 files changed, 104 insertions(+), 60 deletions(-) diff --git a/src/vuln_analysis/tools/test_transitive_code_search.py b/src/vuln_analysis/tools/test_transitive_code_search.py index d2e2e0af9..a6d38defa 100644 --- a/src/vuln_analysis/tools/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/test_transitive_code_search.py @@ -82,6 +82,20 @@ async def test_transitive_search_golang_5(): assert path_found is False assert len(list_path) is 1 +# Test fix of https://issues.redhat.com/browse/APPENG-3435 +@pytest.mark.asyncio +async def test_transitive_search_golang_6(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/open-telemetry/opentelemetry-operator", + git_ref="32443f9b956f7aba768fcc1b350d490a34218939", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("net/http,ListenAndServe") + (path_found, list_path) = result + print(result) + assert path_found is False + assert len(list_path) is 0 + def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], excluded_extensions: list[str]): diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index beadfa125..36c851945 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -122,7 +122,6 @@ def is_same_package(package_name_from_input, package_name_from_tree): def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}() {{}}" - def __trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], callee_package: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict], @@ -298,26 +297,16 @@ def __get_type_kind(self, the_type: Document) -> str: if self.__is_struct_or_interface_type(the_type): parts_of_type_header = the_type.page_content.split() return parts_of_type_header[2] if len(parts_of_type_header) > 2 else "" - # not composite type, either primitive or wrapper + # not composite type, primitive or wrapper type else: - if match := re.search(r"^type\s+ \(", the_type.page_content): - right_bracket = match.group(0).find(")") - left_bracket = match.group(0).find("(") - parts = match.group(0)[left_bracket:right_bracket].strip().split(sep=" ", maxsplit=1) - return parts[1] - else: - parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content - return parts[2] + names = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content + # Make sure that aliased types names are cleaned from '=' + return names[2].replace("=", "") def get_type_name(self, the_type: Document) -> str: if self.__is_struct_or_interface_type(the_type): parts_of_type_header = the_type.page_content.split() return parts_of_type_header[1] if len(parts_of_type_header) > 1 else "" - elif match := re.search(r"^type\s+ \(", the_type.page_content): - right_bracket = match.group(0).find(")") - left_bracket = match.group(0).find("(") - parts = match.group(0)[left_bracket:right_bracket].strip().split(sep=" ", maxsplit=1) - return parts[0] else: parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content return parts[1] @@ -325,49 +314,91 @@ def get_type_name(self, the_type: Document) -> str: def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: types_mapping = dict() for the_type in types: - the_kind = self.__get_type_kind(the_type) - type_name = self.get_type_name(the_type) - type_key = type_name - if the_kind.lower() == "interface": - type_key = f"interface;{type_name}" - - start_index = the_type.page_content.find("{") + 2 - end_index = the_type.page_content.rfind("}") - current_index = start_index - fields_list = list() - if end_index > - 1: - while current_index < end_index: - index_of_eol = the_type.page_content[current_index:].find(os.linesep) - end_of_row = end_index - if index_of_eol > -1: - end_of_row = index_of_eol - current_line = the_type.page_content[current_index:current_index + end_of_row] - if not self.is_comment_line(current_line) and not current_line.strip() == "": - right_bracket_completion = "" - if type_key.startswith("interface"): - parts = current_line.split(sep=")", maxsplit=1) - right_bracket_completion = ")" - else: - parts = current_line.split(sep=" ", maxsplit=1) - if len(parts) == 2: - fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), - parts[1].lstrip()[:parts[1].lstrip().find(" ")])) - elif len(parts) == 1: - fields_list.append((EMBEDDED_TYPE, parts[0])) - elif len(parts) > 2: - fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), - parts[1][:parts[1].find(" ")].strip())) - else: - pass - - current_index = current_index + end_of_row + 1 - if type_key.strip() != "" and len(fields_list) != 0: - types_mapping[(type_key, the_type.metadata['source'])] = fields_list - # Primitive type or wrapper of another type + # multiple types in a single block declaration + if re.search(r"^type\s+\(", the_type.page_content): + right_bracket = the_type.page_content.rfind(")") + left_bracket = the_type.page_content.find("(") + all_types = the_type.page_content[left_bracket + 1:right_bracket - 1] + pos = 0 + + current_line_stripped = all_types.lstrip() + size_of_type_block = len(all_types.lstrip()) + while pos < size_of_type_block: + + next_eol = current_line_stripped.find(os.linesep) + next_struct = current_line_stripped.find("struct") + if next_eol > - 1 and (next_struct == -1 or next_struct > next_eol): + if not self.is_comment_line(current_line_stripped[:next_eol + 1]): + + declaration_parts = current_line_stripped[:next_eol + 1].split() + # ignore alias' "equals" notation + if len(declaration_parts) == 3: + [name, _, type_name] = declaration_parts + elif len(declaration_parts) == 2: + [name, type_name] = declaration_parts + + self.parse_one_type(Document(page_content=f"type {name} {type_name}", + metadata={"source": the_type.metadata['source']}), + types_mapping) + pos += next_eol + 1 + current_line_stripped = current_line_stripped[next_eol + 1:].lstrip() + elif -1 < next_struct < next_eol and next_eol > -1: + right_curly_bracket_ind = current_line_stripped.find("}") + self.parse_one_type( + Document(page_content=f"type {current_line_stripped[:right_curly_bracket_ind + 1]}", + metadata={"source": the_type.metadata['source']}), + types_mapping) + pos += right_curly_bracket_ind + current_line_stripped = current_line_stripped[right_curly_bracket_ind + 1:].lstrip() + # End of multiple types declaration + elif next_eol == -1: + pos = size_of_type_block else: - types_mapping[(type_key.strip(), the_type.metadata['source'])] = [(type_name.strip(), the_kind.strip())] + self.parse_one_type(the_type, types_mapping) return types_mapping + def parse_one_type(self, the_type, types_mapping): + the_kind = self.__get_type_kind(the_type) + type_name = self.get_type_name(the_type) + type_key = type_name + if the_kind.lower() == "interface": + type_key = f"interface;{type_name}" + start_index = the_type.page_content.find("{") + 2 + end_index = the_type.page_content.rfind("}") + current_index = start_index + fields_list = list() + if end_index > - 1: + while current_index < end_index: + index_of_eol = the_type.page_content[current_index:].find(os.linesep) + end_of_row = end_index + if index_of_eol > -1: + end_of_row = index_of_eol + current_line = the_type.page_content[current_index:current_index + end_of_row] + if not self.is_comment_line(current_line) and not current_line.strip() == "": + right_bracket_completion = "" + if type_key.startswith("interface"): + parts = current_line.split(sep=")", maxsplit=1) + right_bracket_completion = ")" + else: + parts = current_line.split(sep=" ", maxsplit=1) + if len(parts) == 2: + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + parts[1].lstrip()[:parts[1].lstrip().find(" ")])) + elif len(parts) == 1: + fields_list.append((EMBEDDED_TYPE, parts[0])) + elif len(parts) > 2: + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + parts[1][:parts[1].find(" ")].strip())) + else: + pass + + current_index = current_index + end_of_row + 1 + if type_key.strip() != "" and len(fields_list) != 0: + types_mapping[(type_key, the_type.metadata['source'])] = fields_list + # Primitive type or wrapper of another type + else: + types_mapping[(type_key.strip(), the_type.metadata['source'])] = [(type_name.strip(), the_kind.strip())] + def get_function_reserved_word(self) -> str: return "func" @@ -426,8 +457,10 @@ def get_function_name(self, function: Document) -> str: # TODO Try to extract anonymous function var # else: - def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, - callee_function_package: str, code_documents: list[Document], type_documents: list[Document], + def search_for_called_function(self, caller_function: Document, callee_function_name: str, + callee_function: Document, + callee_function_package: str, code_documents: list[Document], + type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], functions_local_variables_index: dict[str, dict]) -> bool: index_of_function_opening = caller_function.page_content.index("{") @@ -491,7 +524,6 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum if ")" in identifier: identifier = identifier[:identifier.index(")")] - # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , # if it's the same as callee package name # TODO - reduce list to only files in package @@ -531,7 +563,6 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum functions_local_variables_index=functions_local_variables_index, caller_function_index=caller_function_index) - # parameters = [tuple(e.replace(")", "").replace("(", "").split(",")) for e # in re.findall(regex_arguments, function_header)[:2]] # return check_types_from_callee_package(parameter=identifier, params=parameters, @@ -569,6 +600,5 @@ def get_package_names(self, function: Document) -> list[str]: return package_names - def is_root_package(self, function: Document) -> bool: return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) From f3382e9dc32c5ce921fe3d37abba9320824e3009 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf <98809100+TamarW0@users.noreply.github.com> Date: Sun, 19 Oct 2025 13:38:28 +0300 Subject: [PATCH 085/286] test: add tests for python transitive search (#131) --- src/vuln_analysis/tools/tests/__init__.py | 0 .../tools/tests/mock_documents.py | 245 ++++++++++++++++++ .../test_transitive_code_search.py | 111 ++++++++ 3 files changed, 356 insertions(+) create mode 100644 src/vuln_analysis/tools/tests/__init__.py create mode 100644 src/vuln_analysis/tools/tests/mock_documents.py rename src/vuln_analysis/tools/{ => tests}/test_transitive_code_search.py (56%) diff --git a/src/vuln_analysis/tools/tests/__init__.py b/src/vuln_analysis/tools/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/tools/tests/mock_documents.py b/src/vuln_analysis/tools/tests/mock_documents.py new file mode 100644 index 000000000..e205da6eb --- /dev/null +++ b/src/vuln_analysis/tools/tests/mock_documents.py @@ -0,0 +1,245 @@ +from langchain.docstore.document import Document + +python_script_example = Document( + page_content="from flask import helpers\n" + "from werkzeug.formparser import MultiPartParser\n\n" + "helpers.redirect('/home', 302)\n\n" + "multi_part_parser = MultiPartParser()\n" + "print(multi_part_parser.max_form_parts)", + type='Document', metadata={'source': 'src/example_main.py', + 'content_type': 'simplified_code', + 'language': 'python'}) +python_class_example = Document( + page_content='class MultiPartParser:\n' + ' def __init__(\n' + ' self,\n' + ' stream_factory: TStreamFactory | None = None,\n' + ' max_form_memory_size: int | None = None,\n' + ' cls: type[MultiDict[str, t.Any]] | None = None,\n' + ' buffer_size: int = 64 * 1024,\n' + ' max_form_parts: int | None = None,\n' + ' ) -> None:\n' + ' self.max_form_memory_size = max_form_memory_size\n' + ' self.max_form_parts = max_form_parts\n\n' + ' if stream_factory is None:\n' + ' stream_factory = default_stream_factory\n\n' + ' self.stream_factory = stream_factory\n\n' + ' if cls is None:\n' + ' cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict)\n\n' + ' self.cls = cls\n' + ' self.buffer_size = buffer_size\n\n' + ' def fail(self, message: str) -> te.NoReturn:\n' + ' raise ValueError(message)\n\n' + ' def get_part_charset(self, headers: Headers) -> str:\n' + ' # Figure out input charset for current part\n' + ' content_type = headers.get("content-type")\n\n' + ' if content_type:\n' + ' parameters = parse_options_header(content_type)[1]\n' + ' ct_charset = parameters.get("charset", "").lower()\n\n' + ' # A safe list of encodings. Modern clients should only send ASCII or UTF-8.\n' + ' # This list will not be extended further.\n' + ' if ct_charset in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}:\n' + ' return ct_charset\n\n' + ' return "utf-8"\n\n' + ' def start_file_streaming(\n' + ' self, event: File, total_content_length: int | None\n' + ' ) -> t.IO[bytes]:\n' + ' content_type = event.headers.get("content-type")\n\n' + ' try:\n' + ' content_length = _plain_int(event.headers["content-length"])\n' + ' except (KeyError, ValueError):\n' + ' content_length = 0\n\n' + ' container = self.stream_factory(\n' + ' total_content_length=total_content_length,\n' + ' filename=event.filename,\n' + ' content_type=content_type,\n' + ' content_length=content_length,\n' + ' )\n' + ' return container\n\n' + ' def parse(\n' + ' self, stream: t.IO[bytes], boundary: bytes, content_length: int | None\n' + ' ) -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:\n' + ' current_part: Field | File\n' + ' field_size: int | None = None\n' + ' container: t.IO[bytes] | list[bytes]\n' + ' _write: t.Callable[[bytes], t.Any]\n\n' + ' parser = MultipartDecoder(\n' + ' boundary,\n' + ' max_form_memory_size=self.max_form_memory_size,\n' + ' max_parts=self.max_form_parts,\n' + ' )\n\n' + ' fields = []\n' + ' files = []\n\n' + ' for data in _chunk_iter(stream.read, self.buffer_size):\n' + ' parser.receive_data(data)\n' + ' event = parser.next_event()\n' + ' while not isinstance(event, (Epilogue, NeedData)):\n' + ' if isinstance(event, Field):\n' + ' current_part = event\n' + ' field_size = 0\n' + ' container = []\n' + ' _write = container.append\n' + ' elif isinstance(event, File):\n' + ' current_part = event\n' + ' field_size = None\n' + ' container = self.start_file_streaming(event, content_length)\n' + ' _write = container.write\n' + ' elif isinstance(event, Data):\n' + ' if self.max_form_memory_size is not None and field_size is not None:\n' + ' # Ensure that accumulated data events do not exceed limit.\n' + ' # Also checked within single event in MultipartDecoder.\n' + ' field_size += len(event.data)\n\n' + ' if field_size > self.max_form_memory_size:\n' + ' raise RequestEntityTooLarge()\n\n' + ' _write(event.data)\n' + ' if not event.more_data:\n' + ' if isinstance(current_part, Field):\n' + ' value = b"".join(container).decode(\n' + ' self.get_part_charset(current_part.headers), "replace"\n' + ' )\n' + ' fields.append((current_part.name, value))\n' + ' else:\n' + ' container = t.cast(t.IO[bytes], container)\n' + ' container.seek(0)\n' + ' files.append(\n' + ' (\n' + ' current_part.name,\n' + ' FileStorage(\n' + ' container,\n' + ' current_part.filename,\n' + ' current_part.name,\n' + ' headers=current_part.headers,\n' + ' ),\n' + ' )\n' + ' )\n\n' + ' event = parser.next_event()\n\n' + ' return self.cls(fields), self.cls(files)', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_init_function_example = Document(page_content='def __init__(\n' + ' self,\n' + ' stream_factory: TStreamFactory | None = None,\n' + ' max_form_memory_size: int | None = None,\n' + ' cls: type[MultiDict[str, t.Any]] | None = None,\n' + ' buffer_size: int = 64 * 1024,\n' + ' max_form_parts: int | None = None,\n' + ') -> None:\n' + ' mock_function_in_use(str(stream_factory))\n' + ' self.max_form_memory_size = max_form_memory_size\n' + ' self.max_form_parts = max_form_parts\n\n' + ' if stream_factory is None:\n' + ' stream_factory = default_stream_factory\n\n' + ' self.stream_factory = stream_factory\n\n' + ' if cls is None:\n' + ' cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict)\n\n' + ' self.cls = cls\n' + ' self.buffer_size = buffer_size\n' + '#(class: MultiPartParser)', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_full_document_example = Document(page_content='from __future__ import annotations\n\n' + 'from mock_package import mock_function_in_use' + 'import typing as t\n' + 'from io import BytesIO\n' + 'from urllib.parse import parse_qsl\n\n' + 'from ._internal import _plain_int\nfrom .datastructures import FileStorage\n' + 'from .datastructures import Headers\n' + 'from .datastructures import MultiDict\n' + 'from .exceptions import RequestEntityTooLarge\n' + 'from .http import parse_options_header\n' + 'from .sansio.multipart import Data\n' + 'from .sansio.multipart import Epilogue\n' + 'from .sansio.multipart import Field\n' + 'from .sansio.multipart import File\n' + 'from .sansio.multipart import MultipartDecoder\n' + 'from .sansio.multipart import NeedData\n' + 'from .wsgi import get_content_length\n' + 'from .wsgi import get_input_stream\n\n' + '# there are some platforms where SpooledTemporaryFile is not available.\n' + '# In that case we need to provide a fallback.\n' + 'try:\n' + ' from tempfile import SpooledTemporaryFile\nexcept ImportError:\n' + ' from tempfile import TemporaryFile\n\n SpooledTemporaryFile = None # type: ignore\n\nif t.TYPE_CHECKING:\n import typing as te\n\n from _typeshed.wsgi import WSGIEnvironment\n\n t_parse_result = tuple[\n t.IO[bytes], MultiDict[str, str], MultiDict[str, FileStorage]\n ]\n\n class TStreamFactory(te.Protocol):\n def __call__(\n self,\n total_content_length: int | None,\n content_type: str | None,\n filename: str | None,\n content_length: int | None = None,\n ) -> t.IO[bytes]: ...\n\n\nF = t.TypeVar("F", bound=t.Callable[..., t.Any])\n\n\n# Code for: def default_stream_factory(\n\n\n# Code for: def parse_form_data(\n\n\n# Code for: class FormDataParser:\n\n\n# Code for: class MultiPartParser:\n\n\n# Code for: def _chunk_iter(read: t.Callable[[int], bytes], size: int) -> t.Iterator[bytes | None]:', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'simplified_code', + 'language': 'python'}) +python_parse_function_example = Document(page_content='def parse(\n' + ' self, stream: t.IO[bytes], boundary: bytes, content_length: int | None\n' + ') -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:\n' + ' current_part: Field | File\n field_size: int | None = None\n' + ' container: t.IO[bytes] | list[bytes]\n' + ' _write: t.Callable[[bytes], t.Any]\n\n' + ' parser = MultipartDecoder(\n' + ' boundary,\n' + ' max_form_memory_size=self.max_form_memory_size,\n' + ' max_parts=self.max_form_parts,\n )\n\n' + ' fields = []\n files = []\n\n' + ' for data in _chunk_iter(stream.read, self.buffer_size):\n' + ' parser.receive_data(data)\n' + ' event = parser.next_event()\n' + ' while not isinstance(event, (Epilogue, NeedData)):\n' + ' if isinstance(event, Field):\n' + ' current_part = event\n' + ' field_size = 0\n' + ' container = []\n' + ' _write = container.append\n' + ' elif isinstance(event, File):\n' + ' current_part = event\n' + ' field_size = None\n' + ' container = self.start_file_streaming(event, content_length)\n' + ' _write = container.write\n' + ' elif isinstance(event, Data):\n' + ' if self.max_form_memory_size is not None and field_size is not None:\n' + ' # Ensure that accumulated data events do not exceed limit.\n' + ' # Also checked within single event in MultipartDecoder.\n' + ' field_size += len(event.data)\n\n' + ' if field_size > self.max_form_memory_size:\n' + ' raise RequestEntityTooLarge()\n\n' + ' _write(event.data)\n' + ' if not event.more_data:\n' + ' if isinstance(current_part, Field):\n' + ' value = b"".join(container).decode(\n' + ' self.get_part_charset(current_part.headers), "replace"\n' + ' )\n' + ' fields.append((current_part.name, value))\n' + ' else:\n' + ' container = t.cast(t.IO[bytes], container)\n' + ' container.seek(0)\n' + ' files.append(\n' + ' (\n' + ' current_part.name,\n' + ' FileStorage(\n' + ' container,\n' + ' current_part.filename,\n' + ' current_part.name,\n' + ' headers=current_part.headers,\n' + ' ),\n' + ' )\n )\n\n' + ' event = parser.next_event()\n\n' + ' return self.cls(fields), self.cls(files)\n' + '#(class: MultiPartParser)', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_mock_function_in_use = Document(page_content='def mock_function_in_use(stream_factory: str) -> None:\n' + ' print stream_factory', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/mock_package/mock_file.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_mock_file = Document(page_content='def mock_function_in_use(stream_factory: str) -> None:\n' + ' print stream_factory', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/mock_package/mock_file.py', + 'content_type': 'simplified_code', + 'language': 'python'}) diff --git a/src/vuln_analysis/tools/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py similarity index 56% rename from src/vuln_analysis/tools/test_transitive_code_search.py rename to src/vuln_analysis/tools/tests/test_transitive_code_search.py index a6d38defa..62c9d6d9b 100644 --- a/src/vuln_analysis/tools/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -1,6 +1,7 @@ import logging import pytest +from unittest.mock import patch, MagicMock from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.data_models.state import AgentMorpheusEngineState @@ -9,6 +10,9 @@ ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput , SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.tools.tests.mock_documents import (python_script_example, python_init_function_example, + python_full_document_example, python_parse_function_example, + python_mock_function_in_use, python_mock_file) transitive_code_search_runner_coroutine = None @@ -120,3 +124,110 @@ async def get_transitive_code_runner_function(): transitive_code_search = transitive_search(config=TransitiveCodeSearchToolConfig(), builder=None) async for function in transitive_code_search.gen: return function.single_fn + +python_dependency_tree_mock_output = ( + 'deptree==0.0.12 # deptree\n' + ' importlib-metadata==8.7.0 # importlib-metadata\n' + ' zipp==3.23.0 # zipp>=3.20\n' + ' setuptools==80.9.0 # setuptools\n' + 'flask==3.1.2 # flask\n' + ' blinker==1.9.0 # blinker>=1.9.0\n' + ' click==8.3.0 # click>=8.1.3\n' + ' itsdangerous==2.2.0 # itsdangerous>=2.2.0\n' + ' jinja2==3.1.6 # jinja2>=3.1.2\n' + ' markupsafe==3.0.3 # MarkupSafe>=2.0\n' + ' markupsafe==3.0.3 # markupsafe>=2.1.1\n' + ' werkzeug==3.1.3 # werkzeug>=3.1.0\n' + ' markupsafe==3.0.3 # MarkupSafe>=2.1.1\n' + ' mock-package==1.1.1' +) + +def mock_file_open(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "PYTHON" + elif 'requirements.txt' in str(file_path): + mock_file.__iter__ = MagicMock(return_value=iter([ + "flask==2.0.1\n", + "werkzeug==2.0.1\n", + "requests==2.25.1\n" + ])) + else: + mock_file.read.return_value = "" + return mock_file + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_case", [ + { + "name": "python_1", + "search_query": "werkzeug,formparser.MultiPartParser", + "expected_path_found": True, + "expected_list_length": 2, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example] + }, + { + "name": "python_2", + "search_query": "werkzeug,formparser.MultiPartParser.parse", + "expected_path_found": False, + "expected_list_length": 0, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example] + }, + { + "name": "python_3", + "search_query": "mock_package,mock_function_in_use", + "expected_path_found": True, + "expected_list_length": 3, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file] + } +]) +@patch('vuln_analysis.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) +@patch('builtins.open', side_effect=mock_file_open) +async def test_transitive_search_python_parameterized(mock_open, mock_run_command,test_case): + """Parameterized test that runs all existing test cases with their respective configurations.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/something/something', + git_ref='f792cd138891dc1ead99fd089aa757fbca3aace9', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + with patch('vuln_analysis.utils.document_embedding.retrieve_from_cache', + return_value=(test_case["mock_documents"], True)): + result = await transitive_code_search_runner_coroutine(test_case["search_query"]) + + (path_found, list_path) = result + print(f"DEBUG: Test case {test_case['name']}") + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == test_case["expected_path_found"] + assert len(list_path) == test_case["expected_list_length"] + +@pytest.mark.asyncio +async def test_python_transitive_search(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/TamarW0/example-repo', + git_ref='17af124607ff06fdf95734419b7ecd3af769c2f8', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + result = await transitive_code_search_runner_coroutine("werkzeug,formparser.MultiPartParser") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 \ No newline at end of file From d9b1d6f00f5d4df6f19a2eafbbf7b14dc999e9e1 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 20 Oct 2025 13:01:59 +0300 Subject: [PATCH 086/286] ci: enable unit tests with shared cache storage Signed-off-by: Vladimir Belousov --- .tekton/on-pull-request.yaml | 53 ++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index e23626173..cb0765b22 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -126,9 +126,12 @@ spec: workspaces: - name: source workspace: source + - name: unit-test-cache + workspace: unit-test-cache taskSpec: workspaces: - name: source + - name: unit-test-cache volumes: - name: env-file-volume secret: @@ -137,6 +140,25 @@ spec: configMap: name: integration-tests-reports steps: + - name: setup-cache-link + image: registry.redhat.io/ubi9/ubi-micro:latest + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -eou pipefail + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="$(workspaces.unit-test-cache.path)" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" - name: lint-test env: - name: TARGET_BRANCH_NAME @@ -151,16 +173,16 @@ spec: mountPath: $(workspaces.source.path)/examples script: | #!/bin/bash - set -eou pipefail - # Install uv - curl -LsSf https://astral.sh/uv/install.sh | sh - + print_banner() { echo echo "----------- ${1} -----------" } - + + # Install uv + print_banner "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh print_banner "CHECKING FOR .ENV FILE" if [ -f .env ]; then echo ".env file successfully mounted." @@ -183,7 +205,19 @@ spec: print_banner "INSTALLING DEPENDENCIES" uv sync - + + # Install Go + print_banner "Installing Go" + + GO_VERSION="1.24.1" + ARCH="amd64" + curl -s -L -o go.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz + mkdir -p $HOME/go-sdk + tar -C $HOME/go-sdk -xzf go.tar.gz + export PATH=$HOME/go-sdk/go/bin:$PATH + echo "Go version:" + go version + print_banner "RUNNING LINTER" # Add the current directory to git's safe directories to avoid ownership errors. git config --global --add safe.directory /workspace/source @@ -196,8 +230,7 @@ spec: make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME print_banner "RUNNING UNIT TESTS" - # TODO: Uncomment the following line when unit tests are added to the 'src' directory. - # make test-unit + make test-unit print_banner "LINT AND TEST COMPLETE" workspaces: @@ -227,6 +260,10 @@ spec: resources: requests: storage: 5Gi + + - name: unit-test-cache + persistentVolumeClaim: + claimName: unit-test-shared-cache # This workspace will inject secret to help the git-clone task to be able to # checkout the private repositories - name: basic-auth From 9346e054dd9a8549e42ef42dd8178c257022d497 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 21 Oct 2025 14:34:02 +0300 Subject: [PATCH 087/286] chore: add unit tests and improved git loader logic Signed-off-by: Vladimir Belousov --- .../tests/test_source_code_git_loader.py | 23 +++++++++++++++++++ .../utils/source_code_git_loader.py | 10 ++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 src/vuln_analysis/tools/tests/test_source_code_git_loader.py diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py new file mode 100644 index 000000000..37daa023c --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -0,0 +1,23 @@ +import pytest +import shutil +from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader + +@pytest.mark.parametrize("refs", [ + ("3.1.43", "3.1.42", "tag switch"), + ("ba5c10d3655c4fec714294cbc2ae0829c44dc046", + "5b098ff41266d6990237cbade5b6e27cab23557f", "sha switch"), +]) +def test_load_repo_ref_switch(tmp_path, refs): + repo_url = "https://github.com/gitpython-developers/GitPython" + repo_path = tmp_path / "git_python_test_repo" + ref_a, ref_b, desc = refs + + loader_a = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=ref_a) + repo_a = loader_a.load_repo() + assert repo_a.head.commit.hexsha == repo_a.commit(ref_a).hexsha, f"Initial {desc} failed" + + loader_b = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=ref_b) + repo_b = loader_b.load_repo() + assert repo_b.head.commit.hexsha == repo_b.commit(ref_b).hexsha, f"Ref switch {desc} failed" + + shutil.rmtree(repo_path) diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py index d596374e6..ae2b089dd 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/vuln_analysis/utils/source_code_git_loader.py @@ -20,6 +20,7 @@ from git import Blob as GitBlob from git import Repo +from git.exc import GitCommandError from langchain_community.document_loaders.blob_loaders.schema import BlobLoader from langchain_core.document_loaders.blob_loaders import Blob from tqdm import tqdm @@ -128,8 +129,13 @@ def load_repo(self): logger.info("Using existing Git repo at path: '%s' @ '%s'", self.repo_path, self.ref) # Reliable way to check out the ref using a shallow clone - repo.git.fetch("origin", self.ref, depth=1) - repo.git.checkout("FETCH_HEAD") + repo.git.fetch("origin", self.ref, "--depth=1", "--force") + tag_refspec = f"refs/tags/{self.ref}:refs/tags/{self.ref}" + try: + repo.git.fetch("origin", tag_refspec, "--depth=1" , "--force") + except GitCommandError: + pass + repo.git.checkout(self.ref, "--force") logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) TransitiveCodeSearcher.download_dependencies(self.repo_path) From ce4b45f0370037c71098323eacd95c01195c9d19 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 22 Oct 2025 13:10:25 +0300 Subject: [PATCH 088/286] ci: remove integration test artifacts from Tekton pipeline Signed-off-by: Vladimir Belousov --- .tekton/on-pull-request.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index cb0765b22..54d7d7d44 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -132,13 +132,6 @@ spec: workspaces: - name: source - name: unit-test-cache - volumes: - - name: env-file-volume - secret: - secretName: integration-tests-env-file - - name: examples-volume - configMap: - name: integration-tests-reports steps: - name: setup-cache-link image: registry.redhat.io/ubi9/ubi-micro:latest @@ -165,12 +158,6 @@ spec: value: "{{target_branch}}" image: registry.access.redhat.com/ubi9/python-312:9.6 workingDir: $(workspaces.source.path) - volumeMounts: - - name: env-file-volume - mountPath: $(workspaces.source.path)/.env - subPath: .env - - name: examples-volume - mountPath: $(workspaces.source.path)/examples script: | #!/bin/bash set -eou pipefail @@ -183,21 +170,6 @@ spec: # Install uv print_banner "Installing uv" curl -LsSf https://astral.sh/uv/install.sh | sh - print_banner "CHECKING FOR .ENV FILE" - if [ -f .env ]; then - echo ".env file successfully mounted." - else - echo "ERROR: .env file not found!" - exit 1 - fi - - print_banner "CHECKING FOR EXAMPLES DIR" - if [ -d "examples" ] && [ -n "$(ls -A examples)" ]; then - echo "Examples directory successfully mounted." - else - echo "ERROR: Examples directory not found or is empty!" - exit 1 - fi print_banner "CREATING AND ACTIVATING TEST ENV" uv venv --python 3.12 .venv From 3964f2d658dec7cd8db036a374c2fb33f633036c Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:02:34 +0200 Subject: [PATCH 089/286] APPENG-3181: Feature Add C support to transitive tool (#104) * feat: Add C/C++ support to transitive code search tool - Add C language support and enhance function parsing capabilities - Implement singleton pattern for RPMDependencyManager - Add container source download for C/C++ dependencies - Add C extended segmenter class with comprehensive testing - Add new Function Name Locator tool with fuzzy matching - Enhance transitive code search with assistant tool - Add C/C++ function parsers and language support - Add comprehensive test suite for C code analysis - Improve function name extraction and parsing - Add source RPM downloader for C/C++ dependencies - Update configuration files for NIM and OpenAI endpoints - Fix various bugs and improve code quality - Add proper documentation and type hints - Clean up imports and remove debug prints This commit consolidates 39 individual commits that add comprehensive C/C++ language support to the vulnerability analysis toolchain. * update docs, kustomize service config, tests and rpm downloader support UT * review fix move C_DEP_LIBS_NAME locaiton * fixed review comments * Add cache for std lib names for lang python,go,java,javascript * Update src/vuln_analysis/utils/standard_library_cache.py Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> * fix indentation * add persistance to cache * code review comments * add comments and update docker for bsdtar app to be install in image * update docfile * small improvments * handle invalid utf-8 files --------- Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --- Dockerfile | 4 + README.md | 9 + kustomize/README.md | 27 +- kustomize/base/excludes.json | 9 + kustomize/base/exploit-iq-config.yml | 6 +- kustomize/base/exploit_iq_service.yaml | 10 + kustomize/base/includes.json | 4 + kustomize/config-http-openai-local.yml | 4 + src/vuln_analysis/configs/config-http-nim.yml | 239 +++ .../configs/config-http-openai.yml | 4 + .../standard_libs/standard_libraries.json | 386 +++++ src/vuln_analysis/functions/cve_agent.py | 5 +- .../functions/cve_generate_vdbs.py | 21 +- .../tests/test_transitive_code_search.py | 75 +- .../tools/transitive_code_search.py | 191 ++- src/vuln_analysis/utils/c_segmenter_custom.py | 70 + .../utils/chain_of_calls_retriever.py | 390 ++++- src/vuln_analysis/utils/dep_tree.py | 733 ++++++++- src/vuln_analysis/utils/document_embedding.py | 18 +- .../utils/function_name_extractor.py | 24 +- .../utils/function_name_locator.py | 275 ++++ .../c_lang_function_parsers.py | 740 +++++++++ .../golang_functions_parsers.py | 7 +- .../lang_functions_parsers.py | 71 +- .../lang_functions_parsers_factory.py | 12 +- .../python_functions_parser.py | 10 +- src/vuln_analysis/utils/intel_source_score.py | 6 +- .../utils/source_rpm_downloader.py | 707 +++++++++ .../utils/standard_library_cache.py | 159 ++ .../utils/transitive_code_searcher_tool.py | 30 +- tests/ctests/cms_sd.c | 1191 +++++++++++++++ tests/ctests/ec_asn1.c | 1332 +++++++++++++++++ tests/ctests/err.c | 898 +++++++++++ tests/ctests/ess_lib.c | 368 +++++ tests/ctests/pk7_asn1.c | 256 ++++ tests/ctests/rsa_asn1.c | 128 ++ tests/ctests/test_c_extended.py | 146 ++ 37 files changed, 8376 insertions(+), 189 deletions(-) create mode 100644 src/vuln_analysis/configs/config-http-nim.yml create mode 100644 src/vuln_analysis/data/standard_libs/standard_libraries.json create mode 100644 src/vuln_analysis/utils/c_segmenter_custom.py create mode 100644 src/vuln_analysis/utils/function_name_locator.py create mode 100644 src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py create mode 100644 src/vuln_analysis/utils/source_rpm_downloader.py create mode 100644 src/vuln_analysis/utils/standard_library_cache.py create mode 100644 tests/ctests/cms_sd.c create mode 100644 tests/ctests/ec_asn1.c create mode 100644 tests/ctests/err.c create mode 100644 tests/ctests/ess_lib.c create mode 100644 tests/ctests/pk7_asn1.c create mode 100644 tests/ctests/rsa_asn1.c create mode 100644 tests/ctests/test_c_extended.py diff --git a/Dockerfile b/Dockerfile index 6477049f4..40054c23c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -33,9 +33,13 @@ RUN apt-get update && apt-get install -y \ git \ git-lfs \ wget \ + skopeo \ + libarchive-tools \ && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates + RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24.1.linux-amd64.tar.gz \ && tar -C /usr/local -xzf /tmp/go1.24.1.linux-amd64.tar.gz \ && rm /tmp/go1.24.1.linux-amd64.tar.gz diff --git a/README.md b/README.md index 8290810d8..2cb666b2b 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,15 @@ To run the workflow you need to obtain API keys for the following APIs. These wi - Click on your account in the top right, select "Setup" from the dropdown. - Click the "Generate Personal Key" option and then the "+ Generate Personal Key" button to create your API key. - This will be used in the `NVIDIA_API_KEY` environment variable. + - REDHAT container registry (Recommended but not compulsory) + - To get source images from a Red Hat container registry using registry service account tokens. You will need to create a [registry service account](https://access.redhat.com/terms-based-registry/) + - Steps: + - Sign in to the registry service accout + - Press on New service Account Button + - Fill the Name (ex. 'test-case') and Description fields and Click the Create botton + - Token user name for example '11008101|test-case' for REGISTRY_REDHAT_USERNAME environment variable + - Token password a long string for REGISTRY_REDHAT_PASSWORD environment variable + The workflow can be configured to use other LLM services as well, see the [Customizing the LLM models](#customizing-the-llm-models) section for more info. diff --git a/kustomize/README.md b/kustomize/README.md index df9d5bd29..6fdcc40d4 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -38,17 +38,38 @@ export NVIDIA_API_BASE=http://YOUR_SELF_HOSTED_OPENAI_LLM_ADDRESSS/v1 export PYTHONUNBUFFERED=1 export SERPAPI_API_KEY=YOUR_SERPAPI_KEY export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export REGISTRY_REDHAT_USERNAME="your_username" +export REGISTRY_REDHAT_PASSWORD="your_password_or_token" ``` 6. **Run the Application**: After a successful installation, run the application. - ```shell nat --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 ``` +### Container Source Download Configuration + +For C/C++ projects, you can enable container source download to extract RPM dependencies from Red Hat container registries. This feature uses skopeo to download container source layers and automatically extracts RPM packages, excluding the main application RPMs to focus on dependencies only. +**Required Environment Variables:** +# Enable container source download mode (default behavior ) +export USE_CONTAINER_SOURCES=true + +**How it works:** +1. Downloads container source layers using skopeo +2. Extracts RPM packages from the downloaded layers +3. Filters out main application RPMs (e.g., `postgresql-*` for PostgreSQL containers) +4. Copies dependency RPMs to the standard RPM cache directory + + +**Prerequisites:** +- `skopeo` must be installed on the system +- Valid Red Hat registry credentials +- Network access to `registry.redhat.io` + + ## Deploy And Run On OCP -1. Create a `base/secrets.env` file containing the API keys for external services `ExploitIQ` might use. Not all keys are mandatory. Refer to the main [README](../README.md) for details. +1. Create a `base/secrets.env` file containing the API keys for external services `ExploitIQ` might use. Not all keys are mandatory. Refer to the main [README](../README.md#obtain-api-keys) for details on how to create the Red Hat credentials and other API keys. ```shell cat > base/secrets.env << EOF @@ -56,6 +77,8 @@ nvd_api_key=you_api_key serpapi_api_key=your_api_key nvidia_api_key=your_api_key ghsa_api_key=your_api_key +registry_redhat_username=your_registry_username +registry_redhat_password=your_registry_pass_token EOF ``` diff --git a/kustomize/base/excludes.json b/kustomize/base/excludes.json index e9fc1320f..f3fac3cb6 100644 --- a/kustomize/base/excludes.json +++ b/kustomize/base/excludes.json @@ -17,6 +17,15 @@ "pom.xml", "build.gradle" ], + "C": [ + "**/tests/**/*", + "**/test/**/*", + "**/demos/**/*", + "**/examples/**/*", + "**/samples/**/*", + "**/doc/**/*", + "**/docs/**/*" + ], "JavaScript": [ "node_modules/**/*", "dist/**/*", diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 29994cd92..e0f91dc2a 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -32,7 +32,8 @@ functions: base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git base_vdb_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}vdb base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index - base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -59,6 +60,8 @@ functions: Calling Function Name Extractor: _type: calling_function_name_extractor enable_functions_usage_search: true + Package and Function Locator: + _type: package_and_function_locator Container Image Code QA System: _type: local_vdb_retriever embedder_name: nim_embedder @@ -89,6 +92,7 @@ functions: - Internet Search - Transitive code search tool - Calling Function Name Extractor + - Package and Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 96c7a3d4f..8d33adf3d 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -79,6 +79,16 @@ spec: value: EXPLOIT_IQ - name: NGC_API_KEY value: EXPLOIT_IQ + - name: REGISTRY_REDHAT_PASSWORD + valueFrom: + secretKeyRef: + key: registry_redhat_password + name: exploit-iq-secret + - name: REGISTRY_REDHAT_USERNAME + valueFrom: + secretKeyRef: + key: registry_redhat_username + name: exploit-iq-secret - name: CVE_DETAILS_BASE_URL value: http://nginx-cache:8080/cve-details - name: CWE_DETAILS_BASE_URL diff --git a/kustomize/base/includes.json b/kustomize/base/includes.json index 3ccc429e1..7f1df85f8 100644 --- a/kustomize/base/includes.json +++ b/kustomize/base/includes.json @@ -48,6 +48,10 @@ "public/**/*", "assets/**/*" ], + "C":[ + "**/*.c", + "**/*.h" + ], "Dockerfile": [ "Dockerfile*", "docker-compose.yml", diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 9f7aa7c19..91fc57fa1 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -37,6 +37,7 @@ functions: base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -60,6 +61,8 @@ functions: Calling Function Name Extractor: _type: calling_function_name_extractor enable_functions_usage_search: true + Package and Function Locator: + _type: package_and_function_locator Container Image Code QA System: _type: local_vdb_retriever embedder_name: nim_embedder @@ -90,6 +93,7 @@ functions: - Internet Search - Transitive code search tool - Calling Function Name Extractor + - Package and Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml new file mode 100644 index 000000000..c95cf05f9 --- /dev/null +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: http://localhost:6006/v1/traces + project: cve_agent + +functions: + cve_generate_vdbs: + _type: cve_generate_vdbs + agent_name: cve_agent_executor # Used to determine which tools are enabled + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + cve_process_sbom: + _type: cve_process_sbom + cve_check_vuln_deps : + _type: cve_check_vuln_deps + skip: true + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Transitive code search tool: + _type: transitive_code_search + enable_transitive_search: true + Calling Function Name Extractor: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Package and Function Locator: + _type: package_and_function_locator + Container Image Code QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Container Image Developer Guide QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Lexical Search Container Image Code QA System: + _type: lexical_code_search + top_k: 5 + Internet Search: + _type: serp_wrapper + max_retries: 5 + Container Image Analysis Data: + _type: container_image_analysis_data + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Internet Search + - Transitive code search tool + - Calling Function Name Extractor + - Package and Function Locator + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /reports + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + +llms: + checklist_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + generate_cvss_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + summarize_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + intel_source_score_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_generate_vdbs_name: cve_generate_vdbs + cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score + cve_process_sbom_name: cve_process_sbom + cve_check_vuln_deps_name: cve_check_vuln_deps + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 775515a0f..ccec2190f 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -31,6 +31,7 @@ functions: base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -53,6 +54,8 @@ functions: Calling Function Name Extractor: _type: calling_function_name_extractor enable_functions_usage_search: true + Package and Function Locator: + _type: package_and_function_locator Container Image Code QA System: _type: local_vdb_retriever embedder_name: nim_embedder @@ -83,6 +86,7 @@ functions: - Internet Search - Transitive code search tool - Calling Function Name Extractor + - Package and Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/data/standard_libs/standard_libraries.json b/src/vuln_analysis/data/standard_libs/standard_libraries.json new file mode 100644 index 000000000..0d0d660f6 --- /dev/null +++ b/src/vuln_analysis/data/standard_libs/standard_libraries.json @@ -0,0 +1,386 @@ +{ + "_metadata": { + "_comment": "AUTO-GENERATED FILE - Do not manually edit. Generated using AI. Review changes.", + "_generation_prompt": "For {ecosystem}, generate a comprehensive JSON list of ALL standard library packages/modules. Include built-in modules that come with the language installation. Format as a flat array of strings. Be exhaustive and accurate.", + "_last_updated": "2025-10-21" + }, + "go": [ + "archive/tar", + "archive/zip", + "bufio", + "bytes", + "compress/gzip", + "compress/zlib", + "container/heap", + "container/list", + "context", + "crypto", + "crypto/aes", + "crypto/cipher", + "crypto/des", + "crypto/ecdsa", + "crypto/ed25519", + "crypto/hmac", + "crypto/md5", + "crypto/rand", + "crypto/rsa", + "crypto/sha1", + "crypto/sha256", + "crypto/sha512", + "crypto/tls", + "crypto/x509", + "database/sql", + "database/sql/driver", + "debug/pprof", + "encoding", + "encoding/ascii85", + "encoding/base32", + "encoding/base64", + "encoding/binary", + "encoding/csv", + "encoding/gob", + "encoding/hex", + "encoding/json", + "encoding/pem", + "encoding/xml", + "errors", + "expvar", + "flag", + "fmt", + "go/ast", + "go/parser", + "go/token", + "hash", + "hash/adler32", + "hash/crc32", + "hash/fnv", + "html", + "html/template", + "image", + "image/color", + "image/draw", + "image/gif", + "image/jpeg", + "image/png", + "index/suffixarray", + "io", + "io/fs", + "io/ioutil", + "log", + "log/syslog", + "math", + "math/big", + "math/cmplx", + "math/rand", + "mime", + "mime/multipart", + "net", + "net/http", + "net/http/cgi", + "net/http/cookiejar", + "net/http/httptest", + "net/http/httputil", + "net/http/pprof", + "net/mail", + "net/rpc", + "net/smtp", + "net/textproto", + "net/url", + "os", + "os/exec", + "os/signal", + "os/user", + "path", + "path/filepath", + "reflect", + "regexp", + "regexp/syntax", + "runtime", + "runtime/cgo", + "runtime/debug", + "runtime/pprof", + "sort", + "strconv", + "strings", + "sync", + "sync/atomic", + "syscall", + "testing", + "testing/quick", + "text/scanner", + "text/tabwriter", + "text/template", + "time", + "time/tzdata", + "unicode", + "unicode/utf16", + "unicode/utf8", + "unsafe" + ], + "python": [ + "abc", + "argparse", + "array", + "ast", + "asyncio", + "base64", + "bisect", + "bz2", + "calendar", + "cProfile", + "codecs", + "collections", + "collections.abc", + "concurrent.futures", + "configparser", + "contextlib", + "copy", + "csv", + "ctypes", + "dataclasses", + "datetime", + "decimal", + "difflib", + "dis", + "email", + "enum", + "errno", + "fractions", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "gzip", + "hashlib", + "heapq", + "hmac", + "http", + "http.client", + "http.cookiejar", + "http.cookies", + "http.server", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "logging", + "logging.config", + "logging.handlers", + "lzma", + "math", + "mmap", + "multiprocessing", + "netrc", + "operator", + "os", + "os.path", + "pathlib", + "pdb", + "pickle", + "platform", + "pprint", + "pty", + "queue", + "random", + "re", + "sched", + "secrets", + "select", + "shelve", + "shlex", + "shutil", + "signal", + "socket", + "socketserver", + "sqlite3", + "ssl", + "statistics", + "string", + "struct", + "subprocess", + "sys", + "sysconfig", + "tarfile", + "tempfile", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "traceback", + "tracemalloc", + "types", + "typing", + "unittest", + "urllib", + "urllib.error", + "urllib.parse", + "urllib.request", + "urllib.robotparser", + "uuid", + "warnings", + "weakref", + "webbrowser", + "wsgiref", + "xml.dom.minidom", + "xml.etree.ElementTree", + "xml.parsers.expat", + "xmlrpc.client", + "xmlrpc.server", + "zipapp", + "zipfile", + "zlib" + ], + "java": [ + "java.applet", + "java.awt", + "java.awt.color", + "java.awt.datatransfer", + "java.awt.event", + "java.awt.font", + "java.awt.geom", + "java.awt.im", + "java.awt.image", + "java.awt.print", + "java.beans", + "java.io", + "java.lang", + "java.lang.annotation", + "java.lang.instrument", + "java.lang.invoke", + "java.lang.management", + "java.lang.ref", + "java.lang.reflect", + "java.math", + "java.net", + "java.net.http", + "java.nio", + "java.nio.channels", + "java.nio.channels.spi", + "java.nio.charset", + "java.nio.charset.spi", + "java.nio.file", + "java.nio.file.attribute", + "java.nio.file.spi", + "java.rmi", + "java.rmi.activation", + "java.rmi.registry", + "java.rmi.server", + "java.security", + "java.security.acl", + "java.security.cert", + "java.security.interfaces", + "java.security.spec", + "java.sql", + "java.text", + "java.text.spi", + "java.time", + "java.time.chrono", + "java.time.format", + "java.time.temporal", + "java.time.zone", + "java.util", + "java.util.concurrent", + "java.util.concurrent.atomic", + "java.util.concurrent.locks", + "java.util.function", + "java.util.jar", + "java.util.logging", + "java.util.prefs", + "java.util.regex", + "java.util.spi", + "java.util.stream", + "java.util.zip", + "javax.accessibility", + "javax.crypto", + "javax.crypto.interfaces", + "javax.crypto.spec", + "javax.imageio", + "javax.imageio.event", + "javax.imageio.spi", + "javax.imageio.stream", + "javax.lang.model", + "javax.management", + "javax.naming", + "javax.naming.directory", + "javax.naming.event", + "javax.naming.ldap", + "javax.naming.spi", + "javax.net", + "javax.net.ssl", + "javax.print", + "javax.rmi", + "javax.script", + "javax.security.auth", + "javax.security.auth.callback", + "javax.security.auth.login", + "javax.security.auth.x500", + "javax.security.cert", + "javax.sound.midi", + "javax.sound.sampled", + "javax.sql", + "javax.sql.rowset", + "javax.swing", + "javax.swing.border", + "javax.swing.event", + "javax.swing.filechooser", + "javax.swing.plaf", + "javax.swing.table", + "javax.swing.text", + "javax.swing.tree", + "javax.swing.undo", + "javax.tools", + "javax.transaction.xa", + "javax.xml.parsers", + "javax.xml.transform", + "javax.xml.xpath", + "org.w3c.dom", + "org.xml.sax" + ], + "javascript": [ + "assert", + "async_hooks", + "buffer", + "child_process", + "cluster", + "console", + "crypto", + "dgram", + "dns", + "domain", + "events", + "fs", + "fs/promises", + "http", + "http2", + "https", + "inspector", + "module", + "net", + "os", + "path", + "perf_hooks", + "process", + "querystring", + "readline", + "repl", + "stream", + "string_decoder", + "timers", + "timers/promises", + "tls", + "trace_events", + "tty", + "url", + "util", + "v8", + "vm", + "wasi", + "worker_threads", + "zlib" + ] +} \ No newline at end of file diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 6ea0cc882..a850c1263 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -79,7 +79,10 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, (tool.name == "Transitive code search tool" and (not config.transitive_search_tool_enabled or state.code_index_path is None)) or (tool.name == "Calling Function Name Extractor" and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) + state.code_index_path is None)) or + (tool.name == "Package and Function Locator" and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) + ) ] diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 7772e88d5..54aae31ae 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -26,6 +26,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field +from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) @@ -48,6 +49,8 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): description="The directory used for storing code index files.") base_pickle_dir: str = Field(default=".cache/am_cache/pickle", description="The directory used for storing code index files.") + base_rpm_dir: str = Field(default=".cache/am_cache/rpms", + description="The directory used for storing rpm files.") ignore_errors: bool = Field(default=False, description="Whether to ignore errors during generation.") ignore_code_embedding: bool = Field(default=False, description="Whether to ignore building code vector database.") @@ -64,7 +67,10 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): from vuln_analysis.utils.document_embedding import DocumentEmbedding from vuln_analysis.utils.full_text_search import FullTextSearch from vuln_analysis.utils.git_utils import get_repo_from_path - + from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager + from vuln_analysis.data_models.input import ManualSBOMInfoInput + from vuln_analysis.utils.standard_library_cache import StandardLibraryCache + agent_config = builder.get_function_config(config.agent_name) assert isinstance(agent_config, CVEAgentExecutorToolConfig) @@ -79,6 +85,12 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): config.ignore_code_index = True embedding = await builder.get_embedder(embedder_name=config.embedder_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + # Configure RPM singleton with cache directory from config + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + cache_std = StandardLibraryCache.get_instance() + cache_std.set_cache_directory(config.base_pickle_dir) embedder = DocumentEmbedding(embedding=embedding, vdb_directory=config.base_vdb_dir, @@ -175,6 +187,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: base_image = message.image.name source_infos = message.image.source_info + sbom_infos = message.image.sbom_info try: trace_id.set(message.scan.id) @@ -195,6 +208,12 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: # Build code index if not ignored if not config.ignore_code_index: + logger.info("analysis type: %s", message.image.analysis_type) + if message.image.analysis_type == AnalysisType.IMAGE and isinstance(sbom_infos, ManualSBOMInfoInput): + RPMDependencyManager.get_instance().sbom = sbom_infos.packages + image = f"{message.image.name}:{message.image.tag}" + RPMDependencyManager.get_instance().container_image = image + code_index_path = _build_code_index(source_infos) if code_index_path is None: diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 62c9d6d9b..6589b53ca 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -14,6 +14,8 @@ python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file) +from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager + transitive_code_search_runner_coroutine = None @@ -102,11 +104,14 @@ async def test_transitive_search_golang_6(): def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], - excluded_extensions: list[str]): + excluded_extensions: list[str], sbom_packages: list[SBOMPackage] = None): source_code_info = [SourceDocumentsInfo(type='code', git_repo=git_repository, ref=git_ref, include=included_extensions, exclude=excluded_extensions)] - sbom_info_input = ManualSBOMInfoInput(packages=[SBOMPackage(name="a", version="1.0", system="blabla")]) + if sbom_packages: + sbom_info_input = ManualSBOMInfoInput(packages=sbom_packages) + else: + sbom_info_input = ManualSBOMInfoInput(packages=[SBOMPackage(name="a", version="1.0", system="blabla")]) morpheus_input = AgentMorpheusInput(image=ImageInfoInput(source_info=source_code_info, sbom_info=sbom_info_input, analysis_type=AnalysisType.IMAGE), @@ -230,4 +235,68 @@ async def test_python_transitive_search(): print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") assert path_found == True - assert len(list_path) == 2 \ No newline at end of file + assert len(list_path) == 2 + + + + +@pytest.mark.asyncio +async def test_c_transitive_search(): + """Test that runs with a real postgres repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + #create sample sbom packages + sbom_list = [ + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + + set_input_for_next_run( + git_repository='https://github.com/postgres/postgres', + git_ref='REL_13_14', + included_extensions=['**/*.c','**/*.h'], + excluded_extensions=['**/*test*.c',"**/docs/**/*","**/doc/**/*"] + ) + RPMDependencyManager.get_instance().set_rpm_cache_dir(".cache/am_cache/rpms") + RPMDependencyManager.get_instance().enableUnitTestMode() + RPMDependencyManager.get_instance().sbom = sbom_list + result = await transitive_code_search_runner_coroutine("libxml2,xmlParseComment") + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert len(list_path) == 2 + assert path_found == True + + +@pytest.mark.asyncio +async def test_c_transitive_search_2(): + """Test that runs with a real postgres repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + #create sample sbom packages + sbom_list = [ + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + + set_input_for_next_run( + git_repository='https://github.com/postgres/postgres', + git_ref='REL_13_14', + included_extensions=['**/*.c','**/*.h'], + excluded_extensions=['**/*test*.c',"**/docs/**/*","**/doc/**/*"] + ) + RPMDependencyManager.get_instance().set_rpm_cache_dir(".cache/am_cache/rpms") + RPMDependencyManager.get_instance().enableUnitTestMode() + RPMDependencyManager.get_instance().sbom = sbom_list + result = await transitive_code_search_runner_coroutine("libxslt,xsltParseStylesheetPI") + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert len(list_path) == 1 + assert path_found == False \ No newline at end of file diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index ec093470d..090ae90cd 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -30,13 +30,12 @@ from vuln_analysis.utils.dep_tree import Ecosystem from vuln_analysis.utils.document_embedding import DocumentEmbedding from ..utils.function_name_extractor import FunctionNameExtractor +from ..utils.function_name_locator import FunctionNameLocator from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) - - class TransitiveCodeSearchToolConfig(FunctionBaseConfig, name="transitive_code_search"): """ Transitive code search tool used to search source code. @@ -49,50 +48,78 @@ class CallingFunctionNameExtractorToolConfig(FunctionBaseConfig, name="calling_f """ +class PackageAndFunctionLocatorToolConfig(FunctionBaseConfig, name="package_and_function_locator"): + """ + Package and function locator tool used to validate package names and find function names using fuzzy matching. + """ + + def get_call_of_chains_retriever(documents_embedder, si): documents: list[Document] + git_repo = None for source_info in si: if source_info.type == "code": git_repo = documents_embedder.get_repo_path(source_info) documents = documents_embedder.collect_documents(source_info) - with open(os.path.join(git_repo, 'ecosystem_data.txt'), 'r') as file: + if git_repo is None: + raise ValueError("No code source info found") + with open(os.path.join(git_repo, 'ecosystem_data.txt'), 'r', encoding='utf-8') as file: ecosystem = file.read() ecosystem = Ecosystem[ecosystem] coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=ecosystem, manifest_path=git_repo) return coc_retriever +def get_transitive_code_searcher(): + state: AgentMorpheusEngineState = ctx_state.get() + if state.transitive_code_searcher is None: + si = state.original_input.input.image.source_info + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + state.transitive_code_searcher = transitive_code_searcher + return state.transitive_code_searcher + + @register_function(config_type=TransitiveCodeSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def transitive_search(config: TransitiveCodeSearchToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> tuple: transitive_code_searcher: TransitiveCodeSearcher - state: AgentMorpheusEngineState = ctx_state.get() - if state.transitive_code_searcher is None: - si = state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) - transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) - state.transitive_code_searcher = transitive_code_searcher - else: - transitive_code_searcher = state.transitive_code_searcher + transitive_code_searcher = get_transitive_code_searcher() result = transitive_code_searcher.search(query) return result yield FunctionInfo.from_fn( _arun, - description=("This tool is useful when you need to search the container image's code for " - "a given function or method in a given package or library, if it's being used in" - " application or called from application code base. " - "This requires both package name and function/method name, separated by " - "',' as input. Input example - 'package_name,function_name'." - " It returns a tuple of first boolean element indicating whether the function" - " is being called or reachable from an application code base or not, and " - "the second element is a list of documents comprising the call of " - "hierarchy path, showing the calls hierarchy from application " - "to a function in the package,in case the first element value in the tuple is True." - "Do not use this list of documents in subsequent or later checks")) + description=(""" + + This tool searches the container image's code to determine if a specified function + from a given package is used (directly or transitively) by the application code. + + + Input: package name and function/method name, separated by ','. + + package_name,function_name + package_name,class_name.method_name + + + + CRITICAL: Do not call directly. Mandatory 3-step workflow: + 1. Call 'Package and Function Locator' to get ecosystem type and validate package/function names + 2. Use validated package and function names from locator tool + 3. Select most accurate function name from locator suggestions + Only then call this tool with validated `package_name,function_name`. + WARNING: Guessed names will fail. Workflow is mandatory. + + + Returns a tuple -> (boolean, list_of_documents): + - boolean = True if function is reachable from application code + - list_of_documents = call hierarchy path (only returned if boolean is True) + WARNING: Do not reuse the list_of_documents for further tool calls — + use it only for reporting results to the user. + """)) @register_function(config_type=CallingFunctionNameExtractorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -100,34 +127,104 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, builder: Builder): # pylint: disable=unused-argument async def _arun(query: str) -> list: - - state: AgentMorpheusEngineState = ctx_state.get() - si = state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) coc_retriever: ChainOfCallsRetriever - if state.transitive_code_searcher is None: - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) - else: - transitive_search_temp = state.transitive_code_searcher - coc_retriever = transitive_search_temp.chain_of_calls_retriever + transitive_code_searcher: TransitiveCodeSearcher + transitive_code_searcher = get_transitive_code_searcher() + coc_retriever = transitive_code_searcher.chain_of_calls_retriever function_name_extractor = FunctionNameExtractor(coc_retriever) result = function_name_extractor.fetch_list(query) return result yield FunctionInfo.from_fn( _arun, - description=("This tool is useful when you need to search the container image's code " - "for a specific usage or occurrences of standard library function or method within" - " a specific package, in order " - "to get a list of calling functions within that package." - " It gets one parameter, containing the package name to search in and " - "the function usage string, for example - " - "input_package,sysPackage.functionName(arg, 'textLiteral') " - "It returns list of all functions containing usages in the given package," - " for example - ['input_package,function1','input_package,function2']. " - "If there are no usages returns empty list." - "Each one of the items in the returned list can be checked with another search " - "tool, in order to determine eventually whether usage " - "of a standard library function is called or not from an application code base." - " Even if the package name in the results is not the same as the expected one," - " use it anyway.")) + description=(""" + + This tool searches a container image’s code for occurrences + of standard library functions or methods within a package + to list calling functions in that package. Designed for GO only. + + + Input format: 'package_name,function_usage_string' + input_package,sysPackage.functionName(arg, 'textLiteral') + + + CRITICAL: GO ecosystem ONLY. Mandatory workflow: + 1. Call 'Package and Function Locator' first to get ecosystem type + 2. If NOT GO, use other tools. If GO, proceed to step 3 + 3. Use this tool to find functions calling standard library methods + + github.com/golang-jwt/jwt,errors.New("Invalid Key: Key must be PEM encoded") + ["github.com/golang-jwt/jwt,Valid"] + + WARNING: Must verify ecosystem is GO first. + + + Returns a list of all functions containing such usages in the given package. + ['input_package,function1', 'input_package,function2'] + If none found, returns an empty list. + Each returned item can be checked with the 'Transitive code search tool' + to determine if the standard library usage is called from app code. + Note: Even if the result package differs from the expected one, use it anyway. + + +""")) + + +@register_function(config_type=PackageAndFunctionLocatorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def package_and_function_locator(config: PackageAndFunctionLocatorToolConfig, + builder: Builder): # pylint: disable=unused-argument + + async def _arun(query: str) -> dict: + coc_retriever: ChainOfCallsRetriever + transitive_code_searcher: TransitiveCodeSearcher + transitive_code_searcher = get_transitive_code_searcher() + coc_retriever = transitive_code_searcher.chain_of_calls_retriever + locator = FunctionNameLocator(coc_retriever) + result = await locator.locate_functions(query) + pkg_msg = "Package is valid." + if not locator.is_package_valid and not locator.is_std_package: + pkg_msg = "Package is not valid." + + + return { + "ecosystem": coc_retriever.ecosystem.name, + "package_msg": pkg_msg, + "result": result + } + + yield FunctionInfo.from_fn( + _arun, + description=(""" + + FIRST STEP in code analysis. Validates packages, locates functions via fuzzy matching, provides ecosystem type. + + + Input format: 'package_name,function_name' or 'package_name,class_name.method_name' + + libxml2,xmlParseDocument + requests,Session.get + numpy,array.reshape + + + + **MANDATORY FIRST STEP** for ALL code analysis. Use before ANY other tool. + - Returns ecosystem type (GO, PYTHON, JAVASCRIPT, JAVA, C_CPP) - determines which tools to use + - Validates package names, suggests alternatives if missing + - Finds function matches via fuzzy matching - pick the most contextually correct one + + libxml2,xmlParseDocument + ["docbParseDocument", "test_xmlParseDocument", "startDocument", "xmlParseDoc"] + "xmlParseDocument" doesn't exist. Best match: "xmlParseDoc" + requests,Session.get + ["Session.get", "Session.post", "Session.put", "Session.delete"] + Found class method "Session.get" + + + + Returns dictionary with: + - "ecosystem": ecosystem name (GO, PYTHON, JAVASCRIPT, JAVA, C_CPP) + - "result": suggested function names (if valid package) or error with alternatives (if invalid) + Selected function is input for **'Transitive code search tool' (Step 2)**. + + +""")) diff --git a/src/vuln_analysis/utils/c_segmenter_custom.py b/src/vuln_analysis/utils/c_segmenter_custom.py new file mode 100644 index 000000000..cf34f8084 --- /dev/null +++ b/src/vuln_analysis/utils/c_segmenter_custom.py @@ -0,0 +1,70 @@ +import re +from langchain_community.document_loaders.parsers.language.c import CSegmenter +from typing import List + + +class CSegmenterExtended(CSegmenter): + + def __init__(self, code: str): + # Preprocess code: remove comments and macro blocks + code = self.remove_comments(code) + code = self.remove_macro_blocks(code) + super().__init__(code) + self.structs_enums: List[str] = [] + + @staticmethod + def remove_comments(code: str) -> str: + # Remove all multi-line comments (/* ... */) + code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) + # Remove all single-line comments (//...) + code = re.sub(r'//.*', '', code) + return code + + @staticmethod + def remove_macro_blocks(text: str) -> str: + """ + Remove macro blocks of the form: + MACRO_NAME(args) { ... } + or + MACRO_NAME(args) { ... } MACRO_NAME_END(args); + Only matches ALL UPPERCASE macro names. + """ + lines = text.splitlines() + total = len(lines) + indices_to_remove = set() + i = 0 + + macro_header = re.compile(r'^\s*([A-Z_][A-Z0-9_]*)\s*\(.*\)\s*{') + macro_end_template = r'^\s*{}\s*_END\\s*\\(.*\\)\\s*;?' + + while i < total: + line = lines[i] + match = macro_header.match(line) + if match: + macro_name = match.group(1) + start = i + brace_count = line.count('{') - line.count('}') + i += 1 + # Find the end of the code block + while i < total and brace_count > 0: + brace_count += lines[i].count('{') + brace_count -= lines[i].count('}') + i += 1 + # Optionally, look for a matching _END macro + macro_end = re.compile(macro_end_template.format(macro_name)) + if i < total and macro_end.match(lines[i]): + i += 1 + for idx in range(start, i): + indices_to_remove.add(idx) + else: + i += 1 + + return '\n'.join( + line for idx, line in enumerate(lines) + if idx not in indices_to_remove + ) + + def extract_functions_classes(self) -> List[str]: + segments = super().extract_functions_classes() + return segments + \ No newline at end of file diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 4759df94b..1854f893b 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -1,11 +1,12 @@ import re +import time +from collections import defaultdict, deque from pathlib import Path -from typing import Any, List, Optional +from typing import List, Optional from langchain_core.documents import Document -import logging - +from vuln_analysis.logging.loggers_factory import LoggingFactory from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem from .functions_parsers.lang_functions_parsers import LanguageFunctionsParser from .functions_parsers.lang_functions_parsers_factory import ( @@ -13,10 +14,8 @@ ) PARENTS_INDEX = 0 - EXCLUSIONS_INDEX = 1 -from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") @@ -76,8 +75,40 @@ def function_called_from_caller_body(document, function_to_search, language_pars def document_belongs_to_package(language_parser: LanguageFunctionsParser, document: Document, package_name: str) -> bool: - return (not language_parser.is_root_package(document) and - language_parser.get_package_name(function=document, package_name=package_name)) + return language_parser.is_a_package(package_name, document) + + +def is_likely_macro_block(segment: str) -> bool: + lines = segment.strip().splitlines() + if not lines: + return True + first_line = lines[0].strip() + # If no '{' at all, not a function + if '{' not in segment: + return True + # If segment is very short (less than 3 lines), likely not a function + if len(lines) < 3: + return True + # Macro assignment: MACRO(...) = { ... } + if re.match(r'^[A-Z_][A-Z0-9_]*\s*\(.*\)\s*=\s*{', first_line): + return True + # Macro end: ..._END(...) + if re.match(r'.*_END\s*\(', first_line): + return True + # Macro invocation: ALLCAPS or _ and ( -- but allow if looks like function signature + # If the first line ends with ')' and the next non-empty line is '{', it's a function signature + if re.match(r'^[A-Z_][A-Z0-9_]*\s*\(', first_line): + # Check if it's a function signature + if first_line.rstrip().endswith(')'): + # Look for '{' on the next non-empty line + for line in lines[1:]: + if line.strip() == '{': + return False # It's a function! + # Or, '{' at end of first line + if first_line.rstrip().endswith('){') or first_line.rstrip().endswith(') {'): + return False + return True + return False class ChainOfCallsRetriever: @@ -92,6 +123,7 @@ class ChainOfCallsRetriever: language_parser: Optional[LanguageFunctionsParser] dependency_tree: Optional[DependencyTree] tree_dict: Optional[dict] + supported_packages: Optional[list[str]] ecosystem: Optional[Ecosystem] manifest_path: Optional[Path] package_name: Optional[str] @@ -119,7 +151,7 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat # Build dependency tree based on the parameter programming language/package manager. self.dependency_tree = DependencyTree(ecosystem=ecosystem) logger.debug("Chain of Calls Retriever - get language parser") - self.language_parser = get_language_function_parser(ecosystem) + self.language_parser = get_language_function_parser(ecosystem, self.dependency_tree) self.manifest_path = manifest_path # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. if self.dependency_tree.builder is None: @@ -134,41 +166,77 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat self.tree_dict[package] = list() self.tree_dict[package].append(parents) self.tree_dict[package].append([]) + self.supported_packages = list(self.tree_dict.keys()) logger.debug("Chain of Calls Retriever - populating functions documents") + allowed_files_extensions = self.language_parser.supported_files_extensions() # filter out unsupported files extensions. - self.documents = [doc for doc in documents + filtered_documents = [doc for doc in documents if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) .endswith(ext)])] - # filter out types and full code documents, retaining only functions/methods documents in this attribute. - self.documents_of_functions = [doc for doc in self.documents - if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] - # filter out full code documents and functions/methods docs, retaining only types/classes docs - self.documents_of_types = [doc for doc in self.documents - if doc.page_content.startswith(self.language_parser.get_type_reserved_word())] - # boolean attribute that indicates whether a path was found or not, initially set to False. - self.found_path = False # Filter out all documents but full source docs. - self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in self.documents + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in filtered_documents if doc.metadata.get('content_type') == 'simplified_code'} - if not self.language_parser.is_script_language(): - self.documents = self.documents_of_functions + # filter out full code documents and functions/methods docs, retaining only types/classes docs + self.documents_of_types = [doc for doc in filtered_documents + if self.language_parser.is_doc_type(doc)] + + # Apply macro filtering only for C/C++ languages + if not self.language_parser.is_language_supports_macro(): + no_macro_documents = filtered_documents + logger.debug(f"skipping macro check for {self.ecosystem}, using all filtered documents") + else: + no_macro_documents = [doc for doc in filtered_documents if not is_likely_macro_block(doc.page_content)] + logger.debug(f"no_macro_documents len : {len(no_macro_documents)}") + + if not self.language_parser.is_script_language(): + # filter out types and full code documents, retaining only functions/methods documents in this attribute. + self.documents = [doc for doc in no_macro_documents if self.language_parser.is_function(doc)] + self.documents_of_functions = self.documents + else: + self.documents = filtered_documents + self.documents_of_functions = [doc for doc in self.documents + if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] + + logger.debug(f"self.documents len : {len(self.documents)}") + logger.debug("Chain of Calls Retriever - retaining only types/classes docs " + "documents_of_types len %d", len(self.documents_of_types)) + # boolean attribute that indicates whether a path was found or not, initially set to False. + self.found_path = False logger.debug("Chain of Calls Retriever - after documents_of_full_sources") + self.last_visited_parent_package_indexes = dict() + self.last_visited = dict() # Constructing a map of types and classes to their attributes/members/fields - self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields( - self.documents_of_types) + self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields(self.documents_of_types) # Create a data structure containing dict of key=(function_name@source_file),value = dict of # local variables names mapped to (types, (values or expressions)) self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) logger.debug("Chain of Calls Retriever - after functions_local_variables_index") - - def __find_caller_function(self, document_function: Document, function_package: str) -> Document: + + if not self.language_parser.is_search_algo_dfs(): + self.sort_docs = self.__group_docs_by_pkg() + + def __group_docs_by_pkg(self) -> dict[str, list[Document]]: + """ + sort docs by pkg name + """ + t0 = time.time() + sort_docs = defaultdict(list) + for doc in self.documents: + pkgs = self.language_parser.get_package_names(doc) + pkg_name = pkgs[0] + sort_docs[pkg_name].append(doc) + t1 = time.time() - t0 + logger.debug("PROFILE: sort all docs (%d) elaps %.3f", len(self.documents), t1) + return sort_docs + + def __find_caller_function_dfs(self, document_function: Document, function_package: str) -> Document: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function - :return: a document of a function that is calling document_function + :return: a single document of a function that is calling document_function, or None if not found """ package_names = self.language_parser.get_package_names(document_function) direct_parents = list() @@ -255,6 +323,7 @@ def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: return True return False + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all # excluded functions, and all function that their body doesn't contain the target function name to search for. def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], @@ -272,6 +341,196 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] + def __find_caller_functions_bfs(self, document_function: Document, function_package: str) -> List[Document]: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :return: a list of documents of functions that are calling document_function + """ + + total_start = time.time() + direct_parents = list() + # gets list of all direct parents of function + + list_of_packages = self.tree_dict.get(function_package) + if list_of_packages is not None: + direct_parents.extend(list_of_packages[0]) + # Add same package itself to search path. + # direct_parents.extend([function_package]) + # gets list of documents to search in only from parents of function' package. + function_name_to_search = self.language_parser.get_function_name(document_function) + function_file_name = document_function.metadata.get('source') + relevant_docs_to_search_in = list() + # Search for caller functions only at parents according to dependency tree. + log_entries = [] + loop_start = time.time() + try: + for package in direct_parents: + pkg_docs = self.sort_docs[package] + for doc in pkg_docs: + # for doc in self.documents: + # is_doc_in_pkg = False + # for package in direct_parents: + # if self.language_parser.is_a_package(package, doc): + # is_doc_in_pkg = True + # break + # if not is_doc_in_pkg: + # continue + + file_name = doc.metadata.get('source') + if doc.metadata.get('state') == "invalid": + continue + func_name = self.language_parser.get_function_name(doc) + # check for same doc + if (function_name_to_search == func_name) and (file_name == function_file_name): + continue + # same function name different files ? + # if (function_name_to_search == func_name): + # logger.debug(f"same func name {function_name_to_search}") + + if func_name == "main": + file_path = Path(function_file_name) + # Get all the parts of the path + path_parts = file_path.parts + if self.language_parser.dir_name_for_3rd_party_packages() in path_parts: + continue + if doc.page_content.__contains__(f"{function_name_to_search}("): + last_visited = (self.last_visited.get( + calculate_hashable_string_for_function(file_name, func_name), 0)) + if last_visited == 0: + found = self.language_parser.search_for_called_function( + caller_function=doc, + callee_function=function_name_to_search, + callee_function_package=function_package, + code_documents=self.documents_of_full_sources, + type_documents=self.documents_of_types, + callee_function_file_name=function_file_name, + fields_of_types=self.types_classes_fields_mapping, + functions_local_variables_index=self.functions_local_variables_index) + + if found and self.language_parser.is_call_allowed( pkg_docs, doc, document_function): + log_entries.append((file_name, func_name, function_name_to_search)) + relevant_docs_to_search_in.append(doc) + except ValueError as ex: + logger.error("doc %s / %s", doc, ex) + + loop_elapsed = time.time() - loop_start + logger.debug(f"[PROFILE] __find_caller_functions main for-loop took {loop_elapsed:.3f} seconds") + # logger.debug table-style summary + logger.debug("\nFunction Match Summary:") + logger.debug("-" * 100) + logger.debug(f"{'No.':<5} {'File':<60} {'Function':<30} {'Target'}") + logger.debug("-" * 100) + + for idx, (file_name, func_name, function_name_to_search) in enumerate(log_entries, 1): + logger.debug(f"{idx:<5} {file_name:<60} {func_name:<30} {function_name_to_search}") + + logger.debug("-" * 100) + total_elapsed = time.time() - total_start + logger.debug(f"[PROFILE] __find_caller_functions total time took {total_elapsed:.3f} seconds") + + # If didn't find a matching caller function document, returns None. + return relevant_docs_to_search_in + + + def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + current_package_name: str) -> tuple[List[Document], bool]: + # main loop. + file_counter = 0 + q = deque() + self.last_visited.clear() + q.append(target_function_doc) + loop_start = time.time() + while q: + target_doc = q.popleft() + + target_pkg = self.language_parser.get_package_names(target_doc)[0] + if target_doc.metadata.get('state') == "invalid": + logger.debug("get_relevant_documents: invalid function %s", target_doc.metadata['source']) + continue + function_name = self.language_parser.get_function_name(target_doc) + + function_file = target_doc.metadata.get('source') + hashed_value = calculate_hashable_string_for_function(function_file, function_name) + if hashed_value in self.last_visited: + continue + + self.last_visited[hashed_value] = 1 + logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", + file_counter, target_doc.metadata['source'], function_name, target_pkg, len(q)) + + file_counter += 1 + # Find a caller function and containing package in the dependency tree according to hierarchy + sub_start = time.time() + found_documents = self.__find_caller_functions_bfs(document_function=target_doc, + function_package=target_pkg) + sub_elapsed = time.time() - sub_start + logger.debug(f"[PROFILE] __find_caller_functions took {sub_elapsed:.3f} seconds") + # If found, then add it to path + if found_documents is not None: + for doc_candidate in found_documents: + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(doc_candidate): + matching_documents.append(doc_candidate) + self.found_path = True + break + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + q.append(doc_candidate) + + if self.found_path: + break + loop_elapsed = time.time() - loop_start + logger.debug(f"[PROFILE] Main loop in get_relevant_C_documents took {loop_elapsed:.3f} seconds") + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + logger.debug("get_relevant_documents: result %s", self.found_path) + logger.debug("get_relevant_documents: docs %s", matching_documents) + return matching_documents, self.found_path + + def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + current_package_name: str) -> tuple[List[Document], bool]: + """Execute depth-first search with backtracking strategy.""" + end_loop = False + + # main loop. + while not end_loop: + # Find a caller function and containing package in the dependency tree according to hierarchy + found_document = self.__find_caller_function_dfs(document_function=target_function_doc, + function_package=current_package_name) + # If found, then add it to path + if found_document: + matching_documents.append(found_document) + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(found_document): + end_loop = True + self.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + target_function_doc = found_document + # extract package name from function document + current_package_name = self.__determine_doc_package_name(target_function_doc) + else: + # end loop because didn't find a caller for initial function + if len(matching_documents) == 1: + end_loop = True + # Backtrack - means that we're going back because current function has no callers anywhere, so we + # need to remove it, and continue with other possible potential called functions from its caller + else: + dead_end_node = matching_documents.pop() + # Excludes dead end function node from future searches, as it led to nowhere. + self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + target_function_doc = matching_documents[-1] + current_package_name = self.__determine_doc_package_name(target_function_doc) + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, self.found_path + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of @@ -279,7 +538,7 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" self.found_path = False - query = query.splitlines()[0].strip('"\'') + query = query.splitlines()[0].replace('"', '').replace("'", "").strip() (package_name, function) = tuple(query.split(",")) class_name = None splitters = [splitter for splitter in ['.'] if splitter in function] @@ -293,6 +552,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: package_name = package found_package = True break + # If it's , then create a document for it. if found_package: target_function_doc = self.__find_initial_function(function, package_name=package_name, @@ -305,7 +565,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: documents=self.documents, language_parser=self.language_parser, class_name=function) - + # If not, there is a chance that the package is some standard library in the ecosystem. else: # Try to create dummy package for ecosystem standard library function, in such case, build a document for @@ -316,12 +576,10 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: target_function_doc = Document(page_content=page_content , metadata={"source": package_name, "ecosystem": self.ecosystem}) - escaped_package_name = re.escape(package_name) - importing_docs = [value for (file, value) in self.documents_of_full_sources.items() - if re.search( - rf"(import {escaped_package_name}|(import\s*\(\s*[\w\s\/.\"-]*{escaped_package_name}[\w\s\/.\"-]*\s*\))|from\s+{escaped_package_name}\s+import)" - , value.page_content, flags=re.MULTILINE)] + importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, re.escape(package_name)) + + root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. @@ -344,38 +602,16 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # library else: end_loop = True - logger.error(f"Cannot find initial function=${function}, in package=${package_name}") - # main loop. - while not end_loop: - # Find a caller function and containing package in the dependency tree according to hierarchy - found_document = self.__find_caller_function(document_function=target_function_doc, - function_package=current_package_name) - # If found, then add it to path - if found_document: - matching_documents.append(found_document) - # If the function is in the application ( root package), then we finished and found such a path. - if self.language_parser.is_root_package(found_document): - end_loop = True - self.found_path = True - # Otherwise, we continue to search for callers for the current found function, in order to extend - # the chain of calls and potentially find a path from application to the vulnerable - # function in input package - else: - target_function_doc = found_document - # extract package name from function document - current_package_name = self.__determine_doc_package_name(target_function_doc) - else: - # end loop because didn't find a caller for initial function - if len(matching_documents) == 1: - end_loop = True - # Backtrack - means that we're going back because current function has no callers anywhere, so we - # need to remove it, and continue with other possible potential called functions from its caller - else: - dead_end_node = matching_documents.pop() - # Excludes dead end function node from future searches, as it led to nowhere. - self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) - target_function_doc = matching_documents[-1] - current_package_name = self.__determine_doc_package_name(target_function_doc) + logger.error("Cannot find initial function=%s, in package=%s", function, package_name) + if end_loop: + return matching_documents, self.found_path + + if self.language_parser.is_search_algo_dfs(): + matching_documents, self.found_path = self._depth_first_search( + matching_documents, target_function_doc, current_package_name) + else: + matching_documents, self.found_path = self._breadth_first_search( + matching_documents, target_function_doc, current_package_name) # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was # found or not. @@ -388,20 +624,36 @@ def __determine_doc_package_name(self, target_function_doc): def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], language_parser: LanguageFunctionsParser, class_name: str = None) -> Document: - relevant_docs = [doc for doc in documents if doc.metadata.get('source').__contains__(package_name) and - doc.page_content.__contains__(function_name)] + + if self.language_parser.is_search_algo_dfs(): + pkg_docs = documents + else: + pkg_docs = self.sort_docs[package_name] + + relevant_docs = language_parser.filter_docs_by_func_pkg_name(function_name, package_name, pkg_docs) if class_name: relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] + package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] - for document in get_functions_for_package(package_name, relevant_docs, language_parser): + #for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): + for document in get_functions_for_package(package_name, relevant_docs, language_parser): # document_function_calls_input_function = True if function_name.lower() == language_parser.get_function_name(document).lower(): # if language_parser.search_for_called_function(document, callee_function=function_name): package_exclusions.append(document) return document - else: - return None + + if language_parser.create_dummy_for_standard_lib(package_name): + dummy_function_doc = Document(page_content=f"func {function_name + '()' + '{}'}", + metadata={ + "source": language_parser.dir_name_for_3rd_party_packages() + "/" + package_name + "/dummy.c", + "ecosystem": self.ecosystem, + "func_name": function_name, + "pkg_name": package_name}) + return dummy_function_doc + + return None # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end # (target function). diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 7267e034e..69c8b2e94 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -1,3 +1,7 @@ +import os +import re +import shutil +import time import subprocess from abc import ABC, abstractmethod from collections import defaultdict @@ -10,12 +14,15 @@ from tqdm import tqdm import logging -import os import ast import json -import re from vuln_analysis.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from contextlib import contextmanager + +# C_DEP_LIBS_NAME moved here to avoid circular import +C_DEP_LIBS_NAME = "rpm_libs" +from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -34,19 +41,27 @@ class Ecosystem(str, Enum): PYTHON = "python" JAVASCRIPT = "javascript" JAVA = "java" - C = "c" + C_CPP = "c" GOLANG_MANIFEST = "go.mod" JAVA_MANIFEST = "pom.xml" JS_MANIFEST = "package.json" PYTHON_MANIFEST = "requirements.txt" +C_CPLUSPLUS_MANIFEST_1 = "CMakeLists.txt" +C_CPLUSPLUS_MANIFEST_2 = "meson.build" +C_CPLUSPLUS_MANIFEST_3 = "Makefile" +C_CPLUSPLUS_MANIFEST_4 = "configure" MANIFESTS_TO_ECOSYSTEMS = { "go.mod": Ecosystem.GO, "requirements.txt": Ecosystem.PYTHON, "package.json": Ecosystem.JAVASCRIPT, - "pom.xml": Ecosystem.JAVA + "pom.xml": Ecosystem.JAVA, + "CMakeLists.txt": Ecosystem.C_CPP, + "meson.build": Ecosystem.C_CPP, + "Makefile": Ecosystem.C_CPP, + "configure": Ecosystem.C_CPP } def run_command(cmd: str) -> str: @@ -62,22 +77,653 @@ def run_command(cmd: str) -> str: class DependencyTreeBuilder(ABC): """ - An abstract base class that represents an object that knows how to build a dependency tree for a given ecosystem. - Should be subclassed for each supported ecosystem. + An abstract base class that represents an object that knows how to build a + dependency tree for a given ecosystem. Should be subclassed for each + supported ecosystem. """ @abstractmethod - # Build a sort of "upside down" tree - a dict containing mapping of each package to a list of all consuming packages + # Build a sort of "upside down" tree - a dict containing mapping of each + # package to a list of all consuming packages def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: pass @abstractmethod - # A Method that knows how to install the app dependencies based on the manifest path and ecosystem. + # A Method that knows how to install the app dependencies based on the + # manifest path and ecosystem. def install_dependencies(self, manifest_path: Path): pass +class CCppDependencyTreeBuilder(DependencyTreeBuilder): + # Pre-compiled regex patterns (optimization: compile once, use many times) + INCLUDE_COMBINED_RE = re.compile( + r'#include\s*([<"])([^>"]+)[>"]' + ) # Combined pattern + # Common system headers for fast-path resolution + COMMON_SYSTEM_HEADERS = { + 'stdio.h', 'stdlib.h', 'string.h', 'math.h', 'unistd.h', + 'sys/types.h', 'sys/stat.h', 'errno.h', 'fcntl.h', 'time.h' + } + + KERNEL_LIBS = ["libbpf"] + KERNEL_LIBS_PATH_MAP = { + "libbpf": "tools/lib/bpf/" + } + + def __init__(self): + self.include_paths = [] + self.root_dir = None + self.prj_name: str = "dummy_project" + self.exclude_patterns = [ + "test", "tests", "doc", "docs", "example", "examples", + "bench", "benchmark", "demo", "sample" + ] + self.C_STANDARD_LIB = "glibc" + self.RPM_LIBS_DIR = C_DEP_LIBS_NAME + self.output_json_path = None + self.ccp_dep_tree = None + + def add_include_path(self, path: str): + """Add an include path for header resolution""" + self.include_paths.append(Path(path)) + + def should_exclude_directory(self, dir_path: str) -> bool: + """Check if a directory should be excluded""" + path_lower = dir_path.lower() + return any(pattern in path_lower for pattern in self.exclude_patterns) + + + def relative_matches_end(self, full_path, rel_path): + # Normalize to avoid redundant slashes or ../ + full_path = os.path.normpath(full_path) + rel_path = os.path.normpath(rel_path) + + full_parts = full_path.split(os.sep) + rel_parts = rel_path.split(os.sep) + + return full_parts[-len(rel_parts):] == rel_parts + + def find_all_files(self, root_dir): + exclude_keywords = [ + "test", "tests", "doc", "docs", "example", "examples" + ] + files = [] + for dirpath, _, filenames in os.walk(root_dir): + # Skip any directory that contains excluded keywords + normalized_path = Path(dirpath.lower()) + # Get all the parts of the path + path_parts = normalized_path.parts + if any(kw.lower() in path_parts for kw in exclude_keywords): + continue + kernel_pkg = False + for lib in self.KERNEL_LIBS: + if lib in path_parts: + kernel_pkg = True + rel_path = self.KERNEL_LIBS_PATH_MAP[lib] + # get the files that are part of the package project + if self.relative_matches_end(dirpath, rel_path): + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = os.path.join(dirpath, fname) + files.append(full_path) + else: + # we don't need kernel files which are not part of the + # package + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = Path(dirpath) / fname + new_path = full_path.with_suffix(".k") + try: + full_path.rename(new_path) + except Exception as e: + logging.warning( + "Rename failed: %s → %s: %s", + full_path, new_path, e + ) + + if not kernel_pkg: + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = os.path.join(dirpath, fname) + files.append(full_path) + return files + + def extract_includes_from_file(self, file_path: str) -> list[dict]: + """Extract #include statements from a file (no per-header cache)""" + includes = [] + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line.startswith('#include'): + continue + match = self.INCLUDE_COMBINED_RE.search(line) + if match: + bracket_type = match.group(1) + header = match.group(2) + include_type = ( + 'local' if bracket_type == '"' else 'system' + ) + if (include_type == 'system' and + header in self.COMMON_SYSTEM_HEADERS): + includes.append({ + 'header': header, + 'line': line_num, + 'type': 'system_common' + }) + else: + includes.append({ + 'header': header, + 'line': line_num, + 'type': include_type + }) + except (OSError, UnicodeDecodeError): + pass + return includes + + def extract_includes_from_content(self, content: list[str]) -> list[dict]: + includes = [] + for line_num, line in enumerate(content): + line = line.strip() + if not line.startswith('#include'): + continue + match = self.INCLUDE_COMBINED_RE.search(line) + if match: + bracket_type = match.group(1) + header = match.group(2) + include_type = ('local' if bracket_type == '"' + else 'system') + if (include_type == 'system' and + header in self.COMMON_SYSTEM_HEADERS): + includes.append({ + 'header': header, + 'line': line_num, + 'type': 'system_common' + }) + else: + includes.append({ + 'header': header, + 'line': line_num, + 'type': include_type + }) + return includes + + def module_from_path(self, path: str) -> str: + """ + Enhanced module detection (no cache) + """ + if not self.root_dir: + return "unknown" + rel_path = Path(os.path.relpath( + os.path.dirname(path), self.root_dir + )) + parts = rel_path.parts + if self.RPM_LIBS_DIR in parts: + try: + idx = parts.index(self.RPM_LIBS_DIR) + if idx + 1 < len(parts): + module = parts[idx + 1] + return module + except ValueError: + pass + module = self.prj_name or "ROOT" + return module + + @contextmanager + def read_file(self, file_path: str): + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + yield f + except FileNotFoundError: + yield None + + def build_dependency_tree_enhanced( + self, root_dir: Path, max_workers: int = 1 + ) -> tuple[dict[str, list[str]], dict]: + """ + Build a C/C++ dependency tree for a project directory. + + This function scans all C/C++ source and header files in the given root + directory, extracts #include dependencies, and constructs a module-level + dependency tree. It uses a single-threaded approach for file reading and + parsing, which is optimal for single-core environments. The function also + tracks and reports cache statistics for header resolution. + + Args: + root_dir (Path): The root directory of the project to analyze. + max_workers (int): Ignored in this implementation (kept for + compatibility). + + Returns: + tuple: + - dict[str, list[str]]: The dependency tree as a mapping from + module to list of dependencies. + - list[str]: List of all files processed. + - dict: Cache statistics (hits, misses, cache size). + """ + self.root_dir = root_dir + self.prj_name = self.find_project_name(str(root_dir)) + # Find all C/C++ files in the project + files = self.find_all_files(root_dir) + # Map from filename to all paths with that name + filename_to_paths = defaultdict(list) + for f in files: + filename_to_paths[os.path.basename(f)].append(f) + # Map each file to its module (project, rpm, or glibc) + file_to_module = {} + standard_lib_headers = set() + for f in files: + module = self.module_from_path(f) + file_to_module[f] = module + if module == self.C_STANDARD_LIB and f.endswith('.h'): + standard_lib_headers.add(os.path.basename(f)) + # Extract includes from each file (single-threaded) + file_includes_map = {} + all_includes = set() + for i, f in enumerate(files): + lines = [] + includes = [] + with self.read_file(f) as fileobj: + # file_content = fileobj.read() + if fileobj: + for line in fileobj: + lines.append(line) + includes = self.extract_includes_from_content(lines) + + file_includes_map[f] = includes + including_module = file_to_module[f] + for include_info in includes: + header = include_info['header'] + all_includes.add((header, f, including_module)) + # Build header-to-module ambiguity map + batch_header_cache = {} + ambiguous_headers = set() + header_to_modules = defaultdict(set) + for header, _, _ in all_includes: + candidates = filename_to_paths.get(os.path.basename(header), []) + for candidate in candidates: + mod = self.module_from_path(candidate) + header_to_modules[header].add(mod) + for header, mods in header_to_modules.items(): + if len(mods) > 1: + ambiguous_headers.add(header) + # Resolve all unique (header, including_module) pairs in bulk, with + # cache stats + cache_hits = 0 + cache_misses = 0 + for header, _, including_module in all_includes: + cache_key = ((header, including_module) + if header in ambiguous_headers else header) + if cache_key in batch_header_cache: + cache_hits += 1 + continue + resolved = self._batch_resolve_header( + header, including_module, filename_to_paths + ) + batch_header_cache[cache_key] = resolved + cache_misses += 1 + # Build the dependency tree using the resolved headers + tree = defaultdict(set) + for f in files: + current_module = file_to_module[f] + includes = file_includes_map[f] + for include_info in includes: + header = include_info['header'] + include_type = include_info['type'] + # Skip system headers and standard library + if include_type == 'system_common': + continue + if (include_type == 'system' and + os.path.basename(header) in standard_lib_headers): + continue + cache_key = ((header, current_module) + if header in ambiguous_headers else header) + if cache_key in batch_header_cache: + resolved_module = batch_header_cache[cache_key] + else: + resolved_module = self._batch_resolve_header( + header, current_module, filename_to_paths + ) + if (resolved_module and + resolved_module != current_module): + if (resolved_module != self.C_STANDARD_LIB and + current_module != self.C_STANDARD_LIB): + tree[current_module].add(resolved_module) + tree_as_list = {k: list(v) for k, v in tree.items()} + print(f"[dep_tree] batch_header_cache: {len(batch_header_cache)} " + f"entries, hits: {cache_hits}, misses: {cache_misses}") + cache_stats = { + 'batch_header_cache_size': len(batch_header_cache), + 'cache_hits': cache_hits, + 'cache_misses': cache_misses + } + return tree_as_list, cache_stats + + def _batch_resolve_header( + self, header: str, including_module: str, + filename_to_paths: dict[str, list[str]] + ) -> Optional[str]: + """ + Header resolution logic for batch/bulk mode (same as before, but no + caching here) Adds debug output for ambiguous headers (limited to 10 + examples). Only prints debug info if header or including_module contains + 'openssl' (case-insensitive). + """ + header_basename = os.path.basename(header) + candidates = filename_to_paths.get(header_basename, []) + if not candidates: + return None + if len(candidates) == 1: + return self.module_from_path(candidates[0]) + + # Try to resolve by module context + for candidate in candidates: + if self.module_from_path(candidate) == including_module: + return including_module + # Try to resolve by path match + if '/' in header: + for candidate in candidates: + if candidate.endswith(header): + return self.module_from_path(candidate) + # Prefer non-glibc + for candidate in candidates: + mod = self.module_from_path(candidate) + if mod != self.C_STANDARD_LIB: + return mod + # Fallback to glibc + for candidate in candidates: + mod = self.module_from_path(candidate) + if mod == self.C_STANDARD_LIB: + return self.C_STANDARD_LIB + # Fallback to first + return self.module_from_path(candidates[0]) + + # Build a sort of "upside down" tree - a dict containing mapping of each + # package to a list of all consuming packages + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + + source_path_obj = Path(manifest_path) + for subdir in ["include", "src"]: + candidate = source_path_obj / subdir + if candidate.exists(): + self.add_include_path(str(candidate)) + start_time = time.time() + tree, cache_stats = self.build_dependency_tree_enhanced(source_path_obj) + elapsed = time.time() - start_time + logger.info(f"Dependency tree built with {len(tree)} nodes in {elapsed:.2f} seconds.") + for clib in tree[self.prj_name]: + if clib in tree: + tree[clib].append(self.prj_name) + else: + tree[clib] = [self.prj_name] + tree[self.prj_name].append(ROOT_LEVEL_SENTINEL) + + # add standard lib to all projects + tree[self.C_STANDARD_LIB] = [] + for key in tree: + tree[self.C_STANDARD_LIB].append(key) + + return tree + + # Return only package name, without version + def extract_package_name(self, package_name: str) -> str: + return "libDummy.so" # not needed and not called in C env + + def install_dependencies(self, manifest_path: Path): + """ + Install dependencies for C/C++ projects. + Supports two modes: + 1. Traditional RPM download from public repositories + 2. Container source download from registry.redhat.io (when USE_CONTAINER_SOURCES=true) + """ + self.prj_name = self.find_project_name(str(manifest_path)) + self.root_dir = str(manifest_path) + download_path = os.path.join(manifest_path, C_DEP_LIBS_NAME) + rpm_cache_dir = RPMDependencyManager.get_instance().get_rpm_cache_dir() + #check if container image is already downloaded + container_image_file = os.path.join(download_path, "container_image.txt") + if os.path.exists(container_image_file): + with open(container_image_file, "r") as f: + container_image = f.read() + if container_image == RPMDependencyManager.get_instance().container_image: + logger.info(f"Container image {container_image} already downloaded") + return + + # Check if container source download is enabled + use_container_sources = os.getenv('USE_CONTAINER_SOURCES', 'true').lower() == 'true' + if use_container_sources: + logger.info("Container source download mode enabled") + try: + self._install_dependencies_from_container(manifest_path, rpm_cache_dir) + except Exception as e: + logger.warning(f"Container source download failed: {e}. Falling back to traditional RPM download.") + rpm_downloader = RPMDependencyManager.get_instance() + rpm_downloader.threaded_rpm_download(rpm_downloader.sbom, self.prj_name, download_path, max_workers=3) + #write the image name to file to mark rpms as downloaded + file_data = RPMDependencyManager.get_instance().container_image + with open(os.path.join(download_path, "container_image.txt"), "w") as f: + f.write(file_data) + + + def _install_dependencies_from_container(self, manifest_path: Path, download_path: str): + """Install dependencies from container sources using skopeo""" + # Validate environment variables + container_image = RPMDependencyManager.get_instance().container_image + if not container_image: + raise ValueError("CONTAINER_IMAGE must be set when USE_CONTAINER_SOURCES=true") + + username = os.getenv('REGISTRY_REDHAT_USERNAME') + password = os.getenv('REGISTRY_REDHAT_PASSWORD') + if not username or not password: + raise ValueError( + "REGISTRY_REDHAT_USERNAME and REGISTRY_REDHAT_PASSWORD must be set when USE_CONTAINER_SOURCES=true" + ) + + logger.info(f"Downloading container sources for: {container_image}") + + # Setup authentication + self._setup_skopeo_auth(username, password) + + # Download container sources + container_sources_dir = self._download_container_sources(container_image, manifest_path) + + # Extract and organize RPMs + self._extract_and_organize_rpms(container_sources_dir, download_path,container_image) + + # Cleanup temporary files + self._cleanup_temp_files(container_sources_dir) + + def _setup_skopeo_auth(self, username: str, password: str): + """Setup skopeo authentication for registry.redhat.io""" + logger.info("Setting up skopeo authentication...") + + cmd = [ + 'skopeo', 'login', 'registry.redhat.io', + '--username', username, + '--password', password + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Skopeo login failed: {result.stderr}") + + logger.info("Skopeo authentication successful") + + def _download_container_sources(self, container_image: str, manifest_path: Path) -> Path: + """Download container source using skopeo with caching support""" + # Create source image name (append -source) + source_image = f"{container_image}-source" + + # Create clean directory name + container_dir = source_image.replace('/', '-').replace(':', '-').replace('@', '-') + container_sources_dir = manifest_path / "container_sources" / container_dir + + # Create directory + container_sources_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Downloading container source: {source_image}") + + cmd = [ + 'skopeo', 'copy', + f'docker://{source_image}', + f'dir:{container_sources_dir}' + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Skopeo copy failed: {result.stderr}") + + logger.info(f"Container source downloaded to: {container_sources_dir}") + return container_sources_dir + + def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: str, container_image: str): + """Extract RPMs from container sources and organize them""" + logger.info("Extracting RPMs from container sources...") + + metadata_dir = container_sources_dir / "metadata" + # Create download path if it doesn't exist + os.makedirs(metadata_dir, exist_ok=True) + + # move metadata.json to metadata directory + metadata_file = container_sources_dir / "manifest.json" + shutil.move(metadata_file, metadata_dir / "manifest.json") + version_file = container_sources_dir / "version" + shutil.move(version_file, metadata_dir / "version") + + # Extract all tar files in the container sources directory + for tar_file in container_sources_dir.glob("*"): + if tar_file.is_dir(): + logger.warning(f"Skipping non-file: {tar_file}") + continue + logger.info(f"Extracting: {tar_file.name}") + try: + result = subprocess.run( + ['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], + capture_output=True, text=True, check=False + ) + if result.returncode != 0: + logger.warning(f"Failed to extract {tar_file}: {result.stderr}") + except Exception as e: + logger.warning(f"Error extracting {tar_file}: {e}") + + # Extract app name from container image (same logic as sync_rpms.sh) + app_name = self._extract_app_name_from_container_image(container_image) + logger.info(f"Extracted app name: {app_name}") + + # Find and copy RPMs to the download path + rpm_dir = container_sources_dir / "rpm_dir" + if rpm_dir.exists(): + logger.info(f"Found RPM directory: {rpm_dir}") + + # Copy RPMs to the standard download path, excluding the container app + for rpm_file in rpm_dir.rglob("*.rpm"): + # Skip RPMs that belong to the container app + if rpm_file.name.startswith(f"{app_name}-"): + logger.debug(f"Skipping container app RPM: {rpm_file.name}") + continue + + try: + subprocess.run(['cp', str(rpm_file), str(download_path)], check=True) + logger.debug(f"Copied RPM: {rpm_file.name} -> {download_path}") + except subprocess.CalledProcessError as e: + logger.warning(f"Failed to copy RPM {rpm_file}: {e}") + else: + logger.warning("No rpm_dir found in container sources") + + def _extract_app_name_from_container_image(self, container_image: str) -> str: + """Extract app name from container image (same logic as sync_rpms.sh)""" + # Extract the basename (last part after /) + basename = container_image.split('/')[-1] + # Remove the tag (part after :) + name_without_tag = basename.split(':')[0] + # Extract the first part before the first dash + app_name = name_without_tag.split('-')[0] + return app_name + + def _cleanup_temp_files(self, container_sources_dir: Path): + """Clean up temporary container source files (with cache preservation option)""" + preserve_cache = os.getenv('PRESERVE_CONTAINER_CACHE', 'false').lower() == 'true' + + if preserve_cache: + logger.info("Preserving container source cache (PRESERVE_CONTAINER_CACHE=true)") + return + + logger.info("Cleaning up temporary files...") + try: + # Remove the container sources directory + import shutil + shutil.rmtree(container_sources_dir) + logger.info("Temporary files cleaned up") + except Exception as e: + logger.warning(f"Failed to cleanup temporary files: {e}") + + def clear_container_cache(self, manifest_path: Path): + """Clear all cached container sources for a project""" + container_sources_dir = manifest_path / "container_sources" + if container_sources_dir.exists(): + try: + import shutil + shutil.rmtree(container_sources_dir) + logger.info(f"Cleared container cache: {container_sources_dir}") + except Exception as e: + logger.error(f"Failed to clear container cache: {e}") + else: + logger.info("No container cache found to clear") + + def find_project_name(self, root_dir="."): + cmake_path = os.path.join(root_dir, "CMakeLists.txt") + if os.path.exists(cmake_path): + with open(cmake_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search( + r'project\s*\(\s*([A-Za-z0-9_\-]+)', line, + re.IGNORECASE + ) + if m: + return m.group(1).lower() + meson_path = os.path.join(root_dir, "meson.build") + if os.path.exists(meson_path): + with open(meson_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search( + r'project\s*\(\s*["\']?([A-Za-z0-9_\-]+)', line, + re.IGNORECASE + ) + if m: + return m.group(1).lower() + makefile_path = os.path.join(root_dir, "Makefile") + if os.path.exists(makefile_path): + with open(makefile_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search( + r'^(PROJECT_NAME|PACKAGE_NAME|PROJECT|PACKAGE)\s*[:=]\s*' + r'([A-Za-z0-9_\-]+)', line + ) + if m: + return m.group(2).lower() + configure_ac_path = os.path.join(root_dir, "configure.ac") + if os.path.exists(configure_ac_path): + with open(configure_ac_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r'AC_INIT\(\[([A-Za-z0-9_\-]+)\]', line) + if m: + return m.group(1).lower() + + configure_path = os.path.join(root_dir, "configure") + if os.path.exists(configure_path): + with open(configure_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search( + r"^PACKAGE_NAME\s*[:=]\s*['\"]?([A-Za-z0-9_\-]+)['\"]?", + line + ) + if m: + return m.group(1).lower() + return "dummy_project" + + class GoDependencyTreeBuilder(DependencyTreeBuilder): def install_dependencies(self, manifest_path: Path): @@ -100,7 +746,8 @@ def __init__(self): def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: # Get go version from manifest - # TODO - download go binary of this version and run the go mod graph using it for optimal performance. + # TODO - download go binary of this version and run the go mod graph + # using it for optimal performance. go_version = self.determine_go_version(manifest_path) go_mod_tree = self.get_go_mod_graph_tree(manifest_path) @@ -132,38 +779,52 @@ def determine_go_version(manifest_path): @staticmethod def get_go_mod_graph_tree(manifest_path) -> str: """ - run the go mod graph command to retirive all dependencies from go.mod, and their hirearchy + run the go mod graph command to retirive all dependencies from go.mod, + and their hirearchy :param manifest_path: path to the go.mod manifest. - :return: a string representation of the go.mod graph, including dependencies and their hierarchy. + :return: a string representation of the go.mod graph, including + dependencies and their hierarchy. """ process_object: subprocess.CompletedProcess formatted_error_msg: str try: process_object = subprocess.run( - ["bash", "-c", f" export GOWORK=off ; go mod graph -modfile {manifest_path}/go.mod"], + ["bash", "-c", f" export GOWORK=off ; go mod graph " + f"-modfile {manifest_path}/go.mod"], capture_output=True, text=True) if process_object.returncode > 0: - process_object = subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" - f"go mod graph"], - capture_output=True, text=True) + process_object = subprocess.run([ + "bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" + f"go mod graph" + ], capture_output=True, text=True) if process_object.returncode > 0: - formatted_error_msg = f"Failed to generate dependencies tree from go.mod manifest at {manifest_path}," \ - f" error details => {process_object.stderr}" + formatted_error_msg = ( + f"Failed to generate dependencies tree from go.mod " + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}" + ) raise Exception(formatted_error_msg) except Exception as e: - error_message_exception = f"Failed to generate dependencies tree from because go.mod manifest wasn't found at {manifest_path}," \ - f" error details => {repr(e)}" + error_message_exception = ( + f"Failed to generate dependencies tree from because go.mod " + f"manifest wasn't found at {manifest_path}, error details => " + f"{repr(e)}" + ) logging.error(error_message_exception) raise e return process_object.stdout def download_go_mod_vendor(self, manifest_path): """ - Run the 'go mod vendor' command, in order to download go' app dependencies, in case vendor directory doesn't - already exist in git repo, it downloads it to git repo root dir where the go.mod file located. + Run the 'go mod vendor' command, in order to download go' app + dependencies, in case vendor directory doesn't already exist in git repo, + it downloads it to git repo root dir where the go.mod file located. :param manifest_path: path to go.mod manifest in repo directory. """ - subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; go mod vendor"]) + subprocess.run([ + "bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; " + f"go mod vendor" + ]) def extract_package_name(self, package_name: str) -> str: if package_name.__contains__("@"): @@ -345,22 +1006,30 @@ def install_dependency(self, dependency, repo_path): def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTreeBuilder: """ - A Factory method function that gets an programming language as argument, and returns a DependencyTreeBuilder' - instance related to the ecosystem. - :param programming_language: ecosystem to get the dependency tree builder for. - :return: DependencyTreeBuilder instance for the ecosystem, If not implemented, raise a NotImplementedError. + A Factory method function that gets an programming language as argument, + and returns a DependencyTreeBuilder' instance related to the ecosystem. + :param programming_language: ecosystem to get the dependency tree builder + for. + :return: DependencyTreeBuilder instance for the ecosystem, If not + implemented, raise a NotImplementedError. """ - if programming_language == Ecosystem.GO.value: + if programming_language.value == Ecosystem.GO.value: return GoDependencyTreeBuilder() - elif programming_language == Ecosystem.PYTHON.value: + elif programming_language.value == Ecosystem.PYTHON.value: return PythonDependencyTreeBuilder() + elif programming_language.value == Ecosystem.C_CPP.value: + return CCppDependencyTreeBuilder() else: - raise NotImplementedError(f'Unsupported Programming Language for transitive search -> {programming_language}') + raise NotImplementedError( + f'Unsupported Programming Language for transitive search -> ' + f'{programming_language}' + ) class DependencyTree: """ - A class that represents a dependency tree to access an appropriate dependency tree builder, based on ecosystem. + A class that represents a dependency tree to access an appropriate + dependency tree builder, based on ecosystem. """ builder: DependencyTreeBuilder ecosystem: Ecosystem @@ -373,7 +1042,7 @@ def __init__(self, ecosystem: Ecosystem): self.ecosystem = ecosystem try: # gets a dependency tree builder object based on ecosystem. - self.builder = get_dependency_tree_builder(ecosystem.value) + self.builder = get_dependency_tree_builder(ecosystem) except ValueError as err: - logger.warning(f"Unable to build dependency tree - ${str(err)}") + logger.warning("Unable to build dependency tree - %s", str(err)) self.builder = None diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index f512f6f3a..e01a600a5 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -45,6 +45,8 @@ from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from vuln_analysis.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.c_segmenter_custom import CSegmenterExtended + if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover @@ -144,6 +146,8 @@ class ExtendedLanguageParser(LanguageParser): } additional_segmenters["go"] = GoSegmenterWithMethods additional_segmenters["python"] = PythonSegmenterWithClassesMethods + additional_segmenters["c"] = CSegmenterExtended + LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, **additional_segmenters, @@ -153,8 +157,16 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: try: code = blob.as_string() except Exception as e: - logger.warning("Failed to read code for '%s'. Ignoring this file. Error: %s", blob.source, e) - return + # Minimal fallback: try decoding raw bytes with UTF-8 replacement, then latin-1 + try: + raw_bytes = blob.as_bytes() + try: + code = raw_bytes.decode('utf-8', errors='replace') + except Exception: + code = raw_bytes.decode('latin-1', errors='replace') + except Exception: + logger.warning("Failed to read code for '%s'. Ignoring this file. Error: %s", blob.source, e) + return language = self.language or (self.LANGUAGE_EXTENSIONS.get(blob.source.rsplit(".", 1)[-1]) if isinstance( blob.source, str) else None) @@ -178,7 +190,7 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) return - segmenter = self.LANGUAGE_SEGMENTERS[language](blob.as_string()) + segmenter = self.LANGUAGE_SEGMENTERS[language](code) try: extracted_functions_classes = segmenter.extract_functions_classes() diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py index ca7799a1f..c85ba2c70 100644 --- a/src/vuln_analysis/utils/function_name_extractor.py +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -2,6 +2,11 @@ import re from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from vuln_analysis.utils.dep_tree import Ecosystem +from vuln_analysis.logging.loggers_factory import LoggingFactory + + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") def traverse_all_parameters(function_ending_index_end, function_prefix_index_end, function_string): @@ -39,8 +44,7 @@ def infer_if_short_package_name_match(package: str, package_names: list[str], fi final_package.add(current_package) return True - else: - return False + return False def handle_argument(param: str) -> str: @@ -65,6 +69,8 @@ class FunctionNameExtractor: def __init__(self, coc_retriever: ChainOfCallsRetriever): self.lang_parser = coc_retriever.language_parser self.docs = coc_retriever.documents + self.ecosystem = coc_retriever.ecosystem + self.coc_retriever = coc_retriever def fetch_list(self, query: str) -> list[str]: """ @@ -74,7 +80,21 @@ def fetch_list(self, query: str) -> list[str]: returns list of calling functions in the package, e.g - ['packageName,functionName','packageName,functionName2'] """ + + query = query.splitlines()[0].replace('"', '').replace("'", "").strip() parts_input = query.split(",", 1) + + # Validate input format + if len(parts_input) < 2: + error_msg = ( + f"ERROR: Invalid input format. " + f"received: '{query}'. " + f"Please provide input in the correct format with a comma separating package name and function name. " + f"Example: 'github.com/golang-jwt/jwt,errors.New(\"value1\",\"value2\")'" + ) + logger.error(error_msg) + return [error_msg] # Return error message that LLM can see + package = parts_input[0] function = parts_input[1] final_package = set() diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py new file mode 100644 index 000000000..78c2dffc8 --- /dev/null +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import difflib + + +from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from vuln_analysis.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.dep_tree import Ecosystem +from vuln_analysis.utils.standard_library_cache import StandardLibraryCache +from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + + +class FunctionNameLocator: + """ + A class dedicated to locating function names using fuzzy matching. + Initially designed for C/C++ but can be extended for other languages. + """ + + def __init__(self, coc_retriever: ChainOfCallsRetriever): + self.lang_parser = coc_retriever.language_parser + self.coc_retriever = coc_retriever + self.is_std_package = False + self.is_package_valid = False + self.stdlib_cache = StandardLibraryCache.get_instance() + + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: + logger.info("package %s is not in supported packages", package) + if self.coc_retriever.supported_packages is None: + return [] + pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.3) + std_close_matches = self.stdlib_cache.get_close_match_list(package, self.coc_retriever.ecosystem) + if std_close_matches: + logger.info("package %s is not in supported packages, but close matches are: %s", package, std_close_matches) + return std_close_matches + if pkg_close_matches: + logger.info("package %s is not in supported packages, but close matches are: %s", package, pkg_close_matches) + return pkg_close_matches + else: + return [] + + def python_flow_control(self, input_function: str, package_docs) -> list[str]: + """ + Python flow control + Args: + input_function: The function name to search for + package_docs: The package documents to search in + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + count_of_dots = 0 + splitters = [splitter for splitter in ['.'] if splitter in input_function] + if splitters: + count_of_dots = len(splitters) + + list_of_matching_combinations = set() + for doc in package_docs: + if self.lang_parser: + function_name = self.lang_parser.get_function_name(doc) + module_name = doc.metadata.get('source').split('/')[-1].split('.')[0] + class_name = self.lang_parser.get_class_name_from_class_function(doc) + if count_of_dots == 0: + if function_name: + list_of_matching_combinations.add(f"{function_name}") + if class_name: + list_of_matching_combinations.add(f"{class_name}") + if count_of_dots == 1: + if module_name and class_name: + list_of_matching_combinations.add(f"{module_name}.{class_name}") + if module_name and function_name: + list_of_matching_combinations.add(f"{module_name}.{function_name}") + if class_name and function_name: + list_of_matching_combinations.add(f"{class_name}.{function_name}") + if count_of_dots == 2: + if module_name and class_name and function_name: + list_of_matching_combinations.add(f"{module_name}.{class_name}.{function_name}") + + close_matches = difflib.get_close_matches(input_function, list_of_matching_combinations, n=10, cutoff=0.6) + if len(close_matches) == 0: + close_matches = difflib.get_close_matches(input_function, list_of_matching_combinations, n=10, cutoff=0.3) + return close_matches + + def common_flow_control(self,input_function: str, package_docs) -> list[str]: + """ + C/Go flow control have the same logic for finding close matches for function names in the package + Args: + input_function: The function name to search for + package_docs: The package documents to search in + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + package_functions = [] + for doc in package_docs: + if self.lang_parser: + function_name = self.lang_parser.get_function_name(doc) + package_functions.append(f"{function_name}") + # Use fuzzy matching to find close matches + close_matches = difflib.get_close_matches(input_function, package_functions, n=10, cutoff=0.6) + if len(close_matches) == 0: + close_matches = difflib.get_close_matches(input_function, package_functions, n=10, cutoff=0.3) + logger.info("Close matches found: %s", close_matches) + return close_matches + + + + async def locate_functions(self, query: str) -> list[str]: + """ + Locate function names in a package using fuzzy matching and validate package names. + Expected format: 'package_name,function_name' + + Args: + query: Input string in format 'package_name,function_name' + + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + query = query.splitlines()[0].replace('"', '').replace("'", "").strip() + parts_input = query.split(",", 1) + self.is_std_package = False + self.is_package_valid = False + # Validate input format + if len(parts_input) < 2: + error_msg = ( + f"ERROR: Invalid input format. Expected format: 'package_name,function_name' " + f"but received: '{query}'. " + f"Please provide input in the correct format with a comma separating package name and function name. " + f"Example: 'libxml2,xmlParseDocument'" + ) + logger.error(error_msg) + return [error_msg] # Return error message that LLM can see + + package = parts_input[0] + function = parts_input[1] + + try: + logger.info("Package and Function Locator: searching for (%s,%s)", package, function) + + # First validate package name exists in supported packages + if (self.coc_retriever.supported_packages is None or + package not in self.coc_retriever.supported_packages): + logger.info("Package '%s' not found in supported packages", package) + is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) + if is_standard_lib: + self.is_std_package = True + self.is_package_valid = True + return [ + ( + f"INFO: Package '{package}' is a standard library package. " + f"make call with the Transitive code search tool" + ) + ] + else: + is_standard_lib_api = await quick_standard_lib_check(package, self.coc_retriever.ecosystem) + if is_standard_lib_api: + self.is_std_package = True + self.is_package_valid = True + self.stdlib_cache.add_to_cache(package, self.coc_retriever.ecosystem) + return [ + ( + f"INFO: Package '{package}' is a standard library package. " + f"make call with the Transitive code search tool" + )] + close_package_matches = self.handle_package_not_in_supported_packages(package) + if close_package_matches: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) + else: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"No close matches found. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) + logger.error(error_msg) + return [error_msg] # Return error message that LLM can see + # Fallback: on cache miss, sparsely verify via API + + + # Package is valid, proceed with function search + self.is_package_valid = True + package_docs = [] + if (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.C_CPP.value): + package_docs = self.coc_retriever.sort_docs[package] + else: + # package_docs = self.coc_retriever.documents_of_functions + package_docs = [ + doc + for doc in (self.coc_retriever.documents_of_functions or []) + if ( + self.lang_parser + and self.lang_parser.get_package_name(doc, package) + ) + ] + + match self.coc_retriever.ecosystem.value: + case Ecosystem.PYTHON.value: + return self.python_flow_control(function, package_docs) + case Ecosystem.C_CPP.value: + return self.common_flow_control(function, package_docs) + case Ecosystem.GO.value: + return self.common_flow_control(function, package_docs) + case _: + return self.common_flow_control(function, package_docs) + + + except KeyError: + logger.warning("Package '%s' not found in available packages", package) + close_package_matches = self.handle_package_not_in_supported_packages(package) + if close_package_matches: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names." + ) + else: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"No close matches found." + ) + return [error_msg] + except Exception as e: + logger.error("Error locating functions in package '%s': %s", package, e) + return [] + +async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> bool: + """Quick check if package is standard library + Args: + package_name: The package name to check + ecosystem: The ecosystem to check + Returns: + True if package is standard library, False otherwise + """ + + search = MorpheusSerpAPIWrapper(max_retries=2) + result = await search.arun(f"Is '{package_name}' part of the {ecosystem} standard library?") + logger.info("quick_standard_lib_check Standard library check result: %s", result) +# Normalize result: if list, join into single string + if isinstance(result, list): + text = " ".join(result).lower() + elif isinstance(result, str): + text = result.lower() + else: + text = str(result).lower() + + # Basic positive signals and avoidance of negative phrasing + if "part of the standard library" in text or \ + "is a standard library" in text or \ + ("standard library" in text and "not" not in text): + return True + + return False + diff --git a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py new file mode 100644 index 000000000..ff320b295 --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py @@ -0,0 +1,740 @@ +from langchain_core.documents import Document +from .lang_functions_parsers import LanguageFunctionsParser +import re +import regex +import os +from vuln_analysis.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +# Constants for parsing configuration +REGEX_TIMEOUT_SECONDS = 0.05 # Timeout for regex operations to prevent hanging +MIN_PARTS_FOR_PARSING = 2 # Minimum number of parts required for parsing +STRING_NOT_FOUND = -1 # Return value when string.find() doesn't find the substring + +from ..dep_tree import DependencyTree, CCppDependencyTreeBuilder, C_DEP_LIBS_NAME + +def _remove_c_comments(code: str) -> str: + # Remove all multi-line comments (/* ... */) + code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) + # Remove all single-line comments (//...) + code = re.sub(r'//.*', '', code) + return code + +# Compile the function pattern with recursive param matching +_FUNCTION_PATTERN_REGEX = regex.compile(r''' + ^[ \t]* # Leading indentation + (?P.*?) # Non-greedy return type (can include macros) + \b + (?P[A-Za-z_]\w*) # Function name (C identifier) + \s* + \( + (?P # Named group for parameters + (?: # Repeated nested structure: + [^()]+ # - non-paren characters + | # - or a nested (...) + \((?¶ms)\) # recurse into the same group + )* + ) + \) + \s*(?:const\s*)? # Optional const after params + \s*\{ # Opening brace of function body + ''', regex.VERBOSE | regex.MULTILINE) + + +class CDocumentParser: + """ + Parses a single C code Document and provides methods to extract information + about structs and functions within that document. + """ + + # Greedy patterns to capture full struct definitions + _NAMED_STRUCT_PATTERN = re.compile( + r'^\s*struct\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\{(.*)\};?\s*$', + re.DOTALL | re.MULTILINE + ) + _TYPEDEF_STRUCT_PATTERN = re.compile( + r'^\s*typedef\s+struct\s+(?:[a-zA-Z_][a-zA-Z0-9_]*)?\s*\{(.*)\}\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*;?\s*$', + re.DOTALL | re.MULTILINE + ) + + + _FUNC_POINTER_PATTERN = re.compile( + r''' + ^\s* + (?P[\w\s\*]+?) + \(\s*\*\s* + (?P[A-Za-z_]\w*(?:\s*\[[^\]]+\])?) + \s*\)\s* + \(\s*(?P[^)]*)\)\s* + ; + $ + ''', + re.VERBOSE + ) + _BITFIELD_PATTERN = re.compile( + r''' + ^\s* + (?P[\w\s\*]+?) + \s+ + (?P[A-Za-z_]\w*) + \s*:\s* + (?P\d+) + \s*; + $ + ''', + re.VERBOSE + ) + + # Pattern for inline struct/union/enum definitions + _INLINE_STRUCT_PATTERN = re.compile( + r'((?:struct|union|enum)\s*\{.*?\})\s*([A-Za-z_][A-Za-z0-9_]*)\s*;', + re.DOTALL + ) + + def __init__(self, doc: Document): + self.c_code = doc.page_content.strip() + self.metadata = doc.metadata + self._parsed_struct_data = None + + def is_doc_struct(self) -> bool: + return bool(self._NAMED_STRUCT_PATTERN.search(self.c_code)) or bool(self._TYPEDEF_STRUCT_PATTERN.search(self.c_code)) + + + + def parse_struct_to_fields(self) -> tuple[tuple, list[tuple]]: + + if self._parsed_struct_data: + return self._parsed_struct_data + # Identify struct name and raw body + m_struct = self._NAMED_STRUCT_PATTERN.search(self.c_code) + m_typedef = self._TYPEDEF_STRUCT_PATTERN.search(self.c_code) + if m_struct: + struct_name = m_struct.group(1).strip() + struct_body = m_struct.group(2) + else: + struct_body = m_typedef.group(1) + struct_name = m_typedef.group(2).strip() + + source_file = self.metadata.get('source_file') + members = [] + + # Handle inline struct/union/enum definitions first + def _collect_inline(match): + raw_block = match.group(1).strip() + # Normalize whitespace to single spaces + normalized = ' '.join(raw_block.split()) + name = match.group(2).strip() + members.append((name, normalized)) + return '' # Remove from body + struct_body = self._INLINE_STRUCT_PATTERN.sub(_collect_inline, struct_body) + + # Process each remaining line + for line in struct_body.splitlines(): + # Strip comments + line = re.sub(r'//.*', '', line) + line = re.sub(r'/\*.*?\*/', '', line) + line = line.strip() + # Only simple declarations ending with ';' + if not line.endswith(';'): + continue + # try to parse function pointers and bit-fields this will work for most of the cases not all of them + if '(*' in line or ':' in line: + functionPointer = self._FUNC_POINTER_PATTERN.match(line) + if functionPointer: + name = functionPointer.group('name').strip() + type_str = f"{functionPointer.group('rettype').strip()} (*)( {functionPointer.group('params').strip()} )" + members.append((name, type_str)) + continue + m_bf = self._BITFIELD_PATTERN.match(line) + if m_bf: + name = m_bf.group('name').strip() + type_str = f"{m_bf.group('type').strip()} :{m_bf.group('bits')}" + members.append((name, type_str)) + continue + #could not parse this should be mark for edge cases + continue + content = line[:-1].strip() + parts = re.split(r'\s+', content) + if len(parts) < MIN_PARTS_FOR_PARSING: + continue + raw_name = parts[-1] + raw_type = ' '.join(parts[:-1]) + # Move pointer stars from name to type + star_match = re.match(r'^(\*+)', raw_name) + star = star_match.group(1) if star_match else '' + name = raw_name[len(star):] + type_str = (raw_type + (' ' + star if star else '')).strip() + members.append((name, type_str)) + + self._parsed_struct_data = ((struct_name, source_file), members) + return self._parsed_struct_data + +PARAMETER = "parameter" + +RETURN_TYPES = "return_types" + +LOCAL_VAR_USAGE = "local_var_usage" + +class CLanguageFunctionsParser(LanguageFunctionsParser): + + def init_CParser(self,depTree :DependencyTree): + self.typedef_pattern = re.compile(r'^\s*typedef\s+', re.MULTILINE) + self.fn_ptr_pattern = re.compile(r'\(\s*\*\s*\w+\s*\)') + self.struct_enum_union_pattern = re.compile(r'^\s*(struct|enum|union)\s+\w+\s*{', re.MULTILINE) + self.function_pattern = re.compile( + r'\b(?:static\s+|extern\s+|inline\s+|const\s+)*' + r'[A-Za-z_][\w\s\*]*?' # Return type (minimal matching) + r'\s+[A-Za-z_]\w*\s*\([^;{]*\)\s*(?:{|;)', # Function name and params + re.MULTILINE + ) + + # Pattern for parsing variable segments in multi-declaration lines + self.var_segment_pattern = re.compile( + r"^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*(\[.*?\])?(?:\s*=\s*(.*))?$" + ) + + # Pattern for extracting return types from function signatures + self.return_type_pattern = re.compile( + r'((?:static|inline|\s)*[\w_][\w\d_]*(?:\s*\*+\s*|\s*))?\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(' + ) + + # Pattern for function pointer variable calls + self.fp_var_pattern = re.compile(r'\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\(') + + if isinstance(depTree.builder , CCppDependencyTreeBuilder) : + self.dep_builder_tree = depTree.builder + + self.C_STANDARD_LIB = "glibc" + + + def _optimize_func_key(self, function_name: str, source_path: str) -> str: + """ + Optimize func_key generation by extracting package name and filename from full path. + Uses the dependency tree's module_from_path method for consistent module extraction. + + Examples: + - 'count_trailing_zeros@rpm_libs/gawk/gawk-5.1/gawk-5.1.0/floatcomp.c' + -> 'count_trailing_zeros@gawk/floatcomp.c' + - 'main@src/app/main.c' -> 'main@app/main.c' + - 'process@unknown_source.c' -> 'process@unknown_source.c' + """ + if not source_path or source_path == 'unknown_source.c': + return f"{function_name}@{source_path}" + + # Use the dependency tree's module_from_path method if available + if self.dep_builder_tree: + try: + module_name = self.dep_builder_tree.module_from_path(source_path) + filename = os.path.basename(source_path) + return f"{function_name}@{module_name}/{filename}" + except Exception as e: + logger.debug(f"Failed to get module from path {source_path}: {e}") + # Fall back to original key method + + # Fallback: use original key generation method + return f"{function_name}@{source_path}" + + # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing + # mappings of all local variables defined in these functions/methods, to their type/value/expression. + # Implementation should be consistent with search_for_called_function, as functions_methods_documents is being used + # in this method as functions_local_variables_index + def __parse_function_signature(self,function_signature: str) -> dict: + """ + Parses a C/C++ function signature to extract parameter types and names. + + Returns: + dict mapping each parameter name to {"value": PARAMETER, "type": }. + Supports qualifiers, pointers, arrays, variadic args, and function pointers. + """ + all_vars = {} + + # Extract the parameter substring inside the outermost parentheses + parentheses = re.search(r"\((.*)\)", function_signature) + if not parentheses: + return all_vars + + params_str = parentheses.group(1).strip() + if not params_str or params_str == "void": + return all_vars + + # Split on commas not inside any parentheses (for function-pointer args) + parts = re.split(r',\s*(?![^()]*\))', params_str) + + for param in parts: + param = param.strip() + if not param: + continue + if param == "...": + all_vars["..."] = {"value": PARAMETER, "type": "VARIADIC_ARGS"} + continue + + # 1. Function-pointer: base (*name)(args) + functionPointer = re.match( + r'^(?P.+?)\s*\(\*\s*(?P[A-Za-z_]\w*)\s*\)\s*\((?P.*)\)\s*$', + param + ) + if functionPointer: + base = functionPointer.group('base').strip() + name = functionPointer.group('name').strip() + args = functionPointer.group('args').strip() + p_type = f"{base} (*{name})({args})" + all_vars[name] = {"value": PARAMETER, "type": p_type} + continue + + # 2. Named-anchored regex: separate type and name (with pointers/arrays) + m_name = re.match( + r'^(?P.+?)\s+(?P[\*&]*[A-Za-z_]\w*(?:\[.*?\])?)\s*$', + param + ) + if m_name: + p_type = m_name.group('type').strip() + raw_name = m_name.group('name').strip() + # Move leading pointer/reference symbols onto type + ptr_match = re.match(r'^(?P[\*&]+)(?P[A-Za-z_]\w*(?:\[.*?\])?)$', raw_name) + if ptr_match: + name = ptr_match.group('nm') + p_type = f"{p_type} {ptr_match.group('prefix')}".strip() + else: + # Extract array dimensions from name + arr_match = re.match(r'^(?P[A-Za-z_]\w*)(?P\[.*?\])$', raw_name) + if arr_match: + name = arr_match.group('nm') + p_type = f"{p_type}{arr_match.group('dims')}" + else: + name = raw_name + all_vars[name] = {"value": PARAMETER, "type": p_type} + continue + + # 3. Fallback: last-space split + parts2 = param.rsplit(None, 1) + if len(parts2) == 2: + p_type, name = parts2 + else: + p_type = parts2[0] + name = parts2[0] + all_vars[name] = {"value": PARAMETER, "type": p_type} + + return all_vars + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + + processed_count = 0 + for func_method in functions_methods_documents: + processed_count += 1 + + func_key = self._optimize_func_key( + self.get_function_name(func_method), + func_method.metadata.get('source', 'unknown_source.c') + ) + + + if ("state" in func_method.metadata ) or (func_method.metadata.get("func_name","N/A") == "if"): + continue #means invalid get function name failed + + all_vars = dict() + + # Optimize string operations - avoid multiple .strip() calls + content = func_method.page_content + first_brace_index = content.find('{') + if first_brace_index != STRING_NOT_FOUND: + function_signature = content[:first_brace_index].strip() + function_body = content[first_brace_index:].strip() + else: + function_signature = content.strip() + function_body = None # No function body found + + + # --- Parsing Function Parameters --- + all_vars = self.__parse_function_signature(function_signature) + + + # --- Parsing Return Types --- + # Capture everything before the function name and its opening parenthesis + # This regex is made more specific to ensure it captures the return type accurately. + # It looks for optional leading keywords (static, inline), then the actual type, + # then whitespace, then the function name, then an opening paren. + return_type_match = self.return_type_pattern.match(function_signature) + if return_type_match: + # Group 1 is the potential type including modifiers and pointers (e.g., "static int*", "const char") + # Group 2 is the function name, which we don't need for return type + p_return_type = return_type_match.group(1) + if p_return_type is None: #incase of bad segmentation a case in openssl file quic_wire.c + continue + potential_return_type = p_return_type.strip() + if potential_return_type: + all_vars[RETURN_TYPES] = [potential_return_type] + else: + all_vars[RETURN_TYPES] = [] # Should ideally not happen for valid C functions + else: + all_vars[RETURN_TYPES] = [] + + # --- Parsing Local Variables in Function Body --- + # Skip processing if no function body + if not function_body: + mappings[func_key] = all_vars + continue + + # Pre-filter lines to avoid unnecessary processing + lines = [line.strip() for line in function_body.splitlines() + if line.strip() and not self.is_comment_line(line.strip()) + and not line.strip().startswith("return ")] + + for cleaned_row in lines: + + # Regex to identify a line containing multiple declarations + # Group 1: The base type (e.g., 'int', 'char*') + # Group 2: The rest of the declaration string (e.g., 'i, j = 10') + multi_declaration_line_match = re.match( + r"^\s*([\w_][\w\d_]*(?:\s*[*&]+\s*|\s*))\s*(.+?);\s*$", cleaned_row + ) + + if multi_declaration_line_match: + base_type = multi_declaration_line_match.group(1).strip() + var_list_str = multi_declaration_line_match.group(2).strip() + + # Split the variable list by commas, handling cases like 'var = val' within a segment + # This regex matches: (var_name)( [array_brackets])( = value) + # It's applied to each comma-separated segment. + + for segment in var_list_str.split(','): + segment_match = self.var_segment_pattern.match(segment.strip()) + if segment_match: + var_name = segment_match.group(1).strip() + array_suffix = segment_match.group(2) + initial_value = segment_match.group(3) + + current_var_type = base_type + if array_suffix: + current_var_type += "[]" # Standardize array type representation + + all_vars[var_name] = { + "value": initial_value.strip() if initial_value is not None else "", + "type": current_var_type + } + + else: + # Fallback if a segment doesn't fit the expected var_segment_pattern + # This might indicate a syntax error or a more complex declaration not handled. + pass # Or log a warning + + + else: + # Fallback to single declaration or assignment logic if not a multi-declaration line + # C variable declaration: type var_name [= value]; + declaration_match = re.match(r'^\s*([\w_][\w\d_]*(?:\s*[*&]+\s*|\s*))([a-zA-Z_][a-zA-Z0-9_]*)\s*(\[.*?\])?\s*(?:=\s*(.*?))?\s*;', cleaned_row) + + if declaration_match: + full_type_str = declaration_match.group(1).strip() + var_name = declaration_match.group(2).strip() + array_suffix = declaration_match.group(3) + initial_value = declaration_match.group(4) + + if array_suffix: + full_type_str += "[]" + + all_vars[var_name] = { + "value": initial_value.strip() if initial_value is not None else "", + "type": full_type_str + } + + + # Simple assignments for existing variables (no declaration) + elif '=' in cleaned_row and ';' in cleaned_row: + assignment_match = re.match(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([^=;]+);', cleaned_row) + if assignment_match: + var_name = assignment_match.group(1).strip() + value_expr = assignment_match.group(2).strip() + + if var_name in all_vars: + all_vars[var_name]["value"] = value_expr + else: + all_vars[var_name] = {"value": value_expr, "type": LOCAL_VAR_USAGE} + + + # Store the result + mappings[func_key] = all_vars + + return mappings + + + # This method returns (for the given ecosystem) , a mapping of all types/classes in all packages to a dict + # containing mapping of all fields names in the type/class to their defining types. + # a list of + def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for _type in types: + #parse the document + parser = CDocumentParser(_type) + #get the struct name and fields + struct_name, fields = parser.parse_struct_to_fields() + types_mapping[struct_name] = fields + + return types_mapping + + def is_macro_function_invocation(self,seg: str) -> bool: + """ + Detects if a line is a macro function invocation, e.g. IMPLEMENT_ASN1_FUNCTIONS(...) + or IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(...) + """ + lines = seg.strip().splitlines() + for line in lines: + res = bool(re.match(r'^[A-Z_][A-Z0-9_]*(?:_[A-Z0-9_]+)*(_fname)?\s*\(.*\)\s*;?$', line.strip())) + if res: + return True + + # Match ALLCAPS or ALLCAPS_with_underscores, optionally with trailing _fname, and then '(' + return False + + def get_function_name(self, function: Document) -> str: + try: + if function.metadata.get('func_name') : + return function.metadata.get('func_name') + + seg =function.page_content.strip() + code = seg.replace('\r\n', '\n') + function_pattern = _FUNCTION_PATTERN_REGEX.search(code, timeout=REGEX_TIMEOUT_SECONDS) + if function_pattern: + f_name = function_pattern.group('name') + if f_name: + function.metadata['func_name'] = f_name + return f_name + function.metadata['state'] = "invalid" + return "" + except TimeoutError: + logger.warning(f"Function name extraction timed out for file: {function.metadata.get('source', 'unknown')}") + function.metadata['state'] = "invalid" + return "" + + + def search_for_called_function( + self, + caller_function: Document, + callee_function: str, + callee_function_package: str, # For C, this is usually the file or library name + code_documents: list[Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict] + ) -> bool: + """ + Returns True if caller_function calls callee_function (directly or via struct/function pointer). + """ + # 1. Extract function body (remove comments for easier parsing) + function_body = _remove_c_comments(caller_function.page_content) + # 2. Direct call: callee_function( + direct_call_pattern = re.compile(r'\b' + re.escape(callee_function) + r'\s*\(') + if direct_call_pattern.search(function_body): + return True + + # 3. Struct member or pointer call: obj->callee_function( or obj.callee_function( + member_call_pattern = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(->|\.)\s*' + re.escape(callee_function) + r'\s*\(') + for match in member_call_pattern.finditer(function_body): + var_name = match.group(1) + # Look up var_name in local variables and parameters + func_key = self._optimize_func_key( + self.get_function_name(caller_function), + caller_function.metadata.get('source', 'unknown_source.c') + ) + + local_vars = functions_local_variables_index.get(func_key, {}) + var_type = local_vars.get(var_name, {}).get("type", "") + # Check if var_type is a struct with a function pointer named callee_function + for (struct_name, _), fields in fields_of_types.items(): + if struct_name == var_type: + for field_name, field_type in fields: + if field_name == callee_function and "(*" in field_type: + return True + + # 4. Function pointer variable: foo_ptr( + for match in self.fp_var_pattern.finditer(function_body): + var_name = match.group(1) + if var_name == callee_function: + continue # Already checked direct call + # Is this variable a function pointer to callee_function? + func_key = self._optimize_func_key( + self.get_function_name(caller_function), + caller_function.metadata.get('source', 'unknown_source.c') + ) + local_vars = functions_local_variables_index.get(func_key, {}) + var_type = local_vars.get(var_name, {}).get("type", "") + # Optionally, check if the function pointer is assigned to callee_function somewhere + assignment_pattern = re.compile(rf'{re.escape(var_name)}\s*=\s*&?{re.escape(callee_function)}\s*;') + if assignment_pattern.search(function_body): + return True + + # 5. (Optional) Macro expansion, indirect calls, etc. (advanced, skip for now) + + return False + + def get_package_names(self, function: Document) -> list[str]: + pkgs = [] + full_doc_path = str(function.metadata['source']) + if self.dep_builder_tree: + mod = self.dep_builder_tree.module_from_path(full_doc_path) + pkgs.append(mod) + function.metadata['pkg_name']=mod + else: + pkgs.append(os.path.basename(full_doc_path)) + return pkgs + + def get_package_name(self, function: Document, package_name: str) -> str: + return package_name + + def is_root_package(self, function: Document) -> bool: + + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + stripped = line.strip() + return stripped.startswith("//") or stripped.startswith("/*") + + def get_comment_line_notation(self) -> str: + return "//" + + def is_exported_function(self, function: Document) -> bool: + # In C, functions have external linkage by default unless declared as static + #however internal functions might be called by exported functions + #The analysis needs to understand the complete call graph + #So we return True for all functions + return True + + def is_function(self, function: Document) -> bool: + """ + Check if the code contains a C function declaration/definition using regex. + Focuses on essential function patterns: + 1. Optional modifiers (static, extern, etc) + 2. Any return type (we don't care about specifics) + 3. Function name + 4. Parameter list + 5. Ending with ; or { + + Args: + code: String containing C code + + Returns: + bool: True if code contains a function, False otherwise + """ + # Clean comments first + if function.metadata.get('content_type') == 'functions_classes': + code = _remove_c_comments(function.page_content) + + # Negative checks + if self.typedef_pattern.search(code): + return False + if self.fn_ptr_pattern.search(code): + return False + if self.struct_enum_union_pattern.search(code): + return False + + + return bool(self.function_pattern.search(code)) + + return False + + def dir_name_for_3rd_party_packages(self) -> str: + return C_DEP_LIBS_NAME + + def supported_files_extensions(self) -> list[str]: + return [".c", ".h"] + + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + def is_searchable_file_name(self, function: Document) -> bool: + return function.metadata.get("file_name", "").endswith((".c", ".h")) + + def get_function_reserved_word(self) -> str: + return "" # C doesn't have a consistent reserved word for functions + + def get_type_reserved_word(self) -> str: + return "struct" + + def is_doc_type(self,doc: Document ) -> bool: + try: + if self.is_function(doc): + return False + parser = CDocumentParser(doc) + res = parser.is_doc_struct() + except Exception as e: + res = False + return res + + def filter_docs_by_func_pkg_name(self,function_name:str ,package_name : str, documents : list[Document] ) ->list[Document] : + + relevant_docs = [doc for doc in documents if doc.page_content.__contains__(function_name)] + + return relevant_docs + + def document_imports_package(self,documents:list[Document],package_name:str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if re.search( + rf"(include {package_name}|include\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + , value.page_content, flags=re.MULTILINE)] + return importing_docs + + def is_a_package(self,package_name: str,doc:Document) -> bool: + #check for external pkgs + file = doc.metadata.get('source') + b_3party = doc.metadata.get('source').startswith(self.dir_name_for_3rd_party_packages()) + if self.dep_builder_tree and b_3party: + pkg = self.dep_builder_tree.module_from_path(file) + else: + pkg = self.dep_builder_tree.prj_name + return pkg == package_name + + def is_search_algo_dfs(self)-> bool: + return False + + def is_language_supports_macro(self)-> bool: + return True + + def create_dummy_for_standard_lib(self,package_name: str)-> bool: + if package_name == self.C_STANDARD_LIB: + return True + return False + + def get_dummy_function(self, function_name): + return f"void {function_name}() {{}}" + + def is_same_package(self, package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() == package_name_from_tree.lower() + + def is_call_allowed(self, pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: + caller_pkg = self.get_package_names(caller_function)[0] + callee_pkg = self.get_package_names(callee_function)[0] + + # check if callee is static + first_line = callee_function.page_content.splitlines()[0] + is_callee_static = bool(re.match(r"^\s*static\b", first_line)) + + if is_callee_static: + # static callee: must be same package and same file + if caller_pkg != callee_pkg: + return False + if caller_function.metadata.get('source') != callee_function.metadata.get('source'): + return False + return True + + # non-static callee + if caller_pkg != callee_pkg: + # collect all functions in caller package + caller_functions = {self.get_function_name(doc): doc for doc in pkg_docs} + + callee_name = self.get_function_name(callee_function) + + if callee_name == "do_shell": + print(f"callee_name: {callee_name}") + + if callee_name in caller_functions: + doc = caller_functions[callee_name] + # if static and in same file → call is allowed + if re.match(r"^\s*static\b", doc.page_content.splitlines()[0]): + if doc.metadata.get('source') != caller_function.metadata.get('source'): + return True + # otherwise (global collision) → not allowed + return False + + return True diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 36c851945..2ebb71bc1 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -115,10 +115,8 @@ def handle_imports(code_content: str, identifier: str, callee_package: str) -> b class GoLanguageFunctionsParser(LanguageFunctionsParser): - @staticmethod - def is_same_package(package_name_from_input, package_name_from_tree): - return package_name_from_input.lower() in package_name_from_tree.lower() + def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}() {{}}" @@ -602,3 +600,6 @@ def get_package_names(self, function: Document) -> list[str]: def is_root_package(self, function: Document) -> bool: return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + return line.strip().startswith("//") \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py index bf2503a2e..c6da4ccaf 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py @@ -1,8 +1,10 @@ from abc import ABC, abstractmethod - +import re from langchain_core.documents import Document + + class LanguageFunctionsParser(ABC): # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing @@ -72,9 +74,10 @@ def is_exported_function(self, function: Document) -> bool: # This method get a source document, and returns True only if the document type # is a function ( not type/class/full_source) for the ecosystem. - @abstractmethod + def is_function(self, function: Document) -> bool: - pass + return function.page_content.startswith(self.get_function_reserved_word()) + # This method returns the name of the directory containing the 3rd party packages/libs source code files for the # ecosystem @@ -110,21 +113,69 @@ def get_type_reserved_word(self) -> str: pass @abstractmethod - def get_dummy_function(self, function_name): + def get_dummy_function(self, function_name) -> str: pass @staticmethod def is_script_language(): return False - @staticmethod - @abstractmethod - def is_same_package(package_name_from_input, package_name_from_tree): - pass + def is_same_package(self, package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() in package_name_from_tree.lower() @staticmethod def get_constructor_method_name(): return None - def get_class_name_from_class_function(self): - pass + # This method gets a function document as an argument, and returns the name of the class + # that contains this function. Returns None if the function is not a class method or if + # the class name cannot be determined for the given ecosystem. + def get_class_name_from_class_function(self, func: Document): + return None + + # This method gets a list of documents and a package name as arguments, and returns a list of + # documents that contain import statements for the specified package. It searches for both + # single import statements and grouped import statements that include the package name. + def document_imports_package(self, documents: list[Document], package_name: str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if re.search( + rf"(import {package_name}|import\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + , value.page_content, flags=re.MULTILINE)] + return importing_docs + + def is_doc_type(self, doc: Document) -> bool: + return doc.page_content.startswith(self.get_type_reserved_word()) + + def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: + relevant_docs = [ + doc for doc in documents + if doc.metadata.get('source').__contains__(package_name) and + doc.page_content.__contains__(function_name) + ] + return relevant_docs + + def is_a_package(self, package_name: str, doc: Document) -> bool: + b_3party = doc.metadata.get('source').startswith(self.dir_name_for_3rd_party_packages()) + b_package = (not self.is_root_package(doc) and (self.get_package_name(function=doc, package_name=package_name) != "")) + return b_3party and b_package + + # This method determines if the search algorithm uses Depth-First Search (DFS) for traversing + # function call graphs and dependency trees. Returns True if DFS is used, False for other algorithms. + def is_search_algo_dfs(self) -> bool: + return True + + # This method checks if the programming language supports macro definitions/preprocessor + # capabilities that can affect function parsing and analysis. Returns True for languages + # with macros (like C/C++), False otherwise. + def is_language_supports_macro(self) -> bool: + return False + + # This method determines if a dummy function should be created for standard library packages + # that may not have complete source code available for analysis. Returns True if a dummy + # function should be created for the given package, False if complete source is available. + def create_dummy_for_standard_lib(self, package_name: str) -> bool: + return False + + # is this call allowed by the language? + def is_call_allowed(self,pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: + return True \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py index 72595e337..f6252bd2d 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py @@ -1,14 +1,15 @@ -from ..dep_tree import Ecosystem +from ..dep_tree import Ecosystem, DependencyTree from .golang_functions_parsers import GoLanguageFunctionsParser from .python_functions_parser import PythonLanguageFunctionsParser from .lang_functions_parsers import LanguageFunctionsParser +from .c_lang_function_parsers import CLanguageFunctionsParser - -def get_language_function_parser(ecosystem: Ecosystem) -> LanguageFunctionsParser: +def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | None) -> LanguageFunctionsParser: """ Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated with the ecosystem. :param ecosystem: the desired programming language parser. + :param tree: the dependency tree for the ecosystem, can be None. :return: The right language functions parser associated to the ecosystem, if not exists, return ABC parent that doesn't do anything @@ -17,5 +18,10 @@ def get_language_function_parser(ecosystem: Ecosystem) -> LanguageFunctionsParse return GoLanguageFunctionsParser() elif ecosystem == Ecosystem.PYTHON: return PythonLanguageFunctionsParser() + elif ecosystem == Ecosystem.C_CPP: + parser_obj = CLanguageFunctionsParser() + if tree is not None: + parser_obj.init_CParser(tree) + return parser_obj else: return LanguageFunctionsParser() diff --git a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py index 49a4573c3..0b8500fef 100644 --- a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py +++ b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py @@ -27,8 +27,8 @@ class PythonLanguageFunctionsParser(LanguageFunctionsParser): - @staticmethod - def is_same_package(package_name_from_input, package_name_from_tree): + + def is_same_package(self, package_name_from_input, package_name_from_tree): return package_name_from_input.lower() == package_name_from_tree.lower() def get_dummy_function(self, function_name): @@ -402,4 +402,8 @@ def is_script_language(): @staticmethod def get_constructor_method_name(): - return '__init__' \ No newline at end of file + return '__init__' + + def is_a_package(self, package_name: str, doc: Document) -> bool: + return (not self.is_root_package(doc) and + self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 18f6c0de6..f0715c2d2 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -11,7 +11,7 @@ import json import os import logging - +import re from aiq.builder.builder import Builder from langchain_core.language_models.base import BaseLanguageModel @@ -89,9 +89,9 @@ def __get_calculate_score_prompt(self, intel: CveIntel) -> str: def __extract_score(self, text: str) -> int: text = text.replace("```", "").replace("json", "").strip() if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): - logger.debug("\ntext: %s", str(text)) + logger.debug("\ntext: %s", str(text)) data = json.loads(text) - + total_score = data['total_score'] if isinstance(total_score, dict): total_score = total_score['score'] diff --git a/src/vuln_analysis/utils/source_rpm_downloader.py b/src/vuln_analysis/utils/source_rpm_downloader.py new file mode 100644 index 000000000..051cbb2b6 --- /dev/null +++ b/src/vuln_analysis/utils/source_rpm_downloader.py @@ -0,0 +1,707 @@ + +import os +from pathlib import Path +import gzip +import tarfile +import xml.etree.ElementTree as ET +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from dataclasses import dataclass +from typing import List, Dict, Any, Optional +import threading +from vuln_analysis.data_models.info import SBOMPackage +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + + +@dataclass +class RepoUrl: + url: str + platform: str # e.g., 'el9', 'el10' + + +def extract_archives_in_folder(folder: str): + """ + Extracts all .tar.* files in the given folder to subdirectories. + Supports .tar.gz, .tgz, .tar.xz, .txz, .tar.bz2, .tbz2, and .tar. + + :param folder: Path to the folder to scan. + """ + supported_extensions = { + '.tar.gz', '.tgz', + '.tar.xz', '.txz', + '.tar.bz2', '.tbz2', + '.tar' + } + + folder_path = Path(folder) + + for file in folder_path.iterdir(): + if not file.is_file(): + continue + + suffix = ''.join(file.suffixes[-2:]) if len(file.suffixes) >= 2 else file.suffix + if suffix not in supported_extensions and file.suffix not in supported_extensions: + continue + + # Determine full suffix like .tar.gz or .tgz + full_suffix = suffix if suffix in supported_extensions else file.suffix + + # Determine output directory name + output_dir = file.with_suffix('').with_suffix('').stem if full_suffix.startswith('.tar') else file.stem + output_path = folder_path / output_dir + + logger.debug(f"Extracting: {file.name} → {output_path}") + output_path.mkdir(exist_ok=True) + + try: + with tarfile.open(file, 'r:*') as tar: + tar.extractall(path=output_path) + except Exception as e: + logger.debug(f"Failed to extract {file.name}: {e}") + +class RPMDependencyManager: + """ + Encapsulates RPM download, dependency tree building, pruning, and output writing logic. + Can be used from both scripts and library code. + + Implements singleton pattern to ensure single instance across the application. + """ + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not hasattr(self, '_initialized'): + # Initialize only once + self.base_urls = [ + RepoUrl(url="https://vault.centos.org/8-stream/BaseOS/Source/", platform="el8"), + RepoUrl(url="https://vault.centos.org/8-stream/AppStream/Source/", platform="el8"), + RepoUrl(url="https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/", platform="el9"), + RepoUrl(url="https://mirror.stream.centos.org/9-stream/AppStream/source/tree/", platform="el9"), + RepoUrl(url="https://mirror.stream.centos.org/10-stream/BaseOS/source/tree/", platform="el10"), + RepoUrl(url="https://mirror.stream.centos.org/10-stream/AppStream/source/tree/", platform="el10"), + RepoUrl(url="https://mirror.stream.centos.org/11-stream/BaseOS/source/tree/", platform="el11"), + RepoUrl(url="https://mirror.stream.centos.org/11-stream/AppStream/source/tree/", platform="el11"), + RepoUrl(url="https://mirror.stream.centos.org/12-stream/BaseOS/source/tree/", platform="el12"), + RepoUrl(url="https://mirror.stream.centos.org/12-stream/AppStream/source/tree/", platform="el12") + ] + # Default RPM cache directory + self.rpm_cache_dir = "/morpheus-data/rpms/" + # Unit Test mode toggle (default disabled) + self._unit_test_mode: bool = False + self._initialized = True + self._sbom: list[SBOMPackage] = [] + logger.info(f"RPMDependencyManager singleton initialized with default cache dir: {self.rpm_cache_dir}") + + @classmethod + def get_instance(cls): + """Get the singleton instance of RPMDependencyManager""" + return cls() + + def set_rpm_cache_dir(self, cache_dir: str): + """Set the RPM cache directory for all future operations""" + self.rpm_cache_dir = cache_dir + + # Ensure the cache directory exists + cache_path = Path(cache_dir) + if not cache_path.exists(): + try: + cache_path.mkdir(parents=True, exist_ok=True) + logger.info("Created RPM cache directory: %s", cache_dir) + except Exception as e: + logger.error("Failed to create RPM cache directory %s: %s", cache_dir, e) + # Fall back to default directory if creation fails + self.rpm_cache_dir = "/morpheus-data/rpms/" + logger.warning("Falling back to default cache directory: %s", self.rpm_cache_dir) + else: + logger.info("RPM cache directory updated to: %s", cache_dir) + + def get_rpm_cache_dir(self) -> str: + """Get the current RPM cache directory""" + return self.rpm_cache_dir + + def enableUnitTestMode(self) -> None: + """Enable Unit Test mode to relax version matching during RPM URL resolution.""" + self._unit_test_mode = True + + def disableUnitTestMode(self) -> None: + """Disable Unit Test mode and restore strict version matching.""" + self._unit_test_mode = False + + def isUnitTestMode(self) -> bool: + """Return whether Unit Test mode is enabled.""" + return self._unit_test_mode + + @property + def sbom(self) -> list[SBOMPackage]: + """Get the SBOM list""" + return self._sbom + + @sbom.setter + def sbom(self, value: list[SBOMPackage]) -> None: + """Set the SBOM list""" + if not isinstance(value, list): + raise TypeError("sbom must be a list") + self._sbom = value + + @property + def container_image(self) -> str: + """Get the container image string""" + return getattr(self, '_container_image', '') + + @container_image.setter + def container_image(self, value: str) -> None: + """Set the container image string""" + if not isinstance(value, str): + raise TypeError("container_image must be a string") + self._container_image = value + + def threaded_rpm_download(self, sbom, project_name, download_path, max_workers=1): + """ + Download RPMs in parallel using threads. Each SBOM entry is downloaded in a separate thread. + """ + logger.info(f"Starting threaded RPM download with {max_workers} workers...") + # Use singleton's cache directory instead of hardcoded value + rpm_cache_dir = self.rpm_cache_dir + downloader = SourceRPMDownloader( + project_name, sbom, self.base_urls, + download_path, rpm_cache_dir, max_workers, + unit_test_mode=self._unit_test_mode + ) + downloader.run() + + +class SourceRPMDownloader: + def __init__(self, prj_name, sbom_info, repo_urls: List[RepoUrl], work_dir="ext_libs", rpm_cache_dir=None, max_workers=1, unit_test_mode: bool = False): + self.sbom_packages = sbom_info + self.repo_urls = repo_urls # Now a list of RepoUrl objects + self.work_dir = Path(work_dir) + # Unit Test mode: when enabled, ignore exact version during URL resolution + self.unit_test_mode = unit_test_mode + + # Set up RPM cache directory + if rpm_cache_dir is None: + # No cache directory specified - disable caching + self.rpm_cache_dir = None + logger.info("RPM caching disabled (no cache directory specified)") + else: + self.rpm_cache_dir = Path(rpm_cache_dir) + # Ensure cache directory exists + self.rpm_cache_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"RPM cache directory: {self.rpm_cache_dir}") + + # Build cache index from existing files + self.cache_index = self._build_cache_index() + self.cache_hit =0 + self.cache_miss =0 + self.repo_metadata = {} + self.max_workers = max_workers + self.project_name = prj_name + # Thread-safe session with retries + self.session = requests.Session() + retries = Retry( + total=3, backoff_factor=0.3, status_forcelist=[500, 502, 503, 504], allowed_methods=["GET"] + ) + adapter = HTTPAdapter(max_retries=retries) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + + def _parse_rpm_filename(self, filename: str) -> Optional[Dict[str, str]]: + """ + Parse RPM filename to extract package info. + Args: + filename: RPM filename string + + Returns: + Optional[Dict[str, str]]: Parsed package info dictionary with keys: + 'name', 'version', 'release', 'platform', 'filename', or None if invalid + """ + if not filename.endswith('.src.rpm'): + return None + + #filesystem-3.16-5.el9.src.rpm + #fonts-rpm-macros-2.0.5-7.el9.1.src.rpm + #libsemanage-3.6-5.el9_6.src.rpm + #python-setuptools-53.0.0-13.el9_6.1.src.rpm + + + # Remove .src.rpm extension + base_name = filename[:-8] + + + # Split by dashes to get parts + parts = base_name.split('-') + if len(parts) < 3: + return None + #fonts,rpm,macros,2.0.5,7.el9.1 + if len(parts) == 3: + package_name = parts[0] + elif len(parts) > 3: + # Build package name from all parts except last 2 + package_name_parts = [] + for i in range(len(parts) - 2): + package_name_parts.append(parts[i]) + package_name = '-'.join(package_name_parts) + # Last 2 elements: version and platform + version_part = parts[-2] + full_platform = parts[-1] + + #cases + #5.el9 + #7.el9.1 + #5.el9_6 + #13.el9_6.1 + platform = None + if 'el' in full_platform: + # Extract base platform by removing _ and . and finding el followed by digits + import re + match = re.search(r'el\d+', full_platform) + if match: + platform = match.group() + else: + platform = full_platform + + return { + 'name': package_name, + 'version': version_part, + 'release': full_platform, + 'platform': platform, + 'filename': filename + } + + def _build_cache_index(self): + """ + Build cache index from existing RPM files in cache directory. + + Returns: + dict: Cache index with key format: (package_name, version, platform) + """ + if self.rpm_cache_dir is None: + return {} + + cache_index = {} + + for rpm_file in self.rpm_cache_dir.glob("*.src.rpm"): + parsed = self._parse_rpm_filename(rpm_file.name) + if parsed: + # Key: (package_name, version, platform) + key = (parsed['name'], parsed['version'], parsed['platform']) + cache_index[key] = { + 'filepath': rpm_file, + 'release': parsed['release'], + 'filename': parsed['filename'] + } + #logger.debug(f"Cached RPM: {parsed['name']}-{parsed['version']}-{parsed['platform']}") + + logger.info(f"Cache index built: {len(cache_index)} RPMs found") + return cache_index + + def _get_cache_path(self, package: Dict[str, Any]) -> Path: + """ + Generate cache file path for a package. + + Args: + package: Package dictionary containing name, version, rel information + + Returns: + Path: Full path to cached RPM file + """ + # Create cache filename: package-version-release.src.rpm + cache_filename = f"{package['name']}-{package['version']}-{package['rel']}.src.rpm" + return self.rpm_cache_dir / cache_filename + + def _find_cached_rpm(self, package: Dict[str, Any], platform_version: str) -> Optional[Path]: + """ + Find cached RPM using the cache index. + + Args: + package: Package dictionary containing name, version, rel information + platform_version: Platform version string (e.g., 'el9', 'el10') + + Returns: + Optional[Path]: Path to cached RPM file if found, None otherwise + """ + if self.rpm_cache_dir is None or not self.cache_index: + return None + + package_name = package['name'] + package_version = package['version'] + + # Exact match with platform version + key = (package_name, package_version, platform_version) + if key in self.cache_index: + return self.cache_index[key]['filepath'] + return None + + def _is_cached(self, package, platform_version): + """ + Check if RPM is already cached. + + Args: + package: Package dictionary with name, version, rel + platform_version: Platform version (e.g., 'el9', 'el10') + + Returns: + bool: True if RPM exists in cache + """ + return self._find_cached_rpm(package, platform_version) is not None + + def _download_to_cache(self, package: Dict[str, Any], download_url: str) -> bool: + """ + Download RPM and store in cache. + + Args: + package: Package dictionary containing package information + download_url: URL string to download RPM from + + Returns: + bool: True if download and cache successful, False otherwise + """ + if self.rpm_cache_dir is None: + return False + cache_path = self._get_cache_path(package) + try: + logger.info(f"Downloading {package['name']} to cache: {cache_path}") + data = self.session.get(download_url, timeout=120).content + cache_path.write_bytes(data) + #logger.info(f"Cached {package['name']} ({len(data)} bytes)") + #add file to cache + parsed = self._parse_rpm_filename(cache_path.name) + if parsed: + key = (parsed['name'], parsed['version'], parsed['platform']) + self.cache_index[key] = { + 'filepath': cache_path, + 'release': parsed['release'], + 'filename': parsed['filename'] + } + return True + except Exception as e: + logger.error(f"Failed to download {package['name']} to cache: {e}") + return False + + def _copy_from_cache(self, package: Dict[str, Any], work_dir: Path, platform_version: str) -> Optional[Path]: + """ + Copy RPM from cache to work directory. + + Args: + package: Package dictionary containing package information + work_dir: Target work directory (Path object) + platform_version: Platform version string (e.g., 'el9', 'el10') + + Returns: + Optional[Path]: Path to copied RPM file in work directory, or None if failed + """ + if self.rpm_cache_dir is None: + return None + + # Find the cached RPM (might have different release than expected) + cache_path = self._find_cached_rpm(package, platform_version) + if cache_path is None: + logger.error(f"No cached RPM found for {package['name']}") + return None + + # work_dir is already the package directory, so just use the filename + work_path = work_dir / cache_path.name + + try: + import shutil + # Ensure the target directory exists + work_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(cache_path, work_path) + #logger.info(f"Copied {package['name']} from cache to {work_path}") + return work_path + except Exception as e: + logger.error(f"Failed to copy {package['name']} from cache: {e}") + return None + + def parse_sbom(self): + logger.info("Parsing SBOM input...") + packages = [] + filter_pkgs = ["glibc"] + standard_pkg = None + filter_pkgs.append(self.project_name) + platform_version = None + for pkg in self.sbom_packages: + versions = pkg.version.split('-') + version_no_epoch = versions[0].split(':', 1)[-1] + rel = versions[1] if len(versions) > 1 else '' + # Try to extract el9/el10 from release string + if not platform_version and 'el' in rel: + for part in rel.split('.'): + if part.startswith('el') and part[2:].isdigit(): + platform_version = part + break + skip_pkg = False + for f_pkg in filter_pkgs: + if f_pkg in pkg.name : + skip_pkg = True + if f_pkg == pkg.name: + standard_pkg = {'short_name': pkg.name,'name': pkg.name,'version': version_no_epoch,'rel': rel} + break + if skip_pkg: + continue + + packages.append({ + 'short_name': pkg.name, + 'name': pkg.name, + 'version': version_no_epoch, + 'rel': rel + }) + if standard_pkg : + packages.append(standard_pkg) + logger.info(f"Found {len(packages)} packages in SBOM, platform: {platform_version}") + return packages, platform_version + + def extract_src_rpm(self, rpm_path: Path, extract_dir: Path): + #logger.info(f" Extracting {rpm_path.name} to {extract_dir} ...") + extract_dir.mkdir(parents=True, exist_ok=True) + try: + base_rpm_path = os.path.basename(rpm_path) + # Use bsdtar to extract the RPM + result = subprocess.run([ + "bsdtar", "-xf",base_rpm_path + ], cwd=extract_dir, capture_output=True) + if result.returncode == 0: + # After extracting, extract any tar files in the folder + extract_archives_in_folder(str(extract_dir)) + else: + logger.error(f" Extraction failed with return code {result.returncode}: {result.stderr.decode().strip()}") + except Exception as e: + logger.error(f" Extraction error: {e}") + + def _get_repo_metadata(self, base_url: str): + if base_url in self.repo_metadata: + return self.repo_metadata[base_url] + logger.info(f"Fetching metadata for repo: {base_url}") + try: + repomd_url = base_url.rstrip('/') + "/repodata/repomd.xml" + response = self.session.get(repomd_url, timeout=30) + response.raise_for_status() + root = ET.fromstring(response.content) + primary_href = next( + data.find("{*}location").attrib["href"] + for data in root.findall(".//{*}data") if data.attrib.get("type") == "primary" + ) + primary_url = base_url.rstrip('/') + "/" + primary_href + response = self.session.get(primary_url, timeout=60) + response.raise_for_status() + xml_bytes = gzip.decompress(response.content) + tree = ET.fromstring(xml_bytes) + self.repo_metadata[base_url] = tree + + ######## + #filename ="repo_meta.xml" + #root2 = ET.fromstring(xml_bytes) + #tree2 = ET.ElementTree(root2) + #tree2.write(filename, encoding="utf-8", xml_declaration=True) + #logger.debug(f"XML string saved to {filename} successfully!") + return tree + except Exception as e: + logger.error(f" Could not fetch metadata from {base_url}: {e}") + self.repo_metadata[base_url] = None + return None + + def find_package_url(self, package, repos): + """ + For a given package and list of repos, return (download_url, repo) if found, else (None, None). + When unit_test_mode is enabled, relax match to name + src arch only (ignore exact version). + """ + package_name = package['name'] + package_version = package['version'] + for repo in repos: + base_url = repo.url + metadata = self._get_repo_metadata(base_url) + if not metadata: + continue + # attempt match + try: + if self.unit_test_mode: + pkg_elem = next( + pkg for pkg in metadata.findall(".//{*}package") + if (pkg.find("{*}name") is not None and pkg.find("{*}name").text == package_name) + and (pkg.find("{*}arch") is not None and pkg.find("{*}arch").text == 'src') + ) + else: + pkg_elem = next( + pkg for pkg in metadata.findall(".//{*}package") + if (pkg.find("{*}name") is not None and pkg.find("{*}name").text == package_name) + and (pkg.find("{*}arch") is not None and pkg.find("{*}arch").text == 'src') + and (pkg.find("{*}version") is not None and pkg.find("{*}version").attrib.get("ver") == package_version) + ) + except StopIteration: + continue + href = pkg_elem.find("{*}location").attrib.get("href") + rpm_url = f"{base_url.rstrip('/')}/{href}" + return rpm_url, repo + logger.debug(f"could not find in repo {base_url} : {package_name} - version {package_version}") + return None, None + + def _process_package_from_cache(self, package: Dict[str, Any], package_dir: Path, platform_version: str, can_pkg_download: bool) -> bool: + """ + Try to process package from cache. + + Args: + package: Package dictionary + package_dir: Target package directory + platform_version: Platform version + can_pkg_download: Whether the package can be downloaded if not in cache + + Returns: + bool: True if successfully processed from cache, False otherwise + """ + if self.rpm_cache_dir is None or not platform_version: + return False + + try: + # Ensure we have the RPM in cache (download if needed) + if not self._is_cached(package, platform_version): + self.cache_miss += 1 + if can_pkg_download: + if not self._download_to_cache(package, package['download_url']): + logger.error(f"Failed to download {package['short_name']} to cache") + return False + else: + return False + else: + self.cache_hit += 1 + + # Copy from cache and extract + rpm_path = self._copy_from_cache(package, package_dir, platform_version) + if rpm_path: + self.extract_src_rpm(rpm_path, package_dir) + return True + else: + logger.error(f"Failed to copy {package['short_name']} from cache") + return False + + except Exception as e: + logger.error(f"Failed to process {package['short_name']} from cache: {e}") + return False + + def run(self): + logger.info("Starting Source RPM Download and Extraction Process") + # Track processing statistics + stats = { + 'already_extracted': 0, + 'cache_processed': 0, + 'downloaded': 0, + 'no_download_url': 0, + 'failed': 0, + 'found' : 0 + } + packages, platform_version = self.parse_sbom() + if platform_version: + filtered_repos = [repo for repo in self.repo_urls if platform_version == repo.platform] + if not filtered_repos: + logger.warning(f"No repositories found for platform {platform_version}. Using all repos.") + filtered_repos = self.repo_urls + self.repo_urls = filtered_repos + logger.info(f"Filtered repo URLs to: {[repo.url for repo in self.repo_urls]}") + else: + logger.warning("Could not determine platform version from SBOM; using all repo URLs.") + filtered_repos = self.repo_urls + + # Step 1: Find download URLs for all packages + for pkg in packages: + download_url, repo = self.find_package_url(pkg, filtered_repos) + pkg['download_url'] = download_url + pkg['repo'] = repo.url if repo else None + if download_url : + stats['found'] +=1 + + # Step 2: Download and extract in parallel only for found packages + successful_downloads = 0 + total_packages = len(packages) + not_found = 0 + + def process_package(package): + + can_pkg_download = True + if not package['download_url']: + can_pkg_download = False + + short_name = package['short_name'] + package_dir = self.work_dir / short_name + + try: + # Check if already extracted in work directory + if os.path.exists(package_dir): + return "already_extracted" + + # Try cache first, if successful return + if self._process_package_from_cache(package, package_dir, platform_version, can_pkg_download): + return "cache_processed" + + if not can_pkg_download: + return "no_download_url" + + # Original logic - download directly to work directory + data = self.session.get(package['download_url'], timeout=120).content + package_dir.mkdir(parents=True, exist_ok=True) + out_path = package_dir / Path(package['download_url']).name + out_path.write_bytes(data) + self.extract_src_rpm(out_path, package_dir) + return "downloaded" + + except Exception as e: + logger.error(f"Failed to process package {short_name}: {e}") + return "failed" + + + + # Use ThreadPoolExecutor for I/O bound operations (HTTP downloads, file I/O) + # This is more efficient than multiprocessing for this workload + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = {executor.submit(process_package, pkg): pkg for pkg in packages} + for future in as_completed(futures): + result = future.result() + if result in stats: + stats[result] += 1 + if result in ['already_extracted', 'cache_processed', 'downloaded']: + successful_downloads += 1 + + + not_found = sum(1 for pkg in packages if not pkg['download_url']) + + # Step 3: logger.debug comprehensive summary report + logger.debug("\n" + "="*80) + logger.debug("RPM PROCESSING SUMMARY REPORT") + logger.debug("="*80) + + # Processing statistics + logger.debug(f"\nProcessing Results:") + logger.debug(f" • Found Packages: {stats['found']}") + logger.debug(f" • Already extracted: {stats['already_extracted']}") + logger.debug(f" • Processed from cache: {stats['cache_processed']}") + logger.debug(f" • Downloaded and processed: {stats['downloaded']}") + logger.debug(f" • No download URL available: {stats['no_download_url']}") + logger.debug(f" • Failed to process: {stats['failed']}") + + # Cache statistics + if self.rpm_cache_dir is not None: + logger.debug(f"\nCache Performance:") + logger.debug(f" • Cache hits: {self.cache_hit}") + logger.debug(f" • Cache misses: {self.cache_miss}") + if self.cache_hit + self.cache_miss > 0: + hit_rate = (self.cache_hit / (self.cache_hit + self.cache_miss)) * 100 + logger.debug(f" • Cache hit rate: {hit_rate:.1f}%") + + # Overall summary + logger.debug(f"\nOverall Summary:") + logger.debug(f" • Total packages: {total_packages}") + logger.debug(f" • Successfully processed: {successful_downloads}") + logger.debug(f" • Not found in repositories: {not_found}") + success_rate = (successful_downloads / total_packages) * 100 if total_packages > 0 else 0 + logger.debug(f" • Success rate: {success_rate:.1f}%") + + logger.debug("="*80) \ No newline at end of file diff --git a/src/vuln_analysis/utils/standard_library_cache.py b/src/vuln_analysis/utils/standard_library_cache.py new file mode 100644 index 000000000..7c67a4214 --- /dev/null +++ b/src/vuln_analysis/utils/standard_library_cache.py @@ -0,0 +1,159 @@ +""" +Standard library cache utilities. +Loads an auto-generated JSON of standard libraries for supported ecosystems +and exposes a simple lookup with lightweight hit/miss counters. Every 5 +lookups, a short summary is logged. Designed to minimize external API usage. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import threading +import pickle +import difflib +# No typing imports needed; we use builtin generics + +from vuln_analysis.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.dep_tree import Ecosystem + + +logger = LoggingFactory.get_agent_logger(__name__) + + +class StandardLibraryCache: + """In-memory cache for standard library package/module names per ecosystem. + Implements singleton pattern to ensure single instance across the application. + """ + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self, json_path: str | Path | None = None) -> None: + if not hasattr(self, '_initialized'): + # Initialize only once + base_path = Path(__file__).resolve().parents[1] + default_json = base_path / "data" / "standard_libs" / "standard_libraries.json" + self.json_path = Path(json_path) if json_path else default_json + + self._ecosystem_to_set: dict[str, set[str]] = {} + self._hits: int = 0 + self._misses: int = 0 + self._lookups_since_last_log: int = 0 + self._data_lock = threading.RLock() + self._initialized = True + self.cache_directory = Path(".cache/am_cache/pickle") + self.cache_file_name = "runtime_std_packages_cache.pkl" + self._load() + + @classmethod + def get_instance(cls): + """Get the singleton instance of StandardLibraryCache""" + return cls() + + def _load(self) -> None: + try: + data = json.loads(self.json_path.read_text(encoding="utf-8")) + except FileNotFoundError: + logger.warning("Standard libraries JSON not found at %s", self.json_path) + data = {} + except json.JSONDecodeError as exc: + logger.error("Failed to parse standard libraries JSON: %s", exc) + data = {} + + for eco_key in (Ecosystem.GO.value, Ecosystem.PYTHON.value, Ecosystem.JAVASCRIPT.value, Ecosystem.JAVA.value): + items: list[str] = data.get(eco_key, []) + normalized = {self._normalize_name(name) for name in items if isinstance(name, str)} + self._ecosystem_to_set[eco_key] = normalized + + @staticmethod + def _normalize_name(name: str) -> str: + # Normalize to lowercase; trim whitespace + return name.strip().lower() + + def is_standard_library(self, package_name: str, ecosystem: Ecosystem | None) -> bool: + if not package_name: + return False + if ecosystem is None: + return False + + with self._data_lock: + eco_key = ecosystem.value + candidates = self._ecosystem_to_set.get(eco_key, set()) + found = self._normalize_name(package_name) in candidates + + if found: + self._hits += 1 + else: + self._misses += 1 + logger.info("StdLibCache lookup (%s): hits=%d misses=%d (eco=%s)", + package_name, self._hits, self._misses, eco_key) + return found + + def get_close_match_list(self, package_name:str, ecosystem: Ecosystem)->list[str]: + package_name = self._normalize_name(package_name) + eco_key = ecosystem.value + pkg_close_matches = [] + with self._data_lock: + candidates = self._ecosystem_to_set.get(eco_key, set()) + pkg_close_matches = difflib.get_close_matches(package_name, candidates, n=5, cutoff=0.6) + return pkg_close_matches + + def load_from_cache(self): + try: + path = self.cache_directory / self.cache_file_name + if path.is_file(): + with open(path, 'rb') as file: + cache_ecosystem_to_set = pickle.load(file) + # Only replace if load was successful + self._ecosystem_to_set = cache_ecosystem_to_set + logger.info("Loaded runtime standard library cache from %s", path) + except Exception as e: + logger.error("Failed to load runtime standard library cache from %s: %s", path, e) + + def set_cache_directory(self, base_directory: str | Path) -> None: + """Set the cache directory and create subdirectory if it doesn't exist. + + Args: + base_directory: The base directory where the std_packages_cache subdirectory will be created + """ + base_path = Path(base_directory) + self.cache_directory = base_path / "std_packages_cache" + + # Create the subdirectory if it doesn't exist + self.cache_directory.mkdir(parents=True, exist_ok=True) + logger.info("Cache directory set to: %s", self.cache_directory) + self.load_from_cache() + + def add_to_cache(self, package_name: str, ecosystem: Ecosystem) -> None: + """Add a package name to the cache for the specified ecosystem. + + Args: + package_name: The package/module name to add to the cache + ecosystem: The ecosystem to add the package to + """ + if not package_name: + logger.warning("Cannot add empty package name to cache") + return + + eco_key = ecosystem.value + normalized_name = self._normalize_name(package_name) + with self._data_lock: + # Initialize the set for this ecosystem if it doesn't exist + if eco_key not in self._ecosystem_to_set: + self._ecosystem_to_set[eco_key] = set() + + # Add the normalized name to the set + self._ecosystem_to_set[eco_key].add(normalized_name) + logger.debug("Added '%s' to standard library cache for ecosystem '%s'", + package_name, eco_key) + path = self.cache_directory / self.cache_file_name + with open(path, 'wb') as file: + pickle.dump(self._ecosystem_to_set, file) + logger.info("Saved runtime standard library cache to %s", path) \ No newline at end of file diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py index e43af8d6a..bf65ac3d7 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -27,6 +27,10 @@ PYTHON_MANIFEST, Ecosystem, get_dependency_tree_builder, + C_CPLUSPLUS_MANIFEST_1, + C_CPLUSPLUS_MANIFEST_2, + C_CPLUSPLUS_MANIFEST_3, + C_CPLUSPLUS_MANIFEST_4, ) from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE @@ -70,14 +74,28 @@ def download_dependencies(git_repo_path: Path) -> bool: elif os.path.isfile(git_repo_path / JAVA_MANIFEST): ecosystem = MANIFESTS_TO_ECOSYSTEMS[JAVA_MANIFEST] else: - logger.info("Didn't find manifest to install, skipping.. ") - return False - with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: - file.write(ecosystem.name) + # 1. Direct checks + candidates = [ + Path(git_repo_path / C_CPLUSPLUS_MANIFEST_1), + Path(git_repo_path / C_CPLUSPLUS_MANIFEST_2), + Path(git_repo_path / C_CPLUSPLUS_MANIFEST_3), + Path(git_repo_path / C_CPLUSPLUS_MANIFEST_4), + Path(git_repo_path / "auto" / + C_CPLUSPLUS_MANIFEST_4) + ] + found = [p for p in candidates if p.is_file()] + if found: + ecosystem = MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] + else: + logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") + return False + try: - logger.info(f"Started installing packages for {ecosystem}") - tree_builder = get_dependency_tree_builder(ecosystem.value) + logger.info(f"Started installing packages for {ecosystem}") + tree_builder = get_dependency_tree_builder(ecosystem) tree_builder.install_dependencies(git_repo_path) + with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: + file.write(ecosystem.name) logger.info(f"Finished installing packages for {ecosystem}") return True except NotImplementedError as err: diff --git a/tests/ctests/cms_sd.c b/tests/ctests/cms_sd.c new file mode 100644 index 000000000..8ad94a9ed --- /dev/null +++ b/tests/ctests/cms_sd.c @@ -0,0 +1,1191 @@ +/* + * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "internal/cryptlib.h" +#include +#include +#include +#include +#include +#include +#include +#include "internal/sizes.h" +#include "crypto/asn1.h" +#include "crypto/evp.h" +#include "crypto/ess.h" +#include "crypto/x509.h" /* for ossl_x509_add_cert_new() */ +#include "cms_local.h" + +/* CMS SignedData Utilities */ + +static CMS_SignedData *cms_get0_signed(CMS_ContentInfo *cms) +{ + if (OBJ_obj2nid(cms->contentType) != NID_pkcs7_signed) { + ERR_raise(ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_SIGNED_DATA); + return NULL; + } + return cms->d.signedData; +} + +static CMS_SignedData *cms_signed_data_init(CMS_ContentInfo *cms) +{ + if (cms->d.other == NULL) { + cms->d.signedData = M_ASN1_new_of(CMS_SignedData); + if (!cms->d.signedData) { + ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + return NULL; + } + cms->d.signedData->version = 1; + cms->d.signedData->encapContentInfo->eContentType = + OBJ_nid2obj(NID_pkcs7_data); + cms->d.signedData->encapContentInfo->partial = 1; + ASN1_OBJECT_free(cms->contentType); + cms->contentType = OBJ_nid2obj(NID_pkcs7_signed); + return cms->d.signedData; + } + return cms_get0_signed(cms); +} + +/* Just initialise SignedData e.g. for certs only structure */ +int CMS_SignedData_init(CMS_ContentInfo *cms) +{ + if (cms_signed_data_init(cms)) + return 1; + else + return 0; +} + +/* Check structures and fixup version numbers (if necessary) */ +static void cms_sd_set_version(CMS_SignedData *sd) +{ + int i; + CMS_CertificateChoices *cch; + CMS_RevocationInfoChoice *rch; + CMS_SignerInfo *si; + + for (i = 0; i < sk_CMS_CertificateChoices_num(sd->certificates); i++) { + cch = sk_CMS_CertificateChoices_value(sd->certificates, i); + if (cch->type == CMS_CERTCHOICE_OTHER) { + if (sd->version < 5) + sd->version = 5; + } else if (cch->type == CMS_CERTCHOICE_V2ACERT) { + if (sd->version < 4) + sd->version = 4; + } else if (cch->type == CMS_CERTCHOICE_V1ACERT) { + if (sd->version < 3) + sd->version = 3; + } + } + + for (i = 0; i < sk_CMS_RevocationInfoChoice_num(sd->crls); i++) { + rch = sk_CMS_RevocationInfoChoice_value(sd->crls, i); + if (rch->type == CMS_REVCHOICE_OTHER) { + if (sd->version < 5) + sd->version = 5; + } + } + + if ((OBJ_obj2nid(sd->encapContentInfo->eContentType) != NID_pkcs7_data) + && (sd->version < 3)) + sd->version = 3; + + for (i = 0; i < sk_CMS_SignerInfo_num(sd->signerInfos); i++) { + si = sk_CMS_SignerInfo_value(sd->signerInfos, i); + if (si->sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) { + if (si->version < 3) + si->version = 3; + if (sd->version < 3) + sd->version = 3; + } else if (si->version < 1) { + si->version = 1; + } + } + + if (sd->version < 1) + sd->version = 1; + +} + +/* + * RFC 5652 Section 11.1 Content Type + * The content-type attribute within signed-data MUST + * 1) be present if there are signed attributes + * 2) match the content type in the signed-data, + * 3) be a signed attribute. + * 4) not have more than one copy of the attribute. + * + * Note that since the CMS_SignerInfo_sign() always adds the "signing time" + * attribute, the content type attribute MUST be added also. + * Assumptions: This assumes that the attribute does not already exist. + */ +static int cms_set_si_contentType_attr(CMS_ContentInfo *cms, CMS_SignerInfo *si) +{ + ASN1_OBJECT *ctype = cms->d.signedData->encapContentInfo->eContentType; + + /* Add the contentType attribute */ + return CMS_signed_add1_attr_by_NID(si, NID_pkcs9_contentType, + V_ASN1_OBJECT, ctype, -1) > 0; +} + +/* Copy an existing messageDigest value */ +static int cms_copy_messageDigest(CMS_ContentInfo *cms, CMS_SignerInfo *si) +{ + STACK_OF(CMS_SignerInfo) *sinfos; + CMS_SignerInfo *sitmp; + int i; + + sinfos = CMS_get0_SignerInfos(cms); + for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { + ASN1_OCTET_STRING *messageDigest; + + sitmp = sk_CMS_SignerInfo_value(sinfos, i); + if (sitmp == si) + continue; + if (CMS_signed_get_attr_count(sitmp) < 0) + continue; + if (OBJ_cmp(si->digestAlgorithm->algorithm, + sitmp->digestAlgorithm->algorithm)) + continue; + messageDigest = CMS_signed_get0_data_by_OBJ(sitmp, + OBJ_nid2obj + (NID_pkcs9_messageDigest), + -3, V_ASN1_OCTET_STRING); + if (!messageDigest) { + ERR_raise(ERR_LIB_CMS, CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE); + return 0; + } + + if (CMS_signed_add1_attr_by_NID(si, NID_pkcs9_messageDigest, + V_ASN1_OCTET_STRING, + messageDigest, -1)) + return 1; + else + return 0; + } + ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_DIGEST); + return 0; +} + +int ossl_cms_set1_SignerIdentifier(CMS_SignerIdentifier *sid, X509 *cert, + int type, const CMS_CTX *ctx) +{ + switch (type) { + case CMS_SIGNERINFO_ISSUER_SERIAL: + if (!ossl_cms_set1_ias(&sid->d.issuerAndSerialNumber, cert)) + return 0; + break; + + case CMS_SIGNERINFO_KEYIDENTIFIER: + if (!ossl_cms_set1_keyid(&sid->d.subjectKeyIdentifier, cert)) + return 0; + break; + + default: + ERR_raise(ERR_LIB_CMS, CMS_R_UNKNOWN_ID); + return 0; + } + + sid->type = type; + + return 1; +} + +int ossl_cms_SignerIdentifier_get0_signer_id(CMS_SignerIdentifier *sid, + ASN1_OCTET_STRING **keyid, + X509_NAME **issuer, + ASN1_INTEGER **sno) +{ + if (sid->type == CMS_SIGNERINFO_ISSUER_SERIAL) { + if (issuer) + *issuer = sid->d.issuerAndSerialNumber->issuer; + if (sno) + *sno = sid->d.issuerAndSerialNumber->serialNumber; + } else if (sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) { + if (keyid) + *keyid = sid->d.subjectKeyIdentifier; + } else { + return 0; + } + return 1; +} + +int ossl_cms_SignerIdentifier_cert_cmp(CMS_SignerIdentifier *sid, X509 *cert) +{ + if (sid->type == CMS_SIGNERINFO_ISSUER_SERIAL) + return ossl_cms_ias_cert_cmp(sid->d.issuerAndSerialNumber, cert); + else if (sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) + return ossl_cms_keyid_cert_cmp(sid->d.subjectKeyIdentifier, cert); + else + return -1; +} + +/* Method to map any, incl. provider-implemented PKEY types to OIDs */ +/* (EC)DSA and all provider-delivered signatures implementation is the same */ +static int cms_generic_sign(CMS_SignerInfo *si, int verify) +{ + if (!ossl_assert(verify == 0 || verify == 1)) + return -1; + + if (!verify) { + EVP_PKEY *pkey = si->pkey; + int snid, hnid, pknid = EVP_PKEY_get_id(pkey); + X509_ALGOR *alg1, *alg2; + + CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2); + if (alg1 == NULL || alg1->algorithm == NULL) + return -1; + hnid = OBJ_obj2nid(alg1->algorithm); + if (hnid == NID_undef) + return -1; + if (pknid <= 0) { /* check whether a provider registered a NID */ + const char *typename = EVP_PKEY_get0_type_name(pkey); + + if (typename != NULL) + pknid = OBJ_txt2nid(typename); + } + if (!OBJ_find_sigid_by_algs(&snid, hnid, pknid)) + return -1; + return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL); + } + return 1; +} + +static int cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd) +{ + EVP_PKEY *pkey = si->pkey; + int i; + + if (EVP_PKEY_is_a(pkey, "DSA") || EVP_PKEY_is_a(pkey, "EC")) + return cms_generic_sign(si, cmd) > 0; + else if (EVP_PKEY_is_a(pkey, "RSA") || EVP_PKEY_is_a(pkey, "RSA-PSS")) + return ossl_cms_rsa_sign(si, cmd) > 0; + + /* Now give engines, providers, etc a chance to handle this */ + if (pkey->ameth == NULL || pkey->ameth->pkey_ctrl == NULL) + return cms_generic_sign(si, cmd) > 0; + i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_SIGN, cmd, si); + if (i == -2) { + ERR_raise(ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); + return 0; + } + if (i <= 0) { + ERR_raise(ERR_LIB_CMS, CMS_R_CTRL_FAILURE); + return 0; + } + return 1; +} + +/* Add SigningCertificate signed attribute to the signer info. */ +static int ossl_cms_add1_signing_cert(CMS_SignerInfo *si, + const ESS_SIGNING_CERT *sc) +{ + ASN1_STRING *seq = NULL; + unsigned char *p, *pp = NULL; + int ret, len = i2d_ESS_SIGNING_CERT(sc, NULL); + + if (len <= 0 || (pp = OPENSSL_malloc(len)) == NULL) + return 0; + + p = pp; + i2d_ESS_SIGNING_CERT(sc, &p); + if (!(seq = ASN1_STRING_new()) || !ASN1_STRING_set(seq, pp, len)) { + ASN1_STRING_free(seq); + OPENSSL_free(pp); + return 0; + } + OPENSSL_free(pp); + ret = CMS_signed_add1_attr_by_NID(si, NID_id_smime_aa_signingCertificate, + V_ASN1_SEQUENCE, seq, -1); + ASN1_STRING_free(seq); + return ret; +} + +/* Add SigningCertificateV2 signed attribute to the signer info. */ +static int ossl_cms_add1_signing_cert_v2(CMS_SignerInfo *si, + const ESS_SIGNING_CERT_V2 *sc) +{ + ASN1_STRING *seq = NULL; + unsigned char *p, *pp = NULL; + int ret, len = i2d_ESS_SIGNING_CERT_V2(sc, NULL); + + if (len <= 0 || (pp = OPENSSL_malloc(len)) == NULL) + return 0; + + p = pp; + i2d_ESS_SIGNING_CERT_V2(sc, &p); + if (!(seq = ASN1_STRING_new()) || !ASN1_STRING_set(seq, pp, len)) { + ASN1_STRING_free(seq); + OPENSSL_free(pp); + return 0; + } + OPENSSL_free(pp); + ret = CMS_signed_add1_attr_by_NID(si, NID_id_smime_aa_signingCertificateV2, + V_ASN1_SEQUENCE, seq, -1); + ASN1_STRING_free(seq); + return ret; +} + +CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, + X509 *signer, EVP_PKEY *pk, const EVP_MD *md, + unsigned int flags) +{ + CMS_SignedData *sd; + CMS_SignerInfo *si = NULL; + X509_ALGOR *alg; + int i, type; + const CMS_CTX *ctx = ossl_cms_get0_cmsctx(cms); + + if (!X509_check_private_key(signer, pk)) { + ERR_raise(ERR_LIB_CMS, CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE); + return NULL; + } + sd = cms_signed_data_init(cms); + if (!sd) + goto err; + si = M_ASN1_new_of(CMS_SignerInfo); + if (!si) { + ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + goto err; + } + /* Call for side-effect of computing hash and caching extensions */ + X509_check_purpose(signer, -1, -1); + + X509_up_ref(signer); + EVP_PKEY_up_ref(pk); + + si->cms_ctx = ctx; + si->pkey = pk; + si->signer = signer; + si->mctx = EVP_MD_CTX_new(); + si->pctx = NULL; + + if (si->mctx == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + goto err; + } + + if (flags & CMS_USE_KEYID) { + si->version = 3; + if (sd->version < 3) + sd->version = 3; + type = CMS_SIGNERINFO_KEYIDENTIFIER; + } else { + type = CMS_SIGNERINFO_ISSUER_SERIAL; + si->version = 1; + } + + if (!ossl_cms_set1_SignerIdentifier(si->sid, signer, type, ctx)) + goto err; + + if (md == NULL) { + int def_nid; + + if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0) { + ERR_raise_data(ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST, + "pkey nid=%d", EVP_PKEY_get_id(pk)); + goto err; + } + md = EVP_get_digestbynid(def_nid); + if (md == NULL) { + ERR_raise_data(ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST, + "default md nid=%d", def_nid); + goto err; + } + } + + X509_ALGOR_set_md(si->digestAlgorithm, md); + + /* See if digest is present in digestAlgorithms */ + for (i = 0; i < sk_X509_ALGOR_num(sd->digestAlgorithms); i++) { + const ASN1_OBJECT *aoid; + char name[OSSL_MAX_NAME_SIZE]; + + alg = sk_X509_ALGOR_value(sd->digestAlgorithms, i); + X509_ALGOR_get0(&aoid, NULL, NULL, alg); + OBJ_obj2txt(name, sizeof(name), aoid, 0); + if (EVP_MD_is_a(md, name)) + break; + } + + if (i == sk_X509_ALGOR_num(sd->digestAlgorithms)) { + if ((alg = X509_ALGOR_new()) == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + goto err; + } + X509_ALGOR_set_md(alg, md); + if (!sk_X509_ALGOR_push(sd->digestAlgorithms, alg)) { + X509_ALGOR_free(alg); + ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); + goto err; + } + } + + if (!(flags & CMS_KEY_PARAM) && !cms_sd_asn1_ctrl(si, 0)) { + ERR_raise_data(ERR_LIB_CMS, CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM, + "pkey nid=%d", EVP_PKEY_get_id(pk)); + goto err; + } + if (!(flags & CMS_NOATTR)) { + /* + * Initialize signed attributes structure so other attributes + * such as signing time etc are added later even if we add none here. + */ + if (!si->signedAttrs) { + si->signedAttrs = sk_X509_ATTRIBUTE_new_null(); + if (!si->signedAttrs) { + ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); + goto err; + } + } + + if (!(flags & CMS_NOSMIMECAP)) { + STACK_OF(X509_ALGOR) *smcap = NULL; + + i = CMS_add_standard_smimecap(&smcap); + if (i) + i = CMS_add_smimecap(si, smcap); + sk_X509_ALGOR_pop_free(smcap, X509_ALGOR_free); + if (!i) { + ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + goto err; + } + } + if (flags & CMS_CADES) { + ESS_SIGNING_CERT *sc = NULL; + ESS_SIGNING_CERT_V2 *sc2 = NULL; + int add_sc; + + if (md == NULL || EVP_MD_is_a(md, SN_sha1)) { + if ((sc = OSSL_ESS_signing_cert_new_init(signer, + NULL, 1)) == NULL) + goto err; + add_sc = ossl_cms_add1_signing_cert(si, sc); + ESS_SIGNING_CERT_free(sc); + } else { + if ((sc2 = OSSL_ESS_signing_cert_v2_new_init(md, signer, + NULL, 1)) == NULL) + goto err; + add_sc = ossl_cms_add1_signing_cert_v2(si, sc2); + ESS_SIGNING_CERT_V2_free(sc2); + } + if (!add_sc) + goto err; + } + if (flags & CMS_REUSE_DIGEST) { + if (!cms_copy_messageDigest(cms, si)) + goto err; + if (!cms_set_si_contentType_attr(cms, si)) + goto err; + if (!(flags & (CMS_PARTIAL | CMS_KEY_PARAM)) && + !CMS_SignerInfo_sign(si)) + goto err; + } + } + + if (!(flags & CMS_NOCERTS)) { + /* NB ignore -1 return for duplicate cert */ + if (!CMS_add1_cert(cms, signer)) { + ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + goto err; + } + } + + if (flags & CMS_KEY_PARAM) { + if (flags & CMS_NOATTR) { + si->pctx = EVP_PKEY_CTX_new_from_pkey(ossl_cms_ctx_get0_libctx(ctx), + si->pkey, + ossl_cms_ctx_get0_propq(ctx)); + if (si->pctx == NULL) + goto err; + if (EVP_PKEY_sign_init(si->pctx) <= 0) + goto err; + if (EVP_PKEY_CTX_set_signature_md(si->pctx, md) <= 0) + goto err; + } else if (EVP_DigestSignInit_ex(si->mctx, &si->pctx, + EVP_MD_get0_name(md), + ossl_cms_ctx_get0_libctx(ctx), + ossl_cms_ctx_get0_propq(ctx), + pk, NULL) <= 0) { + si->pctx = NULL; + goto err; + } + else { + EVP_MD_CTX_set_flags(si->mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); + } + } + + if (sd->signerInfos == NULL) + sd->signerInfos = sk_CMS_SignerInfo_new_null(); + if (sd->signerInfos == NULL || !sk_CMS_SignerInfo_push(sd->signerInfos, si)) { + ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); + goto err; + } + + return si; + + err: + M_ASN1_free_of(si, CMS_SignerInfo); + return NULL; + +} + +void ossl_cms_SignerInfos_set_cmsctx(CMS_ContentInfo *cms) +{ + int i; + CMS_SignerInfo *si; + STACK_OF(CMS_SignerInfo) *sinfos; + const CMS_CTX *ctx = ossl_cms_get0_cmsctx(cms); + + ERR_set_mark(); + sinfos = CMS_get0_SignerInfos(cms); + ERR_pop_to_mark(); /* removes error in case sinfos == NULL */ + + for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { + si = sk_CMS_SignerInfo_value(sinfos, i); + if (si != NULL) + si->cms_ctx = ctx; + } +} + +static int cms_add1_signingTime(CMS_SignerInfo *si, ASN1_TIME *t) +{ + ASN1_TIME *tt; + int r = 0; + + if (t != NULL) + tt = t; + else + tt = X509_gmtime_adj(NULL, 0); + + if (tt == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB); + goto err; + } + + if (CMS_signed_add1_attr_by_NID(si, NID_pkcs9_signingTime, + tt->type, tt, -1) <= 0) { + ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + goto err; + } + + r = 1; + err: + if (t == NULL) + ASN1_TIME_free(tt); + + return r; + +} + +EVP_PKEY_CTX *CMS_SignerInfo_get0_pkey_ctx(CMS_SignerInfo *si) +{ + return si->pctx; +} + +EVP_MD_CTX *CMS_SignerInfo_get0_md_ctx(CMS_SignerInfo *si) +{ + return si->mctx; +} + +STACK_OF(CMS_SignerInfo) *CMS_get0_SignerInfos(CMS_ContentInfo *cms) +{ + CMS_SignedData *sd = cms_get0_signed(cms); + + return sd != NULL ? sd->signerInfos : NULL; +} + +STACK_OF(X509) *CMS_get0_signers(CMS_ContentInfo *cms) +{ + STACK_OF(X509) *signers = NULL; + STACK_OF(CMS_SignerInfo) *sinfos; + CMS_SignerInfo *si; + int i; + + sinfos = CMS_get0_SignerInfos(cms); + for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { + si = sk_CMS_SignerInfo_value(sinfos, i); + if (si->signer != NULL) { + if (!ossl_x509_add_cert_new(&signers, si->signer, + X509_ADD_FLAG_DEFAULT)) { + sk_X509_free(signers); + return NULL; + } + } + } + return signers; +} + +void CMS_SignerInfo_set1_signer_cert(CMS_SignerInfo *si, X509 *signer) +{ + if (signer != NULL) { + X509_up_ref(signer); + EVP_PKEY_free(si->pkey); + si->pkey = X509_get_pubkey(signer); + } + X509_free(si->signer); + si->signer = signer; +} + +int CMS_SignerInfo_get0_signer_id(CMS_SignerInfo *si, + ASN1_OCTET_STRING **keyid, + X509_NAME **issuer, ASN1_INTEGER **sno) +{ + return ossl_cms_SignerIdentifier_get0_signer_id(si->sid, keyid, issuer, sno); +} + +int CMS_SignerInfo_cert_cmp(CMS_SignerInfo *si, X509 *cert) +{ + return ossl_cms_SignerIdentifier_cert_cmp(si->sid, cert); +} + +int CMS_set1_signers_certs(CMS_ContentInfo *cms, STACK_OF(X509) *scerts, + unsigned int flags) +{ + CMS_SignedData *sd; + CMS_SignerInfo *si; + CMS_CertificateChoices *cch; + STACK_OF(CMS_CertificateChoices) *certs; + X509 *x; + int i, j; + int ret = 0; + + sd = cms_get0_signed(cms); + if (sd == NULL) + return -1; + certs = sd->certificates; + for (i = 0; i < sk_CMS_SignerInfo_num(sd->signerInfos); i++) { + si = sk_CMS_SignerInfo_value(sd->signerInfos, i); + if (si->signer != NULL) + continue; + + for (j = 0; j < sk_X509_num(scerts); j++) { + x = sk_X509_value(scerts, j); + if (CMS_SignerInfo_cert_cmp(si, x) == 0) { + CMS_SignerInfo_set1_signer_cert(si, x); + ret++; + break; + } + } + + if (si->signer != NULL || (flags & CMS_NOINTERN)) + continue; + + for (j = 0; j < sk_CMS_CertificateChoices_num(certs); j++) { + cch = sk_CMS_CertificateChoices_value(certs, j); + if (cch->type != CMS_CERTCHOICE_CERT) + continue; + x = cch->d.certificate; + if (CMS_SignerInfo_cert_cmp(si, x) == 0) { + CMS_SignerInfo_set1_signer_cert(si, x); + ret++; + break; + } + } + } + return ret; +} + +void CMS_SignerInfo_get0_algs(CMS_SignerInfo *si, EVP_PKEY **pk, + X509 **signer, X509_ALGOR **pdig, + X509_ALGOR **psig) +{ + if (pk != NULL) + *pk = si->pkey; + if (signer != NULL) + *signer = si->signer; + if (pdig != NULL) + *pdig = si->digestAlgorithm; + if (psig != NULL) + *psig = si->signatureAlgorithm; +} + +ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si) +{ + return si->signature; +} + +static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, + CMS_SignerInfo *si, BIO *chain, + const unsigned char *md, + unsigned int mdlen) +{ + EVP_MD_CTX *mctx = EVP_MD_CTX_new(); + int r = 0; + EVP_PKEY_CTX *pctx = NULL; + const CMS_CTX *ctx = ossl_cms_get0_cmsctx(cms); + + if (mctx == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + return 0; + } + + if (si->pkey == NULL) { + ERR_raise(ERR_LIB_CMS, CMS_R_NO_PRIVATE_KEY); + goto err; + } + + if (!ossl_cms_DigestAlgorithm_find_ctx(mctx, chain, si->digestAlgorithm)) + goto err; + /* Set SignerInfo algorithm details if we used custom parameter */ + if (si->pctx && !cms_sd_asn1_ctrl(si, 0)) + goto err; + + /* + * If any signed attributes calculate and add messageDigest attribute + */ + if (CMS_signed_get_attr_count(si) >= 0) { + unsigned char computed_md[EVP_MAX_MD_SIZE]; + + if (md == NULL) { + if (!EVP_DigestFinal_ex(mctx, computed_md, &mdlen)) + goto err; + md = computed_md; + } + if (!CMS_signed_add1_attr_by_NID(si, NID_pkcs9_messageDigest, + V_ASN1_OCTET_STRING, md, mdlen)) + goto err; + /* Copy content type across */ + if (!cms_set_si_contentType_attr(cms, si)) + goto err; + + if (!CMS_SignerInfo_sign(si)) + goto err; + } else if (si->pctx) { + unsigned char *sig; + size_t siglen; + unsigned char computed_md[EVP_MAX_MD_SIZE]; + + pctx = si->pctx; + si->pctx = NULL; + if (md == NULL) { + if (!EVP_DigestFinal_ex(mctx, computed_md, &mdlen)) + goto err; + md = computed_md; + } + siglen = EVP_PKEY_get_size(si->pkey); + if (siglen == 0 || (sig = OPENSSL_malloc(siglen)) == NULL) + goto err; + if (EVP_PKEY_sign(pctx, sig, &siglen, md, mdlen) <= 0) { + OPENSSL_free(sig); + goto err; + } + ASN1_STRING_set0(si->signature, sig, siglen); + } else { + unsigned char *sig; + unsigned int siglen; + + if (md != NULL) { + ERR_raise(ERR_LIB_CMS, CMS_R_OPERATION_UNSUPPORTED); + goto err; + } + siglen = EVP_PKEY_get_size(si->pkey); + if (siglen == 0 || (sig = OPENSSL_malloc(siglen)) == NULL) + goto err; + if (!EVP_SignFinal_ex(mctx, sig, &siglen, si->pkey, + ossl_cms_ctx_get0_libctx(ctx), + ossl_cms_ctx_get0_propq(ctx))) { + ERR_raise(ERR_LIB_CMS, CMS_R_SIGNFINAL_ERROR); + OPENSSL_free(sig); + goto err; + } + ASN1_STRING_set0(si->signature, sig, siglen); + } + + r = 1; + + err: + EVP_MD_CTX_free(mctx); + EVP_PKEY_CTX_free(pctx); + return r; + +} + +int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain, + const unsigned char *precomp_md, + unsigned int precomp_mdlen) +{ + STACK_OF(CMS_SignerInfo) *sinfos; + CMS_SignerInfo *si; + int i; + + sinfos = CMS_get0_SignerInfos(cms); + for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { + si = sk_CMS_SignerInfo_value(sinfos, i); + if (!cms_SignerInfo_content_sign(cms, si, chain, + precomp_md, precomp_mdlen)) + return 0; + } + cms->d.signedData->encapContentInfo->partial = 0; + return 1; +} + +int CMS_SignerInfo_sign(CMS_SignerInfo *si) +{ + EVP_MD_CTX *mctx = si->mctx; + EVP_PKEY_CTX *pctx = NULL; + unsigned char *abuf = NULL; + int alen; + size_t siglen; + const CMS_CTX *ctx = si->cms_ctx; + char md_name[OSSL_MAX_NAME_SIZE]; + + if (OBJ_obj2txt(md_name, sizeof(md_name), + si->digestAlgorithm->algorithm, 0) <= 0) + return 0; + + if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) { + if (!cms_add1_signingTime(si, NULL)) + goto err; + } + + if (!ossl_cms_si_check_attributes(si)) + goto err; + + if (si->pctx) { + pctx = si->pctx; + } else { + EVP_MD_CTX_reset(mctx); + if (EVP_DigestSignInit_ex(mctx, &pctx, md_name, + ossl_cms_ctx_get0_libctx(ctx), + ossl_cms_ctx_get0_propq(ctx), si->pkey, + NULL) <= 0) + goto err; + EVP_MD_CTX_set_flags(mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); + si->pctx = pctx; + } + + alen = ASN1_item_i2d((ASN1_VALUE *)si->signedAttrs, &abuf, + ASN1_ITEM_rptr(CMS_Attributes_Sign)); + if (!abuf) + goto err; + if (EVP_DigestSignUpdate(mctx, abuf, alen) <= 0) + goto err; + if (EVP_DigestSignFinal(mctx, NULL, &siglen) <= 0) + goto err; + OPENSSL_free(abuf); + abuf = OPENSSL_malloc(siglen); + if (abuf == NULL) + goto err; + if (EVP_DigestSignFinal(mctx, abuf, &siglen) <= 0) + goto err; + + EVP_MD_CTX_reset(mctx); + + ASN1_STRING_set0(si->signature, abuf, siglen); + + return 1; + + err: + OPENSSL_free(abuf); + EVP_MD_CTX_reset(mctx); + return 0; +} + +int CMS_SignerInfo_verify(CMS_SignerInfo *si) +{ + EVP_MD_CTX *mctx = NULL; + unsigned char *abuf = NULL; + int alen, r = -1; + char name[OSSL_MAX_NAME_SIZE]; + const EVP_MD *md; + EVP_MD *fetched_md = NULL; + const CMS_CTX *ctx = si->cms_ctx; + OSSL_LIB_CTX *libctx = ossl_cms_ctx_get0_libctx(ctx); + const char *propq = ossl_cms_ctx_get0_propq(ctx); + + if (si->pkey == NULL) { + ERR_raise(ERR_LIB_CMS, CMS_R_NO_PUBLIC_KEY); + return -1; + } + + if (!ossl_cms_si_check_attributes(si)) + return -1; + + OBJ_obj2txt(name, sizeof(name), si->digestAlgorithm->algorithm, 0); + + (void)ERR_set_mark(); + fetched_md = EVP_MD_fetch(libctx, name, propq); + + if (fetched_md != NULL) + md = fetched_md; + else + md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm); + if (md == NULL) { + (void)ERR_clear_last_mark(); + ERR_raise(ERR_LIB_CMS, CMS_R_UNKNOWN_DIGEST_ALGORITHM); + return -1; + } + (void)ERR_pop_to_mark(); + + if (si->mctx == NULL && (si->mctx = EVP_MD_CTX_new()) == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + goto err; + } + mctx = si->mctx; + if (si->pctx != NULL) { + EVP_PKEY_CTX_free(si->pctx); + si->pctx = NULL; + } + if (EVP_DigestVerifyInit_ex(mctx, &si->pctx, EVP_MD_get0_name(md), libctx, + propq, si->pkey, NULL) <= 0) { + si->pctx = NULL; + goto err; + } + EVP_MD_CTX_set_flags(mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); + + if (!cms_sd_asn1_ctrl(si, 1)) + goto err; + + alen = ASN1_item_i2d((ASN1_VALUE *)si->signedAttrs, &abuf, + ASN1_ITEM_rptr(CMS_Attributes_Verify)); + if (abuf == NULL || alen < 0) + goto err; + r = EVP_DigestVerifyUpdate(mctx, abuf, alen); + OPENSSL_free(abuf); + if (r <= 0) { + r = -1; + goto err; + } + r = EVP_DigestVerifyFinal(mctx, + si->signature->data, si->signature->length); + if (r <= 0) + ERR_raise(ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE); + err: + EVP_MD_free(fetched_md); + EVP_MD_CTX_reset(mctx); + return r; +} + +/* Create a chain of digest BIOs from a CMS ContentInfo */ +BIO *ossl_cms_SignedData_init_bio(CMS_ContentInfo *cms) +{ + int i; + CMS_SignedData *sd; + BIO *chain = NULL; + + sd = cms_get0_signed(cms); + if (sd == NULL) + return NULL; + if (cms->d.signedData->encapContentInfo->partial) + cms_sd_set_version(sd); + for (i = 0; i < sk_X509_ALGOR_num(sd->digestAlgorithms); i++) { + X509_ALGOR *digestAlgorithm; + BIO *mdbio; + + digestAlgorithm = sk_X509_ALGOR_value(sd->digestAlgorithms, i); + mdbio = ossl_cms_DigestAlgorithm_init_bio(digestAlgorithm, + ossl_cms_get0_cmsctx(cms)); + if (mdbio == NULL) + goto err; + if (chain != NULL) + BIO_push(chain, mdbio); + else + chain = mdbio; + } + return chain; + err: + BIO_free_all(chain); + return NULL; +} + +int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) +{ + ASN1_OCTET_STRING *os = NULL; + EVP_MD_CTX *mctx = EVP_MD_CTX_new(); + EVP_PKEY_CTX *pkctx = NULL; + int r = -1; + unsigned char mval[EVP_MAX_MD_SIZE]; + unsigned int mlen; + + if (mctx == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + goto err; + } + /* If we have any signed attributes look for messageDigest value */ + if (CMS_signed_get_attr_count(si) >= 0) { + os = CMS_signed_get0_data_by_OBJ(si, + OBJ_nid2obj(NID_pkcs9_messageDigest), + -3, V_ASN1_OCTET_STRING); + if (os == NULL) { + ERR_raise(ERR_LIB_CMS, CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE); + goto err; + } + } + + if (!ossl_cms_DigestAlgorithm_find_ctx(mctx, chain, si->digestAlgorithm)) + goto err; + + if (EVP_DigestFinal_ex(mctx, mval, &mlen) <= 0) { + ERR_raise(ERR_LIB_CMS, CMS_R_UNABLE_TO_FINALIZE_CONTEXT); + goto err; + } + + /* If messageDigest found compare it */ + if (os != NULL) { + if (mlen != (unsigned int)os->length) { + ERR_raise(ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH); + goto err; + } + + if (memcmp(mval, os->data, mlen)) { + ERR_raise(ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE); + r = 0; + } else { + r = 1; + } + } else { + const EVP_MD *md = EVP_MD_CTX_get0_md(mctx); + const CMS_CTX *ctx = si->cms_ctx; + + pkctx = EVP_PKEY_CTX_new_from_pkey(ossl_cms_ctx_get0_libctx(ctx), + si->pkey, + ossl_cms_ctx_get0_propq(ctx)); + if (pkctx == NULL) + goto err; + if (EVP_PKEY_verify_init(pkctx) <= 0) + goto err; + if (EVP_PKEY_CTX_set_signature_md(pkctx, md) <= 0) + goto err; + si->pctx = pkctx; + if (!cms_sd_asn1_ctrl(si, 1)) { + si->pctx = NULL; + goto err; + } + si->pctx = NULL; + r = EVP_PKEY_verify(pkctx, si->signature->data, + si->signature->length, mval, mlen); + if (r <= 0) { + ERR_raise(ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE); + r = 0; + } + } + + err: + EVP_PKEY_CTX_free(pkctx); + EVP_MD_CTX_free(mctx); + return r; + +} + +BIO *CMS_SignedData_verify(CMS_SignedData *sd, BIO *detached_data, + STACK_OF(X509) *scerts, X509_STORE *store, + STACK_OF(X509) *extra, STACK_OF(X509_CRL) *crls, + unsigned int flags, + OSSL_LIB_CTX *libctx, const char *propq) +{ + CMS_ContentInfo *ci; + BIO *bio = NULL; + int i, res = 0; + + if (sd == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if ((ci = CMS_ContentInfo_new_ex(libctx, propq)) == NULL) + return NULL; + if ((bio = BIO_new(BIO_s_mem())) == NULL) + goto end; + ci->contentType = OBJ_nid2obj(NID_pkcs7_signed); + ci->d.signedData = sd; + + for (i = 0; i < sk_X509_num(extra); i++) + if (!CMS_add1_cert(ci, sk_X509_value(extra, i))) + goto end; + for (i = 0; i < sk_X509_CRL_num(crls); i++) + if (!CMS_add1_crl(ci, sk_X509_CRL_value(crls, i))) + goto end; + res = CMS_verify(ci, scerts, store, detached_data, bio, flags); + + end: + if (ci != NULL) + ci->d.signedData = NULL; /* do not indirectly free |sd| */ + CMS_ContentInfo_free(ci); + if (!res) { + BIO_free(bio); + bio = NULL; + } + return bio; +} + +int CMS_add_smimecap(CMS_SignerInfo *si, STACK_OF(X509_ALGOR) *algs) +{ + unsigned char *smder = NULL; + int smderlen, r; + + smderlen = i2d_X509_ALGORS(algs, &smder); + if (smderlen <= 0) + return 0; + r = CMS_signed_add1_attr_by_NID(si, NID_SMIMECapabilities, + V_ASN1_SEQUENCE, smder, smderlen); + OPENSSL_free(smder); + return r; +} + +int CMS_add_simple_smimecap(STACK_OF(X509_ALGOR) **algs, + int algnid, int keysize) +{ + X509_ALGOR *alg; + ASN1_INTEGER *key = NULL; + + if (keysize > 0) { + key = ASN1_INTEGER_new(); + if (key == NULL || !ASN1_INTEGER_set(key, keysize)) { + ASN1_INTEGER_free(key); + return 0; + } + } + alg = ossl_X509_ALGOR_from_nid(algnid, key != NULL ? V_ASN1_INTEGER : + V_ASN1_UNDEF, key); + if (alg == NULL) { + ASN1_INTEGER_free(key); + return 0; + } + + if (*algs == NULL) + *algs = sk_X509_ALGOR_new_null(); + if (*algs == NULL || !sk_X509_ALGOR_push(*algs, alg)) { + X509_ALGOR_free(alg); + return 0; + } + return 1; +} + +/* Check to see if a cipher exists and if so add S/MIME capabilities */ +static int cms_add_cipher_smcap(STACK_OF(X509_ALGOR) **sk, int nid, int arg) +{ + if (EVP_get_cipherbynid(nid)) + return CMS_add_simple_smimecap(sk, nid, arg); + return 1; +} + +static int cms_add_digest_smcap(STACK_OF(X509_ALGOR) **sk, int nid, int arg) +{ + if (EVP_get_digestbynid(nid)) + return CMS_add_simple_smimecap(sk, nid, arg); + return 1; +} + +int CMS_add_standard_smimecap(STACK_OF(X509_ALGOR) **smcap) +{ + if (!cms_add_cipher_smcap(smcap, NID_aes_256_cbc, -1) + || !cms_add_digest_smcap(smcap, NID_id_GostR3411_2012_256, -1) + || !cms_add_digest_smcap(smcap, NID_id_GostR3411_2012_512, -1) + || !cms_add_digest_smcap(smcap, NID_id_GostR3411_94, -1) + || !cms_add_cipher_smcap(smcap, NID_id_Gost28147_89, -1) + || !cms_add_cipher_smcap(smcap, NID_aes_192_cbc, -1) + || !cms_add_cipher_smcap(smcap, NID_aes_128_cbc, -1) + || !cms_add_cipher_smcap(smcap, NID_des_ede3_cbc, -1) + || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 128) + || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 64) + || !cms_add_cipher_smcap(smcap, NID_des_cbc, -1) + || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 40)) + return 0; + return 1; +} diff --git a/tests/ctests/ec_asn1.c b/tests/ctests/ec_asn1.c new file mode 100644 index 000000000..b32697fb8 --- /dev/null +++ b/tests/ctests/ec_asn1.c @@ -0,0 +1,1332 @@ +/* + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* + * ECDSA low level APIs are deprecated for public use, but still ok for + * internal use. + */ +#include "internal/deprecated.h" + +#include +#include "ec_local.h" +#include +#include +#include +#include "internal/nelem.h" +#include "crypto/asn1.h" +#include "crypto/asn1_dsa.h" + +#ifndef FIPS_MODULE + +/* some structures needed for the asn1 encoding */ +typedef struct x9_62_pentanomial_st { + int32_t k1; + int32_t k2; + int32_t k3; +} X9_62_PENTANOMIAL; + +typedef struct x9_62_characteristic_two_st { + int32_t m; + ASN1_OBJECT *type; + union { + char *ptr; + /* NID_X9_62_onBasis */ + ASN1_NULL *onBasis; + /* NID_X9_62_tpBasis */ + ASN1_INTEGER *tpBasis; + /* NID_X9_62_ppBasis */ + X9_62_PENTANOMIAL *ppBasis; + /* anything else */ + ASN1_TYPE *other; + } p; +} X9_62_CHARACTERISTIC_TWO; + +typedef struct x9_62_fieldid_st { + ASN1_OBJECT *fieldType; + union { + char *ptr; + /* NID_X9_62_prime_field */ + ASN1_INTEGER *prime; + /* NID_X9_62_characteristic_two_field */ + X9_62_CHARACTERISTIC_TWO *char_two; + /* anything else */ + ASN1_TYPE *other; + } p; +} X9_62_FIELDID; + +typedef struct x9_62_curve_st { + ASN1_OCTET_STRING *a; + ASN1_OCTET_STRING *b; + ASN1_BIT_STRING *seed; +} X9_62_CURVE; + +struct ec_parameters_st { + int32_t version; + X9_62_FIELDID *fieldID; + X9_62_CURVE *curve; + ASN1_OCTET_STRING *base; + ASN1_INTEGER *order; + ASN1_INTEGER *cofactor; +} /* ECPARAMETERS */ ; + +typedef enum { + ECPKPARAMETERS_TYPE_NAMED = 0, + ECPKPARAMETERS_TYPE_EXPLICIT, + ECPKPARAMETERS_TYPE_IMPLICIT +} ecpk_parameters_type_t; + +struct ecpk_parameters_st { + int type; + union { + ASN1_OBJECT *named_curve; + ECPARAMETERS *parameters; + ASN1_NULL *implicitlyCA; + } value; +} /* ECPKPARAMETERS */ ; + +/* SEC1 ECPrivateKey */ +typedef struct ec_privatekey_st { + int32_t version; + ASN1_OCTET_STRING *privateKey; + ECPKPARAMETERS *parameters; + ASN1_BIT_STRING *publicKey; +} EC_PRIVATEKEY; + +/* the OpenSSL ASN.1 definitions */ +ASN1_SEQUENCE(X9_62_PENTANOMIAL) = { + ASN1_EMBED(X9_62_PENTANOMIAL, k1, INT32), + ASN1_EMBED(X9_62_PENTANOMIAL, k2, INT32), + ASN1_EMBED(X9_62_PENTANOMIAL, k3, INT32) +} static_ASN1_SEQUENCE_END(X9_62_PENTANOMIAL) + +DECLARE_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) +IMPLEMENT_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) + +ASN1_ADB_TEMPLATE(char_two_def) = ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, p.other, ASN1_ANY); + +ASN1_ADB(X9_62_CHARACTERISTIC_TWO) = { + ADB_ENTRY(NID_X9_62_onBasis, ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, p.onBasis, ASN1_NULL)), + ADB_ENTRY(NID_X9_62_tpBasis, ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, p.tpBasis, ASN1_INTEGER)), + ADB_ENTRY(NID_X9_62_ppBasis, ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, p.ppBasis, X9_62_PENTANOMIAL)) +} ASN1_ADB_END(X9_62_CHARACTERISTIC_TWO, 0, type, 0, &char_two_def_tt, NULL); + +ASN1_SEQUENCE(X9_62_CHARACTERISTIC_TWO) = { + ASN1_EMBED(X9_62_CHARACTERISTIC_TWO, m, INT32), + ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, type, ASN1_OBJECT), + ASN1_ADB_OBJECT(X9_62_CHARACTERISTIC_TWO) +} static_ASN1_SEQUENCE_END(X9_62_CHARACTERISTIC_TWO) + +DECLARE_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) +IMPLEMENT_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) + +ASN1_ADB_TEMPLATE(fieldID_def) = ASN1_SIMPLE(X9_62_FIELDID, p.other, ASN1_ANY); + +ASN1_ADB(X9_62_FIELDID) = { + ADB_ENTRY(NID_X9_62_prime_field, ASN1_SIMPLE(X9_62_FIELDID, p.prime, ASN1_INTEGER)), + ADB_ENTRY(NID_X9_62_characteristic_two_field, ASN1_SIMPLE(X9_62_FIELDID, p.char_two, X9_62_CHARACTERISTIC_TWO)) +} ASN1_ADB_END(X9_62_FIELDID, 0, fieldType, 0, &fieldID_def_tt, NULL); + +ASN1_SEQUENCE(X9_62_FIELDID) = { + ASN1_SIMPLE(X9_62_FIELDID, fieldType, ASN1_OBJECT), + ASN1_ADB_OBJECT(X9_62_FIELDID) +} static_ASN1_SEQUENCE_END(X9_62_FIELDID) + +ASN1_SEQUENCE(X9_62_CURVE) = { + ASN1_SIMPLE(X9_62_CURVE, a, ASN1_OCTET_STRING), + ASN1_SIMPLE(X9_62_CURVE, b, ASN1_OCTET_STRING), + ASN1_OPT(X9_62_CURVE, seed, ASN1_BIT_STRING) +} static_ASN1_SEQUENCE_END(X9_62_CURVE) + +ASN1_SEQUENCE(ECPARAMETERS) = { + ASN1_EMBED(ECPARAMETERS, version, INT32), + ASN1_SIMPLE(ECPARAMETERS, fieldID, X9_62_FIELDID), + ASN1_SIMPLE(ECPARAMETERS, curve, X9_62_CURVE), + ASN1_SIMPLE(ECPARAMETERS, base, ASN1_OCTET_STRING), + ASN1_SIMPLE(ECPARAMETERS, order, ASN1_INTEGER), + ASN1_OPT(ECPARAMETERS, cofactor, ASN1_INTEGER) +} ASN1_SEQUENCE_END(ECPARAMETERS) + +DECLARE_ASN1_ALLOC_FUNCTIONS(ECPARAMETERS) +IMPLEMENT_ASN1_ALLOC_FUNCTIONS(ECPARAMETERS) + +ASN1_CHOICE(ECPKPARAMETERS) = { + ASN1_SIMPLE(ECPKPARAMETERS, value.named_curve, ASN1_OBJECT), + ASN1_SIMPLE(ECPKPARAMETERS, value.parameters, ECPARAMETERS), + ASN1_SIMPLE(ECPKPARAMETERS, value.implicitlyCA, ASN1_NULL) +} ASN1_CHOICE_END(ECPKPARAMETERS) + +DECLARE_ASN1_FUNCTIONS(ECPKPARAMETERS) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(ECPKPARAMETERS, ECPKPARAMETERS) +IMPLEMENT_ASN1_FUNCTIONS(ECPKPARAMETERS) + +ASN1_SEQUENCE(EC_PRIVATEKEY) = { + ASN1_EMBED(EC_PRIVATEKEY, version, INT32), + ASN1_SIMPLE(EC_PRIVATEKEY, privateKey, ASN1_OCTET_STRING), + ASN1_EXP_OPT(EC_PRIVATEKEY, parameters, ECPKPARAMETERS, 0), + ASN1_EXP_OPT(EC_PRIVATEKEY, publicKey, ASN1_BIT_STRING, 1) +} static_ASN1_SEQUENCE_END(EC_PRIVATEKEY) + +DECLARE_ASN1_FUNCTIONS(EC_PRIVATEKEY) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(EC_PRIVATEKEY, EC_PRIVATEKEY) +IMPLEMENT_ASN1_FUNCTIONS(EC_PRIVATEKEY) + +/* some declarations of internal function */ + +/* ec_asn1_group2field() sets the values in a X9_62_FIELDID object */ +static int ec_asn1_group2fieldid(const EC_GROUP *, X9_62_FIELDID *); +/* ec_asn1_group2curve() sets the values in a X9_62_CURVE object */ +static int ec_asn1_group2curve(const EC_GROUP *, X9_62_CURVE *); + +/* the function definitions */ + +static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) +{ + int ok = 0, nid; + BIGNUM *tmp = NULL; + + if (group == NULL || field == NULL) + return 0; + + /* clear the old values (if necessary) */ + ASN1_OBJECT_free(field->fieldType); + ASN1_TYPE_free(field->p.other); + + nid = EC_GROUP_get_field_type(group); + /* set OID for the field */ + if ((field->fieldType = OBJ_nid2obj(nid)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_OBJ_LIB); + goto err; + } + + if (nid == NID_X9_62_prime_field) { + if ((tmp = BN_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + /* the parameters are specified by the prime number p */ + if (!EC_GROUP_get_curve(group, tmp, NULL, NULL, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + /* set the prime number */ + field->p.prime = BN_to_ASN1_INTEGER(tmp, NULL); + if (field->p.prime == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } else if (nid == NID_X9_62_characteristic_two_field) +#ifdef OPENSSL_NO_EC2M + { + ERR_raise(ERR_LIB_EC, EC_R_GF2M_NOT_SUPPORTED); + goto err; + } +#else + { + int field_type; + X9_62_CHARACTERISTIC_TWO *char_two; + + field->p.char_two = X9_62_CHARACTERISTIC_TWO_new(); + char_two = field->p.char_two; + + if (char_two == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + char_two->m = (long)EC_GROUP_get_degree(group); + + field_type = EC_GROUP_get_basis_type(group); + + if (field_type == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + /* set base type OID */ + if ((char_two->type = OBJ_nid2obj(field_type)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_OBJ_LIB); + goto err; + } + + if (field_type == NID_X9_62_tpBasis) { + unsigned int k; + + if (!EC_GROUP_get_trinomial_basis(group, &k)) + goto err; + + char_two->p.tpBasis = ASN1_INTEGER_new(); + if (char_two->p.tpBasis == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + if (!ASN1_INTEGER_set(char_two->p.tpBasis, (long)k)) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } else if (field_type == NID_X9_62_ppBasis) { + unsigned int k1, k2, k3; + + if (!EC_GROUP_get_pentanomial_basis(group, &k1, &k2, &k3)) + goto err; + + char_two->p.ppBasis = X9_62_PENTANOMIAL_new(); + if (char_two->p.ppBasis == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + /* set k? values */ + char_two->p.ppBasis->k1 = (long)k1; + char_two->p.ppBasis->k2 = (long)k2; + char_two->p.ppBasis->k3 = (long)k3; + } else { /* field_type == NID_X9_62_onBasis */ + + /* for ONB the parameters are (asn1) NULL */ + char_two->p.onBasis = ASN1_NULL_new(); + if (char_two->p.onBasis == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } + } +#endif + else { + ERR_raise(ERR_LIB_EC, EC_R_UNSUPPORTED_FIELD); + goto err; + } + + ok = 1; + + err: + BN_free(tmp); + return ok; +} + +static int ec_asn1_group2curve(const EC_GROUP *group, X9_62_CURVE *curve) +{ + int ok = 0; + BIGNUM *tmp_1 = NULL, *tmp_2 = NULL; + unsigned char *a_buf = NULL, *b_buf = NULL; + size_t len; + + if (!group || !curve || !curve->a || !curve->b) + return 0; + + if ((tmp_1 = BN_new()) == NULL || (tmp_2 = BN_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + + /* get a and b */ + if (!EC_GROUP_get_curve(group, NULL, tmp_1, tmp_2, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* + * Per SEC 1, the curve coefficients must be padded up to size. See C.2's + * definition of Curve, C.1's definition of FieldElement, and 2.3.5's + * definition of how to encode the field elements. + */ + len = ((size_t)EC_GROUP_get_degree(group) + 7) / 8; + if ((a_buf = OPENSSL_malloc(len)) == NULL + || (b_buf = OPENSSL_malloc(len)) == NULL) + goto err; + if (BN_bn2binpad(tmp_1, a_buf, len) < 0 + || BN_bn2binpad(tmp_2, b_buf, len) < 0) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + + /* set a and b */ + if (!ASN1_OCTET_STRING_set(curve->a, a_buf, len) + || !ASN1_OCTET_STRING_set(curve->b, b_buf, len)) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + /* set the seed (optional) */ + if (group->seed) { + if (!curve->seed) + if ((curve->seed = ASN1_BIT_STRING_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + ossl_asn1_string_set_bits_left(curve->seed, 0); + if (!ASN1_BIT_STRING_set(curve->seed, group->seed, + (int)group->seed_len)) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } else { + ASN1_BIT_STRING_free(curve->seed); + curve->seed = NULL; + } + + ok = 1; + + err: + OPENSSL_free(a_buf); + OPENSSL_free(b_buf); + BN_free(tmp_1); + BN_free(tmp_2); + return ok; +} + +ECPARAMETERS *EC_GROUP_get_ecparameters(const EC_GROUP *group, + ECPARAMETERS *params) +{ + size_t len = 0; + ECPARAMETERS *ret = NULL; + const BIGNUM *tmp; + unsigned char *buffer = NULL; + const EC_POINT *point = NULL; + point_conversion_form_t form; + ASN1_INTEGER *orig; + + if (params == NULL) { + if ((ret = ECPARAMETERS_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } else + ret = params; + + /* set the version (always one) */ + ret->version = (long)0x1; + + /* set the fieldID */ + if (!ec_asn1_group2fieldid(group, ret->fieldID)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* set the curve */ + if (!ec_asn1_group2curve(group, ret->curve)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* set the base point */ + if ((point = EC_GROUP_get0_generator(group)) == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_UNDEFINED_GENERATOR); + goto err; + } + + form = EC_GROUP_get_point_conversion_form(group); + + len = EC_POINT_point2buf(group, point, form, &buffer, NULL); + if (len == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + if (ret->base == NULL && (ret->base = ASN1_OCTET_STRING_new()) == NULL) { + OPENSSL_free(buffer); + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + ASN1_STRING_set0(ret->base, buffer, len); + + /* set the order */ + tmp = EC_GROUP_get0_order(group); + if (tmp == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + ret->order = BN_to_ASN1_INTEGER(tmp, orig = ret->order); + if (ret->order == NULL) { + ret->order = orig; + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + /* set the cofactor (optional) */ + tmp = EC_GROUP_get0_cofactor(group); + if (tmp != NULL) { + ret->cofactor = BN_to_ASN1_INTEGER(tmp, orig = ret->cofactor); + if (ret->cofactor == NULL) { + ret->cofactor = orig; + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + } + + return ret; + + err: + if (params == NULL) + ECPARAMETERS_free(ret); + return NULL; +} + +ECPKPARAMETERS *EC_GROUP_get_ecpkparameters(const EC_GROUP *group, + ECPKPARAMETERS *params) +{ + int ok = 1, tmp; + ECPKPARAMETERS *ret = params; + + if (ret == NULL) { + if ((ret = ECPKPARAMETERS_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + return NULL; + } + } else { + if (ret->type == ECPKPARAMETERS_TYPE_NAMED) + ASN1_OBJECT_free(ret->value.named_curve); + else if (ret->type == ECPKPARAMETERS_TYPE_EXPLICIT + && ret->value.parameters != NULL) + ECPARAMETERS_free(ret->value.parameters); + } + + if (EC_GROUP_get_asn1_flag(group) == OPENSSL_EC_NAMED_CURVE) { + /* + * use the asn1 OID to describe the elliptic curve parameters + */ + tmp = EC_GROUP_get_curve_name(group); + if (tmp) { + ASN1_OBJECT *asn1obj = OBJ_nid2obj(tmp); + + if (asn1obj == NULL || OBJ_length(asn1obj) == 0) { + ASN1_OBJECT_free(asn1obj); + ERR_raise(ERR_LIB_EC, EC_R_MISSING_OID); + ok = 0; + } else { + ret->type = ECPKPARAMETERS_TYPE_NAMED; + ret->value.named_curve = asn1obj; + } + } else + /* we don't know the nid => ERROR */ + ok = 0; + } else { + /* use the ECPARAMETERS structure */ + ret->type = ECPKPARAMETERS_TYPE_EXPLICIT; + if ((ret->value.parameters = + EC_GROUP_get_ecparameters(group, NULL)) == NULL) + ok = 0; + } + + if (!ok) { + ECPKPARAMETERS_free(ret); + return NULL; + } + return ret; +} + +EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) +{ + int ok = 0, tmp; + EC_GROUP *ret = NULL, *dup = NULL; + BIGNUM *p = NULL, *a = NULL, *b = NULL; + EC_POINT *point = NULL; + long field_bits; + int curve_name = NID_undef; + BN_CTX *ctx = NULL; + + if (params->fieldID == NULL + || params->fieldID->fieldType == NULL + || params->fieldID->p.ptr == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + + /* + * Now extract the curve parameters a and b. Note that, although SEC 1 + * specifies the length of their encodings, historical versions of OpenSSL + * encoded them incorrectly, so we must accept any length for backwards + * compatibility. + */ + if (params->curve == NULL + || params->curve->a == NULL || params->curve->a->data == NULL + || params->curve->b == NULL || params->curve->b->data == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + a = BN_bin2bn(params->curve->a->data, params->curve->a->length, NULL); + if (a == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + b = BN_bin2bn(params->curve->b->data, params->curve->b->length, NULL); + if (b == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + + /* get the field parameters */ + tmp = OBJ_obj2nid(params->fieldID->fieldType); + if (tmp == NID_X9_62_characteristic_two_field) +#ifdef OPENSSL_NO_EC2M + { + ERR_raise(ERR_LIB_EC, EC_R_GF2M_NOT_SUPPORTED); + goto err; + } +#else + { + X9_62_CHARACTERISTIC_TWO *char_two; + + char_two = params->fieldID->p.char_two; + + field_bits = char_two->m; + if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS) { + ERR_raise(ERR_LIB_EC, EC_R_FIELD_TOO_LARGE); + goto err; + } + + if ((p = BN_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + + /* get the base type */ + tmp = OBJ_obj2nid(char_two->type); + + if (tmp == NID_X9_62_tpBasis) { + long tmp_long; + + if (!char_two->p.tpBasis) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + + tmp_long = ASN1_INTEGER_get(char_two->p.tpBasis); + + if (!(char_two->m > tmp_long && tmp_long > 0)) { + ERR_raise(ERR_LIB_EC, EC_R_INVALID_TRINOMIAL_BASIS); + goto err; + } + + /* create the polynomial */ + if (!BN_set_bit(p, (int)char_two->m)) + goto err; + if (!BN_set_bit(p, (int)tmp_long)) + goto err; + if (!BN_set_bit(p, 0)) + goto err; + } else if (tmp == NID_X9_62_ppBasis) { + X9_62_PENTANOMIAL *penta; + + penta = char_two->p.ppBasis; + if (penta == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + + if (! + (char_two->m > penta->k3 && penta->k3 > penta->k2 + && penta->k2 > penta->k1 && penta->k1 > 0)) { + ERR_raise(ERR_LIB_EC, EC_R_INVALID_PENTANOMIAL_BASIS); + goto err; + } + + /* create the polynomial */ + if (!BN_set_bit(p, (int)char_two->m)) + goto err; + if (!BN_set_bit(p, (int)penta->k1)) + goto err; + if (!BN_set_bit(p, (int)penta->k2)) + goto err; + if (!BN_set_bit(p, (int)penta->k3)) + goto err; + if (!BN_set_bit(p, 0)) + goto err; + } else if (tmp == NID_X9_62_onBasis) { + ERR_raise(ERR_LIB_EC, EC_R_NOT_IMPLEMENTED); + goto err; + } else { /* error */ + + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + + /* create the EC_GROUP structure */ + ret = EC_GROUP_new_curve_GF2m(p, a, b, NULL); + } +#endif + else if (tmp == NID_X9_62_prime_field) { + /* we have a curve over a prime field */ + /* extract the prime number */ + if (params->fieldID->p.prime == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + p = ASN1_INTEGER_to_BN(params->fieldID->p.prime, NULL); + if (p == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + if (BN_is_negative(p) || BN_is_zero(p)) { + ERR_raise(ERR_LIB_EC, EC_R_INVALID_FIELD); + goto err; + } + + field_bits = BN_num_bits(p); + if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS) { + ERR_raise(ERR_LIB_EC, EC_R_FIELD_TOO_LARGE); + goto err; + } + + /* create the EC_GROUP structure */ + ret = EC_GROUP_new_curve_GFp(p, a, b, NULL); + } else { + ERR_raise(ERR_LIB_EC, EC_R_INVALID_FIELD); + goto err; + } + + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* extract seed (optional) */ + if (params->curve->seed != NULL) { + /* + * This happens for instance with + * fuzz/corpora/asn1/65cf44e85614c62f10cf3b7a7184c26293a19e4a + * and causes the OPENSSL_malloc below to choke on the + * zero length allocation request. + */ + if (params->curve->seed->length == 0) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + OPENSSL_free(ret->seed); + if ((ret->seed = OPENSSL_malloc(params->curve->seed->length)) == NULL) + goto err; + memcpy(ret->seed, params->curve->seed->data, + params->curve->seed->length); + ret->seed_len = params->curve->seed->length; + } + + if (params->order == NULL + || params->base == NULL + || params->base->data == NULL + || params->base->length == 0) { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + goto err; + } + + if ((point = EC_POINT_new(ret)) == NULL) + goto err; + + /* set the point conversion form */ + EC_GROUP_set_point_conversion_form(ret, (point_conversion_form_t) + (params->base->data[0] & ~0x01)); + + /* extract the ec point */ + if (!EC_POINT_oct2point(ret, point, params->base->data, + params->base->length, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* extract the order */ + if (ASN1_INTEGER_to_BN(params->order, a) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + if (BN_is_negative(a) || BN_is_zero(a)) { + ERR_raise(ERR_LIB_EC, EC_R_INVALID_GROUP_ORDER); + goto err; + } + if (BN_num_bits(a) > (int)field_bits + 1) { /* Hasse bound */ + ERR_raise(ERR_LIB_EC, EC_R_INVALID_GROUP_ORDER); + goto err; + } + + /* extract the cofactor (optional) */ + if (params->cofactor == NULL) { + BN_free(b); + b = NULL; + } else if (ASN1_INTEGER_to_BN(params->cofactor, b) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + /* set the generator, order and cofactor (if present) */ + if (!EC_GROUP_set_generator(ret, point, a, b)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + /* + * Check if the explicit parameters group just created matches one of the + * built-in curves. + * + * We create a copy of the group just built, so that we can remove optional + * fields for the lookup: we do this to avoid the possibility that one of + * the optional parameters is used to force the library into using a less + * performant and less secure EC_METHOD instead of the specialized one. + * In any case, `seed` is not really used in any computation, while a + * cofactor different from the one in the built-in table is just + * mathematically wrong anyway and should not be used. + */ + if ((ctx = BN_CTX_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + if ((dup = EC_GROUP_dup(ret)) == NULL + || EC_GROUP_set_seed(dup, NULL, 0) != 1 + || !EC_GROUP_set_generator(dup, point, a, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + if ((curve_name = ossl_ec_curve_nid_from_params(dup, ctx)) != NID_undef) { + /* + * The input explicit parameters successfully matched one of the + * built-in curves: often for built-in curves we have specialized + * methods with better performance and hardening. + * + * In this case we replace the `EC_GROUP` created through explicit + * parameters with one created from a named group. + */ + EC_GROUP *named_group = NULL; + +#ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 + /* + * NID_wap_wsg_idm_ecid_wtls12 and NID_secp224r1 are both aliases for + * the same curve, we prefer the SECP nid when matching explicit + * parameters as that is associated with a specialized EC_METHOD. + */ + if (curve_name == NID_wap_wsg_idm_ecid_wtls12) + curve_name = NID_secp224r1; +#endif /* !def(OPENSSL_NO_EC_NISTP_64_GCC_128) */ + + if ((named_group = EC_GROUP_new_by_curve_name(curve_name)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + EC_GROUP_free(ret); + ret = named_group; + + /* + * Set the flag so that EC_GROUPs created from explicit parameters are + * serialized using explicit parameters by default. + */ + EC_GROUP_set_asn1_flag(ret, OPENSSL_EC_EXPLICIT_CURVE); + + /* + * If the input params do not contain the optional seed field we make + * sure it is not added to the returned group. + * + * The seed field is not really used inside libcrypto anyway, and + * adding it to parsed explicit parameter keys would alter their DER + * encoding output (because of the extra field) which could impact + * applications fingerprinting keys by their DER encoding. + */ + if (params->curve->seed == NULL) { + if (EC_GROUP_set_seed(ret, NULL, 0) != 1) + goto err; + } + } + + ok = 1; + + err: + if (!ok) { + EC_GROUP_free(ret); + ret = NULL; + } + EC_GROUP_free(dup); + + BN_free(p); + BN_free(a); + BN_free(b); + EC_POINT_free(point); + + BN_CTX_free(ctx); + + return ret; +} + +EC_GROUP *EC_GROUP_new_from_ecpkparameters(const ECPKPARAMETERS *params) +{ + EC_GROUP *ret = NULL; + int tmp = 0; + + if (params == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_MISSING_PARAMETERS); + return NULL; + } + + if (params->type == ECPKPARAMETERS_TYPE_NAMED) { + /* the curve is given by an OID */ + tmp = OBJ_obj2nid(params->value.named_curve); + if ((ret = EC_GROUP_new_by_curve_name(tmp)) == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_EC_GROUP_NEW_BY_NAME_FAILURE); + return NULL; + } + EC_GROUP_set_asn1_flag(ret, OPENSSL_EC_NAMED_CURVE); + } else if (params->type == ECPKPARAMETERS_TYPE_EXPLICIT) { + /* the parameters are given by an ECPARAMETERS structure */ + ret = EC_GROUP_new_from_ecparameters(params->value.parameters); + if (!ret) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + return NULL; + } + EC_GROUP_set_asn1_flag(ret, OPENSSL_EC_EXPLICIT_CURVE); + } else if (params->type == ECPKPARAMETERS_TYPE_IMPLICIT) { + /* implicit parameters inherited from CA - unsupported */ + return NULL; + } else { + ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); + return NULL; + } + + return ret; +} + +/* EC_GROUP <-> DER encoding of ECPKPARAMETERS */ + +EC_GROUP *d2i_ECPKParameters(EC_GROUP **a, const unsigned char **in, long len) +{ + EC_GROUP *group = NULL; + ECPKPARAMETERS *params = NULL; + const unsigned char *p = *in; + + if ((params = d2i_ECPKPARAMETERS(NULL, &p, len)) == NULL) { + ECPKPARAMETERS_free(params); + return NULL; + } + + if ((group = EC_GROUP_new_from_ecpkparameters(params)) == NULL) { + ECPKPARAMETERS_free(params); + return NULL; + } + + if (params->type == ECPKPARAMETERS_TYPE_EXPLICIT) + group->decoded_from_explicit_params = 1; + + if (a) { + EC_GROUP_free(*a); + *a = group; + } + + ECPKPARAMETERS_free(params); + *in = p; + return group; +} + +int i2d_ECPKParameters(const EC_GROUP *a, unsigned char **out) +{ + int ret = 0; + ECPKPARAMETERS *tmp = EC_GROUP_get_ecpkparameters(a, NULL); + if (tmp == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_GROUP2PKPARAMETERS_FAILURE); + return 0; + } + if ((ret = i2d_ECPKPARAMETERS(tmp, out)) == 0) { + ERR_raise(ERR_LIB_EC, EC_R_I2D_ECPKPARAMETERS_FAILURE); + ECPKPARAMETERS_free(tmp); + return 0; + } + ECPKPARAMETERS_free(tmp); + return ret; +} + +/* some EC_KEY functions */ + +EC_KEY *d2i_ECPrivateKey(EC_KEY **a, const unsigned char **in, long len) +{ + EC_KEY *ret = NULL; + EC_PRIVATEKEY *priv_key = NULL; + const unsigned char *p = *in; + + if ((priv_key = d2i_EC_PRIVATEKEY(NULL, &p, len)) == NULL) + return NULL; + + if (a == NULL || *a == NULL) { + if ((ret = EC_KEY_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + } else + ret = *a; + + if (priv_key->parameters) { + EC_GROUP_free(ret->group); + ret->group = EC_GROUP_new_from_ecpkparameters(priv_key->parameters); + if (ret->group != NULL + && priv_key->parameters->type == ECPKPARAMETERS_TYPE_EXPLICIT) + ret->group->decoded_from_explicit_params = 1; + } + + if (ret->group == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + ret->version = priv_key->version; + + if (priv_key->privateKey) { + ASN1_OCTET_STRING *pkey = priv_key->privateKey; + if (EC_KEY_oct2priv(ret, ASN1_STRING_get0_data(pkey), + ASN1_STRING_length(pkey)) == 0) + goto err; + } else { + ERR_raise(ERR_LIB_EC, EC_R_MISSING_PRIVATE_KEY); + goto err; + } + + if (EC_GROUP_get_curve_name(ret->group) == NID_sm2) + EC_KEY_set_flags(ret, EC_FLAG_SM2_RANGE); + + EC_POINT_clear_free(ret->pub_key); + ret->pub_key = EC_POINT_new(ret->group); + if (ret->pub_key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + if (priv_key->publicKey) { + const unsigned char *pub_oct; + int pub_oct_len; + + pub_oct = ASN1_STRING_get0_data(priv_key->publicKey); + pub_oct_len = ASN1_STRING_length(priv_key->publicKey); + if (!EC_KEY_oct2key(ret, pub_oct, pub_oct_len, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + } else { + if (ret->group->meth->keygenpub == NULL + || ret->group->meth->keygenpub(ret) == 0) + goto err; + /* Remember the original private-key-only encoding. */ + ret->enc_flag |= EC_PKEY_NO_PUBKEY; + } + + if (a) + *a = ret; + EC_PRIVATEKEY_free(priv_key); + *in = p; + ret->dirty_cnt++; + return ret; + + err: + if (a == NULL || *a != ret) + EC_KEY_free(ret); + EC_PRIVATEKEY_free(priv_key); + return NULL; +} + +int i2d_ECPrivateKey(const EC_KEY *a, unsigned char **out) +{ + int ret = 0, ok = 0; + unsigned char *priv= NULL, *pub= NULL; + size_t privlen = 0, publen = 0; + + EC_PRIVATEKEY *priv_key = NULL; + + if (a == NULL || a->group == NULL || + (!(a->enc_flag & EC_PKEY_NO_PUBKEY) && a->pub_key == NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + goto err; + } + + if ((priv_key = EC_PRIVATEKEY_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + priv_key->version = a->version; + + privlen = EC_KEY_priv2buf(a, &priv); + + if (privlen == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + ASN1_STRING_set0(priv_key->privateKey, priv, privlen); + priv = NULL; + + if (!(a->enc_flag & EC_PKEY_NO_PARAMETERS)) { + if ((priv_key->parameters = + EC_GROUP_get_ecpkparameters(a->group, + priv_key->parameters)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + } + + if (!(a->enc_flag & EC_PKEY_NO_PUBKEY)) { + priv_key->publicKey = ASN1_BIT_STRING_new(); + if (priv_key->publicKey == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + goto err; + } + + publen = EC_KEY_key2buf(a, a->conv_form, &pub, NULL); + + if (publen == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + + ossl_asn1_string_set_bits_left(priv_key->publicKey, 0); + ASN1_STRING_set0(priv_key->publicKey, pub, publen); + pub = NULL; + } + + if ((ret = i2d_EC_PRIVATEKEY(priv_key, out)) == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + goto err; + } + ok = 1; + err: + OPENSSL_clear_free(priv, privlen); + OPENSSL_free(pub); + EC_PRIVATEKEY_free(priv_key); + return (ok ? ret : 0); +} + +int i2d_ECParameters(const EC_KEY *a, unsigned char **out) +{ + if (a == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + return i2d_ECPKParameters(a->group, out); +} + +EC_KEY *d2i_ECParameters(EC_KEY **a, const unsigned char **in, long len) +{ + EC_KEY *ret; + + if (in == NULL || *in == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (a == NULL || *a == NULL) { + if ((ret = EC_KEY_new()) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + return NULL; + } + } else + ret = *a; + + if (!d2i_ECPKParameters(&ret->group, in, len)) { + if (a == NULL || *a != ret) + EC_KEY_free(ret); + else + ret->dirty_cnt++; + return NULL; + } + + if (EC_GROUP_get_curve_name(ret->group) == NID_sm2) + EC_KEY_set_flags(ret, EC_FLAG_SM2_RANGE); + + ret->dirty_cnt++; + + if (a) + *a = ret; + + return ret; +} + +EC_KEY *o2i_ECPublicKey(EC_KEY **a, const unsigned char **in, long len) +{ + EC_KEY *ret = NULL; + + if (a == NULL || (*a) == NULL || (*a)->group == NULL) { + /* + * sorry, but a EC_GROUP-structure is necessary to set the public key + */ + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + ret = *a; + /* EC_KEY_opt2key updates dirty_cnt */ + if (!EC_KEY_oct2key(ret, *in, len, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + return 0; + } + *in += len; + return ret; +} + +int i2o_ECPublicKey(const EC_KEY *a, unsigned char **out) +{ + size_t buf_len = 0; + int new_buffer = 0; + + if (a == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + buf_len = EC_POINT_point2oct(a->group, a->pub_key, + a->conv_form, NULL, 0, NULL); + + if (out == NULL || buf_len == 0) + /* out == NULL => just return the length of the octet string */ + return buf_len; + + if (*out == NULL) { + if ((*out = OPENSSL_malloc(buf_len)) == NULL) + return 0; + new_buffer = 1; + } + if (!EC_POINT_point2oct(a->group, a->pub_key, a->conv_form, + *out, buf_len, NULL)) { + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + if (new_buffer) { + OPENSSL_free(*out); + *out = NULL; + } + return 0; + } + if (!new_buffer) + *out += buf_len; + return buf_len; +} + +DECLARE_ASN1_FUNCTIONS(ECDSA_SIG) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(ECDSA_SIG, ECDSA_SIG) + +#endif /* FIPS_MODULE */ + +ECDSA_SIG *ECDSA_SIG_new(void) +{ + ECDSA_SIG *sig = OPENSSL_zalloc(sizeof(*sig)); + + return sig; +} + +void ECDSA_SIG_free(ECDSA_SIG *sig) +{ + if (sig == NULL) + return; + BN_clear_free(sig->r); + BN_clear_free(sig->s); + OPENSSL_free(sig); +} + +ECDSA_SIG *d2i_ECDSA_SIG(ECDSA_SIG **psig, const unsigned char **ppin, long len) +{ + ECDSA_SIG *sig; + + if (len < 0) + return NULL; + if (psig != NULL && *psig != NULL) { + sig = *psig; + } else { + sig = ECDSA_SIG_new(); + if (sig == NULL) + return NULL; + } + if (sig->r == NULL) + sig->r = BN_new(); + if (sig->s == NULL) + sig->s = BN_new(); + if (sig->r == NULL || sig->s == NULL + || ossl_decode_der_dsa_sig(sig->r, sig->s, ppin, (size_t)len) == 0) { + if (psig == NULL || *psig == NULL) + ECDSA_SIG_free(sig); + return NULL; + } + if (psig != NULL && *psig == NULL) + *psig = sig; + return sig; +} + +int i2d_ECDSA_SIG(const ECDSA_SIG *sig, unsigned char **ppout) +{ + BUF_MEM *buf = NULL; + size_t encoded_len; + WPACKET pkt; + + if (ppout == NULL) { + if (!WPACKET_init_null(&pkt, 0)) + return -1; + } else if (*ppout == NULL) { + if ((buf = BUF_MEM_new()) == NULL + || !WPACKET_init_len(&pkt, buf, 0)) { + BUF_MEM_free(buf); + return -1; + } + } else { + if (!WPACKET_init_static_len(&pkt, *ppout, SIZE_MAX, 0)) + return -1; + } + + if (!ossl_encode_der_dsa_sig(&pkt, sig->r, sig->s) + || !WPACKET_get_total_written(&pkt, &encoded_len) + || !WPACKET_finish(&pkt)) { + BUF_MEM_free(buf); + WPACKET_cleanup(&pkt); + return -1; + } + + if (ppout != NULL) { + if (*ppout == NULL) { + *ppout = (unsigned char *)buf->data; + buf->data = NULL; + BUF_MEM_free(buf); + } else { + *ppout += encoded_len; + } + } + + return (int)encoded_len; +} + +void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps) +{ + if (pr != NULL) + *pr = sig->r; + if (ps != NULL) + *ps = sig->s; +} + +const BIGNUM *ECDSA_SIG_get0_r(const ECDSA_SIG *sig) +{ + return sig->r; +} + +const BIGNUM *ECDSA_SIG_get0_s(const ECDSA_SIG *sig) +{ + return sig->s; +} + +int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s) +{ + if (r == NULL || s == NULL) + return 0; + BN_clear_free(sig->r); + BN_clear_free(sig->s); + sig->r = r; + sig->s = s; + return 1; +} + +int ECDSA_size(const EC_KEY *ec) +{ + int ret; + ECDSA_SIG sig; + const EC_GROUP *group; + const BIGNUM *bn; + + if (ec == NULL) + return 0; + group = EC_KEY_get0_group(ec); + if (group == NULL) + return 0; + + bn = EC_GROUP_get0_order(group); + if (bn == NULL) + return 0; + + sig.r = sig.s = (BIGNUM *)bn; + ret = i2d_ECDSA_SIG(&sig, NULL); + + if (ret < 0) + ret = 0; + return ret; +} diff --git a/tests/ctests/err.c b/tests/ctests/err.c new file mode 100644 index 000000000..b95182d70 --- /dev/null +++ b/tests/ctests/err.c @@ -0,0 +1,898 @@ +/* + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#define OSSL_FORCE_ERR_STATE + +#include +#include +#include +#include "crypto/cryptlib.h" +#include "internal/err.h" +#include "crypto/err.h" +#include +#include +#include +#include +#include +#include "internal/thread_once.h" +#include "crypto/ctype.h" +#include "internal/constant_time.h" +#include "internal/e_os.h" +#include "err_local.h" + +/* Forward declaration in case it's not published because of configuration */ +ERR_STATE *ERR_get_state(void); + +#ifndef OPENSSL_NO_ERR +static int err_load_strings(const ERR_STRING_DATA *str); +#endif + +#ifndef OPENSSL_NO_ERR +static ERR_STRING_DATA ERR_str_libraries[] = { + {ERR_PACK(ERR_LIB_NONE, 0, 0), "unknown library"}, + {ERR_PACK(ERR_LIB_SYS, 0, 0), "system library"}, + {ERR_PACK(ERR_LIB_BN, 0, 0), "bignum routines"}, + {ERR_PACK(ERR_LIB_RSA, 0, 0), "rsa routines"}, + {ERR_PACK(ERR_LIB_DH, 0, 0), "Diffie-Hellman routines"}, + {ERR_PACK(ERR_LIB_EVP, 0, 0), "digital envelope routines"}, + {ERR_PACK(ERR_LIB_BUF, 0, 0), "memory buffer routines"}, + {ERR_PACK(ERR_LIB_OBJ, 0, 0), "object identifier routines"}, + {ERR_PACK(ERR_LIB_PEM, 0, 0), "PEM routines"}, + {ERR_PACK(ERR_LIB_DSA, 0, 0), "dsa routines"}, + {ERR_PACK(ERR_LIB_X509, 0, 0), "x509 certificate routines"}, + {ERR_PACK(ERR_LIB_ASN1, 0, 0), "asn1 encoding routines"}, + {ERR_PACK(ERR_LIB_CONF, 0, 0), "configuration file routines"}, + {ERR_PACK(ERR_LIB_CRYPTO, 0, 0), "common libcrypto routines"}, + {ERR_PACK(ERR_LIB_EC, 0, 0), "elliptic curve routines"}, + {ERR_PACK(ERR_LIB_ECDSA, 0, 0), "ECDSA routines"}, + {ERR_PACK(ERR_LIB_ECDH, 0, 0), "ECDH routines"}, + {ERR_PACK(ERR_LIB_SSL, 0, 0), "SSL routines"}, + {ERR_PACK(ERR_LIB_BIO, 0, 0), "BIO routines"}, + {ERR_PACK(ERR_LIB_PKCS7, 0, 0), "PKCS7 routines"}, + {ERR_PACK(ERR_LIB_X509V3, 0, 0), "X509 V3 routines"}, + {ERR_PACK(ERR_LIB_PKCS12, 0, 0), "PKCS12 routines"}, + {ERR_PACK(ERR_LIB_RAND, 0, 0), "random number generator"}, + {ERR_PACK(ERR_LIB_DSO, 0, 0), "DSO support routines"}, + {ERR_PACK(ERR_LIB_TS, 0, 0), "time stamp routines"}, + {ERR_PACK(ERR_LIB_ENGINE, 0, 0), "engine routines"}, + {ERR_PACK(ERR_LIB_OCSP, 0, 0), "OCSP routines"}, + {ERR_PACK(ERR_LIB_UI, 0, 0), "UI routines"}, + {ERR_PACK(ERR_LIB_FIPS, 0, 0), "FIPS routines"}, + {ERR_PACK(ERR_LIB_CMS, 0, 0), "CMS routines"}, + {ERR_PACK(ERR_LIB_CRMF, 0, 0), "CRMF routines"}, + {ERR_PACK(ERR_LIB_CMP, 0, 0), "CMP routines"}, + {ERR_PACK(ERR_LIB_HMAC, 0, 0), "HMAC routines"}, + {ERR_PACK(ERR_LIB_CT, 0, 0), "CT routines"}, + {ERR_PACK(ERR_LIB_ASYNC, 0, 0), "ASYNC routines"}, + {ERR_PACK(ERR_LIB_KDF, 0, 0), "KDF routines"}, + {ERR_PACK(ERR_LIB_OSSL_STORE, 0, 0), "STORE routines"}, + {ERR_PACK(ERR_LIB_SM2, 0, 0), "SM2 routines"}, + {ERR_PACK(ERR_LIB_ESS, 0, 0), "ESS routines"}, + {ERR_PACK(ERR_LIB_PROV, 0, 0), "Provider routines"}, + {ERR_PACK(ERR_LIB_OSSL_ENCODER, 0, 0), "ENCODER routines"}, + {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, 0), "DECODER routines"}, + {ERR_PACK(ERR_LIB_HTTP, 0, 0), "HTTP routines"}, + {0, NULL}, +}; + +/* + * Should make sure that all ERR_R_ reasons defined in include/openssl/err.h.in + * are listed. For maintainability, please keep all reasons in the same order. + */ +static ERR_STRING_DATA ERR_str_reasons[] = { + {ERR_R_SYS_LIB, "system lib"}, + {ERR_R_BN_LIB, "BN lib"}, + {ERR_R_RSA_LIB, "RSA lib"}, + {ERR_R_DH_LIB, "DH lib"}, + {ERR_R_EVP_LIB, "EVP lib"}, + {ERR_R_BUF_LIB, "BUF lib"}, + {ERR_R_OBJ_LIB, "OBJ lib"}, + {ERR_R_PEM_LIB, "PEM lib"}, + {ERR_R_DSA_LIB, "DSA lib"}, + {ERR_R_X509_LIB, "X509 lib"}, + {ERR_R_ASN1_LIB, "ASN1 lib"}, + {ERR_R_CRYPTO_LIB, "CRYPTO lib"}, + {ERR_R_EC_LIB, "EC lib"}, + {ERR_R_BIO_LIB, "BIO lib"}, + {ERR_R_PKCS7_LIB, "PKCS7 lib"}, + {ERR_R_X509V3_LIB, "X509V3 lib"}, + {ERR_R_ENGINE_LIB, "ENGINE lib"}, + {ERR_R_UI_LIB, "UI lib"}, + {ERR_R_ECDSA_LIB, "ECDSA lib"}, + {ERR_R_OSSL_STORE_LIB, "OSSL_STORE lib"}, + {ERR_R_OSSL_DECODER_LIB, "OSSL_DECODER lib"}, + + {ERR_R_FATAL, "fatal"}, + {ERR_R_MALLOC_FAILURE, "malloc failure"}, + {ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, + "called a function you should not call"}, + {ERR_R_PASSED_NULL_PARAMETER, "passed a null parameter"}, + {ERR_R_INTERNAL_ERROR, "internal error"}, + {ERR_R_DISABLED, "called a function that was disabled at compile-time"}, + {ERR_R_INIT_FAIL, "init fail"}, + {ERR_R_PASSED_INVALID_ARGUMENT, "passed invalid argument"}, + {ERR_R_OPERATION_FAIL, "operation fail"}, + {ERR_R_INVALID_PROVIDER_FUNCTIONS, "invalid provider functions"}, + {ERR_R_INTERRUPTED_OR_CANCELLED, "interrupted or cancelled"}, + {ERR_R_NESTED_ASN1_ERROR, "nested asn1 error"}, + {ERR_R_MISSING_ASN1_EOS, "missing asn1 eos"}, + /* + * Something is unsupported, exactly what is expressed with additional data + */ + {ERR_R_UNSUPPORTED, "unsupported"}, + /* + * A fetch failed for other reasons than the name to be fetched being + * unsupported. + */ + {ERR_R_FETCH_FAILED, "fetch failed"}, + {ERR_R_INVALID_PROPERTY_DEFINITION, "invalid property definition"}, + {ERR_R_UNABLE_TO_GET_READ_LOCK, "unable to get read lock"}, + {ERR_R_UNABLE_TO_GET_WRITE_LOCK, "unable to get write lock"}, + {0, NULL}, +}; +#endif + +static CRYPTO_ONCE err_init = CRYPTO_ONCE_STATIC_INIT; +static int set_err_thread_local; +static CRYPTO_THREAD_LOCAL err_thread_local; + +static CRYPTO_ONCE err_string_init = CRYPTO_ONCE_STATIC_INIT; +static CRYPTO_RWLOCK *err_string_lock = NULL; + +#ifndef OPENSSL_NO_ERR +static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *); +#endif + +/* + * The internal state + */ + +#ifndef OPENSSL_NO_ERR +static LHASH_OF(ERR_STRING_DATA) *int_error_hash = NULL; +#endif +static int int_err_library_number = ERR_LIB_USER; + +typedef enum ERR_GET_ACTION_e { + EV_POP, EV_PEEK, EV_PEEK_LAST +} ERR_GET_ACTION; + +static unsigned long get_error_values(ERR_GET_ACTION g, + const char **file, int *line, + const char **func, const char **data, + int *flags); + +#ifndef OPENSSL_NO_ERR +static unsigned long err_string_data_hash(const ERR_STRING_DATA *a) +{ + unsigned long ret, l; + + l = a->error; + ret = l ^ ERR_GET_LIB(l); + return (ret ^ ret % 19 * 13); +} + +static int err_string_data_cmp(const ERR_STRING_DATA *a, + const ERR_STRING_DATA *b) +{ + if (a->error == b->error) + return 0; + return a->error > b->error ? 1 : -1; +} + +static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *d) +{ + ERR_STRING_DATA *p = NULL; + + if (!CRYPTO_THREAD_read_lock(err_string_lock)) + return NULL; + p = lh_ERR_STRING_DATA_retrieve(int_error_hash, d); + CRYPTO_THREAD_unlock(err_string_lock); + + return p; +} +#endif + +void OSSL_ERR_STATE_free(ERR_STATE *state) +{ + int i; + + if (state == NULL) + return; + for (i = 0; i < ERR_NUM_ERRORS; i++) { + err_clear(state, i, 1); + } + CRYPTO_free(state, OPENSSL_FILE, OPENSSL_LINE); +} + +DEFINE_RUN_ONCE_STATIC(do_err_strings_init) +{ + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return 0; + err_string_lock = CRYPTO_THREAD_lock_new(); + if (err_string_lock == NULL) + return 0; +#ifndef OPENSSL_NO_ERR + int_error_hash = lh_ERR_STRING_DATA_new(err_string_data_hash, + err_string_data_cmp); + if (int_error_hash == NULL) { + CRYPTO_THREAD_lock_free(err_string_lock); + err_string_lock = NULL; + return 0; + } +#endif + return 1; +} + +void err_cleanup(void) +{ + if (set_err_thread_local != 0) + CRYPTO_THREAD_cleanup_local(&err_thread_local); + CRYPTO_THREAD_lock_free(err_string_lock); + err_string_lock = NULL; +#ifndef OPENSSL_NO_ERR + lh_ERR_STRING_DATA_free(int_error_hash); + int_error_hash = NULL; +#endif +} + +#ifndef OPENSSL_NO_ERR +/* + * Legacy; pack in the library. + */ +static void err_patch(int lib, ERR_STRING_DATA *str) +{ + unsigned long plib = ERR_PACK(lib, 0, 0); + + for (; str->error != 0; str++) + str->error |= plib; +} + +/* + * Hash in |str| error strings. Assumes the RUN_ONCE was done. + */ +static int err_load_strings(const ERR_STRING_DATA *str) +{ + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + for (; str->error; str++) + (void)lh_ERR_STRING_DATA_insert(int_error_hash, + (ERR_STRING_DATA *)str); + CRYPTO_THREAD_unlock(err_string_lock); + return 1; +} +#endif + +int ossl_err_load_ERR_strings(void) +{ +#ifndef OPENSSL_NO_ERR + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + err_load_strings(ERR_str_libraries); + err_load_strings(ERR_str_reasons); +#endif + return 1; +} + +int ERR_load_strings(int lib, ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (ossl_err_load_ERR_strings() == 0) + return 0; + + err_patch(lib, str); + err_load_strings(str); +#endif + + return 1; +} + +int ERR_load_strings_const(const ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (ossl_err_load_ERR_strings() == 0) + return 0; + err_load_strings(str); +#endif + + return 1; +} + +int ERR_unload_strings(int lib, ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + /* + * We don't need to ERR_PACK the lib, since that was done (to + * the table) when it was loaded. + */ + for (; str->error; str++) + (void)lh_ERR_STRING_DATA_delete(int_error_hash, str); + CRYPTO_THREAD_unlock(err_string_lock); +#endif + + return 1; +} + +void err_free_strings_int(void) +{ + /* obsolete */ +} + +/********************************************************/ + +void ERR_clear_error(void) +{ + int i; + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return; + + for (i = 0; i < ERR_NUM_ERRORS; i++) { + err_clear(es, i, 0); + } + es->top = es->bottom = 0; +} + +unsigned long ERR_get_error(void) +{ + return get_error_values(EV_POP, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_get_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_POP, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_get_error_line(const char **file, int *line) +{ + return get_error_values(EV_POP, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_get_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_POP, file, line, NULL, data, flags); +} +#endif + +unsigned long ERR_peek_error(void) +{ + return get_error_values(EV_PEEK, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_peek_error_line(const char **file, int *line) +{ + return get_error_values(EV_PEEK, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_peek_error_func(const char **func) +{ + return get_error_values(EV_PEEK, NULL, NULL, func, NULL, NULL); +} + +unsigned long ERR_peek_error_data(const char **data, int *flags) +{ + return get_error_values(EV_PEEK, NULL, NULL, NULL, data, flags); +} + +unsigned long ERR_peek_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_peek_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK, file, line, NULL, data, flags); +} +#endif + +unsigned long ERR_peek_last_error(void) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_peek_last_error_line(const char **file, int *line) +{ + return get_error_values(EV_PEEK_LAST, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_peek_last_error_func(const char **func) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, func, NULL, NULL); +} + +unsigned long ERR_peek_last_error_data(const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, NULL, data, flags); +} + +unsigned long ERR_peek_last_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_peek_last_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, file, line, NULL, data, flags); +} +#endif + +static unsigned long get_error_values(ERR_GET_ACTION g, + const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + int i = 0; + ERR_STATE *es; + unsigned long ret; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + /* + * Clear anything that should have been cleared earlier. We do this + * here because this doesn't have constant-time issues. + */ + while (es->bottom != es->top) { + if (es->err_flags[es->top] & ERR_FLAG_CLEAR) { + err_clear(es, es->top, 0); + es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1; + continue; + } + i = (es->bottom + 1) % ERR_NUM_ERRORS; + if (es->err_flags[i] & ERR_FLAG_CLEAR) { + es->bottom = i; + err_clear(es, es->bottom, 0); + continue; + } + break; + } + + /* If everything has been cleared, the stack is empty. */ + if (es->bottom == es->top) + return 0; + + /* Which error, the top of stack (latest one) or the first one? */ + if (g == EV_PEEK_LAST) + i = es->top; + else + i = (es->bottom + 1) % ERR_NUM_ERRORS; + + ret = es->err_buffer[i]; + if (g == EV_POP) { + es->bottom = i; + es->err_buffer[i] = 0; + } + + if (file != NULL) { + *file = es->err_file[i]; + if (*file == NULL) + *file = ""; + } + if (line != NULL) + *line = es->err_line[i]; + if (func != NULL) { + *func = es->err_func[i]; + if (*func == NULL) + *func = ""; + } + if (flags != NULL) + *flags = es->err_data_flags[i]; + if (data == NULL) { + if (g == EV_POP) { + err_clear_data(es, i, 0); + } + } else { + *data = es->err_data[i]; + if (*data == NULL) { + *data = ""; + if (flags != NULL) + *flags = 0; + } + } + return ret; +} + +void ossl_err_string_int(unsigned long e, const char *func, + char *buf, size_t len) +{ + char lsbuf[64], rsbuf[256]; + const char *ls, *rs = NULL; + unsigned long l, r; + + if (len == 0) + return; + + l = ERR_GET_LIB(e); + ls = ERR_lib_error_string(e); + if (ls == NULL) { + BIO_snprintf(lsbuf, sizeof(lsbuf), "lib(%lu)", l); + ls = lsbuf; + } + + /* + * ERR_reason_error_string() can't safely return system error strings, + * since it would call openssl_strerror_r(), which needs a buffer for + * thread safety. So for system errors, we call openssl_strerror_r() + * directly instead. + */ + r = ERR_GET_REASON(e); +#ifndef OPENSSL_NO_ERR + if (ERR_SYSTEM_ERROR(e)) { + if (openssl_strerror_r(r, rsbuf, sizeof(rsbuf))) + rs = rsbuf; + } else { + rs = ERR_reason_error_string(e); + } +#endif + if (rs == NULL) { + BIO_snprintf(rsbuf, sizeof(rsbuf), "reason(%lu)", + r & ~(ERR_RFLAGS_MASK << ERR_RFLAGS_OFFSET)); + rs = rsbuf; + } + + BIO_snprintf(buf, len, "error:%08lX:%s:%s:%s", e, ls, func, rs); + if (strlen(buf) == len - 1) { + /* Didn't fit; use a minimal format. */ + BIO_snprintf(buf, len, "err:%lx:%lx:%lx:%lx", e, l, 0L, r); + } +} + + +void ERR_error_string_n(unsigned long e, char *buf, size_t len) +{ + ossl_err_string_int(e, "", buf, len); +} + +/* + * ERR_error_string_n should be used instead for ret != NULL as + * ERR_error_string cannot know how large the buffer is + */ +char *ERR_error_string(unsigned long e, char *ret) +{ + static char buf[256]; + + if (ret == NULL) + ret = buf; + ERR_error_string_n(e, ret, (int)sizeof(buf)); + return ret; +} + +const char *ERR_lib_error_string(unsigned long e) +{ +#ifndef OPENSSL_NO_ERR + ERR_STRING_DATA d, *p; + unsigned long l; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) { + return NULL; + } + + l = ERR_GET_LIB(e); + d.error = ERR_PACK(l, 0, 0); + p = int_err_get_item(&d); + return ((p == NULL) ? NULL : p->string); +#else + return NULL; +#endif +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +const char *ERR_func_error_string(unsigned long e) +{ + return NULL; +} +#endif + +const char *ERR_reason_error_string(unsigned long e) +{ +#ifndef OPENSSL_NO_ERR + ERR_STRING_DATA d, *p = NULL; + unsigned long l, r; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) { + return NULL; + } + + /* + * ERR_reason_error_string() can't safely return system error strings, + * since openssl_strerror_r() needs a buffer for thread safety, and we + * haven't got one that would serve any sensible purpose. + */ + if (ERR_SYSTEM_ERROR(e)) + return NULL; + + l = ERR_GET_LIB(e); + r = ERR_GET_REASON(e); + d.error = ERR_PACK(l, 0, r); + p = int_err_get_item(&d); + if (p == NULL) { + d.error = ERR_PACK(0, 0, r); + p = int_err_get_item(&d); + } + return ((p == NULL) ? NULL : p->string); +#else + return NULL; +#endif +} + +static void err_delete_thread_state(void *unused) +{ + ERR_STATE *state = CRYPTO_THREAD_get_local(&err_thread_local); + if (state == NULL) + return; + + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + OSSL_ERR_STATE_free(state); +} + +#ifndef OPENSSL_NO_DEPRECATED_1_1_0 +void ERR_remove_thread_state(void *dummy) +{ +} +#endif + +#ifndef OPENSSL_NO_DEPRECATED_1_0_0 +void ERR_remove_state(unsigned long pid) +{ +} +#endif + +DEFINE_RUN_ONCE_STATIC(err_do_init) +{ + set_err_thread_local = 1; + return CRYPTO_THREAD_init_local(&err_thread_local, NULL); +} + +ERR_STATE *ossl_err_get_state_int(void) +{ + ERR_STATE *state; + int saveerrno = get_last_sys_error(); + + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return NULL; + + if (!RUN_ONCE(&err_init, err_do_init)) + return NULL; + + state = CRYPTO_THREAD_get_local(&err_thread_local); + if (state == (ERR_STATE*)-1) + return NULL; + + if (state == NULL) { + if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)-1)) + return NULL; + + state = OSSL_ERR_STATE_new(); + if (state == NULL) { + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + return NULL; + } + + if (!ossl_init_thread_start(NULL, NULL, err_delete_thread_state) + || !CRYPTO_THREAD_set_local(&err_thread_local, state)) { + OSSL_ERR_STATE_free(state); + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + return NULL; + } + + /* Ignore failures from these */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); + } + + set_sys_error(saveerrno); + return state; +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +ERR_STATE *ERR_get_state(void) +{ + return ossl_err_get_state_int(); +} +#endif + + +/* + * err_shelve_state returns the current thread local error state + * and freezes the error module until err_unshelve_state is called. + */ +int err_shelve_state(void **state) +{ + int saveerrno = get_last_sys_error(); + + /* + * Note, at present our only caller is OPENSSL_init_crypto(), indirectly + * via ossl_init_load_crypto_nodelete(), by which point the requested + * "base" initialization has already been performed, so the below call is a + * NOOP, that re-enters OPENSSL_init_crypto() only to quickly return. + * + * If are no other valid callers of this function, the call below can be + * removed, avoiding the re-entry into OPENSSL_init_crypto(). If there are + * potential uses that are not from inside OPENSSL_init_crypto(), then this + * call is needed, but some care is required to make sure that the re-entry + * remains a NOOP. + */ + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return 0; + + if (!RUN_ONCE(&err_init, err_do_init)) + return 0; + + *state = CRYPTO_THREAD_get_local(&err_thread_local); + if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)-1)) + return 0; + + set_sys_error(saveerrno); + return 1; +} + +/* + * err_unshelve_state restores the error state that was returned + * by err_shelve_state previously. + */ +void err_unshelve_state(void* state) +{ + if (state != (void*)-1) + CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)state); +} + +int ERR_get_next_error_library(void) +{ + int ret; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + ret = int_err_library_number++; + CRYPTO_THREAD_unlock(err_string_lock); + return ret; +} + +static int err_set_error_data_int(char *data, size_t size, int flags, + int deallocate) +{ + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + err_clear_data(es, es->top, deallocate); + err_set_data(es, es->top, data, size, flags); + + return 1; +} + +void ERR_set_error_data(char *data, int flags) +{ + /* + * This function is void so we cannot propagate the error return. Since it + * is also in the public API we can't change the return type. + * + * We estimate the size of the data. If it's not flagged as allocated, + * then this is safe, and if it is flagged as allocated, then our size + * may be smaller than the actual allocation, but that doesn't matter + * too much, the buffer will remain untouched or will eventually be + * reallocated to a new size. + * + * callers should be advised that this function takes over ownership of + * the allocated memory, i.e. they can't count on the pointer to remain + * valid. + */ + err_set_error_data_int(data, strlen(data) + 1, flags, 1); +} + +void ERR_add_error_data(int num, ...) +{ + va_list args; + va_start(args, num); + ERR_add_error_vdata(num, args); + va_end(args); +} + +void ERR_add_error_vdata(int num, va_list args) +{ + int i, len, size; + int flags = ERR_TXT_MALLOCED | ERR_TXT_STRING; + char *str, *arg; + ERR_STATE *es; + + /* Get the current error data; if an allocated string get it. */ + es = ossl_err_get_state_int(); + if (es == NULL) + return; + i = es->top; + + /* + * If err_data is allocated already, reuse the space. + * Otherwise, allocate a small new buffer. + */ + if ((es->err_data_flags[i] & flags) == flags + && ossl_assert(es->err_data[i] != NULL)) { + str = es->err_data[i]; + size = es->err_data_size[i]; + + /* + * To protect the string we just grabbed from tampering by other + * functions we may call, or to protect them from freeing a pointer + * that may no longer be valid at that point, we clear away the + * data pointer and the flags. We will set them again at the end + * of this function. + */ + es->err_data[i] = NULL; + es->err_data_flags[i] = 0; + } else if ((str = OPENSSL_malloc(size = 81)) == NULL) { + return; + } else { + str[0] = '\0'; + } + len = strlen(str); + + while (--num >= 0) { + arg = va_arg(args, char *); + if (arg == NULL) + arg = ""; + len += strlen(arg); + if (len >= size) { + char *p; + + size = len + 20; + p = OPENSSL_realloc(str, size); + if (p == NULL) { + OPENSSL_free(str); + return; + } + str = p; + } + OPENSSL_strlcat(str, arg, (size_t)size); + } + if (!err_set_error_data_int(str, size, flags, 0)) + OPENSSL_free(str); +} + +void err_clear_last_constant_time(int clear) +{ + ERR_STATE *es; + int top; + + es = ossl_err_get_state_int(); + if (es == NULL) + return; + + top = es->top; + + /* + * Flag error as cleared but remove it elsewhere to avoid two errors + * accessing the same error stack location, revealing timing information. + */ + clear = constant_time_select_int(constant_time_eq_int(clear, 0), + 0, ERR_FLAG_CLEAR); + es->err_flags[top] |= clear; +} diff --git a/tests/ctests/ess_lib.c b/tests/ctests/ess_lib.c new file mode 100644 index 000000000..ff174470d --- /dev/null +++ b/tests/ctests/ess_lib.c @@ -0,0 +1,368 @@ +/* + * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include +#include +#include "internal/sizes.h" +#include "crypto/ess.h" +#include "crypto/x509.h" + +static ESS_CERT_ID *ESS_CERT_ID_new_init(const X509 *cert, + int set_issuer_serial); +static ESS_CERT_ID_V2 *ESS_CERT_ID_V2_new_init(const EVP_MD *hash_alg, + const X509 *cert, + int set_issuer_serial); + +ESS_SIGNING_CERT *OSSL_ESS_signing_cert_new_init(const X509 *signcert, + const STACK_OF(X509) *certs, + int set_issuer_serial) +{ + ESS_CERT_ID *cid = NULL; + ESS_SIGNING_CERT *sc; + int i; + + if ((sc = ESS_SIGNING_CERT_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (sc->cert_ids == NULL + && (sc->cert_ids = sk_ESS_CERT_ID_new_null()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + + if ((cid = ESS_CERT_ID_new_init(signcert, set_issuer_serial)) == NULL + || !sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + for (i = 0; i < sk_X509_num(certs); ++i) { + X509 *cert = sk_X509_value(certs, i); + + if ((cid = ESS_CERT_ID_new_init(cert, 1)) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + } + + return sc; + err: + ESS_SIGNING_CERT_free(sc); + ESS_CERT_ID_free(cid); + return NULL; +} + +static ESS_CERT_ID *ESS_CERT_ID_new_init(const X509 *cert, + int set_issuer_serial) +{ + ESS_CERT_ID *cid = NULL; + GENERAL_NAME *name = NULL; + unsigned char cert_sha1[SHA_DIGEST_LENGTH]; + + if ((cid = ESS_CERT_ID_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!X509_digest(cert, EVP_sha1(), cert_sha1, NULL)) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + if (!ASN1_OCTET_STRING_set(cid->hash, cert_sha1, SHA_DIGEST_LENGTH)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + /* Setting the issuer/serial if requested. */ + if (!set_issuer_serial) + return cid; + + if (cid->issuer_serial == NULL + && (cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if ((name = GENERAL_NAME_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + name->type = GEN_DIRNAME; + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + name = NULL; /* Ownership is lost. */ + ASN1_INTEGER_free(cid->issuer_serial->serial); + if ((cid->issuer_serial->serial + = ASN1_INTEGER_dup(X509_get0_serialNumber(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + return cid; + err: + GENERAL_NAME_free(name); + ESS_CERT_ID_free(cid); + return NULL; +} + +ESS_SIGNING_CERT_V2 *OSSL_ESS_signing_cert_v2_new_init(const EVP_MD *hash_alg, + const X509 *signcert, + const + STACK_OF(X509) *certs, + int set_issuer_serial) +{ + ESS_CERT_ID_V2 *cid = NULL; + ESS_SIGNING_CERT_V2 *sc; + int i; + + if ((sc = ESS_SIGNING_CERT_V2_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + cid = ESS_CERT_ID_V2_new_init(hash_alg, signcert, set_issuer_serial); + if (cid == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + cid = NULL; + + for (i = 0; i < sk_X509_num(certs); ++i) { + X509 *cert = sk_X509_value(certs, i); + + if ((cid = ESS_CERT_ID_V2_new_init(hash_alg, cert, 1)) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + cid = NULL; + } + + return sc; + err: + ESS_SIGNING_CERT_V2_free(sc); + ESS_CERT_ID_V2_free(cid); + return NULL; +} + +static ESS_CERT_ID_V2 *ESS_CERT_ID_V2_new_init(const EVP_MD *hash_alg, + const X509 *cert, + int set_issuer_serial) +{ + ESS_CERT_ID_V2 *cid; + GENERAL_NAME *name = NULL; + unsigned char hash[EVP_MAX_MD_SIZE]; + unsigned int hash_len = sizeof(hash); + X509_ALGOR *alg = NULL; + + memset(hash, 0, sizeof(hash)); + + if ((cid = ESS_CERT_ID_V2_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + + if (!EVP_MD_is_a(hash_alg, SN_sha256)) { + alg = X509_ALGOR_new(); + if (alg == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + X509_ALGOR_set_md(alg, hash_alg); + if (alg->algorithm == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + cid->hash_alg = alg; + alg = NULL; + } else { + cid->hash_alg = NULL; + } + + if (!X509_digest(cert, hash_alg, hash, &hash_len)) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + + if (!ASN1_OCTET_STRING_set(cid->hash, hash, hash_len)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + if (!set_issuer_serial) + return cid; + + if ((cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if ((name = GENERAL_NAME_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + name->type = GEN_DIRNAME; + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + name = NULL; /* Ownership is lost. */ + ASN1_INTEGER_free(cid->issuer_serial->serial); + cid->issuer_serial->serial = ASN1_INTEGER_dup(X509_get0_serialNumber(cert)); + if (cid->issuer_serial->serial == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + return cid; + err: + X509_ALGOR_free(alg); + GENERAL_NAME_free(name); + ESS_CERT_ID_V2_free(cid); + return NULL; +} + +static int ess_issuer_serial_cmp(const ESS_ISSUER_SERIAL *is, const X509 *cert) +{ + GENERAL_NAME *issuer; + + if (is == NULL || cert == NULL || sk_GENERAL_NAME_num(is->issuer) != 1) + return -1; + + issuer = sk_GENERAL_NAME_value(is->issuer, 0); + if (issuer->type != GEN_DIRNAME + || X509_NAME_cmp(issuer->d.dirn, X509_get_issuer_name(cert)) != 0) + return -1; + + return ASN1_INTEGER_cmp(is->serial, X509_get0_serialNumber(cert)); +} + +/* + * Find the cert in |certs| referenced by |cid| if not NULL, else by |cid_v2|. + * The cert must be the first one in |certs| if and only if |index| is 0. + * Return 0 on not found, -1 on error, else 1 + the position in |certs|. + */ +static int find(const ESS_CERT_ID *cid, const ESS_CERT_ID_V2 *cid_v2, + int index, const STACK_OF(X509) *certs) +{ + const X509 *cert; + EVP_MD *md = NULL; + char name[OSSL_MAX_NAME_SIZE]; + unsigned char cert_digest[EVP_MAX_MD_SIZE]; + unsigned int len, cid_hash_len; + const ESS_ISSUER_SERIAL *is; + int i; + int ret = -1; + + if (cid == NULL && cid_v2 == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_PASSED_INVALID_ARGUMENT); + return -1; + } + + if (cid != NULL) + strcpy(name, "SHA1"); + else if (cid_v2->hash_alg == NULL) + strcpy(name, "SHA256"); + else + OBJ_obj2txt(name, sizeof(name), cid_v2->hash_alg->algorithm, 0); + + (void)ERR_set_mark(); + md = EVP_MD_fetch(NULL, name, NULL); + + if (md == NULL) + md = (EVP_MD *)EVP_get_digestbyname(name); + + if (md == NULL) { + (void)ERR_clear_last_mark(); + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_DIGEST_ALG_UNKNOWN); + goto end; + } + (void)ERR_pop_to_mark(); + + for (i = 0; i < sk_X509_num(certs); ++i) { + cert = sk_X509_value(certs, i); + + cid_hash_len = cid != NULL ? cid->hash->length : cid_v2->hash->length; + if (!X509_digest(cert, md, cert_digest, &len) + || cid_hash_len != len) { + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_DIGEST_ERROR); + goto end; + } + + if (memcmp(cid != NULL ? cid->hash->data : cid_v2->hash->data, + cert_digest, len) == 0) { + is = cid != NULL ? cid->issuer_serial : cid_v2->issuer_serial; + /* Well, it's not really required to match the serial numbers. */ + if (is == NULL || ess_issuer_serial_cmp(is, cert) == 0) { + if ((i == 0) == (index == 0)) { + ret = i + 1; + goto end; + } + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_ID_WRONG_ORDER); + goto end; + } + } + } + + ret = 0; + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_ID_NOT_FOUND); +end: + EVP_MD_free(md); + return ret; +} + +int OSSL_ESS_check_signing_certs(const ESS_SIGNING_CERT *ss, + const ESS_SIGNING_CERT_V2 *ssv2, + const STACK_OF(X509) *chain, + int require_signing_cert) +{ + int n_v1 = ss == NULL ? -1 : sk_ESS_CERT_ID_num(ss->cert_ids); + int n_v2 = ssv2 == NULL ? -1 : sk_ESS_CERT_ID_V2_num(ssv2->cert_ids); + int i, ret; + + if (require_signing_cert && ss == NULL && ssv2 == NULL) { + ERR_raise(ERR_LIB_ESS, ESS_R_MISSING_SIGNING_CERTIFICATE_ATTRIBUTE); + return -1; + } + if (n_v1 == 0 || n_v2 == 0) { + ERR_raise(ERR_LIB_ESS, ESS_R_EMPTY_ESS_CERT_ID_LIST); + return -1; + } + /* If both ss and ssv2 exist, as required evaluate them independently. */ + for (i = 0; i < n_v1; i++) { + ret = find(sk_ESS_CERT_ID_value(ss->cert_ids, i), NULL, i, chain); + if (ret <= 0) + return ret; + } + for (i = 0; i < n_v2; i++) { + ret = find(NULL, sk_ESS_CERT_ID_V2_value(ssv2->cert_ids, i), i, chain); + if (ret <= 0) + return ret; + } + return 1; +} diff --git a/tests/ctests/pk7_asn1.c b/tests/ctests/pk7_asn1.c new file mode 100644 index 000000000..3abcc3dc8 --- /dev/null +++ b/tests/ctests/pk7_asn1.c @@ -0,0 +1,256 @@ +/* + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include "internal/cryptlib.h" +#include +#include +#include +#include "pk7_local.h" + +/* PKCS#7 ASN1 module */ + +/* This is the ANY DEFINED BY table for the top level PKCS#7 structure */ + +ASN1_ADB_TEMPLATE(p7default) = ASN1_EXP_OPT(PKCS7, d.other, ASN1_ANY, 0); + +ASN1_ADB(PKCS7) = { + ADB_ENTRY(NID_pkcs7_data, ASN1_NDEF_EXP_OPT(PKCS7, d.data, ASN1_OCTET_STRING_NDEF, 0)), + ADB_ENTRY(NID_pkcs7_signed, ASN1_NDEF_EXP_OPT(PKCS7, d.sign, PKCS7_SIGNED, 0)), + ADB_ENTRY(NID_pkcs7_enveloped, ASN1_NDEF_EXP_OPT(PKCS7, d.enveloped, PKCS7_ENVELOPE, 0)), + ADB_ENTRY(NID_pkcs7_signedAndEnveloped, ASN1_NDEF_EXP_OPT(PKCS7, d.signed_and_enveloped, PKCS7_SIGN_ENVELOPE, 0)), + ADB_ENTRY(NID_pkcs7_digest, ASN1_NDEF_EXP_OPT(PKCS7, d.digest, PKCS7_DIGEST, 0)), + ADB_ENTRY(NID_pkcs7_encrypted, ASN1_NDEF_EXP_OPT(PKCS7, d.encrypted, PKCS7_ENCRYPT, 0)) +} ASN1_ADB_END(PKCS7, 0, type, 0, &p7default_tt, NULL); + +/* PKCS#7 streaming support */ +static int pk7_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + ASN1_STREAM_ARG *sarg = exarg; + PKCS7 **pp7 = (PKCS7 **)pval; + + switch (operation) { + + case ASN1_OP_STREAM_PRE: + if (PKCS7_stream(&sarg->boundary, *pp7) <= 0) + return 0; + /* fall through */ + case ASN1_OP_DETACHED_PRE: + sarg->ndef_bio = PKCS7_dataInit(*pp7, sarg->out); + if (!sarg->ndef_bio) + return 0; + break; + + case ASN1_OP_STREAM_POST: + case ASN1_OP_DETACHED_POST: + if (PKCS7_dataFinal(*pp7, sarg->ndef_bio) <= 0) + return 0; + break; + + } + return 1; +} + +ASN1_NDEF_SEQUENCE_cb(PKCS7, pk7_cb) = { + ASN1_SIMPLE(PKCS7, type, ASN1_OBJECT), + ASN1_ADB_OBJECT(PKCS7) +}ASN1_NDEF_SEQUENCE_END_cb(PKCS7, PKCS7) + +PKCS7 *d2i_PKCS7(PKCS7 **a, const unsigned char **in, long len) +{ + PKCS7 *ret; + OSSL_LIB_CTX *libctx = NULL; + const char *propq = NULL; + + if (a != NULL && *a != NULL) { + libctx = (*a)->ctx.libctx; + propq = (*a)->ctx.propq; + } + + ret = (PKCS7 *)ASN1_item_d2i_ex((ASN1_VALUE **)a, in, len, (PKCS7_it()), + libctx, propq); + if (ret != NULL) + ossl_pkcs7_resolve_libctx(ret); + return ret; +} + +int i2d_PKCS7(const PKCS7 *a, unsigned char **out) +{ + return ASN1_item_i2d((const ASN1_VALUE *)a, out, (PKCS7_it()));\ +} + +PKCS7 *PKCS7_new(void) +{ + return (PKCS7 *)ASN1_item_new(ASN1_ITEM_rptr(PKCS7)); +} + +PKCS7 *PKCS7_new_ex(OSSL_LIB_CTX *libctx, const char *propq) +{ + PKCS7 *pkcs7 = (PKCS7 *)ASN1_item_new_ex(ASN1_ITEM_rptr(PKCS7), libctx, + propq); + + if (pkcs7 != NULL) { + pkcs7->ctx.libctx = libctx; + pkcs7->ctx.propq = NULL; + if (propq != NULL) { + pkcs7->ctx.propq = OPENSSL_strdup(propq); + if (pkcs7->ctx.propq == NULL) { + PKCS7_free(pkcs7); + pkcs7 = NULL; + } + } + } + return pkcs7; +} + +void PKCS7_free(PKCS7 *p7) +{ + if (p7 != NULL) { + OPENSSL_free(p7->ctx.propq); + ASN1_item_free((ASN1_VALUE *)p7, ASN1_ITEM_rptr(PKCS7)); + } +} + +IMPLEMENT_ASN1_NDEF_FUNCTION(PKCS7) + +IMPLEMENT_ASN1_DUP_FUNCTION(PKCS7) + +ASN1_NDEF_SEQUENCE(PKCS7_SIGNED) = { + ASN1_SIMPLE(PKCS7_SIGNED, version, ASN1_INTEGER), + ASN1_SET_OF(PKCS7_SIGNED, md_algs, X509_ALGOR), + ASN1_SIMPLE(PKCS7_SIGNED, contents, PKCS7), + ASN1_IMP_SEQUENCE_OF_OPT(PKCS7_SIGNED, cert, X509, 0), + ASN1_IMP_SET_OF_OPT(PKCS7_SIGNED, crl, X509_CRL, 1), + ASN1_SET_OF(PKCS7_SIGNED, signer_info, PKCS7_SIGNER_INFO) +} ASN1_NDEF_SEQUENCE_END(PKCS7_SIGNED) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_SIGNED) + +/* Minor tweak to operation: free up EVP_PKEY */ +static int si_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + if (operation == ASN1_OP_FREE_POST) { + PKCS7_SIGNER_INFO *si = (PKCS7_SIGNER_INFO *)*pval; + EVP_PKEY_free(si->pkey); + } + return 1; +} + +ASN1_SEQUENCE_cb(PKCS7_SIGNER_INFO, si_cb) = { + ASN1_SIMPLE(PKCS7_SIGNER_INFO, version, ASN1_INTEGER), + ASN1_SIMPLE(PKCS7_SIGNER_INFO, issuer_and_serial, PKCS7_ISSUER_AND_SERIAL), + ASN1_SIMPLE(PKCS7_SIGNER_INFO, digest_alg, X509_ALGOR), + /* NB this should be a SET OF but we use a SEQUENCE OF so the + * original order * is retained when the structure is reencoded. + * Since the attributes are implicitly tagged this will not affect + * the encoding. + */ + ASN1_IMP_SEQUENCE_OF_OPT(PKCS7_SIGNER_INFO, auth_attr, X509_ATTRIBUTE, 0), + ASN1_SIMPLE(PKCS7_SIGNER_INFO, digest_enc_alg, X509_ALGOR), + ASN1_SIMPLE(PKCS7_SIGNER_INFO, enc_digest, ASN1_OCTET_STRING), + ASN1_IMP_SET_OF_OPT(PKCS7_SIGNER_INFO, unauth_attr, X509_ATTRIBUTE, 1) +} ASN1_SEQUENCE_END_cb(PKCS7_SIGNER_INFO, PKCS7_SIGNER_INFO) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_SIGNER_INFO) + +ASN1_SEQUENCE(PKCS7_ISSUER_AND_SERIAL) = { + ASN1_SIMPLE(PKCS7_ISSUER_AND_SERIAL, issuer, X509_NAME), + ASN1_SIMPLE(PKCS7_ISSUER_AND_SERIAL, serial, ASN1_INTEGER) +} ASN1_SEQUENCE_END(PKCS7_ISSUER_AND_SERIAL) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_ISSUER_AND_SERIAL) + +ASN1_NDEF_SEQUENCE(PKCS7_ENVELOPE) = { + ASN1_SIMPLE(PKCS7_ENVELOPE, version, ASN1_INTEGER), + ASN1_SET_OF(PKCS7_ENVELOPE, recipientinfo, PKCS7_RECIP_INFO), + ASN1_SIMPLE(PKCS7_ENVELOPE, enc_data, PKCS7_ENC_CONTENT) +} ASN1_NDEF_SEQUENCE_END(PKCS7_ENVELOPE) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_ENVELOPE) + +/* Minor tweak to operation: free up X509 */ +static int ri_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + if (operation == ASN1_OP_FREE_POST) { + PKCS7_RECIP_INFO *ri = (PKCS7_RECIP_INFO *)*pval; + X509_free(ri->cert); + } + return 1; +} + +ASN1_SEQUENCE_cb(PKCS7_RECIP_INFO, ri_cb) = { + ASN1_SIMPLE(PKCS7_RECIP_INFO, version, ASN1_INTEGER), + ASN1_SIMPLE(PKCS7_RECIP_INFO, issuer_and_serial, PKCS7_ISSUER_AND_SERIAL), + ASN1_SIMPLE(PKCS7_RECIP_INFO, key_enc_algor, X509_ALGOR), + ASN1_SIMPLE(PKCS7_RECIP_INFO, enc_key, ASN1_OCTET_STRING) +} ASN1_SEQUENCE_END_cb(PKCS7_RECIP_INFO, PKCS7_RECIP_INFO) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_RECIP_INFO) + +ASN1_NDEF_SEQUENCE(PKCS7_ENC_CONTENT) = { + ASN1_SIMPLE(PKCS7_ENC_CONTENT, content_type, ASN1_OBJECT), + ASN1_SIMPLE(PKCS7_ENC_CONTENT, algorithm, X509_ALGOR), + ASN1_IMP_OPT(PKCS7_ENC_CONTENT, enc_data, ASN1_OCTET_STRING_NDEF, 0) +} ASN1_NDEF_SEQUENCE_END(PKCS7_ENC_CONTENT) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_ENC_CONTENT) + +ASN1_NDEF_SEQUENCE(PKCS7_SIGN_ENVELOPE) = { + ASN1_SIMPLE(PKCS7_SIGN_ENVELOPE, version, ASN1_INTEGER), + ASN1_SET_OF(PKCS7_SIGN_ENVELOPE, recipientinfo, PKCS7_RECIP_INFO), + ASN1_SET_OF(PKCS7_SIGN_ENVELOPE, md_algs, X509_ALGOR), + ASN1_SIMPLE(PKCS7_SIGN_ENVELOPE, enc_data, PKCS7_ENC_CONTENT), + ASN1_IMP_SET_OF_OPT(PKCS7_SIGN_ENVELOPE, cert, X509, 0), + ASN1_IMP_SET_OF_OPT(PKCS7_SIGN_ENVELOPE, crl, X509_CRL, 1), + ASN1_SET_OF(PKCS7_SIGN_ENVELOPE, signer_info, PKCS7_SIGNER_INFO) +} ASN1_NDEF_SEQUENCE_END(PKCS7_SIGN_ENVELOPE) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_SIGN_ENVELOPE) + +ASN1_NDEF_SEQUENCE(PKCS7_ENCRYPT) = { + ASN1_SIMPLE(PKCS7_ENCRYPT, version, ASN1_INTEGER), + ASN1_SIMPLE(PKCS7_ENCRYPT, enc_data, PKCS7_ENC_CONTENT) +} ASN1_NDEF_SEQUENCE_END(PKCS7_ENCRYPT) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_ENCRYPT) + +ASN1_NDEF_SEQUENCE(PKCS7_DIGEST) = { + ASN1_SIMPLE(PKCS7_DIGEST, version, ASN1_INTEGER), + ASN1_SIMPLE(PKCS7_DIGEST, md, X509_ALGOR), + ASN1_SIMPLE(PKCS7_DIGEST, contents, PKCS7), + ASN1_SIMPLE(PKCS7_DIGEST, digest, ASN1_OCTET_STRING) +} ASN1_NDEF_SEQUENCE_END(PKCS7_DIGEST) + +IMPLEMENT_ASN1_FUNCTIONS(PKCS7_DIGEST) + +/* Specials for authenticated attributes */ + +/* + * When signing attributes we want to reorder them to match the sorted + * encoding. + */ + +ASN1_ITEM_TEMPLATE(PKCS7_ATTR_SIGN) = + ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SET_ORDER, 0, PKCS7_ATTRIBUTES, X509_ATTRIBUTE) +ASN1_ITEM_TEMPLATE_END(PKCS7_ATTR_SIGN) + +/* + * When verifying attributes we need to use the received order. So we use + * SEQUENCE OF and tag it to SET OF + */ + +ASN1_ITEM_TEMPLATE(PKCS7_ATTR_VERIFY) = + ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF | ASN1_TFLG_IMPTAG | ASN1_TFLG_UNIVERSAL, + V_ASN1_SET, PKCS7_ATTRIBUTES, X509_ATTRIBUTE) +ASN1_ITEM_TEMPLATE_END(PKCS7_ATTR_VERIFY) + +IMPLEMENT_ASN1_PRINT_FUNCTION(PKCS7) diff --git a/tests/ctests/rsa_asn1.c b/tests/ctests/rsa_asn1.c new file mode 100644 index 000000000..c14cdf415 --- /dev/null +++ b/tests/ctests/rsa_asn1.c @@ -0,0 +1,128 @@ +/* + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* + * RSA low level APIs are deprecated for public use, but still ok for + * internal use. + */ +#include "internal/deprecated.h" + +#include +#include "internal/cryptlib.h" +#include +#include +#include +#include "rsa_local.h" + +/* + * Override the default free and new methods, + * and calculate helper products for multi-prime + * RSA keys. + */ +static int rsa_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + if (operation == ASN1_OP_NEW_PRE) { + *pval = (ASN1_VALUE *)RSA_new(); + if (*pval != NULL) + return 2; + return 0; + } else if (operation == ASN1_OP_FREE_PRE) { + RSA_free((RSA *)*pval); + *pval = NULL; + return 2; + } else if (operation == ASN1_OP_D2I_POST) { + if (((RSA *)*pval)->version != RSA_ASN1_VERSION_MULTI) { + /* not a multi-prime key, skip */ + return 1; + } + return (ossl_rsa_multip_calc_product((RSA *)*pval) == 1) ? 2 : 0; + } + return 1; +} + +/* Based on definitions in RFC 8017 appendix A.1.2 */ +ASN1_SEQUENCE(RSA_PRIME_INFO) = { + ASN1_SIMPLE(RSA_PRIME_INFO, r, CBIGNUM), + ASN1_SIMPLE(RSA_PRIME_INFO, d, CBIGNUM), + ASN1_SIMPLE(RSA_PRIME_INFO, t, CBIGNUM), +} ASN1_SEQUENCE_END(RSA_PRIME_INFO) + +ASN1_SEQUENCE_cb(RSAPrivateKey, rsa_cb) = { + ASN1_EMBED(RSA, version, INT32), + ASN1_SIMPLE(RSA, n, BIGNUM), + ASN1_SIMPLE(RSA, e, BIGNUM), + ASN1_SIMPLE(RSA, d, CBIGNUM), + ASN1_SIMPLE(RSA, p, CBIGNUM), + ASN1_SIMPLE(RSA, q, CBIGNUM), + ASN1_SIMPLE(RSA, dmp1, CBIGNUM), + ASN1_SIMPLE(RSA, dmq1, CBIGNUM), + ASN1_SIMPLE(RSA, iqmp, CBIGNUM), + ASN1_SEQUENCE_OF_OPT(RSA, prime_infos, RSA_PRIME_INFO) +} ASN1_SEQUENCE_END_cb(RSA, RSAPrivateKey) + + +ASN1_SEQUENCE_cb(RSAPublicKey, rsa_cb) = { + ASN1_SIMPLE(RSA, n, BIGNUM), + ASN1_SIMPLE(RSA, e, BIGNUM), +} ASN1_SEQUENCE_END_cb(RSA, RSAPublicKey) + +/* Free up maskHash */ +static int rsa_pss_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + if (operation == ASN1_OP_FREE_PRE) { + RSA_PSS_PARAMS *pss = (RSA_PSS_PARAMS *)*pval; + X509_ALGOR_free(pss->maskHash); + } + return 1; +} + +ASN1_SEQUENCE_cb(RSA_PSS_PARAMS, rsa_pss_cb) = { + ASN1_EXP_OPT(RSA_PSS_PARAMS, hashAlgorithm, X509_ALGOR,0), + ASN1_EXP_OPT(RSA_PSS_PARAMS, maskGenAlgorithm, X509_ALGOR,1), + ASN1_EXP_OPT(RSA_PSS_PARAMS, saltLength, ASN1_INTEGER,2), + ASN1_EXP_OPT(RSA_PSS_PARAMS, trailerField, ASN1_INTEGER,3) +} ASN1_SEQUENCE_END_cb(RSA_PSS_PARAMS, RSA_PSS_PARAMS) + +IMPLEMENT_ASN1_FUNCTIONS(RSA_PSS_PARAMS) +IMPLEMENT_ASN1_DUP_FUNCTION(RSA_PSS_PARAMS) + +/* Free up maskHash */ +static int rsa_oaep_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + if (operation == ASN1_OP_FREE_PRE) { + RSA_OAEP_PARAMS *oaep = (RSA_OAEP_PARAMS *)*pval; + X509_ALGOR_free(oaep->maskHash); + } + return 1; +} + +ASN1_SEQUENCE_cb(RSA_OAEP_PARAMS, rsa_oaep_cb) = { + ASN1_EXP_OPT(RSA_OAEP_PARAMS, hashFunc, X509_ALGOR, 0), + ASN1_EXP_OPT(RSA_OAEP_PARAMS, maskGenFunc, X509_ALGOR, 1), + ASN1_EXP_OPT(RSA_OAEP_PARAMS, pSourceFunc, X509_ALGOR, 2), +} ASN1_SEQUENCE_END_cb(RSA_OAEP_PARAMS, RSA_OAEP_PARAMS) + +IMPLEMENT_ASN1_FUNCTIONS(RSA_OAEP_PARAMS) + +IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(RSA, RSAPrivateKey, RSAPrivateKey) + +IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(RSA, RSAPublicKey, RSAPublicKey) + +RSA *RSAPublicKey_dup(const RSA *rsa) +{ + return ASN1_item_dup(ASN1_ITEM_rptr(RSAPublicKey), rsa); +} + +RSA *RSAPrivateKey_dup(const RSA *rsa) +{ + return ASN1_item_dup(ASN1_ITEM_rptr(RSAPrivateKey), rsa); +} diff --git a/tests/ctests/test_c_extended.py b/tests/ctests/test_c_extended.py new file mode 100644 index 000000000..264ef9b2c --- /dev/null +++ b/tests/ctests/test_c_extended.py @@ -0,0 +1,146 @@ +import re +import regex +from langchain_community.document_loaders.parsers.language.c import CSegmenter +from typing import List +from pathlib import Path + + +class Document: + def __init__(self, page_content: str, metadata: dict | None = None): + self.page_content = page_content + self.metadata = metadata if metadata is not None else {} + + +def _read_single_file(file_path): + """Helper function to read a single file, handles errors internally.""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return f.read() + except FileNotFoundError: + print(f"Error: The file '{file_path}' was not found.") + return None + except Exception as e: + print(f"Error reading '{file_path}': {e}") + return None + + +class CSegmenterExtended(CSegmenter): + + def __init__(self, code: str): + # Preprocess code: remove comments and macro blocks + code = self.remove_comments(code) + code = self.remove_macro_blocks(code) + super().__init__(code) + self.structs_enums: List[str] = [] + + @staticmethod + def remove_comments(code: str) -> str: + # Remove all multi-line comments (/* ... */) + code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) + # Remove all single-line comments (//...) + code = re.sub(r'//.*', '', code) + return code + + @staticmethod + def remove_macro_blocks(text: str) -> str: + """ + Remove macro blocks of the form: + MACRO_NAME(args) { ... } + or + MACRO_NAME(args) { ... } MACRO_NAME_END(args); + Only matches ALL UPPERCASE macro names. + """ + lines = text.splitlines() + total = len(lines) + indices_to_remove = set() + i = 0 + + macro_header = re.compile(r'^\s*([A-Z_][A-Z0-9_]*)\s*\(.*\)\s*{') + macro_end_template = r'^\s*{}\s*_END\\s*\\(.*\\)\\s*;?' + + while i < total: + line = lines[i] + match = macro_header.match(line) + if match: + macro_name = match.group(1) + start = i + brace_count = line.count('{') - line.count('}') + i += 1 + # Find the end of the code block + while i < total and brace_count > 0: + brace_count += lines[i].count('{') + brace_count -= lines[i].count('}') + i += 1 + # Optionally, look for a matching _END macro + macro_end = re.compile(macro_end_template.format(macro_name)) + if i < total and macro_end.match(lines[i]): + i += 1 + for idx in range(start, i): + indices_to_remove.add(idx) + else: + i += 1 + + return '\n'.join( + line for idx, line in enumerate(lines) + if idx not in indices_to_remove + ) + + def extract_functions_classes(self) -> List[str]: + segments = super().extract_functions_classes() + return segments + + +# Compile the function pattern with recursive param matching +_FUNCTION_PATTERN_REGEX = regex.compile(r''' + ^[ \t]* # Leading indentation + (?P.*?) # Non-greedy return type + \b + (?P[A-Za-z_]\w*) # Function name (C identifier) + \s* + \( + (?P # Named group for parameters + (?: # Repeated nested structure: + [^()]+ # - non-paren characters + | # - or a nested (...) + \((?¶ms)\) # recurse into same group + )* + ) + \) + \s*(?:const\s*)? # Optional const after params + \s*\{ # Opening brace of function body + ''', regex.VERBOSE | regex.MULTILINE) + + +def get_function_name(c_code: str) -> str: + # Normalize line endings + code = c_code.replace('\r\n', '\n') + # res = [m.group('name') for m in self._FUNCTION_PATTERN.finditer(code)] + m = _FUNCTION_PATTERN_REGEX.search(code, timeout=0.05) + return m.group('name') if m else "" + + +def test_function_extraction_regex(str, file_name): + segmenter = CSegmenterExtended(str) + functions = segmenter.extract_functions_classes() + for seg in functions: + print("-----Seg ---- ") + print(seg) + f_name = get_function_name(seg) + print(f"function name : {f_name} @ {file_name}") + + +def test_files(): + c_files = [ + "err.c", "ess_lib.c", "cms_sd.c", "ec_asn1.c", + "rsa_asn1.c", "pk7_asn1.c" + ] + dir_for_code = Path(__file__).resolve().parent + for _file in c_files: + print(f"Process file : {_file}") + f1_path = dir_for_code / _file + str_body = _read_single_file(f1_path) + test_function_extraction_regex(str_body, _file) + + +print("\n\n\n ------ TEST Files ") +test_files() From 3b8d8509cec0c53019005394eb35f6026f5858e1 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 28 Oct 2025 13:49:33 +0200 Subject: [PATCH 090/286] fix: prevent crash for backslashes in the checklist content function as escape chars. Originally was fixed in old git commit 710ab46629d6bdd6ec8c3e9dafc7c7d621aa7f6b, which belongs to commits tree of the previous version Signed-off-by: Zvi Grinberg --- src/vuln_analysis/utils/checklist_prompt_generator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 0078b1e26..77b6199a8 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -15,6 +15,7 @@ import ast import logging +import re from jinja2 import Template from langchain_core.language_models.base import BaseLanguageModel @@ -25,6 +26,7 @@ from vuln_analysis.utils.string_utils import attempt_fix_list_string from vuln_analysis.logging.loggers_factory import LoggingFactory + logger = LoggingFactory.get_agent_logger(__name__) DEFAULT_CHECKLIST_PROMPT = MOD_FEW_SHOT.format(examples=get_mod_examples()) @@ -32,6 +34,10 @@ cve_prompt2 = """Parse the following numbered checklist into a python list in the format ["x", "y", "z"], a comma separated list surrounded by square braces: {{template}}""" +def return_escaped_content_backslashes(match) -> str: + return match.group(0).replace("\\", "\\\\") + + async def _parse_list(text: list[str]) -> list[list[str]]: """ Asynchronously parse a list of strings, each representing a list, into a list of lists. @@ -67,8 +73,8 @@ async def _parse_list(text: list[str]) -> list[list[str]]: # Remove newline characters that can cause incorrect string escaping in the next step x = x.replace("\n", "") - # Ensure backslashes are escaped - x = x.replace("\\", "\\\\") + # Ensure backslashes are escaped only when they are not already escaping other characters + x = re.sub(r'\\[^"\'\\nrtbf]', return_escaped_content_backslashes, x) # Try to do some very basic string cleanup to fix unescaped quotes x = attempt_fix_list_string(x) From 75a1116652ab46bd2954261aaf348ca49fd12f06 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 28 Oct 2025 17:29:34 +0200 Subject: [PATCH 091/286] chore: enlarge size of temp buildah storage pvc Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 2 +- .tekton/on-push.yaml | 2 +- .tekton/on-tag.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 54d7d7d44..afad6c5bc 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -231,7 +231,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 5Gi + storage: 10Gi - name: unit-test-cache persistentVolumeClaim: diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml index 283b90350..ffcb58960 100644 --- a/.tekton/on-push.yaml +++ b/.tekton/on-push.yaml @@ -142,7 +142,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 5Gi + storage: 10Gi # This workspace will inject secret to help the git-clone task to be able to # checkout the private repositories - name: basic-auth diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml index 00d0d0458..fd27734f6 100644 --- a/.tekton/on-tag.yaml +++ b/.tekton/on-tag.yaml @@ -171,7 +171,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 5Gi + storage: 10Gi # This workspace will inject secret to help the git-clone task to be able to # checkout the private repositories From 2943f18f3a18bb4a2f5c6e2b39853e191bbe0266 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 19 Oct 2025 17:49:33 +0300 Subject: [PATCH 092/286] fix: fix golang transitive code search test case 6 Signed-off-by: Zvi Grinberg --- .../utils/functions_parsers/golang_functions_parsers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 2ebb71bc1..8cca8bc35 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -327,8 +327,14 @@ def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[t next_struct = current_line_stripped.find("struct") if next_eol > - 1 and (next_struct == -1 or next_struct > next_eol): if not self.is_comment_line(current_line_stripped[:next_eol + 1]): - declaration_parts = current_line_stripped[:next_eol + 1].split() + # If row inside block contains func, then it's a function type and need to parse it in a + # special way. + if current_line_stripped[:next_eol + 1].__contains__("func"): + declaration_parts = current_line_stripped[:next_eol + 1].split("func") + declaration_parts = [part.strip() for part in declaration_parts] + if len(declaration_parts) == 2: + declaration_parts[1] = f"func {declaration_parts[1]}" # ignore alias' "equals" notation if len(declaration_parts) == 3: [name, _, type_name] = declaration_parts From 63d9c88a2bf778fbefe2e428e04632f7bde42726 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 31 Oct 2025 00:47:58 +0200 Subject: [PATCH 093/286] fix: transitive code search - some go std libs funcs wrongly weren't check Signed-off-by: Zvi Grinberg --- .../tools/tests/test_transitive_code_search.py | 6 +++--- .../utils/chain_of_calls_retriever.py | 16 ++++++++++------ .../golang_functions_parsers.py | 14 +++++++------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 6589b53ca..5622ed206 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -99,8 +99,8 @@ async def test_transitive_search_golang_6(): result = await transitive_code_search_runner_coroutine("net/http,ListenAndServe") (path_found, list_path) = result print(result) - assert path_found is False - assert len(list_path) is 0 + assert path_found is True + assert len(list_path) is 2 def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], @@ -299,4 +299,4 @@ async def test_c_transitive_search_2(): print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") assert len(list_path) == 1 - assert path_found == False \ No newline at end of file + assert path_found == False diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 1854f893b..a9957957b 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -12,6 +12,7 @@ from .functions_parsers.lang_functions_parsers_factory import ( get_language_function_parser, ) +from .standard_library_cache import StandardLibraryCache PARENTS_INDEX = 0 EXCLUSIONS_INDEX = 1 @@ -546,12 +547,15 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: class_name, function = function.split(splitters[0]) found_package = False matching_documents = [] - # Check if input package is in dependency tree - for package in self.tree_dict: - if self.language_parser.is_same_package(package_name, package): - package_name = package - found_package = True - break + standard_libs_cache = StandardLibraryCache.get_instance() + # If it's a standard library package, then skip checking the package in dependency tree. + if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): + # Check if input package is in dependency tree + for package in self.tree_dict: + if self.language_parser.is_same_package(package_name, package): + package_name = package + found_package = True + break # If it's , then create a document for it. if found_package: diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 8cca8bc35..94a677b08 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -4,6 +4,8 @@ from langchain_core.documents import Document from .lang_functions_parsers import LanguageFunctionsParser +from ..dep_tree import Ecosystem +from ..standard_library_cache import StandardLibraryCache EMBEDDED_TYPE = "embedded_type" @@ -115,8 +117,6 @@ def handle_imports(code_content: str, identifier: str, callee_package: str) -> b class GoLanguageFunctionsParser(LanguageFunctionsParser): - - def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}() {{}}" @@ -340,10 +340,10 @@ def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[t [name, _, type_name] = declaration_parts elif len(declaration_parts) == 2: [name, type_name] = declaration_parts - - self.parse_one_type(Document(page_content=f"type {name} {type_name}", - metadata={"source": the_type.metadata['source']}), - types_mapping) + if len(declaration_parts) == (2 or 3): + self.parse_one_type(Document(page_content=f"type {name} {type_name}", + metadata={"source": the_type.metadata['source']}), + types_mapping) pos += next_eol + 1 current_line_stripped = current_line_stripped[next_eol + 1:].lstrip() elif -1 < next_struct < next_eol and next_eol > -1: @@ -608,4 +608,4 @@ def is_root_package(self, function: Document) -> bool: return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) def is_comment_line(self, line: str) -> bool: - return line.strip().startswith("//") \ No newline at end of file + return line.strip().startswith("//") From f722058c618613ee11751e20896311dda2a78542 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:14:19 +0200 Subject: [PATCH 094/286] Fix: csegmenter handle cases where segment contains two functions (#139) * fix: csegmenter sometimes place two functions in segment * create UT for c segmenter --- .../tools/tests/test_segmenter.py | 182 + .../tools/tests/tests_data}/err.c | 0 .../tools/tests/tests_data}/ess_lib.c | 0 .../tools/tests/tests_data/parser.c | 15570 ++++++++++++++++ .../tools/tests/tests_data}/pk7_asn1.c | 0 .../tools/transitive_code_search.py | 3 - src/vuln_analysis/utils/c_segmenter_custom.py | 120 +- tests/ctests/cms_sd.c | 1191 -- tests/ctests/ec_asn1.c | 1332 -- tests/ctests/rsa_asn1.c | 128 - tests/ctests/test_c_extended.py | 146 - 11 files changed, 15871 insertions(+), 2801 deletions(-) create mode 100644 src/vuln_analysis/tools/tests/test_segmenter.py rename {tests/ctests => src/vuln_analysis/tools/tests/tests_data}/err.c (100%) rename {tests/ctests => src/vuln_analysis/tools/tests/tests_data}/ess_lib.c (100%) create mode 100644 src/vuln_analysis/tools/tests/tests_data/parser.c rename {tests/ctests => src/vuln_analysis/tools/tests/tests_data}/pk7_asn1.c (100%) delete mode 100644 tests/ctests/cms_sd.c delete mode 100644 tests/ctests/ec_asn1.c delete mode 100644 tests/ctests/rsa_asn1.c delete mode 100644 tests/ctests/test_c_extended.py diff --git a/src/vuln_analysis/tools/tests/test_segmenter.py b/src/vuln_analysis/tools/tests/test_segmenter.py new file mode 100644 index 000000000..45f32766b --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_segmenter.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import pytest +from vuln_analysis.utils.c_segmenter_custom import CSegmenterExtended +from vuln_analysis.utils.functions_parsers.c_lang_function_parsers import _FUNCTION_PATTERN_REGEX +from langchain_community.document_loaders.parsers.language.c import CSegmenter + +def _get_function_name_from_segment(segment: str) -> str: + """ + Extract function name from C code segment using regex pattern. + This parser function can be used for testing. + """ + # Normalize line endings + code = segment.replace('\r\n', '\n') + m = _FUNCTION_PATTERN_REGEX.search(code, timeout=0.05) + return m.group('name') if m else "" + + +# --- Test Data Fixtures --- +# These fixtures load the test data once per test session. +# They assume the .c files are in the same directory as this test script. + +@pytest.fixture(scope="module") +def c_test_files_dir(): + """Returns the directory of this test file.""" + return Path(__file__).parent / "tests_data" + +@pytest.fixture(scope="module") +def err_c_code(c_test_files_dir): + file_path = c_test_files_dir / "err.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def ess_lib_c_code(c_test_files_dir): + file_path = c_test_files_dir / "ess_lib.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def parser_c_code(c_test_files_dir): + file_path = c_test_files_dir / "parser.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def pk7_asn1_c_code(c_test_files_dir): + file_path = c_test_files_dir / "pk7_asn1.c" + return file_path.read_text(encoding="utf-8") + +# --- Integration Tests for CSegmenterExtended --- + +def test_integration_err_c(err_c_code: str): + """ + Tests segmentation on err.c. + Checks for specific functions and ensures comments are gone. + """ + segmenter = CSegmenterExtended(err_c_code) + segments = segmenter.extract_functions_classes() + + # Preprocess code: remove comments and macro blocks + original_segmenter = CSegmenter(err_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in err.c, found {len(segments)}" + + names = set() + for seg in segments: + assert "//" not in seg + assert "/*" not in seg + name = _get_function_name_from_segment(seg) + if name: + names.add(name) + + assert "OSSL_ERR_STATE_free" in names + assert "err_cleanup" in names + assert "ossl_err_get_state_int" in names + assert "err_load_strings" in names + assert "err_string_data_hash" in names + assert "ERR_get_error" in names + assert "ERR_get_state" in names + + +def test_integration_parser_c(parser_c_code): + """ + Tests segmentation on the large parser.c file. + This validates the "hidden segment" logic by ensuring + it can find many functions in a complex file. + """ + segmenter = CSegmenterExtended(parser_c_code) + segments = segmenter.extract_functions_classes() + # Preprocess code: remove comments and macro blocks + original_segmenter = CSegmenter(parser_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in parser.c, found {len(segments)}" + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + # Check for a mix of public and static functions + assert "xmlParseDocument" in names + assert "xmlParseStartTag" in names + assert "xmlParsePI" in names + assert "xmlParserEntityCheck" in names # static function + assert "xmlParseCommentComplex" in names # static function + assert "xmlCtxtResetPush" in names # public function + assert "xmlParseMemory" in names # parses XML from memory buffer + assert "xmlReadMemory" in names # reads XML from memory + assert "xmlCreateMemoryParserCtxt" in names # creates parser context from memory + assert "xmlParseFile" in names # popular public API + assert "xmlReadFile" in names # popular public API + assert "xmlParseEntityRef" in names # important parsing function + assert "xmlParseElement" in names # core parsing function + assert "xmlParseAttribute" in names # common parsing function + + +def test_integration_pk7_asn1_c_bug(pk7_asn1_c_code): + """ + Tests segmentation on pk7_asn1.c. + + This test *should fail* with the original implementation. + The `remove_macro_blocks` function incorrectly removes + definitions like `ASN1_ADB(PKCS7) = { ... }` and + `IMPLEMENT_ASN1_FUNCTIONS(PKCS7)`. + + This test now PASSES because the bug is fixed. + """ + segmenter = CSegmenterExtended(pk7_asn1_c_code) + segments = segmenter.extract_functions_classes() + + original_segmenter = CSegmenter(pk7_asn1_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in pk7_asn1.c, found {len(segments)}" + + + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + # These functions will be missed if remove_macro_blocks is buggy + assert "d2i_PKCS7" in names + assert "i2d_PKCS7" in names + assert "PKCS7_new" in names + assert "PKCS7_free" in names + assert "si_cb" in names # static function + + +def test_integration_ess_lib_c(ess_lib_c_code): + """ + Tests segmentation on ess_lib.c. + Checks for public and static functions. + """ + segmenter = CSegmenterExtended(ess_lib_c_code) + segments = segmenter.extract_functions_classes() + + original_segmenter = CSegmenter(ess_lib_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in ess_lib.c, found {len(segments)}" + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + assert "OSSL_ESS_signing_cert_new_init" in names + assert "ESS_CERT_ID_new_init" in names # static function + assert "find" in names # static function + assert "OSSL_ESS_check_signing_certs" in names + assert "OSSL_ESS_signing_cert_v2_new_init" in names # public function (v2 version) + assert "ESS_CERT_ID_V2_new_init" in names # static function (v2 version) + assert "ess_issuer_serial_cmp" in names # static comparison function + + \ No newline at end of file diff --git a/tests/ctests/err.c b/src/vuln_analysis/tools/tests/tests_data/err.c similarity index 100% rename from tests/ctests/err.c rename to src/vuln_analysis/tools/tests/tests_data/err.c diff --git a/tests/ctests/ess_lib.c b/src/vuln_analysis/tools/tests/tests_data/ess_lib.c similarity index 100% rename from tests/ctests/ess_lib.c rename to src/vuln_analysis/tools/tests/tests_data/ess_lib.c diff --git a/src/vuln_analysis/tools/tests/tests_data/parser.c b/src/vuln_analysis/tools/tests/tests_data/parser.c new file mode 100644 index 000000000..1c5e036ea --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/parser.c @@ -0,0 +1,15570 @@ +/* + * parser.c : an XML 1.0 parser, namespaces and validity support are mostly + * implemented on top of the SAX interfaces + * + * References: + * The XML specification: + * http://www.w3.org/TR/REC-xml + * Original 1.0 version: + * http://www.w3.org/TR/1998/REC-xml-19980210 + * XML second edition working draft + * http://www.w3.org/TR/2000/WD-xml-2e-20000814 + * + * Okay this is a big file, the parser core is around 7000 lines, then it + * is followed by the progressive parser top routines, then the various + * high level APIs to call the parser and a few miscellaneous functions. + * A number of helper functions and deprecated ones have been moved to + * parserInternals.c to reduce this file size. + * As much as possible the functions are associated with their relative + * production in the XML specification. A few productions defining the + * different ranges of character are actually implanted either in + * parserInternals.h or parserInternals.c + * The DOM tree build is realized from the default SAX callbacks in + * the module SAX.c. + * The routines doing the validation checks are in valid.c and called either + * from the SAX callbacks or as standalone functions using a preparsed + * document. + * + * See Copyright for the status of this software. + * + * daniel@veillard.com + */ + +/* To avoid EBCDIC trouble when parsing on zOS */ +#if defined(__MVS__) +#pragma convert("ISO8859-1") +#endif + +#define IN_LIBXML +#include "libxml.h" + +#if defined(_WIN32) && !defined (__CYGWIN__) +#define XML_DIR_SEP '\\' +#else +#define XML_DIR_SEP '/' +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef LIBXML_CATALOG_ENABLED +#include +#endif +#ifdef LIBXML_SCHEMAS_ENABLED +#include +#include +#endif +#ifdef HAVE_CTYPE_H +#include +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_ZLIB_H +#include +#endif +#ifdef HAVE_LZMA_H +#include +#endif + +#include "buf.h" +#include "enc.h" + +static void +xmlFatalErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, const char *info); + +static xmlParserCtxtPtr +xmlCreateEntityParserCtxtInternal(const xmlChar *URL, const xmlChar *ID, + const xmlChar *base, xmlParserCtxtPtr pctx); + +static void xmlHaltParser(xmlParserCtxtPtr ctxt); + +/************************************************************************ + * * + * Arbitrary limits set in the parser. See XML_PARSE_HUGE * + * * + ************************************************************************/ + +#define XML_PARSER_BIG_ENTITY 1000 +#define XML_PARSER_LOT_ENTITY 5000 + +/* + * XML_PARSER_NON_LINEAR is the threshold where the ratio of parsed entity + * replacement over the size in byte of the input indicates that you have + * and eponential behaviour. A value of 10 correspond to at least 3 entity + * replacement per byte of input. + */ +#define XML_PARSER_NON_LINEAR 10 + +/* + * xmlParserEntityCheck + * + * Function to check non-linear entity expansion behaviour + * This is here to detect and stop exponential linear entity expansion + * This is not a limitation of the parser but a safety + * boundary feature. It can be disabled with the XML_PARSE_HUGE + * parser option. + */ +static int +xmlParserEntityCheck(xmlParserCtxtPtr ctxt, size_t size, + xmlEntityPtr ent, size_t replacement) +{ + size_t consumed = 0; + + if ((ctxt == NULL) || (ctxt->options & XML_PARSE_HUGE)) + return (0); + if (ctxt->lastError.code == XML_ERR_ENTITY_LOOP) + return (1); + + /* + * This may look absurd but is needed to detect + * entities problems + */ + if ((ent != NULL) && (ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) && + (ent->content != NULL) && (ent->checked == 0) && + (ctxt->errNo != XML_ERR_ENTITY_LOOP)) { + unsigned long oldnbent = ctxt->nbentities; + xmlChar *rep; + + ent->checked = 1; + + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, 0, 0, 0); + --ctxt->depth; + if (ctxt->errNo == XML_ERR_ENTITY_LOOP) { + ent->content[0] = 0; + } + + ent->checked = (ctxt->nbentities - oldnbent + 1) * 2; + if (rep != NULL) { + if (xmlStrchr(rep, '<')) + ent->checked |= 1; + xmlFree(rep); + rep = NULL; + } + } + if (replacement != 0) { + if (replacement < XML_MAX_TEXT_LENGTH) + return(0); + + /* + * If the volume of entity copy reaches 10 times the + * amount of parsed data and over the large text threshold + * then that's very likely to be an abuse. + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + if (replacement < XML_PARSER_NON_LINEAR * consumed) + return(0); + } else if (size != 0) { + /* + * Do the check based on the replacement size of the entity + */ + if (size < XML_PARSER_BIG_ENTITY) + return(0); + + /* + * A limit on the amount of text data reasonably used + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + if ((size < XML_PARSER_NON_LINEAR * consumed) && + (ctxt->nbentities * 3 < XML_PARSER_NON_LINEAR * consumed)) + return (0); + } else if (ent != NULL) { + /* + * use the number of parsed entities in the replacement + */ + size = ent->checked / 2; + + /* + * The amount of data parsed counting entities size only once + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + /* + * Check the density of entities for the amount of data + * knowing an entity reference will take at least 3 bytes + */ + if (size * 3 < consumed * XML_PARSER_NON_LINEAR) + return (0); + } else { + /* + * strange we got no data for checking + */ + if (((ctxt->lastError.code != XML_ERR_UNDECLARED_ENTITY) && + (ctxt->lastError.code != XML_WAR_UNDECLARED_ENTITY)) || + (ctxt->nbentities <= 10000)) + return (0); + } + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + return (1); +} + +/** + * xmlParserMaxDepth: + * + * arbitrary depth limit for the XML documents that we allow to + * process. This is not a limitation of the parser but a safety + * boundary feature. It can be disabled with the XML_PARSE_HUGE + * parser option. + */ +unsigned int xmlParserMaxDepth = 256; + + + +#define SAX2 1 +#define XML_PARSER_BIG_BUFFER_SIZE 300 +#define XML_PARSER_BUFFER_SIZE 100 +#define SAX_COMPAT_MODE BAD_CAST "SAX compatibility mode document" + +/** + * XML_PARSER_CHUNK_SIZE + * + * When calling GROW that's the minimal amount of data + * the parser expected to have received. It is not a hard + * limit but an optimization when reading strings like Names + * It is not strictly needed as long as inputs available characters + * are followed by 0, which should be provided by the I/O level + */ +#define XML_PARSER_CHUNK_SIZE 100 + +/* + * List of XML prefixed PI allowed by W3C specs + */ + +static const char *xmlW3CPIs[] = { + "xml-stylesheet", + "xml-model", + NULL +}; + + +/* DEPR void xmlParserHandleReference(xmlParserCtxtPtr ctxt); */ +static xmlEntityPtr xmlParseStringPEReference(xmlParserCtxtPtr ctxt, + const xmlChar **str); + +static xmlParserErrors +xmlParseExternalEntityPrivate(xmlDocPtr doc, xmlParserCtxtPtr oldctxt, + xmlSAXHandlerPtr sax, + void *user_data, int depth, const xmlChar *URL, + const xmlChar *ID, xmlNodePtr *list); + +static int +xmlCtxtUseOptionsInternal(xmlParserCtxtPtr ctxt, int options, + const char *encoding); +#ifdef LIBXML_LEGACY_ENABLED +static void +xmlAddEntityReference(xmlEntityPtr ent, xmlNodePtr firstNode, + xmlNodePtr lastNode); +#endif /* LIBXML_LEGACY_ENABLED */ + +static xmlParserErrors +xmlParseBalancedChunkMemoryInternal(xmlParserCtxtPtr oldctxt, + const xmlChar *string, void *user_data, xmlNodePtr *lst); + +static int +xmlLoadEntityContent(xmlParserCtxtPtr ctxt, xmlEntityPtr entity); + +/************************************************************************ + * * + * Some factorized error routines * + * * + ************************************************************************/ + +/** + * xmlErrAttributeDup: + * @ctxt: an XML parser context + * @prefix: the attribute prefix + * @localname: the attribute localname + * + * Handle a redefinition of attribute error + */ +static void +xmlErrAttributeDup(xmlParserCtxtPtr ctxt, const xmlChar * prefix, + const xmlChar * localname) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = XML_ERR_ATTRIBUTE_REDEFINED; + + if (prefix == NULL) + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, + XML_ERR_ATTRIBUTE_REDEFINED, XML_ERR_FATAL, NULL, 0, + (const char *) localname, NULL, NULL, 0, 0, + "Attribute %s redefined\n", localname); + else + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, + XML_ERR_ATTRIBUTE_REDEFINED, XML_ERR_FATAL, NULL, 0, + (const char *) prefix, (const char *) localname, + NULL, 0, 0, "Attribute %s:%s redefined\n", prefix, + localname); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErr: + * @ctxt: an XML parser context + * @error: the error number + * @extra: extra information string + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void +xmlFatalErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, const char *info) +{ + const char *errmsg; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + switch (error) { + case XML_ERR_INVALID_HEX_CHARREF: + errmsg = "CharRef: invalid hexadecimal value"; + break; + case XML_ERR_INVALID_DEC_CHARREF: + errmsg = "CharRef: invalid decimal value"; + break; + case XML_ERR_INVALID_CHARREF: + errmsg = "CharRef: invalid value"; + break; + case XML_ERR_INTERNAL_ERROR: + errmsg = "internal error"; + break; + case XML_ERR_PEREF_AT_EOF: + errmsg = "PEReference at end of document"; + break; + case XML_ERR_PEREF_IN_PROLOG: + errmsg = "PEReference in prolog"; + break; + case XML_ERR_PEREF_IN_EPILOG: + errmsg = "PEReference in epilog"; + break; + case XML_ERR_PEREF_NO_NAME: + errmsg = "PEReference: no name"; + break; + case XML_ERR_PEREF_SEMICOL_MISSING: + errmsg = "PEReference: expecting ';'"; + break; + case XML_ERR_ENTITY_LOOP: + errmsg = "Detected an entity reference loop"; + break; + case XML_ERR_ENTITY_NOT_STARTED: + errmsg = "EntityValue: \" or ' expected"; + break; + case XML_ERR_ENTITY_PE_INTERNAL: + errmsg = "PEReferences forbidden in internal subset"; + break; + case XML_ERR_ENTITY_NOT_FINISHED: + errmsg = "EntityValue: \" or ' expected"; + break; + case XML_ERR_ATTRIBUTE_NOT_STARTED: + errmsg = "AttValue: \" or ' expected"; + break; + case XML_ERR_LT_IN_ATTRIBUTE: + errmsg = "Unescaped '<' not allowed in attributes values"; + break; + case XML_ERR_LITERAL_NOT_STARTED: + errmsg = "SystemLiteral \" or ' expected"; + break; + case XML_ERR_LITERAL_NOT_FINISHED: + errmsg = "Unfinished System or Public ID \" or ' expected"; + break; + case XML_ERR_MISPLACED_CDATA_END: + errmsg = "Sequence ']]>' not allowed in content"; + break; + case XML_ERR_URI_REQUIRED: + errmsg = "SYSTEM or PUBLIC, the URI is missing"; + break; + case XML_ERR_PUBID_REQUIRED: + errmsg = "PUBLIC, the Public Identifier is missing"; + break; + case XML_ERR_HYPHEN_IN_COMMENT: + errmsg = "Comment must not contain '--' (double-hyphen)"; + break; + case XML_ERR_PI_NOT_STARTED: + errmsg = "xmlParsePI : no target name"; + break; + case XML_ERR_RESERVED_XML_NAME: + errmsg = "Invalid PI name"; + break; + case XML_ERR_NOTATION_NOT_STARTED: + errmsg = "NOTATION: Name expected here"; + break; + case XML_ERR_NOTATION_NOT_FINISHED: + errmsg = "'>' required to close NOTATION declaration"; + break; + case XML_ERR_VALUE_REQUIRED: + errmsg = "Entity value required"; + break; + case XML_ERR_URI_FRAGMENT: + errmsg = "Fragment not allowed"; + break; + case XML_ERR_ATTLIST_NOT_STARTED: + errmsg = "'(' required to start ATTLIST enumeration"; + break; + case XML_ERR_NMTOKEN_REQUIRED: + errmsg = "NmToken expected in ATTLIST enumeration"; + break; + case XML_ERR_ATTLIST_NOT_FINISHED: + errmsg = "')' required to finish ATTLIST enumeration"; + break; + case XML_ERR_MIXED_NOT_STARTED: + errmsg = "MixedContentDecl : '|' or ')*' expected"; + break; + case XML_ERR_PCDATA_REQUIRED: + errmsg = "MixedContentDecl : '#PCDATA' expected"; + break; + case XML_ERR_ELEMCONTENT_NOT_STARTED: + errmsg = "ContentDecl : Name or '(' expected"; + break; + case XML_ERR_ELEMCONTENT_NOT_FINISHED: + errmsg = "ContentDecl : ',' '|' or ')' expected"; + break; + case XML_ERR_PEREF_IN_INT_SUBSET: + errmsg = + "PEReference: forbidden within markup decl in internal subset"; + break; + case XML_ERR_GT_REQUIRED: + errmsg = "expected '>'"; + break; + case XML_ERR_CONDSEC_INVALID: + errmsg = "XML conditional section '[' expected"; + break; + case XML_ERR_EXT_SUBSET_NOT_FINISHED: + errmsg = "Content error in the external subset"; + break; + case XML_ERR_CONDSEC_INVALID_KEYWORD: + errmsg = + "conditional section INCLUDE or IGNORE keyword expected"; + break; + case XML_ERR_CONDSEC_NOT_FINISHED: + errmsg = "XML conditional section not closed"; + break; + case XML_ERR_XMLDECL_NOT_STARTED: + errmsg = "Text declaration '' expected"; + break; + case XML_ERR_EXT_ENTITY_STANDALONE: + errmsg = "external parsed entities cannot be standalone"; + break; + case XML_ERR_ENTITYREF_SEMICOL_MISSING: + errmsg = "EntityRef: expecting ';'"; + break; + case XML_ERR_DOCTYPE_NOT_FINISHED: + errmsg = "DOCTYPE improperly terminated"; + break; + case XML_ERR_LTSLASH_REQUIRED: + errmsg = "EndTag: 'errNo = error; + if (info == NULL) { + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, info, NULL, NULL, 0, 0, "%s\n", + errmsg); + } else { + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, info, NULL, NULL, 0, 0, "%s: %s\n", + errmsg, info); + } + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsg: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsg(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, NULL, NULL, NULL, 0, 0, "%s", msg); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlWarningMsg: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: extra data + * @str2: extra data + * + * Handle a warning. + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlWarningMsg(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, const xmlChar *str2) +{ + xmlStructuredErrorFunc schannel = NULL; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if ((ctxt != NULL) && (ctxt->sax != NULL) && + (ctxt->sax->initialized == XML_SAX2_MAGIC)) + schannel = ctxt->sax->serror; + if (ctxt != NULL) { + __xmlRaiseError(schannel, + (ctxt->sax) ? ctxt->sax->warning : NULL, + ctxt->userData, + ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_WARNING, NULL, 0, + (const char *) str1, (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } else { + __xmlRaiseError(schannel, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_WARNING, NULL, 0, + (const char *) str1, (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } +} + +/** + * xmlValidityError: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: extra data + * + * Handle a validity error. + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlValidityError(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, const xmlChar *str2) +{ + xmlStructuredErrorFunc schannel = NULL; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) { + ctxt->errNo = error; + if ((ctxt->sax != NULL) && (ctxt->sax->initialized == XML_SAX2_MAGIC)) + schannel = ctxt->sax->serror; + } + if (ctxt != NULL) { + __xmlRaiseError(schannel, + ctxt->vctxt.error, ctxt->vctxt.userData, + ctxt, NULL, XML_FROM_DTD, error, + XML_ERR_ERROR, NULL, 0, (const char *) str1, + (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + ctxt->valid = 0; + } else { + __xmlRaiseError(schannel, NULL, NULL, + ctxt, NULL, XML_FROM_DTD, error, + XML_ERR_ERROR, NULL, 0, (const char *) str1, + (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } +} + +/** + * xmlFatalErrMsgInt: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: an integer value + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgInt(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, int val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, NULL, NULL, NULL, val, 0, msg, val); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsgStrIntStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: an string info + * @val: an integer value + * @str2: an string info + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgStrIntStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, int val, + const xmlChar *str2) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, (const char *) str1, (const char *) str2, + NULL, val, 0, msg, str1, val, str2); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsgStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: a string value + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar * val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, + XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, (const char *) val, NULL, NULL, 0, 0, msg, + val); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlErrMsgStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: a string value + * + * Handle a non fatal parser error + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlErrMsgStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar * val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, + XML_FROM_PARSER, error, XML_ERR_ERROR, + NULL, 0, (const char *) val, NULL, NULL, 0, 0, msg, + val); +} + +/** + * xmlNsErr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the message + * @info1: extra information string + * @info2: extra information string + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlNsErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, + const xmlChar * info1, const xmlChar * info2, + const xmlChar * info3) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_NAMESPACE, error, + XML_ERR_ERROR, NULL, 0, (const char *) info1, + (const char *) info2, (const char *) info3, 0, 0, msg, + info1, info2, info3); + if (ctxt != NULL) + ctxt->nsWellFormed = 0; +} + +/** + * xmlNsWarn + * @ctxt: an XML parser context + * @error: the error number + * @msg: the message + * @info1: extra information string + * @info2: extra information string + * + * Handle a namespace warning error + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlNsWarn(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, + const xmlChar * info1, const xmlChar * info2, + const xmlChar * info3) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_NAMESPACE, error, + XML_ERR_WARNING, NULL, 0, (const char *) info1, + (const char *) info2, (const char *) info3, 0, 0, msg, + info1, info2, info3); +} + +/************************************************************************ + * * + * Library wide options * + * * + ************************************************************************/ + +/** + * xmlHasFeature: + * @feature: the feature to be examined + * + * Examines if the library has been compiled with a given feature. + * + * Returns a non-zero value if the feature exist, otherwise zero. + * Returns zero (0) if the feature does not exist or an unknown + * unknown feature is requested, non-zero otherwise. + */ +int +xmlHasFeature(xmlFeature feature) +{ + switch (feature) { + case XML_WITH_THREAD: +#ifdef LIBXML_THREAD_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_TREE: +#ifdef LIBXML_TREE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_OUTPUT: +#ifdef LIBXML_OUTPUT_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_PUSH: +#ifdef LIBXML_PUSH_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_READER: +#ifdef LIBXML_READER_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_PATTERN: +#ifdef LIBXML_PATTERN_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_WRITER: +#ifdef LIBXML_WRITER_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SAX1: +#ifdef LIBXML_SAX1_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_FTP: +#ifdef LIBXML_FTP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_HTTP: +#ifdef LIBXML_HTTP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_VALID: +#ifdef LIBXML_VALID_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_HTML: +#ifdef LIBXML_HTML_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_LEGACY: +#ifdef LIBXML_LEGACY_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_C14N: +#ifdef LIBXML_C14N_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_CATALOG: +#ifdef LIBXML_CATALOG_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XPATH: +#ifdef LIBXML_XPATH_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XPTR: +#ifdef LIBXML_XPTR_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XINCLUDE: +#ifdef LIBXML_XINCLUDE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ICONV: +#ifdef LIBXML_ICONV_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ISO8859X: +#ifdef LIBXML_ISO8859X_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_UNICODE: +#ifdef LIBXML_UNICODE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_REGEXP: +#ifdef LIBXML_REGEXP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_AUTOMATA: +#ifdef LIBXML_AUTOMATA_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_EXPR: +#ifdef LIBXML_EXPR_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SCHEMAS: +#ifdef LIBXML_SCHEMAS_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SCHEMATRON: +#ifdef LIBXML_SCHEMATRON_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_MODULES: +#ifdef LIBXML_MODULES_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG: +#ifdef LIBXML_DEBUG_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG_MEM: +#ifdef DEBUG_MEMORY_LOCATION + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG_RUN: +#ifdef LIBXML_DEBUG_RUNTIME + return(1); +#else + return(0); +#endif + case XML_WITH_ZLIB: +#ifdef LIBXML_ZLIB_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_LZMA: +#ifdef LIBXML_LZMA_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ICU: +#ifdef LIBXML_ICU_ENABLED + return(1); +#else + return(0); +#endif + default: + break; + } + return(0); +} + +/************************************************************************ + * * + * SAX2 defaulted attributes handling * + * * + ************************************************************************/ + +/** + * xmlDetectSAX2: + * @ctxt: an XML parser context + * + * Do the SAX2 detection and specific intialization + */ +static void +xmlDetectSAX2(xmlParserCtxtPtr ctxt) { + if (ctxt == NULL) return; +#ifdef LIBXML_SAX1_ENABLED + if ((ctxt->sax) && (ctxt->sax->initialized == XML_SAX2_MAGIC) && + ((ctxt->sax->startElementNs != NULL) || + (ctxt->sax->endElementNs != NULL))) ctxt->sax2 = 1; +#else + ctxt->sax2 = 1; +#endif /* LIBXML_SAX1_ENABLED */ + + ctxt->str_xml = xmlDictLookup(ctxt->dict, BAD_CAST "xml", 3); + ctxt->str_xmlns = xmlDictLookup(ctxt->dict, BAD_CAST "xmlns", 5); + ctxt->str_xml_ns = xmlDictLookup(ctxt->dict, XML_XML_NAMESPACE, 36); + if ((ctxt->str_xml==NULL) || (ctxt->str_xmlns==NULL) || + (ctxt->str_xml_ns == NULL)) { + xmlErrMemory(ctxt, NULL); + } +} + +typedef struct _xmlDefAttrs xmlDefAttrs; +typedef xmlDefAttrs *xmlDefAttrsPtr; +struct _xmlDefAttrs { + int nbAttrs; /* number of defaulted attributes on that element */ + int maxAttrs; /* the size of the array */ +#if __STDC_VERSION__ >= 199901L + /* Using a C99 flexible array member avoids UBSan errors. */ + const xmlChar *values[]; /* array of localname/prefix/values/external */ +#else + const xmlChar *values[5]; +#endif +}; + +/** + * xmlAttrNormalizeSpace: + * @src: the source string + * @dst: the target string + * + * Normalize the space in non CDATA attribute values: + * If the attribute type is not CDATA, then the XML processor MUST further + * process the normalized attribute value by discarding any leading and + * trailing space (#x20) characters, and by replacing sequences of space + * (#x20) characters by a single space (#x20) character. + * Note that the size of dst need to be at least src, and if one doesn't need + * to preserve dst (and it doesn't come from a dictionary or read-only) then + * passing src as dst is just fine. + * + * Returns a pointer to the normalized value (dst) or NULL if no conversion + * is needed. + */ +static xmlChar * +xmlAttrNormalizeSpace(const xmlChar *src, xmlChar *dst) +{ + if ((src == NULL) || (dst == NULL)) + return(NULL); + + while (*src == 0x20) src++; + while (*src != 0) { + if (*src == 0x20) { + while (*src == 0x20) src++; + if (*src != 0) + *dst++ = 0x20; + } else { + *dst++ = *src++; + } + } + *dst = 0; + if (dst == src) + return(NULL); + return(dst); +} + +/** + * xmlAttrNormalizeSpace2: + * @src: the source string + * + * Normalize the space in non CDATA attribute values, a slightly more complex + * front end to avoid allocation problems when running on attribute values + * coming from the input. + * + * Returns a pointer to the normalized value (dst) or NULL if no conversion + * is needed. + */ +static const xmlChar * +xmlAttrNormalizeSpace2(xmlParserCtxtPtr ctxt, xmlChar *src, int *len) +{ + int i; + int remove_head = 0; + int need_realloc = 0; + const xmlChar *cur; + + if ((ctxt == NULL) || (src == NULL) || (len == NULL)) + return(NULL); + i = *len; + if (i <= 0) + return(NULL); + + cur = src; + while (*cur == 0x20) { + cur++; + remove_head++; + } + while (*cur != 0) { + if (*cur == 0x20) { + cur++; + if ((*cur == 0x20) || (*cur == 0)) { + need_realloc = 1; + break; + } + } else + cur++; + } + if (need_realloc) { + xmlChar *ret; + + ret = xmlStrndup(src + remove_head, i - remove_head + 1); + if (ret == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + xmlAttrNormalizeSpace(ret, ret); + *len = (int) strlen((const char *)ret); + return(ret); + } else if (remove_head) { + *len -= remove_head; + memmove(src, src + remove_head, 1 + *len); + return(src); + } + return(NULL); +} + +/** + * xmlAddDefAttrs: + * @ctxt: an XML parser context + * @fullname: the element fullname + * @fullattr: the attribute fullname + * @value: the attribute value + * + * Add a defaulted attribute for an element + */ +static void +xmlAddDefAttrs(xmlParserCtxtPtr ctxt, + const xmlChar *fullname, + const xmlChar *fullattr, + const xmlChar *value) { + xmlDefAttrsPtr defaults; + int len; + const xmlChar *name; + const xmlChar *prefix; + + /* + * Allows to detect attribute redefinitions + */ + if (ctxt->attsSpecial != NULL) { + if (xmlHashLookup2(ctxt->attsSpecial, fullname, fullattr) != NULL) + return; + } + + if (ctxt->attsDefault == NULL) { + ctxt->attsDefault = xmlHashCreateDict(10, ctxt->dict); + if (ctxt->attsDefault == NULL) + goto mem_error; + } + + /* + * split the element name into prefix:localname , the string found + * are within the DTD and then not associated to namespace names. + */ + name = xmlSplitQName3(fullname, &len); + if (name == NULL) { + name = xmlDictLookup(ctxt->dict, fullname, -1); + prefix = NULL; + } else { + name = xmlDictLookup(ctxt->dict, name, -1); + prefix = xmlDictLookup(ctxt->dict, fullname, len); + } + + /* + * make sure there is some storage + */ + defaults = xmlHashLookup2(ctxt->attsDefault, name, prefix); + if (defaults == NULL) { + defaults = (xmlDefAttrsPtr) xmlMalloc(sizeof(xmlDefAttrs) + + (4 * 5) * sizeof(const xmlChar *)); + if (defaults == NULL) + goto mem_error; + defaults->nbAttrs = 0; + defaults->maxAttrs = 4; + if (xmlHashUpdateEntry2(ctxt->attsDefault, name, prefix, + defaults, NULL) < 0) { + xmlFree(defaults); + goto mem_error; + } + } else if (defaults->nbAttrs >= defaults->maxAttrs) { + xmlDefAttrsPtr temp; + + temp = (xmlDefAttrsPtr) xmlRealloc(defaults, sizeof(xmlDefAttrs) + + (2 * defaults->maxAttrs * 5) * sizeof(const xmlChar *)); + if (temp == NULL) + goto mem_error; + defaults = temp; + defaults->maxAttrs *= 2; + if (xmlHashUpdateEntry2(ctxt->attsDefault, name, prefix, + defaults, NULL) < 0) { + xmlFree(defaults); + goto mem_error; + } + } + + /* + * Split the element name into prefix:localname , the string found + * are within the DTD and hen not associated to namespace names. + */ + name = xmlSplitQName3(fullattr, &len); + if (name == NULL) { + name = xmlDictLookup(ctxt->dict, fullattr, -1); + prefix = NULL; + } else { + name = xmlDictLookup(ctxt->dict, name, -1); + prefix = xmlDictLookup(ctxt->dict, fullattr, len); + } + + defaults->values[5 * defaults->nbAttrs] = name; + defaults->values[5 * defaults->nbAttrs + 1] = prefix; + /* intern the string and precompute the end */ + len = xmlStrlen(value); + value = xmlDictLookup(ctxt->dict, value, len); + defaults->values[5 * defaults->nbAttrs + 2] = value; + defaults->values[5 * defaults->nbAttrs + 3] = value + len; + if (ctxt->external) + defaults->values[5 * defaults->nbAttrs + 4] = BAD_CAST "external"; + else + defaults->values[5 * defaults->nbAttrs + 4] = NULL; + defaults->nbAttrs++; + + return; + +mem_error: + xmlErrMemory(ctxt, NULL); + return; +} + +/** + * xmlAddSpecialAttr: + * @ctxt: an XML parser context + * @fullname: the element fullname + * @fullattr: the attribute fullname + * @type: the attribute type + * + * Register this attribute type + */ +static void +xmlAddSpecialAttr(xmlParserCtxtPtr ctxt, + const xmlChar *fullname, + const xmlChar *fullattr, + int type) +{ + if (ctxt->attsSpecial == NULL) { + ctxt->attsSpecial = xmlHashCreateDict(10, ctxt->dict); + if (ctxt->attsSpecial == NULL) + goto mem_error; + } + + if (xmlHashLookup2(ctxt->attsSpecial, fullname, fullattr) != NULL) + return; + + xmlHashAddEntry2(ctxt->attsSpecial, fullname, fullattr, + (void *) (ptrdiff_t) type); + return; + +mem_error: + xmlErrMemory(ctxt, NULL); + return; +} + +/** + * xmlCleanSpecialAttrCallback: + * + * Removes CDATA attributes from the special attribute table + */ +static void +xmlCleanSpecialAttrCallback(void *payload, void *data, + const xmlChar *fullname, const xmlChar *fullattr, + const xmlChar *unused ATTRIBUTE_UNUSED) { + xmlParserCtxtPtr ctxt = (xmlParserCtxtPtr) data; + + if (((ptrdiff_t) payload) == XML_ATTRIBUTE_CDATA) { + xmlHashRemoveEntry2(ctxt->attsSpecial, fullname, fullattr, NULL); + } +} + +/** + * xmlCleanSpecialAttr: + * @ctxt: an XML parser context + * + * Trim the list of attributes defined to remove all those of type + * CDATA as they are not special. This call should be done when finishing + * to parse the DTD and before starting to parse the document root. + */ +static void +xmlCleanSpecialAttr(xmlParserCtxtPtr ctxt) +{ + if (ctxt->attsSpecial == NULL) + return; + + xmlHashScanFull(ctxt->attsSpecial, xmlCleanSpecialAttrCallback, ctxt); + + if (xmlHashSize(ctxt->attsSpecial) == 0) { + xmlHashFree(ctxt->attsSpecial, NULL); + ctxt->attsSpecial = NULL; + } + return; +} + +/** + * xmlCheckLanguageID: + * @lang: pointer to the string value + * + * Checks that the value conforms to the LanguageID production: + * + * NOTE: this is somewhat deprecated, those productions were removed from + * the XML Second edition. + * + * [33] LanguageID ::= Langcode ('-' Subcode)* + * [34] Langcode ::= ISO639Code | IanaCode | UserCode + * [35] ISO639Code ::= ([a-z] | [A-Z]) ([a-z] | [A-Z]) + * [36] IanaCode ::= ('i' | 'I') '-' ([a-z] | [A-Z])+ + * [37] UserCode ::= ('x' | 'X') '-' ([a-z] | [A-Z])+ + * [38] Subcode ::= ([a-z] | [A-Z])+ + * + * The current REC reference the sucessors of RFC 1766, currently 5646 + * + * http://www.rfc-editor.org/rfc/rfc5646.txt + * langtag = language + * ["-" script] + * ["-" region] + * *("-" variant) + * *("-" extension) + * ["-" privateuse] + * language = 2*3ALPHA ; shortest ISO 639 code + * ["-" extlang] ; sometimes followed by + * ; extended language subtags + * / 4ALPHA ; or reserved for future use + * / 5*8ALPHA ; or registered language subtag + * + * extlang = 3ALPHA ; selected ISO 639 codes + * *2("-" 3ALPHA) ; permanently reserved + * + * script = 4ALPHA ; ISO 15924 code + * + * region = 2ALPHA ; ISO 3166-1 code + * / 3DIGIT ; UN M.49 code + * + * variant = 5*8alphanum ; registered variants + * / (DIGIT 3alphanum) + * + * extension = singleton 1*("-" (2*8alphanum)) + * + * ; Single alphanumerics + * ; "x" reserved for private use + * singleton = DIGIT ; 0 - 9 + * / %x41-57 ; A - W + * / %x59-5A ; Y - Z + * / %x61-77 ; a - w + * / %x79-7A ; y - z + * + * it sounds right to still allow Irregular i-xxx IANA and user codes too + * The parser below doesn't try to cope with extension or privateuse + * that could be added but that's not interoperable anyway + * + * Returns 1 if correct 0 otherwise + **/ +int +xmlCheckLanguageID(const xmlChar * lang) +{ + const xmlChar *cur = lang, *nxt; + + if (cur == NULL) + return (0); + if (((cur[0] == 'i') && (cur[1] == '-')) || + ((cur[0] == 'I') && (cur[1] == '-')) || + ((cur[0] == 'x') && (cur[1] == '-')) || + ((cur[0] == 'X') && (cur[1] == '-'))) { + /* + * Still allow IANA code and user code which were coming + * from the previous version of the XML-1.0 specification + * it's deprecated but we should not fail + */ + cur += 2; + while (((cur[0] >= 'A') && (cur[0] <= 'Z')) || + ((cur[0] >= 'a') && (cur[0] <= 'z'))) + cur++; + return(cur[0] == 0); + } + nxt = cur; + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur >= 4) { + /* + * Reserved + */ + if ((nxt - cur > 8) || (nxt[0] != 0)) + return(0); + return(1); + } + if (nxt - cur < 2) + return(0); + /* we got an ISO 639 code */ + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have extlang or script or region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur == 4) + goto script; + if (nxt - cur == 2) + goto region; + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 3) + return(0); + /* we parsed an extlang */ + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have script or region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur == 2) + goto region; + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 4) + return(0); + /* we parsed a script */ +script: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 2) + return(0); + /* we parsed a region */ +region: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can just have a variant */ + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + + if ((nxt - cur < 5) || (nxt - cur > 8)) + return(0); + + /* we parsed a variant */ +variant: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + /* extensions and private use subtags not checked */ + return (1); + +region_m49: + if (((nxt[1] >= '0') && (nxt[1] <= '9')) && + ((nxt[2] >= '0') && (nxt[2] <= '9'))) { + nxt += 3; + goto region; + } + return(0); +} + +/************************************************************************ + * * + * Parser stacks related functions and macros * + * * + ************************************************************************/ + +static xmlEntityPtr xmlParseStringEntityRef(xmlParserCtxtPtr ctxt, + const xmlChar ** str); + +#ifdef SAX2 +/** + * nsPush: + * @ctxt: an XML parser context + * @prefix: the namespace prefix or NULL + * @URL: the namespace name + * + * Pushes a new parser namespace on top of the ns stack + * + * Returns -1 in case of error, -2 if the namespace should be discarded + * and the index in the stack otherwise. + */ +static int +nsPush(xmlParserCtxtPtr ctxt, const xmlChar *prefix, const xmlChar *URL) +{ + if (ctxt->options & XML_PARSE_NSCLEAN) { + int i; + for (i = ctxt->nsNr - 2;i >= 0;i -= 2) { + if (ctxt->nsTab[i] == prefix) { + /* in scope */ + if (ctxt->nsTab[i + 1] == URL) + return(-2); + /* out of scope keep it */ + break; + } + } + } + if ((ctxt->nsMax == 0) || (ctxt->nsTab == NULL)) { + ctxt->nsMax = 10; + ctxt->nsNr = 0; + ctxt->nsTab = (const xmlChar **) + xmlMalloc(ctxt->nsMax * sizeof(xmlChar *)); + if (ctxt->nsTab == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->nsMax = 0; + return (-1); + } + } else if (ctxt->nsNr >= ctxt->nsMax) { + const xmlChar ** tmp; + ctxt->nsMax *= 2; + tmp = (const xmlChar **) xmlRealloc((char *) ctxt->nsTab, + ctxt->nsMax * sizeof(ctxt->nsTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->nsMax /= 2; + return (-1); + } + ctxt->nsTab = tmp; + } + ctxt->nsTab[ctxt->nsNr++] = prefix; + ctxt->nsTab[ctxt->nsNr++] = URL; + return (ctxt->nsNr); +} +/** + * nsPop: + * @ctxt: an XML parser context + * @nr: the number to pop + * + * Pops the top @nr parser prefix/namespace from the ns stack + * + * Returns the number of namespaces removed + */ +static int +nsPop(xmlParserCtxtPtr ctxt, int nr) +{ + int i; + + if (ctxt->nsTab == NULL) return(0); + if (ctxt->nsNr < nr) { + xmlGenericError(xmlGenericErrorContext, "Pbm popping %d NS\n", nr); + nr = ctxt->nsNr; + } + if (ctxt->nsNr <= 0) + return (0); + + for (i = 0;i < nr;i++) { + ctxt->nsNr--; + ctxt->nsTab[ctxt->nsNr] = NULL; + } + return(nr); +} +#endif + +static int +xmlCtxtGrowAttrs(xmlParserCtxtPtr ctxt, int nr) { + const xmlChar **atts; + int *attallocs; + int maxatts; + + if (ctxt->atts == NULL) { + maxatts = 55; /* allow for 10 attrs by default */ + atts = (const xmlChar **) + xmlMalloc(maxatts * sizeof(xmlChar *)); + if (atts == NULL) goto mem_error; + ctxt->atts = atts; + attallocs = (int *) xmlMalloc((maxatts / 5) * sizeof(int)); + if (attallocs == NULL) goto mem_error; + ctxt->attallocs = attallocs; + ctxt->maxatts = maxatts; + } else if (nr + 5 > ctxt->maxatts) { + maxatts = (nr + 5) * 2; + atts = (const xmlChar **) xmlRealloc((void *) ctxt->atts, + maxatts * sizeof(const xmlChar *)); + if (atts == NULL) goto mem_error; + ctxt->atts = atts; + attallocs = (int *) xmlRealloc((void *) ctxt->attallocs, + (maxatts / 5) * sizeof(int)); + if (attallocs == NULL) goto mem_error; + ctxt->attallocs = attallocs; + ctxt->maxatts = maxatts; + } + return(ctxt->maxatts); +mem_error: + xmlErrMemory(ctxt, NULL); + return(-1); +} + +/** + * inputPush: + * @ctxt: an XML parser context + * @value: the parser input + * + * Pushes a new parser input on top of the input stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +inputPush(xmlParserCtxtPtr ctxt, xmlParserInputPtr value) +{ + if ((ctxt == NULL) || (value == NULL)) + return(-1); + if (ctxt->inputNr >= ctxt->inputMax) { + ctxt->inputMax *= 2; + ctxt->inputTab = + (xmlParserInputPtr *) xmlRealloc(ctxt->inputTab, + ctxt->inputMax * + sizeof(ctxt->inputTab[0])); + if (ctxt->inputTab == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFreeInputStream(value); + ctxt->inputMax /= 2; + value = NULL; + return (-1); + } + } + ctxt->inputTab[ctxt->inputNr] = value; + ctxt->input = value; + return (ctxt->inputNr++); +} +/** + * inputPop: + * @ctxt: an XML parser context + * + * Pops the top parser input from the input stack + * + * Returns the input just removed + */ +xmlParserInputPtr +inputPop(xmlParserCtxtPtr ctxt) +{ + xmlParserInputPtr ret; + + if (ctxt == NULL) + return(NULL); + if (ctxt->inputNr <= 0) + return (NULL); + ctxt->inputNr--; + if (ctxt->inputNr > 0) + ctxt->input = ctxt->inputTab[ctxt->inputNr - 1]; + else + ctxt->input = NULL; + ret = ctxt->inputTab[ctxt->inputNr]; + ctxt->inputTab[ctxt->inputNr] = NULL; + return (ret); +} +/** + * nodePush: + * @ctxt: an XML parser context + * @value: the element node + * + * Pushes a new element node on top of the node stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +nodePush(xmlParserCtxtPtr ctxt, xmlNodePtr value) +{ + if (ctxt == NULL) return(0); + if (ctxt->nodeNr >= ctxt->nodeMax) { + xmlNodePtr *tmp; + + tmp = (xmlNodePtr *) xmlRealloc(ctxt->nodeTab, + ctxt->nodeMax * 2 * + sizeof(ctxt->nodeTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + return (-1); + } + ctxt->nodeTab = tmp; + ctxt->nodeMax *= 2; + } + if ((((unsigned int) ctxt->nodeNr) > xmlParserMaxDepth) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INTERNAL_ERROR, + "Excessive depth in document: %d use XML_PARSE_HUGE option\n", + xmlParserMaxDepth); + xmlHaltParser(ctxt); + return(-1); + } + ctxt->nodeTab[ctxt->nodeNr] = value; + ctxt->node = value; + return (ctxt->nodeNr++); +} + +/** + * nodePop: + * @ctxt: an XML parser context + * + * Pops the top element node from the node stack + * + * Returns the node just removed + */ +xmlNodePtr +nodePop(xmlParserCtxtPtr ctxt) +{ + xmlNodePtr ret; + + if (ctxt == NULL) return(NULL); + if (ctxt->nodeNr <= 0) + return (NULL); + ctxt->nodeNr--; + if (ctxt->nodeNr > 0) + ctxt->node = ctxt->nodeTab[ctxt->nodeNr - 1]; + else + ctxt->node = NULL; + ret = ctxt->nodeTab[ctxt->nodeNr]; + ctxt->nodeTab[ctxt->nodeNr] = NULL; + return (ret); +} + +#ifdef LIBXML_PUSH_ENABLED +/** + * nameNsPush: + * @ctxt: an XML parser context + * @value: the element name + * @prefix: the element prefix + * @URI: the element namespace name + * + * Pushes a new element name/prefix/URL on top of the name stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +static int +nameNsPush(xmlParserCtxtPtr ctxt, const xmlChar * value, + const xmlChar *prefix, const xmlChar *URI, int nsNr) +{ + if (ctxt->nameNr >= ctxt->nameMax) { + const xmlChar * *tmp; + void **tmp2; + ctxt->nameMax *= 2; + tmp = (const xmlChar * *) xmlRealloc((xmlChar * *)ctxt->nameTab, + ctxt->nameMax * + sizeof(ctxt->nameTab[0])); + if (tmp == NULL) { + ctxt->nameMax /= 2; + goto mem_error; + } + ctxt->nameTab = tmp; + tmp2 = (void **) xmlRealloc((void * *)ctxt->pushTab, + ctxt->nameMax * 3 * + sizeof(ctxt->pushTab[0])); + if (tmp2 == NULL) { + ctxt->nameMax /= 2; + goto mem_error; + } + ctxt->pushTab = tmp2; + } + ctxt->nameTab[ctxt->nameNr] = value; + ctxt->name = value; + ctxt->pushTab[ctxt->nameNr * 3] = (void *) prefix; + ctxt->pushTab[ctxt->nameNr * 3 + 1] = (void *) URI; + ctxt->pushTab[ctxt->nameNr * 3 + 2] = (void *) (ptrdiff_t) nsNr; + return (ctxt->nameNr++); +mem_error: + xmlErrMemory(ctxt, NULL); + return (-1); +} +/** + * nameNsPop: + * @ctxt: an XML parser context + * + * Pops the top element/prefix/URI name from the name stack + * + * Returns the name just removed + */ +static const xmlChar * +nameNsPop(xmlParserCtxtPtr ctxt) +{ + const xmlChar *ret; + + if (ctxt->nameNr <= 0) + return (NULL); + ctxt->nameNr--; + if (ctxt->nameNr > 0) + ctxt->name = ctxt->nameTab[ctxt->nameNr - 1]; + else + ctxt->name = NULL; + ret = ctxt->nameTab[ctxt->nameNr]; + ctxt->nameTab[ctxt->nameNr] = NULL; + return (ret); +} +#endif /* LIBXML_PUSH_ENABLED */ + +/** + * namePush: + * @ctxt: an XML parser context + * @value: the element name + * + * Pushes a new element name on top of the name stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +namePush(xmlParserCtxtPtr ctxt, const xmlChar * value) +{ + if (ctxt == NULL) return (-1); + + if (ctxt->nameNr >= ctxt->nameMax) { + const xmlChar * *tmp; + tmp = (const xmlChar * *) xmlRealloc((xmlChar * *)ctxt->nameTab, + ctxt->nameMax * 2 * + sizeof(ctxt->nameTab[0])); + if (tmp == NULL) { + goto mem_error; + } + ctxt->nameTab = tmp; + ctxt->nameMax *= 2; + } + ctxt->nameTab[ctxt->nameNr] = value; + ctxt->name = value; + return (ctxt->nameNr++); +mem_error: + xmlErrMemory(ctxt, NULL); + return (-1); +} +/** + * namePop: + * @ctxt: an XML parser context + * + * Pops the top element name from the name stack + * + * Returns the name just removed + */ +const xmlChar * +namePop(xmlParserCtxtPtr ctxt) +{ + const xmlChar *ret; + + if ((ctxt == NULL) || (ctxt->nameNr <= 0)) + return (NULL); + ctxt->nameNr--; + if (ctxt->nameNr > 0) + ctxt->name = ctxt->nameTab[ctxt->nameNr - 1]; + else + ctxt->name = NULL; + ret = ctxt->nameTab[ctxt->nameNr]; + ctxt->nameTab[ctxt->nameNr] = NULL; + return (ret); +} + +static int spacePush(xmlParserCtxtPtr ctxt, int val) { + if (ctxt->spaceNr >= ctxt->spaceMax) { + int *tmp; + + ctxt->spaceMax *= 2; + tmp = (int *) xmlRealloc(ctxt->spaceTab, + ctxt->spaceMax * sizeof(ctxt->spaceTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->spaceMax /=2; + return(-1); + } + ctxt->spaceTab = tmp; + } + ctxt->spaceTab[ctxt->spaceNr] = val; + ctxt->space = &ctxt->spaceTab[ctxt->spaceNr]; + return(ctxt->spaceNr++); +} + +static int spacePop(xmlParserCtxtPtr ctxt) { + int ret; + if (ctxt->spaceNr <= 0) return(0); + ctxt->spaceNr--; + if (ctxt->spaceNr > 0) + ctxt->space = &ctxt->spaceTab[ctxt->spaceNr - 1]; + else + ctxt->space = &ctxt->spaceTab[0]; + ret = ctxt->spaceTab[ctxt->spaceNr]; + ctxt->spaceTab[ctxt->spaceNr] = -1; + return(ret); +} + +/* + * Macros for accessing the content. Those should be used only by the parser, + * and not exported. + * + * Dirty macros, i.e. one often need to make assumption on the context to + * use them + * + * CUR_PTR return the current pointer to the xmlChar to be parsed. + * To be used with extreme caution since operations consuming + * characters may move the input buffer to a different location ! + * CUR returns the current xmlChar value, i.e. a 8 bit value if compiled + * This should be used internally by the parser + * only to compare to ASCII values otherwise it would break when + * running with UTF-8 encoding. + * RAW same as CUR but in the input buffer, bypass any token + * extraction that may have been done + * NXT(n) returns the n'th next xmlChar. Same as CUR is should be used only + * to compare on ASCII based substring. + * SKIP(n) Skip n xmlChar, and must also be used only to skip ASCII defined + * strings without newlines within the parser. + * NEXT1(l) Skip 1 xmlChar, and must also be used only to skip 1 non-newline ASCII + * defined char within the parser. + * Clean macros, not dependent of an ASCII context, expect UTF-8 encoding + * + * NEXT Skip to the next character, this does the proper decoding + * in UTF-8 mode. It also pop-up unfinished entities on the fly. + * NEXTL(l) Skip the current unicode character of l xmlChars long. + * CUR_CHAR(l) returns the current unicode character (int), set l + * to the number of xmlChars used for the encoding [0-5]. + * CUR_SCHAR same but operate on a string instead of the context + * COPY_BUF copy the current unicode char to the target buffer, increment + * the index + * GROW, SHRINK handling of input buffers + */ + +#define RAW (*ctxt->input->cur) +#define CUR (*ctxt->input->cur) +#define NXT(val) ctxt->input->cur[(val)] +#define CUR_PTR ctxt->input->cur +#define BASE_PTR ctxt->input->base + +#define CMP4( s, c1, c2, c3, c4 ) \ + ( ((unsigned char *) s)[ 0 ] == c1 && ((unsigned char *) s)[ 1 ] == c2 && \ + ((unsigned char *) s)[ 2 ] == c3 && ((unsigned char *) s)[ 3 ] == c4 ) +#define CMP5( s, c1, c2, c3, c4, c5 ) \ + ( CMP4( s, c1, c2, c3, c4 ) && ((unsigned char *) s)[ 4 ] == c5 ) +#define CMP6( s, c1, c2, c3, c4, c5, c6 ) \ + ( CMP5( s, c1, c2, c3, c4, c5 ) && ((unsigned char *) s)[ 5 ] == c6 ) +#define CMP7( s, c1, c2, c3, c4, c5, c6, c7 ) \ + ( CMP6( s, c1, c2, c3, c4, c5, c6 ) && ((unsigned char *) s)[ 6 ] == c7 ) +#define CMP8( s, c1, c2, c3, c4, c5, c6, c7, c8 ) \ + ( CMP7( s, c1, c2, c3, c4, c5, c6, c7 ) && ((unsigned char *) s)[ 7 ] == c8 ) +#define CMP9( s, c1, c2, c3, c4, c5, c6, c7, c8, c9 ) \ + ( CMP8( s, c1, c2, c3, c4, c5, c6, c7, c8 ) && \ + ((unsigned char *) s)[ 8 ] == c9 ) +#define CMP10( s, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 ) \ + ( CMP9( s, c1, c2, c3, c4, c5, c6, c7, c8, c9 ) && \ + ((unsigned char *) s)[ 9 ] == c10 ) + +#define SKIP(val) do { \ + ctxt->nbChars += (val),ctxt->input->cur += (val),ctxt->input->col+=(val); \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } while (0) + +#define SKIPL(val) do { \ + int skipl; \ + for(skipl=0; skiplinput->cur) == '\n') { \ + ctxt->input->line++; ctxt->input->col = 1; \ + } else ctxt->input->col++; \ + ctxt->nbChars++; \ + ctxt->input->cur++; \ + } \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } while (0) + +#define SHRINK if ((ctxt->progressive == 0) && \ + (ctxt->input->cur - ctxt->input->base > 2 * INPUT_CHUNK) && \ + (ctxt->input->end - ctxt->input->cur < 2 * INPUT_CHUNK)) \ + xmlSHRINK (ctxt); + +static void xmlSHRINK (xmlParserCtxtPtr ctxt) { + xmlParserInputShrink(ctxt->input); + if (*ctxt->input->cur == 0) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); +} + +#define GROW if ((ctxt->progressive == 0) && \ + (ctxt->input->end - ctxt->input->cur < INPUT_CHUNK)) \ + xmlGROW (ctxt); + +static void xmlGROW (xmlParserCtxtPtr ctxt) { + unsigned long curEnd = ctxt->input->end - ctxt->input->cur; + unsigned long curBase = ctxt->input->cur - ctxt->input->base; + + if (((curEnd > (unsigned long) XML_MAX_LOOKUP_LIMIT) || + (curBase > (unsigned long) XML_MAX_LOOKUP_LIMIT)) && + ((ctxt->input->buf) && (ctxt->input->buf->readcallback != (xmlInputReadCallback) xmlNop)) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, "Huge input lookup"); + xmlHaltParser(ctxt); + return; + } + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + if ((ctxt->input->cur > ctxt->input->end) || + (ctxt->input->cur < ctxt->input->base)) { + xmlHaltParser(ctxt); + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, "cur index out of bound"); + return; + } + if ((ctxt->input->cur != NULL) && (*ctxt->input->cur == 0)) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); +} + +#define SKIP_BLANKS xmlSkipBlankChars(ctxt) + +#define NEXT xmlNextChar(ctxt) + +#define NEXT1 { \ + ctxt->input->col++; \ + ctxt->input->cur++; \ + ctxt->nbChars++; \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } + +#define NEXTL(l) do { \ + if (*(ctxt->input->cur) == '\n') { \ + ctxt->input->line++; ctxt->input->col = 1; \ + } else ctxt->input->col++; \ + ctxt->input->cur += l; \ + } while (0) + +#define CUR_CHAR(l) xmlCurrentChar(ctxt, &l) +#define CUR_SCHAR(s, l) xmlStringCurrentChar(ctxt, s, &l) + +#define COPY_BUF(l,b,i,v) \ + if (l == 1) b[i++] = (xmlChar) v; \ + else i += xmlCopyCharMultiByte(&b[i],v) + +/** + * xmlSkipBlankChars: + * @ctxt: the XML parser context + * + * skip all blanks character found at that point in the input streams. + * It pops up finished entities in the process if allowable at that point. + * + * Returns the number of space chars skipped + */ + +int +xmlSkipBlankChars(xmlParserCtxtPtr ctxt) { + int res = 0; + + /* + * It's Okay to use CUR/NEXT here since all the blanks are on + * the ASCII range. + */ + if ((ctxt->inputNr == 1) && (ctxt->instate != XML_PARSER_DTD)) { + const xmlChar *cur; + /* + * if we are in the document content, go really fast + */ + cur = ctxt->input->cur; + while (IS_BLANK_CH(*cur)) { + if (*cur == '\n') { + ctxt->input->line++; ctxt->input->col = 1; + } else { + ctxt->input->col++; + } + cur++; + res++; + if (*cur == 0) { + ctxt->input->cur = cur; + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + cur = ctxt->input->cur; + } + } + ctxt->input->cur = cur; + } else { + int expandPE = ((ctxt->external != 0) || (ctxt->inputNr != 1)); + + while (1) { + if (IS_BLANK_CH(CUR)) { /* CHECKED tstblanks.xml */ + NEXT; + } else if (CUR == '%') { + /* + * Need to handle support of entities branching here + */ + if ((expandPE == 0) || (IS_BLANK_CH(NXT(1))) || (NXT(1) == 0)) + break; + xmlParsePEReference(ctxt); + } else if (CUR == 0) { + if (ctxt->inputNr <= 1) + break; + xmlPopInput(ctxt); + } else { + break; + } + + /* + * Also increase the counter when entering or exiting a PERef. + * The spec says: "When a parameter-entity reference is recognized + * in the DTD and included, its replacement text MUST be enlarged + * by the attachment of one leading and one following space (#x20) + * character." + */ + res++; + } + } + return(res); +} + +/************************************************************************ + * * + * Commodity functions to handle entities * + * * + ************************************************************************/ + +/** + * xmlPopInput: + * @ctxt: an XML parser context + * + * xmlPopInput: the current input pointed by ctxt->input came to an end + * pop it and return the next char. + * + * Returns the current xmlChar in the parser context + */ +xmlChar +xmlPopInput(xmlParserCtxtPtr ctxt) { + if ((ctxt == NULL) || (ctxt->inputNr <= 1)) return(0); + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "Popping input %d\n", ctxt->inputNr); + if ((ctxt->inputNr > 1) && (ctxt->inSubset == 0) && + (ctxt->instate != XML_PARSER_EOF)) + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, + "Unfinished entity outside the DTD"); + xmlFreeInputStream(inputPop(ctxt)); + if (*ctxt->input->cur == 0) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + return(CUR); +} + +/** + * xmlPushInput: + * @ctxt: an XML parser context + * @input: an XML parser input fragment (entity, XML fragment ...). + * + * xmlPushInput: switch to a new input stream which is stacked on top + * of the previous one(s). + * Returns -1 in case of error or the index in the input stack + */ +int +xmlPushInput(xmlParserCtxtPtr ctxt, xmlParserInputPtr input) { + int ret; + if (input == NULL) return(-1); + + if (xmlParserDebugEntities) { + if ((ctxt->input != NULL) && (ctxt->input->filename)) + xmlGenericError(xmlGenericErrorContext, + "%s(%d): ", ctxt->input->filename, + ctxt->input->line); + xmlGenericError(xmlGenericErrorContext, + "Pushing input %d : %.30s\n", ctxt->inputNr+1, input->cur); + } + if (((ctxt->inputNr > 40) && ((ctxt->options & XML_PARSE_HUGE) == 0)) || + (ctxt->inputNr > 1024)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + while (ctxt->inputNr > 1) + xmlFreeInputStream(inputPop(ctxt)); + return(-1); + } + ret = inputPush(ctxt, input); + if (ctxt->instate == XML_PARSER_EOF) + return(-1); + GROW; + return(ret); +} + +/** + * xmlParseCharRef: + * @ctxt: an XML parser context + * + * parse Reference declarations + * + * [66] CharRef ::= '&#' [0-9]+ ';' | + * '&#x' [0-9a-fA-F]+ ';' + * + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + * + * Returns the value parsed (as an int), 0 in case of error + */ +int +xmlParseCharRef(xmlParserCtxtPtr ctxt) { + unsigned int val = 0; + int count = 0; + unsigned int outofrange = 0; + + /* + * Using RAW/CUR/NEXT is okay since we are working on ASCII range here + */ + if ((RAW == '&') && (NXT(1) == '#') && + (NXT(2) == 'x')) { + SKIP(3); + GROW; + while (RAW != ';') { /* loop blocked by count */ + if (count++ > 20) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(0); + } + if ((RAW >= '0') && (RAW <= '9')) + val = val * 16 + (CUR - '0'); + else if ((RAW >= 'a') && (RAW <= 'f') && (count < 20)) + val = val * 16 + (CUR - 'a') + 10; + else if ((RAW >= 'A') && (RAW <= 'F') && (count < 20)) + val = val * 16 + (CUR - 'A') + 10; + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_HEX_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + NEXT; + count++; + } + if (RAW == ';') { + /* on purpose to avoid reentrancy problems with NEXT and SKIP */ + ctxt->input->col++; + ctxt->nbChars ++; + ctxt->input->cur++; + } + } else if ((RAW == '&') && (NXT(1) == '#')) { + SKIP(2); + GROW; + while (RAW != ';') { /* loop blocked by count */ + if (count++ > 20) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(0); + } + if ((RAW >= '0') && (RAW <= '9')) + val = val * 10 + (CUR - '0'); + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_DEC_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + NEXT; + count++; + } + if (RAW == ';') { + /* on purpose to avoid reentrancy problems with NEXT and SKIP */ + ctxt->input->col++; + ctxt->nbChars ++; + ctxt->input->cur++; + } + } else { + xmlFatalErr(ctxt, XML_ERR_INVALID_CHARREF, NULL); + } + + /* + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + */ + if ((IS_CHAR(val) && (outofrange == 0))) { + return(val); + } else { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseCharRef: invalid xmlChar value %d\n", + val); + } + return(0); +} + +/** + * xmlParseStringCharRef: + * @ctxt: an XML parser context + * @str: a pointer to an index in the string + * + * parse Reference declarations, variant parsing from a string rather + * than an an input flow. + * + * [66] CharRef ::= '&#' [0-9]+ ';' | + * '&#x' [0-9a-fA-F]+ ';' + * + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + * + * Returns the value parsed (as an int), 0 in case of error, str will be + * updated to the current value of the index + */ +static int +xmlParseStringCharRef(xmlParserCtxtPtr ctxt, const xmlChar **str) { + const xmlChar *ptr; + xmlChar cur; + unsigned int val = 0; + unsigned int outofrange = 0; + + if ((str == NULL) || (*str == NULL)) return(0); + ptr = *str; + cur = *ptr; + if ((cur == '&') && (ptr[1] == '#') && (ptr[2] == 'x')) { + ptr += 3; + cur = *ptr; + while (cur != ';') { /* Non input consuming loop */ + if ((cur >= '0') && (cur <= '9')) + val = val * 16 + (cur - '0'); + else if ((cur >= 'a') && (cur <= 'f')) + val = val * 16 + (cur - 'a') + 10; + else if ((cur >= 'A') && (cur <= 'F')) + val = val * 16 + (cur - 'A') + 10; + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_HEX_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + ptr++; + cur = *ptr; + } + if (cur == ';') + ptr++; + } else if ((cur == '&') && (ptr[1] == '#')){ + ptr += 2; + cur = *ptr; + while (cur != ';') { /* Non input consuming loops */ + if ((cur >= '0') && (cur <= '9')) + val = val * 10 + (cur - '0'); + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_DEC_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + ptr++; + cur = *ptr; + } + if (cur == ';') + ptr++; + } else { + xmlFatalErr(ctxt, XML_ERR_INVALID_CHARREF, NULL); + return(0); + } + *str = ptr; + + /* + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + */ + if ((IS_CHAR(val) && (outofrange == 0))) { + return(val); + } else { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseStringCharRef: invalid xmlChar value %d\n", + val); + } + return(0); +} + +/** + * xmlParserHandlePEReference: + * @ctxt: the parser context + * + * [69] PEReference ::= '%' Name ';' + * + * [ WFC: No Recursion ] + * A parsed entity must not contain a recursive + * reference to itself, either directly or indirectly. + * + * [ WFC: Entity Declared ] + * In a document without any DTD, a document with only an internal DTD + * subset which contains no parameter entity references, or a document + * with "standalone='yes'", ... ... The declaration of a parameter + * entity must precede any reference to it... + * + * [ VC: Entity Declared ] + * In a document with an external subset or external parameter entities + * with "standalone='no'", ... ... The declaration of a parameter entity + * must precede any reference to it... + * + * [ WFC: In DTD ] + * Parameter-entity references may only appear in the DTD. + * NOTE: misleading but this is handled. + * + * A PEReference may have been detected in the current input stream + * the handling is done accordingly to + * http://www.w3.org/TR/REC-xml#entproc + * i.e. + * - Included in literal in entity values + * - Included as Parameter Entity reference within DTDs + */ +void +xmlParserHandlePEReference(xmlParserCtxtPtr ctxt) { + switch(ctxt->instate) { + case XML_PARSER_CDATA_SECTION: + return; + case XML_PARSER_COMMENT: + return; + case XML_PARSER_START_TAG: + return; + case XML_PARSER_END_TAG: + return; + case XML_PARSER_EOF: + xmlFatalErr(ctxt, XML_ERR_PEREF_AT_EOF, NULL); + return; + case XML_PARSER_PROLOG: + case XML_PARSER_START: + case XML_PARSER_MISC: + xmlFatalErr(ctxt, XML_ERR_PEREF_IN_PROLOG, NULL); + return; + case XML_PARSER_ENTITY_DECL: + case XML_PARSER_CONTENT: + case XML_PARSER_ATTRIBUTE_VALUE: + case XML_PARSER_PI: + case XML_PARSER_SYSTEM_LITERAL: + case XML_PARSER_PUBLIC_LITERAL: + /* we just ignore it there */ + return; + case XML_PARSER_EPILOG: + xmlFatalErr(ctxt, XML_ERR_PEREF_IN_EPILOG, NULL); + return; + case XML_PARSER_ENTITY_VALUE: + /* + * NOTE: in the case of entity values, we don't do the + * substitution here since we need the literal + * entity value to be able to save the internal + * subset of the document. + * This will be handled by xmlStringDecodeEntities + */ + return; + case XML_PARSER_DTD: + /* + * [WFC: Well-Formedness Constraint: PEs in Internal Subset] + * In the internal DTD subset, parameter-entity references + * can occur only where markup declarations can occur, not + * within markup declarations. + * In that case this is handled in xmlParseMarkupDecl + */ + if ((ctxt->external == 0) && (ctxt->inputNr == 1)) + return; + if (IS_BLANK_CH(NXT(1)) || NXT(1) == 0) + return; + break; + case XML_PARSER_IGNORE: + return; + } + + xmlParsePEReference(ctxt); +} + +/* + * Macro used to grow the current buffer. + * buffer##_size is expected to be a size_t + * mem_error: is expected to handle memory allocation failures + */ +#define growBuffer(buffer, n) { \ + xmlChar *tmp; \ + size_t new_size = buffer##_size * 2 + n; \ + if (new_size < buffer##_size) goto mem_error; \ + tmp = (xmlChar *) xmlRealloc(buffer, new_size); \ + if (tmp == NULL) goto mem_error; \ + buffer = tmp; \ + buffer##_size = new_size; \ +} + +/** + * xmlStringLenDecodeEntities: + * @ctxt: the parser context + * @str: the input string + * @len: the string length + * @what: combination of XML_SUBSTITUTE_REF and XML_SUBSTITUTE_PEREF + * @end: an end marker xmlChar, 0 if none + * @end2: an end marker xmlChar, 0 if none + * @end3: an end marker xmlChar, 0 if none + * + * Takes a entity string content and process to do the adequate substitutions. + * + * [67] Reference ::= EntityRef | CharRef + * + * [69] PEReference ::= '%' Name ';' + * + * Returns A newly allocated string with the substitution done. The caller + * must deallocate it ! + */ +xmlChar * +xmlStringLenDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, + int what, xmlChar end, xmlChar end2, xmlChar end3) { + xmlChar *buffer = NULL; + size_t buffer_size = 0; + size_t nbchars = 0; + + xmlChar *current = NULL; + xmlChar *rep = NULL; + const xmlChar *last; + xmlEntityPtr ent; + int c,l; + + if ((ctxt == NULL) || (str == NULL) || (len < 0)) + return(NULL); + last = str + len; + + if (((ctxt->depth > 40) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) || + (ctxt->depth > 1024)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + return(NULL); + } + + /* + * allocate a translation buffer. + */ + buffer_size = XML_PARSER_BIG_BUFFER_SIZE; + buffer = (xmlChar *) xmlMallocAtomic(buffer_size); + if (buffer == NULL) goto mem_error; + + /* + * OK loop until we reach one of the ending char or a size limit. + * we are operating on already parsed values. + */ + if (str < last) + c = CUR_SCHAR(str, l); + else + c = 0; + while ((c != 0) && (c != end) && /* non input consuming loop */ + (c != end2) && (c != end3)) { + + if (c == 0) break; + if ((c == '&') && (str[1] == '#')) { + int val = xmlParseStringCharRef(ctxt, &str); + if (val == 0) + goto int_error; + COPY_BUF(0,buffer,nbchars,val); + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } else if ((c == '&') && (what & XML_SUBSTITUTE_REF)) { + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "String decoding Entity Reference: %.30s\n", + str); + ent = xmlParseStringEntityRef(ctxt, &str); + xmlParserEntityCheck(ctxt, 0, ent, 0); + if (ent != NULL) + ctxt->nbentities += ent->checked / 2; + if ((ent != NULL) && + (ent->etype == XML_INTERNAL_PREDEFINED_ENTITY)) { + if (ent->content != NULL) { + COPY_BUF(0,buffer,nbchars,ent->content[0]); + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } else { + xmlFatalErrMsg(ctxt, XML_ERR_INTERNAL_ERROR, + "predefined entity has no content\n"); + goto int_error; + } + } else if ((ent != NULL) && (ent->content != NULL)) { + ctxt->depth++; + rep = xmlStringDecodeEntities(ctxt, ent->content, what, + 0, 0, 0); + ctxt->depth--; + if (rep == NULL) + goto int_error; + + current = rep; + while (*current != 0) { /* non input consuming loop */ + buffer[nbchars++] = *current++; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + if (xmlParserEntityCheck(ctxt, nbchars, ent, 0)) + goto int_error; + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + xmlFree(rep); + rep = NULL; + } else if (ent != NULL) { + int i = xmlStrlen(ent->name); + const xmlChar *cur = ent->name; + + buffer[nbchars++] = '&'; + if (nbchars + i + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, i + XML_PARSER_BUFFER_SIZE); + } + for (;i > 0;i--) + buffer[nbchars++] = *cur++; + buffer[nbchars++] = ';'; + } + } else if (c == '%' && (what & XML_SUBSTITUTE_PEREF)) { + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "String decoding PE Reference: %.30s\n", str); + ent = xmlParseStringPEReference(ctxt, &str); + xmlParserEntityCheck(ctxt, 0, ent, 0); + if (ent != NULL) + ctxt->nbentities += ent->checked / 2; + if (ent != NULL) { + if (ent->content == NULL) { + /* + * Note: external parsed entities will not be loaded, + * it is not required for a non-validating parser to + * complete external PEreferences coming from the + * internal subset + */ + if (((ctxt->options & XML_PARSE_NOENT) != 0) || + ((ctxt->options & XML_PARSE_DTDVALID) != 0) || + (ctxt->validate != 0)) { + xmlLoadEntityContent(ctxt, ent); + } else { + xmlWarningMsg(ctxt, XML_ERR_ENTITY_PROCESSING, + "not validating will not read content for PE entity %s\n", + ent->name, NULL); + } + } + ctxt->depth++; + rep = xmlStringDecodeEntities(ctxt, ent->content, what, + 0, 0, 0); + ctxt->depth--; + if (rep == NULL) + goto int_error; + current = rep; + while (*current != 0) { /* non input consuming loop */ + buffer[nbchars++] = *current++; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + if (xmlParserEntityCheck(ctxt, nbchars, ent, 0)) + goto int_error; + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + xmlFree(rep); + rep = NULL; + } + } else { + COPY_BUF(l,buffer,nbchars,c); + str += l; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + if (str < last) + c = CUR_SCHAR(str, l); + else + c = 0; + } + buffer[nbchars] = 0; + return(buffer); + +mem_error: + xmlErrMemory(ctxt, NULL); +int_error: + if (rep != NULL) + xmlFree(rep); + if (buffer != NULL) + xmlFree(buffer); + return(NULL); +} + +/** + * xmlStringDecodeEntities: + * @ctxt: the parser context + * @str: the input string + * @what: combination of XML_SUBSTITUTE_REF and XML_SUBSTITUTE_PEREF + * @end: an end marker xmlChar, 0 if none + * @end2: an end marker xmlChar, 0 if none + * @end3: an end marker xmlChar, 0 if none + * + * Takes a entity string content and process to do the adequate substitutions. + * + * [67] Reference ::= EntityRef | CharRef + * + * [69] PEReference ::= '%' Name ';' + * + * Returns A newly allocated string with the substitution done. The caller + * must deallocate it ! + */ +xmlChar * +xmlStringDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int what, + xmlChar end, xmlChar end2, xmlChar end3) { + if ((ctxt == NULL) || (str == NULL)) return(NULL); + return(xmlStringLenDecodeEntities(ctxt, str, xmlStrlen(str), what, + end, end2, end3)); +} + +/************************************************************************ + * * + * Commodity functions, cleanup needed ? * + * * + ************************************************************************/ + +/** + * areBlanks: + * @ctxt: an XML parser context + * @str: a xmlChar * + * @len: the size of @str + * @blank_chars: we know the chars are blanks + * + * Is this a sequence of blank chars that one can ignore ? + * + * Returns 1 if ignorable 0 otherwise. + */ + +static int areBlanks(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, + int blank_chars) { + int i, ret; + xmlNodePtr lastChild; + + /* + * Don't spend time trying to differentiate them, the same callback is + * used ! + */ + if (ctxt->sax->ignorableWhitespace == ctxt->sax->characters) + return(0); + + /* + * Check for xml:space value. + */ + if ((ctxt->space == NULL) || (*(ctxt->space) == 1) || + (*(ctxt->space) == -2)) + return(0); + + /* + * Check that the string is made of blanks + */ + if (blank_chars == 0) { + for (i = 0;i < len;i++) + if (!(IS_BLANK_CH(str[i]))) return(0); + } + + /* + * Look if the element is mixed content in the DTD if available + */ + if (ctxt->node == NULL) return(0); + if (ctxt->myDoc != NULL) { + ret = xmlIsMixedElement(ctxt->myDoc, ctxt->node->name); + if (ret == 0) return(1); + if (ret == 1) return(0); + } + + /* + * Otherwise, heuristic :-\ + */ + if ((RAW != '<') && (RAW != 0xD)) return(0); + if ((ctxt->node->children == NULL) && + (RAW == '<') && (NXT(1) == '/')) return(0); + + lastChild = xmlGetLastChild(ctxt->node); + if (lastChild == NULL) { + if ((ctxt->node->type != XML_ELEMENT_NODE) && + (ctxt->node->content != NULL)) return(0); + } else if (xmlNodeIsText(lastChild)) + return(0); + else if ((ctxt->node->children != NULL) && + (xmlNodeIsText(ctxt->node->children))) + return(0); + return(1); +} + +/************************************************************************ + * * + * Extra stuff for namespace support * + * Relates to http://www.w3.org/TR/WD-xml-names * + * * + ************************************************************************/ + +/** + * xmlSplitQName: + * @ctxt: an XML parser context + * @name: an XML parser context + * @prefix: a xmlChar ** + * + * parse an UTF8 encoded XML qualified name string + * + * [NS 5] QName ::= (Prefix ':')? LocalPart + * + * [NS 6] Prefix ::= NCName + * + * [NS 7] LocalPart ::= NCName + * + * Returns the local part, and prefix is updated + * to get the Prefix if any. + */ + +xmlChar * +xmlSplitQName(xmlParserCtxtPtr ctxt, const xmlChar *name, xmlChar **prefix) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + xmlChar *buffer = NULL; + int len = 0; + int max = XML_MAX_NAMELEN; + xmlChar *ret = NULL; + const xmlChar *cur = name; + int c; + + if (prefix == NULL) return(NULL); + *prefix = NULL; + + if (cur == NULL) return(NULL); + +#ifndef XML_XML_NAMESPACE + /* xml: prefix is not really a namespace */ + if ((cur[0] == 'x') && (cur[1] == 'm') && + (cur[2] == 'l') && (cur[3] == ':')) + return(xmlStrdup(name)); +#endif + + /* nasty but well=formed */ + if (cur[0] == ':') + return(xmlStrdup(name)); + + c = *cur++; + while ((c != 0) && (c != ':') && (len < max)) { /* tested bigname.xml */ + buf[len++] = c; + c = *cur++; + } + if (len >= max) { + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while ((c != 0) && (c != ':')) { /* tested bigname.xml */ + if (len + 10 > max) { + xmlChar *tmp; + + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlFree(buffer); + xmlErrMemory(ctxt, NULL); + return(NULL); + } + buffer = tmp; + } + buffer[len++] = c; + c = *cur++; + } + buffer[len] = 0; + } + + if ((c == ':') && (*cur == 0)) { + if (buffer != NULL) + xmlFree(buffer); + *prefix = NULL; + return(xmlStrdup(name)); + } + + if (buffer == NULL) + ret = xmlStrndup(buf, len); + else { + ret = buffer; + buffer = NULL; + max = XML_MAX_NAMELEN; + } + + + if (c == ':') { + c = *cur; + *prefix = ret; + if (c == 0) { + return(xmlStrndup(BAD_CAST "", 0)); + } + len = 0; + + /* + * Check that the first character is proper to start + * a new name + */ + if (!(((c >= 0x61) && (c <= 0x7A)) || + ((c >= 0x41) && (c <= 0x5A)) || + (c == '_') || (c == ':'))) { + int l; + int first = CUR_SCHAR(cur, l); + + if (!IS_LETTER(first) && (first != '_')) { + xmlFatalErrMsgStr(ctxt, XML_NS_ERR_QNAME, + "Name %s is not XML Namespace compliant\n", + name); + } + } + cur++; + + while ((c != 0) && (len < max)) { /* tested bigname2.xml */ + buf[len++] = c; + c = *cur++; + } + if (len >= max) { + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (c != 0) { /* tested bigname2.xml */ + if (len + 10 > max) { + xmlChar *tmp; + + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + buffer[len++] = c; + c = *cur++; + } + buffer[len] = 0; + } + + if (buffer == NULL) + ret = xmlStrndup(buf, len); + else { + ret = buffer; + } + } + + return(ret); +} + +/************************************************************************ + * * + * The parser itself * + * Relates to http://www.w3.org/TR/REC-xml * + * * + ************************************************************************/ + +/************************************************************************ + * * + * Routines to parse Name, NCName and NmToken * + * * + ************************************************************************/ +#ifdef DEBUG +static unsigned long nbParseName = 0; +static unsigned long nbParseNmToken = 0; +static unsigned long nbParseNCName = 0; +static unsigned long nbParseNCNameComplex = 0; +static unsigned long nbParseNameComplex = 0; +static unsigned long nbParseStringName = 0; +#endif + +/* + * The two following functions are related to the change of accepted + * characters for Name and NmToken in the Revision 5 of XML-1.0 + * They correspond to the modified production [4] and the new production [4a] + * changes in that revision. Also note that the macros used for the + * productions Letter, Digit, CombiningChar and Extender are not needed + * anymore. + * We still keep compatibility to pre-revision5 parsing semantic if the + * new XML_PARSE_OLD10 option is given to the parser. + */ +static int +xmlIsNameStartChar(xmlParserCtxtPtr ctxt, int c) { + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + (c == '_') || (c == ':') || + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)))) + return(1); + } else { + if (IS_LETTER(c) || (c == '_') || (c == ':')) + return(1); + } + return(0); +} + +static int +xmlIsNameChar(xmlParserCtxtPtr ctxt, int c) { + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + ((c >= '0') && (c <= '9')) || /* !start */ + (c == '_') || (c == ':') || + (c == '-') || (c == '.') || (c == 0xB7) || /* !start */ + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x300) && (c <= 0x36F)) || /* !start */ + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x203F) && (c <= 0x2040)) || /* !start */ + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)))) + return(1); + } else { + if ((IS_LETTER(c)) || (IS_DIGIT(c)) || + (c == '.') || (c == '-') || + (c == '_') || (c == ':') || + (IS_COMBINING(c)) || + (IS_EXTENDER(c))) + return(1); + } + return(0); +} + +static xmlChar * xmlParseAttValueInternal(xmlParserCtxtPtr ctxt, + int *len, int *alloc, int normalize); + +static const xmlChar * +xmlParseNameComplex(xmlParserCtxtPtr ctxt) { + int len = 0, l; + int c; + int count = 0; + +#ifdef DEBUG + nbParseNameComplex++; +#endif + + /* + * Handler for more complex cases + */ + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!(((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + (c == '_') || (c == ':') || + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF))))) { + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + while ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + ((c >= '0') && (c <= '9')) || /* !start */ + (c == '_') || (c == ':') || + (c == '-') || (c == '.') || (c == 0xB7) || /* !start */ + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x300) && (c <= 0x36F)) || /* !start */ + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x203F) && (c <= 0x2040)) || /* !start */ + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)) + )) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + } + } else { + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!IS_LETTER(c) && (c != '_') && + (c != ':'))) { + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + + while ((c != ' ') && (c != '>') && (c != '/') && /* test bigname.xml */ + ((IS_LETTER(c)) || (IS_DIGIT(c)) || + (c == '.') || (c == '-') || + (c == '_') || (c == ':') || + (IS_COMBINING(c)) || + (IS_EXTENDER(c)))) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Name"); + return(NULL); + } + if (ctxt->input->cur - ctxt->input->base < len) { + /* + * There were a couple of bugs where PERefs lead to to a change + * of the buffer. Check the buffer size to avoid passing an invalid + * pointer to xmlDictLookup. + */ + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, + "unexpected change of input buffer"); + return (NULL); + } + if ((*ctxt->input->cur == '\n') && (ctxt->input->cur[-1] == '\r')) + return(xmlDictLookup(ctxt->dict, ctxt->input->cur - (len + 1), len)); + return(xmlDictLookup(ctxt->dict, ctxt->input->cur - len, len)); +} + +/** + * xmlParseName: + * @ctxt: an XML parser context + * + * parse an XML name. + * + * [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | + * CombiningChar | Extender + * + * [5] Name ::= (Letter | '_' | ':') (NameChar)* + * + * [6] Names ::= Name (#x20 Name)* + * + * Returns the Name parsed or NULL + */ + +const xmlChar * +xmlParseName(xmlParserCtxtPtr ctxt) { + const xmlChar *in; + const xmlChar *ret; + int count = 0; + + GROW; + +#ifdef DEBUG + nbParseName++; +#endif + + /* + * Accelerator for simple ASCII names + */ + in = ctxt->input->cur; + if (((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + (*in == '_') || (*in == ':')) { + in++; + while (((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + ((*in >= 0x30) && (*in <= 0x39)) || + (*in == '_') || (*in == '-') || + (*in == ':') || (*in == '.')) + in++; + if ((*in > 0) && (*in < 0x80)) { + count = in - ctxt->input->cur; + if ((count > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Name"); + return(NULL); + } + ret = xmlDictLookup(ctxt->dict, ctxt->input->cur, count); + ctxt->input->cur = in; + ctxt->nbChars += count; + ctxt->input->col += count; + if (ret == NULL) + xmlErrMemory(ctxt, NULL); + return(ret); + } + } + /* accelerator for special cases */ + return(xmlParseNameComplex(ctxt)); +} + +static const xmlChar * +xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + int len = 0, l; + int c; + int count = 0; + size_t startPosition = 0; + +#ifdef DEBUG + nbParseNCNameComplex++; +#endif + + /* + * Handler for more complex cases + */ + GROW; + startPosition = CUR_PTR - BASE_PTR; + c = CUR_CHAR(l); + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!xmlIsNameStartChar(ctxt, c) || (c == ':'))) { + return(NULL); + } + + while ((c != ' ') && (c != '>') && (c != '/') && /* test bigname.xml */ + (xmlIsNameChar(ctxt, c) && (c != ':'))) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + if (c == 0) { + count = 0; + /* + * when shrinking to extend the buffer we really need to preserve + * the part of the name we already parsed. Hence rolling back + * by current lenght. + */ + ctxt->input->cur -= l; + GROW; + ctxt->input->cur += l; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + return(xmlDictLookup(ctxt->dict, (BASE_PTR + startPosition), len)); +} + +/** + * xmlParseNCName: + * @ctxt: an XML parser context + * @len: length of the string parsed + * + * parse an XML name. + * + * [4NS] NCNameChar ::= Letter | Digit | '.' | '-' | '_' | + * CombiningChar | Extender + * + * [5NS] NCName ::= (Letter | '_') (NCNameChar)* + * + * Returns the Name parsed or NULL + */ + +static const xmlChar * +xmlParseNCName(xmlParserCtxtPtr ctxt) { + const xmlChar *in, *e; + const xmlChar *ret; + int count = 0; + +#ifdef DEBUG + nbParseNCName++; +#endif + + /* + * Accelerator for simple ASCII names + */ + in = ctxt->input->cur; + e = ctxt->input->end; + if ((((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + (*in == '_')) && (in < e)) { + in++; + while ((((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + ((*in >= 0x30) && (*in <= 0x39)) || + (*in == '_') || (*in == '-') || + (*in == '.')) && (in < e)) + in++; + if (in >= e) + goto complex; + if ((*in > 0) && (*in < 0x80)) { + count = in - ctxt->input->cur; + if ((count > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + ret = xmlDictLookup(ctxt->dict, ctxt->input->cur, count); + ctxt->input->cur = in; + ctxt->nbChars += count; + ctxt->input->col += count; + if (ret == NULL) { + xmlErrMemory(ctxt, NULL); + } + return(ret); + } + } +complex: + return(xmlParseNCNameComplex(ctxt)); +} + +/** + * xmlParseNameAndCompare: + * @ctxt: an XML parser context + * + * parse an XML name and compares for match + * (specialized for endtag parsing) + * + * Returns NULL for an illegal name, (xmlChar*) 1 for success + * and the name for mismatch + */ + +static const xmlChar * +xmlParseNameAndCompare(xmlParserCtxtPtr ctxt, xmlChar const *other) { + register const xmlChar *cmp = other; + register const xmlChar *in; + const xmlChar *ret; + + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + + in = ctxt->input->cur; + while (*in != 0 && *in == *cmp) { + ++in; + ++cmp; + ctxt->input->col++; + } + if (*cmp == 0 && (*in == '>' || IS_BLANK_CH (*in))) { + /* success */ + ctxt->input->cur = in; + return (const xmlChar*) 1; + } + /* failure (or end of input buffer), check with full function */ + ret = xmlParseName (ctxt); + /* strings coming from the dictionary direct compare possible */ + if (ret == other) { + return (const xmlChar*) 1; + } + return ret; +} + +/** + * xmlParseStringName: + * @ctxt: an XML parser context + * @str: a pointer to the string pointer (IN/OUT) + * + * parse an XML name. + * + * [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | + * CombiningChar | Extender + * + * [5] Name ::= (Letter | '_' | ':') (NameChar)* + * + * [6] Names ::= Name (#x20 Name)* + * + * Returns the Name parsed or NULL. The @str pointer + * is updated to the current location in the string. + */ + +static xmlChar * +xmlParseStringName(xmlParserCtxtPtr ctxt, const xmlChar** str) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + const xmlChar *cur = *str; + int len = 0, l; + int c; + +#ifdef DEBUG + nbParseStringName++; +#endif + + c = CUR_SCHAR(cur, l); + if (!xmlIsNameStartChar(ctxt, c)) { + return(NULL); + } + + COPY_BUF(l,buf,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + while (xmlIsNameChar(ctxt, c)) { + COPY_BUF(l,buf,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + if (len >= XML_MAX_NAMELEN) { /* test bigentname.xml */ + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + xmlChar *buffer; + int max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (xmlIsNameChar(ctxt, c)) { + if (len + 10 > max) { + xmlChar *tmp; + + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + xmlFree(buffer); + return(NULL); + } + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + COPY_BUF(l,buffer,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + } + buffer[len] = 0; + *str = cur; + return(buffer); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + *str = cur; + return(xmlStrndup(buf, len)); +} + +/** + * xmlParseNmtoken: + * @ctxt: an XML parser context + * + * parse an XML Nmtoken. + * + * [7] Nmtoken ::= (NameChar)+ + * + * [8] Nmtokens ::= Nmtoken (#x20 Nmtoken)* + * + * Returns the Nmtoken parsed or NULL + */ + +xmlChar * +xmlParseNmtoken(xmlParserCtxtPtr ctxt) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + int len = 0, l; + int c; + int count = 0; + +#ifdef DEBUG + nbParseNmToken++; +#endif + + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + + while (xmlIsNameChar(ctxt, c)) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + } + COPY_BUF(l,buf,len,c); + NEXTL(l); + c = CUR_CHAR(l); + if (c == 0) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + } + if (len >= XML_MAX_NAMELEN) { + /* + * Okay someone managed to make a huge token, so he's ready to pay + * for the processing speed. + */ + xmlChar *buffer; + int max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (xmlIsNameChar(ctxt, c)) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buffer); + return(NULL); + } + } + if (len + 10 > max) { + xmlChar *tmp; + + if ((max > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NmToken"); + xmlFree(buffer); + return(NULL); + } + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + COPY_BUF(l,buffer,len,c); + NEXTL(l); + c = CUR_CHAR(l); + } + buffer[len] = 0; + return(buffer); + } + } + if (len == 0) + return(NULL); + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NmToken"); + return(NULL); + } + return(xmlStrndup(buf, len)); +} + +/** + * xmlParseEntityValue: + * @ctxt: an XML parser context + * @orig: if non-NULL store a copy of the original entity value + * + * parse a value for ENTITY declarations + * + * [9] EntityValue ::= '"' ([^%&"] | PEReference | Reference)* '"' | + * "'" ([^%&'] | PEReference | Reference)* "'" + * + * Returns the EntityValue parsed with reference substituted or NULL + */ + +xmlChar * +xmlParseEntityValue(xmlParserCtxtPtr ctxt, xmlChar **orig) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + int c, l; + xmlChar stop; + xmlChar *ret = NULL; + const xmlChar *cur = NULL; + xmlParserInputPtr input; + + if (RAW == '"') stop = '"'; + else if (RAW == '\'') stop = '\''; + else { + xmlFatalErr(ctxt, XML_ERR_ENTITY_NOT_STARTED, NULL); + return(NULL); + } + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + + /* + * The content of the entity definition is copied in a buffer. + */ + + ctxt->instate = XML_PARSER_ENTITY_VALUE; + input = ctxt->input; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + goto error; + NEXT; + c = CUR_CHAR(l); + /* + * NOTE: 4.4.5 Included in Literal + * When a parameter entity reference appears in a literal entity + * value, ... a single or double quote character in the replacement + * text is always treated as a normal data character and will not + * terminate the literal. + * In practice it means we stop the loop only when back at parsing + * the initial entity and the quote is found + */ + while (((IS_CHAR(c)) && ((c != stop) || /* checked */ + (ctxt->input != input))) && (ctxt->instate != XML_PARSER_EOF)) { + if (len + 5 >= size) { + xmlChar *tmp; + + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + goto error; + } + buf = tmp; + } + COPY_BUF(l,buf,len,c); + NEXTL(l); + + GROW; + c = CUR_CHAR(l); + if (c == 0) { + GROW; + c = CUR_CHAR(l); + } + } + buf[len] = 0; + if (ctxt->instate == XML_PARSER_EOF) + goto error; + if (c != stop) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_NOT_FINISHED, NULL); + goto error; + } + NEXT; + + /* + * Raise problem w.r.t. '&' and '%' being used in non-entities + * reference constructs. Note Charref will be handled in + * xmlStringDecodeEntities() + */ + cur = buf; + while (*cur != 0) { /* non input consuming */ + if ((*cur == '%') || ((*cur == '&') && (cur[1] != '#'))) { + xmlChar *name; + xmlChar tmp = *cur; + int nameOk = 0; + + cur++; + name = xmlParseStringName(ctxt, &cur); + if (name != NULL) { + nameOk = 1; + xmlFree(name); + } + if ((nameOk == 0) || (*cur != ';')) { + xmlFatalErrMsgInt(ctxt, XML_ERR_ENTITY_CHAR_ERROR, + "EntityValue: '%c' forbidden except for entities references\n", + tmp); + goto error; + } + if ((tmp == '%') && (ctxt->inSubset == 1) && + (ctxt->inputNr == 1)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_PE_INTERNAL, NULL); + goto error; + } + if (*cur == 0) + break; + } + cur++; + } + + /* + * Then PEReference entities are substituted. + * + * NOTE: 4.4.7 Bypassed + * When a general entity reference appears in the EntityValue in + * an entity declaration, it is bypassed and left as is. + * so XML_SUBSTITUTE_REF is not set here. + */ + ++ctxt->depth; + ret = xmlStringDecodeEntities(ctxt, buf, XML_SUBSTITUTE_PEREF, + 0, 0, 0); + --ctxt->depth; + if (orig != NULL) { + *orig = buf; + buf = NULL; + } + +error: + if (buf != NULL) + xmlFree(buf); + return(ret); +} + +/** + * xmlParseAttValueComplex: + * @ctxt: an XML parser context + * @len: the resulting attribute len + * @normalize: wether to apply the inner normalization + * + * parse a value for an attribute, this is the fallback function + * of xmlParseAttValue() when the attribute parsing requires handling + * of non-ASCII characters, or normalization compaction. + * + * Returns the AttValue parsed or NULL. The value has to be freed by the caller. + */ +static xmlChar * +xmlParseAttValueComplex(xmlParserCtxtPtr ctxt, int *attlen, int normalize) { + xmlChar limit = 0; + xmlChar *buf = NULL; + xmlChar *rep = NULL; + size_t len = 0; + size_t buf_size = 0; + int c, l, in_space = 0; + xmlChar *current = NULL; + xmlEntityPtr ent; + + if (NXT(0) == '"') { + ctxt->instate = XML_PARSER_ATTRIBUTE_VALUE; + limit = '"'; + NEXT; + } else if (NXT(0) == '\'') { + limit = '\''; + ctxt->instate = XML_PARSER_ATTRIBUTE_VALUE; + NEXT; + } else { + xmlFatalErr(ctxt, XML_ERR_ATTRIBUTE_NOT_STARTED, NULL); + return(NULL); + } + + /* + * allocate a translation buffer. + */ + buf_size = XML_PARSER_BUFFER_SIZE; + buf = (xmlChar *) xmlMallocAtomic(buf_size); + if (buf == NULL) goto mem_error; + + /* + * OK loop until we reach one of the ending char or a size limit. + */ + c = CUR_CHAR(l); + while (((NXT(0) != limit) && /* checked */ + (IS_CHAR(c)) && (c != '<')) && + (ctxt->instate != XML_PARSER_EOF)) { + /* + * Impose a reasonable limit on attribute size, unless XML_PARSE_HUGE + * special option is given + */ + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue length too long\n"); + goto mem_error; + } + if (c == 0) break; + if (c == '&') { + in_space = 0; + if (NXT(1) == '#') { + int val = xmlParseCharRef(ctxt); + + if (val == '&') { + if (ctxt->replaceEntities) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + buf[len++] = '&'; + } else { + /* + * The reparsing will be done in xmlStringGetNodeList() + * called by the attribute() function in SAX.c + */ + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + buf[len++] = '&'; + buf[len++] = '#'; + buf[len++] = '3'; + buf[len++] = '8'; + buf[len++] = ';'; + } + } else if (val != 0) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + len += xmlCopyChar(0, &buf[len], val); + } + } else { + ent = xmlParseEntityRef(ctxt); + ctxt->nbentities++; + if (ent != NULL) + ctxt->nbentities += ent->owner; + if ((ent != NULL) && + (ent->etype == XML_INTERNAL_PREDEFINED_ENTITY)) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + if ((ctxt->replaceEntities == 0) && + (ent->content[0] == '&')) { + buf[len++] = '&'; + buf[len++] = '#'; + buf[len++] = '3'; + buf[len++] = '8'; + buf[len++] = ';'; + } else { + buf[len++] = ent->content[0]; + } + } else if ((ent != NULL) && + (ctxt->replaceEntities != 0)) { + if (ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) { + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, + 0, 0, 0); + --ctxt->depth; + if (rep != NULL) { + current = rep; + while (*current != 0) { /* non input consuming */ + if ((*current == 0xD) || (*current == 0xA) || + (*current == 0x9)) { + buf[len++] = 0x20; + current++; + } else + buf[len++] = *current++; + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + xmlFree(rep); + rep = NULL; + } + } else { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + if (ent->content != NULL) + buf[len++] = ent->content[0]; + } + } else if (ent != NULL) { + int i = xmlStrlen(ent->name); + const xmlChar *cur = ent->name; + + /* + * This may look absurd but is needed to detect + * entities problems + */ + if ((ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) && + (ent->content != NULL) && (ent->checked == 0)) { + unsigned long oldnbent = ctxt->nbentities; + + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, 0, 0, 0); + --ctxt->depth; + + ent->checked = (ctxt->nbentities - oldnbent + 1) * 2; + if (rep != NULL) { + if (xmlStrchr(rep, '<')) + ent->checked |= 1; + xmlFree(rep); + rep = NULL; + } else { + ent->content[0] = 0; + } + } + + /* + * Just output the reference + */ + buf[len++] = '&'; + while (len + i + 10 > buf_size) { + growBuffer(buf, i + 10); + } + for (;i > 0;i--) + buf[len++] = *cur++; + buf[len++] = ';'; + } + } + } else { + if ((c == 0x20) || (c == 0xD) || (c == 0xA) || (c == 0x9)) { + if ((len != 0) || (!normalize)) { + if ((!normalize) || (!in_space)) { + COPY_BUF(l,buf,len,0x20); + while (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + in_space = 1; + } + } else { + in_space = 0; + COPY_BUF(l,buf,len,c); + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + NEXTL(l); + } + GROW; + c = CUR_CHAR(l); + } + if (ctxt->instate == XML_PARSER_EOF) + goto error; + + if ((in_space) && (normalize)) { + while ((len > 0) && (buf[len - 1] == 0x20)) len--; + } + buf[len] = 0; + if (RAW == '<') { + xmlFatalErr(ctxt, XML_ERR_LT_IN_ATTRIBUTE, NULL); + } else if (RAW != limit) { + if ((c != 0) && (!IS_CHAR(c))) { + xmlFatalErrMsg(ctxt, XML_ERR_INVALID_CHAR, + "invalid character in attribute value\n"); + } else { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue: ' expected\n"); + } + } else + NEXT; + + /* + * There we potentially risk an overflow, don't allow attribute value of + * length more than INT_MAX it is a very reasonnable assumption ! + */ + if (len >= INT_MAX) { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue length too long\n"); + goto mem_error; + } + + if (attlen != NULL) *attlen = (int) len; + return(buf); + +mem_error: + xmlErrMemory(ctxt, NULL); +error: + if (buf != NULL) + xmlFree(buf); + if (rep != NULL) + xmlFree(rep); + return(NULL); +} + +/** + * xmlParseAttValue: + * @ctxt: an XML parser context + * + * parse a value for an attribute + * Note: the parser won't do substitution of entities here, this + * will be handled later in xmlStringGetNodeList + * + * [10] AttValue ::= '"' ([^<&"] | Reference)* '"' | + * "'" ([^<&'] | Reference)* "'" + * + * 3.3.3 Attribute-Value Normalization: + * Before the value of an attribute is passed to the application or + * checked for validity, the XML processor must normalize it as follows: + * - a character reference is processed by appending the referenced + * character to the attribute value + * - an entity reference is processed by recursively processing the + * replacement text of the entity + * - a whitespace character (#x20, #xD, #xA, #x9) is processed by + * appending #x20 to the normalized value, except that only a single + * #x20 is appended for a "#xD#xA" sequence that is part of an external + * parsed entity or the literal entity value of an internal parsed entity + * - other characters are processed by appending them to the normalized value + * If the declared value is not CDATA, then the XML processor must further + * process the normalized attribute value by discarding any leading and + * trailing space (#x20) characters, and by replacing sequences of space + * (#x20) characters by a single space (#x20) character. + * All attributes for which no declaration has been read should be treated + * by a non-validating parser as if declared CDATA. + * + * Returns the AttValue parsed or NULL. The value has to be freed by the caller. + */ + + +xmlChar * +xmlParseAttValue(xmlParserCtxtPtr ctxt) { + if ((ctxt == NULL) || (ctxt->input == NULL)) return(NULL); + return(xmlParseAttValueInternal(ctxt, NULL, NULL, 0)); +} + +/** + * xmlParseSystemLiteral: + * @ctxt: an XML parser context + * + * parse an XML Literal + * + * [11] SystemLiteral ::= ('"' [^"]* '"') | ("'" [^']* "'") + * + * Returns the SystemLiteral parsed or NULL + */ + +xmlChar * +xmlParseSystemLiteral(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + int cur, l; + xmlChar stop; + int state = ctxt->instate; + int count = 0; + + SHRINK; + if (RAW == '"') { + NEXT; + stop = '"'; + } else if (RAW == '\'') { + NEXT; + stop = '\''; + } else { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_STARTED, NULL); + return(NULL); + } + + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + ctxt->instate = XML_PARSER_SYSTEM_LITERAL; + cur = CUR_CHAR(l); + while ((IS_CHAR(cur)) && (cur != stop)) { /* checked */ + if (len + 5 >= size) { + xmlChar *tmp; + + if ((size > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "SystemLiteral"); + xmlFree(buf); + ctxt->instate = (xmlParserInputState) state; + return(NULL); + } + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlFree(buf); + xmlErrMemory(ctxt, NULL); + ctxt->instate = (xmlParserInputState) state; + return(NULL); + } + buf = tmp; + } + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return(NULL); + } + } + COPY_BUF(l,buf,len,cur); + NEXTL(l); + cur = CUR_CHAR(l); + if (cur == 0) { + GROW; + SHRINK; + cur = CUR_CHAR(l); + } + } + buf[len] = 0; + ctxt->instate = (xmlParserInputState) state; + if (!IS_CHAR(cur)) { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_FINISHED, NULL); + } else { + NEXT; + } + return(buf); +} + +/** + * xmlParsePubidLiteral: + * @ctxt: an XML parser context + * + * parse an XML public literal + * + * [12] PubidLiteral ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'" + * + * Returns the PubidLiteral parsed or NULL. + */ + +xmlChar * +xmlParsePubidLiteral(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + xmlChar cur; + xmlChar stop; + int count = 0; + xmlParserInputState oldstate = ctxt->instate; + + SHRINK; + if (RAW == '"') { + NEXT; + stop = '"'; + } else if (RAW == '\'') { + NEXT; + stop = '\''; + } else { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_STARTED, NULL); + return(NULL); + } + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + ctxt->instate = XML_PARSER_PUBLIC_LITERAL; + cur = CUR; + while ((IS_PUBIDCHAR_CH(cur)) && (cur != stop)) { /* checked */ + if (len + 1 >= size) { + xmlChar *tmp; + + if ((size > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Public ID"); + xmlFree(buf); + return(NULL); + } + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buf); + return(NULL); + } + buf = tmp; + } + buf[len++] = cur; + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return(NULL); + } + } + NEXT; + cur = CUR; + if (cur == 0) { + GROW; + SHRINK; + cur = CUR; + } + } + buf[len] = 0; + if (cur != stop) { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_FINISHED, NULL); + } else { + NEXT; + } + ctxt->instate = oldstate; + return(buf); +} + +static void xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata); + +/* + * used for the test in the inner loop of the char data testing + */ +static const unsigned char test_char_data[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9, CR/LF separated */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x00, 0x27, /* & */ + 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3A, 0x3B, 0x00, 0x3D, 0x3E, 0x3F, /* < */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x00, 0x5E, 0x5F, /* ] */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* non-ascii */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/** + * xmlParseCharData: + * @ctxt: an XML parser context + * @cdata: int indicating whether we are within a CDATA section + * + * parse a CharData section. + * if we are within a CDATA section ']]>' marks an end of section. + * + * The right angle bracket (>) may be represented using the string ">", + * and must, for compatibility, be escaped using ">" or a character + * reference when it appears in the string "]]>" in content, when that + * string is not marking the end of a CDATA section. + * + * [14] CharData ::= [^<&]* - ([^<&]* ']]>' [^<&]*) + */ + +void +xmlParseCharData(xmlParserCtxtPtr ctxt, int cdata) { + const xmlChar *in; + int nbchar = 0; + int line = ctxt->input->line; + int col = ctxt->input->col; + int ccol; + + SHRINK; + GROW; + /* + * Accelerated common case where input don't need to be + * modified before passing it to the handler. + */ + if (!cdata) { + in = ctxt->input->cur; + do { +get_more_space: + while (*in == 0x20) { in++; ctxt->input->col++; } + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more_space; + } + if (*in == '<') { + nbchar = in - ctxt->input->cur; + if (nbchar > 0) { + const xmlChar *tmp = ctxt->input->cur; + ctxt->input->cur = in; + + if ((ctxt->sax != NULL) && + (ctxt->sax->ignorableWhitespace != + ctxt->sax->characters)) { + if (areBlanks(ctxt, tmp, nbchar, 1)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + tmp, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + if (*ctxt->space == -1) + *ctxt->space = -2; + } + } else if ((ctxt->sax != NULL) && + (ctxt->sax->characters != NULL)) { + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + } + } + return; + } + +get_more: + ccol = ctxt->input->col; + while (test_char_data[*in]) { + in++; + ccol++; + } + ctxt->input->col = ccol; + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more; + } + if (*in == ']') { + if ((in[1] == ']') && (in[2] == '>')) { + xmlFatalErr(ctxt, XML_ERR_MISPLACED_CDATA_END, NULL); + ctxt->input->cur = in + 1; + return; + } + in++; + ctxt->input->col++; + goto get_more; + } + nbchar = in - ctxt->input->cur; + if (nbchar > 0) { + if ((ctxt->sax != NULL) && + (ctxt->sax->ignorableWhitespace != + ctxt->sax->characters) && + (IS_BLANK_CH(*ctxt->input->cur))) { + const xmlChar *tmp = ctxt->input->cur; + ctxt->input->cur = in; + + if (areBlanks(ctxt, tmp, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + tmp, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + if (*ctxt->space == -1) + *ctxt->space = -2; + } + line = ctxt->input->line; + col = ctxt->input->col; + } else if (ctxt->sax != NULL) { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + ctxt->input->cur, nbchar); + line = ctxt->input->line; + col = ctxt->input->col; + } + /* something really bad happened in the SAX callback */ + if (ctxt->instate != XML_PARSER_CONTENT) + return; + } + ctxt->input->cur = in; + if (*in == 0xD) { + in++; + if (*in == 0xA) { + ctxt->input->cur = in; + in++; + ctxt->input->line++; ctxt->input->col = 1; + continue; /* while */ + } + in--; + } + if (*in == '<') { + return; + } + if (*in == '&') { + return; + } + SHRINK; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return; + in = ctxt->input->cur; + } while (((*in >= 0x20) && (*in <= 0x7F)) || (*in == 0x09)); + nbchar = 0; + } + ctxt->input->line = line; + ctxt->input->col = col; + xmlParseCharDataComplex(ctxt, cdata); +} + +/** + * xmlParseCharDataComplex: + * @ctxt: an XML parser context + * @cdata: int indicating whether we are within a CDATA section + * + * parse a CharData section.this is the fallback function + * of xmlParseCharData() when the parsing requires handling + * of non-ASCII characters. + */ +static void +xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata) { + xmlChar buf[XML_PARSER_BIG_BUFFER_SIZE + 5]; + int nbchar = 0; + int cur, l; + int count = 0; + + SHRINK; + GROW; + cur = CUR_CHAR(l); + while ((cur != '<') && /* checked */ + (cur != '&') && + (IS_CHAR(cur))) /* test also done in xmlCurrentChar() */ { + if ((cur == ']') && (NXT(1) == ']') && + (NXT(2) == '>')) { + if (cdata) break; + else { + xmlFatalErr(ctxt, XML_ERR_MISPLACED_CDATA_END, NULL); + } + } + COPY_BUF(l,buf,nbchar,cur); + if (nbchar >= XML_PARSER_BIG_BUFFER_SIZE) { + buf[nbchar] = 0; + + /* + * OK the segment is to be consumed as chars. + */ + if ((ctxt->sax != NULL) && (!ctxt->disableSAX)) { + if (areBlanks(ctxt, buf, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + buf, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, buf, nbchar); + if ((ctxt->sax->characters != + ctxt->sax->ignorableWhitespace) && + (*ctxt->space == -1)) + *ctxt->space = -2; + } + } + nbchar = 0; + /* something really bad happened in the SAX callback */ + if (ctxt->instate != XML_PARSER_CONTENT) + return; + } + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) + return; + } + NEXTL(l); + cur = CUR_CHAR(l); + } + if (nbchar != 0) { + buf[nbchar] = 0; + /* + * OK the segment is to be consumed as chars. + */ + if ((ctxt->sax != NULL) && (!ctxt->disableSAX)) { + if (areBlanks(ctxt, buf, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, buf, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, buf, nbchar); + if ((ctxt->sax->characters != ctxt->sax->ignorableWhitespace) && + (*ctxt->space == -1)) + *ctxt->space = -2; + } + } + } + if ((cur != 0) && (!IS_CHAR(cur))) { + /* Generate the error and skip the offending character */ + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "PCDATA invalid Char value %d\n", + cur); + NEXTL(l); + } +} + +/** + * xmlParseExternalID: + * @ctxt: an XML parser context + * @publicID: a xmlChar** receiving PubidLiteral + * @strict: indicate whether we should restrict parsing to only + * production [75], see NOTE below + * + * Parse an External ID or a Public ID + * + * NOTE: Productions [75] and [83] interact badly since [75] can generate + * 'PUBLIC' S PubidLiteral S SystemLiteral + * + * [75] ExternalID ::= 'SYSTEM' S SystemLiteral + * | 'PUBLIC' S PubidLiteral S SystemLiteral + * + * [83] PublicID ::= 'PUBLIC' S PubidLiteral + * + * Returns the function returns SystemLiteral and in the second + * case publicID receives PubidLiteral, is strict is off + * it is possible to return NULL and have publicID set. + */ + +xmlChar * +xmlParseExternalID(xmlParserCtxtPtr ctxt, xmlChar **publicID, int strict) { + xmlChar *URI = NULL; + + SHRINK; + + *publicID = NULL; + if (CMP6(CUR_PTR, 'S', 'Y', 'S', 'T', 'E', 'M')) { + SKIP(6); + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after 'SYSTEM'\n"); + } + URI = xmlParseSystemLiteral(ctxt); + if (URI == NULL) { + xmlFatalErr(ctxt, XML_ERR_URI_REQUIRED, NULL); + } + } else if (CMP6(CUR_PTR, 'P', 'U', 'B', 'L', 'I', 'C')) { + SKIP(6); + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after 'PUBLIC'\n"); + } + *publicID = xmlParsePubidLiteral(ctxt); + if (*publicID == NULL) { + xmlFatalErr(ctxt, XML_ERR_PUBID_REQUIRED, NULL); + } + if (strict) { + /* + * We don't handle [83] so "S SystemLiteral" is required. + */ + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after the Public Identifier\n"); + } + } else { + /* + * We handle [83] so we return immediately, if + * "S SystemLiteral" is not detected. We skip blanks if no + * system literal was found, but this is harmless since we must + * be at the end of a NotationDecl. + */ + if (SKIP_BLANKS == 0) return(NULL); + if ((CUR != '\'') && (CUR != '"')) return(NULL); + } + URI = xmlParseSystemLiteral(ctxt); + if (URI == NULL) { + xmlFatalErr(ctxt, XML_ERR_URI_REQUIRED, NULL); + } + } + return(URI); +} + +/** + * xmlParseCommentComplex: + * @ctxt: an XML parser context + * @buf: the already parsed part of the buffer + * @len: number of bytes filles in the buffer + * @size: allocated size of the buffer + * + * Skip an XML (SGML) comment + * The spec says that "For compatibility, the string "--" (double-hyphen) + * must not occur within comments. " + * This is the slow routine in case the accelerator for ascii didn't work + * + * [15] Comment ::= '' + */ +static void +xmlParseCommentComplex(xmlParserCtxtPtr ctxt, xmlChar *buf, + size_t len, size_t size) { + int q, ql; + int r, rl; + int cur, l; + size_t count = 0; + int inputid; + + inputid = ctxt->input->id; + + if (buf == NULL) { + len = 0; + size = XML_PARSER_BUFFER_SIZE; + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return; + } + } + GROW; /* Assure there's enough input data */ + q = CUR_CHAR(ql); + if (q == 0) + goto not_terminated; + if (!IS_CHAR(q)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseComment: invalid xmlChar value %d\n", + q); + xmlFree (buf); + return; + } + NEXTL(ql); + r = CUR_CHAR(rl); + if (r == 0) + goto not_terminated; + if (!IS_CHAR(r)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseComment: invalid xmlChar value %d\n", + q); + xmlFree (buf); + return; + } + NEXTL(rl); + cur = CUR_CHAR(l); + if (cur == 0) + goto not_terminated; + while (IS_CHAR(cur) && /* checked */ + ((cur != '>') || + (r != '-') || (q != '-'))) { + if ((r == '-') && (q == '-')) { + xmlFatalErr(ctxt, XML_ERR_HYPHEN_IN_COMMENT, NULL); + } + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment too big found", NULL); + xmlFree (buf); + return; + } + if (len + 5 >= size) { + xmlChar *new_buf; + size_t new_size; + + new_size = size * 2; + new_buf = (xmlChar *) xmlRealloc(buf, new_size); + if (new_buf == NULL) { + xmlFree (buf); + xmlErrMemory(ctxt, NULL); + return; + } + buf = new_buf; + size = new_size; + } + COPY_BUF(ql,buf,len,q); + q = r; + ql = rl; + r = cur; + rl = l; + + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return; + } + } + NEXTL(l); + cur = CUR_CHAR(l); + if (cur == 0) { + SHRINK; + GROW; + cur = CUR_CHAR(l); + } + } + buf[len] = 0; + if (cur == 0) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment not terminated \n + * The spec says that "For compatibility, the string "--" (double-hyphen) + * must not occur within comments. " + * + * [15] Comment ::= '' + */ +void +xmlParseComment(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + size_t size = XML_PARSER_BUFFER_SIZE; + size_t len = 0; + xmlParserInputState state; + const xmlChar *in; + size_t nbchar = 0; + int ccol; + int inputid; + + /* + * Check that there is a comment right here. + */ + if ((RAW != '<') || (NXT(1) != '!') || + (NXT(2) != '-') || (NXT(3) != '-')) return; + state = ctxt->instate; + ctxt->instate = XML_PARSER_COMMENT; + inputid = ctxt->input->id; + SKIP(4); + SHRINK; + GROW; + + /* + * Accelerated common case where input don't need to be + * modified before passing it to the handler. + */ + in = ctxt->input->cur; + do { + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + } +get_more: + ccol = ctxt->input->col; + while (((*in > '-') && (*in <= 0x7F)) || + ((*in >= 0x20) && (*in < '-')) || + (*in == 0x09)) { + in++; + ccol++; + } + ctxt->input->col = ccol; + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more; + } + nbchar = in - ctxt->input->cur; + /* + * save current set of data + */ + if (nbchar > 0) { + if ((ctxt->sax != NULL) && + (ctxt->sax->comment != NULL)) { + if (buf == NULL) { + if ((*in == '-') && (in[1] == '-')) + size = nbchar + 1; + else + size = XML_PARSER_BUFFER_SIZE + nbchar; + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->instate = state; + return; + } + len = 0; + } else if (len + nbchar + 1 >= size) { + xmlChar *new_buf; + size += len + nbchar + XML_PARSER_BUFFER_SIZE; + new_buf = (xmlChar *) xmlRealloc(buf, + size * sizeof(xmlChar)); + if (new_buf == NULL) { + xmlFree (buf); + xmlErrMemory(ctxt, NULL); + ctxt->instate = state; + return; + } + buf = new_buf; + } + memcpy(&buf[len], ctxt->input->cur, nbchar); + len += nbchar; + buf[len] = 0; + } + } + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment too big found", NULL); + xmlFree (buf); + return; + } + ctxt->input->cur = in; + if (*in == 0xA) { + in++; + ctxt->input->line++; ctxt->input->col = 1; + } + if (*in == 0xD) { + in++; + if (*in == 0xA) { + ctxt->input->cur = in; + in++; + ctxt->input->line++; ctxt->input->col = 1; + continue; /* while */ + } + in--; + } + SHRINK; + GROW; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return; + } + in = ctxt->input->cur; + if (*in == '-') { + if (in[1] == '-') { + if (in[2] == '>') { + if (ctxt->input->id != inputid) { + xmlFatalErrMsg(ctxt, XML_ERR_ENTITY_BOUNDARY, + "comment doesn't start and stop in the" + " same entity\n"); + } + SKIP(3); + if ((ctxt->sax != NULL) && (ctxt->sax->comment != NULL) && + (!ctxt->disableSAX)) { + if (buf != NULL) + ctxt->sax->comment(ctxt->userData, buf); + else + ctxt->sax->comment(ctxt->userData, BAD_CAST ""); + } + if (buf != NULL) + xmlFree(buf); + if (ctxt->instate != XML_PARSER_EOF) + ctxt->instate = state; + return; + } + if (buf != NULL) { + xmlFatalErrMsgStr(ctxt, XML_ERR_HYPHEN_IN_COMMENT, + "Double hyphen within comment: " + " \n, Aborting " - f"pipeline.... %s", func.__name__, - func.__module__, exc, extra=MULTI_LINE_MESSAGE_TRUE) + f"pipeline.... %s", {func.__name__}, + {func.__module__}, exc, extra=MULTI_LINE_MESSAGE_TRUE) raise return catch_errors + + +def catch_tool_errors(tool_name): + def catch_tool_errors_impl(func): + @functools.wraps(func) + async def catch_errors(self, *args, **kwargs): + try: + logger.debug(f"Before running tool {func.__name__} in file {func.__module__}") + if args == () and not kwargs == {}: + return await func(self, **kwargs) + elif kwargs == {} and not args == (): + return await func(self, *args) + else: + return await func(self, *args, **kwargs) + + except Exception as exc: + raise ToolRaisedException(message=f"Intercepted an error from tool , in function=>{func.__name__} on file=>" + f"{func.__module__}", tool_name=tool_name, original_exception=exc) + return catch_errors + + return catch_tool_errors_impl From 9b4538c97596f58c29c7973b19fa7e4583654aa0 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 9 Nov 2025 11:12:45 +0200 Subject: [PATCH 097/286] style: some tiny logging improvements Signed-off-by: Zvi Grinberg --- src/vuln_analysis/functions/cve_agent.py | 2 +- src/vuln_analysis/utils/error_handling_decorator.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 6f1a03e8f..f86833de1 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -156,7 +156,7 @@ def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, re tool_raised_exception: ToolRaisedException = answer logger.warning(f"An exception encountered during tool execution, in result [{i}][{j}]. for " f"question : {checklist_questions[i][j]}" - f", tool raised exception: {tool_raised_exception}" + f".tool raised exception details=> {tool_raised_exception}" f", replacing with default output -> {replace_exceptions_value}") else: logger.warning( diff --git a/src/vuln_analysis/utils/error_handling_decorator.py b/src/vuln_analysis/utils/error_handling_decorator.py index b5adb8340..afabc02ed 100644 --- a/src/vuln_analysis/utils/error_handling_decorator.py +++ b/src/vuln_analysis/utils/error_handling_decorator.py @@ -47,7 +47,7 @@ def catch_tool_errors_impl(func): @functools.wraps(func) async def catch_errors(self, *args, **kwargs): try: - logger.debug(f"Before running tool {func.__name__} in file {func.__module__}") + logger.debug(f"Before running tool {tool_name} in file {func.__module__}") if args == () and not kwargs == {}: return await func(self, **kwargs) elif kwargs == {} and not args == (): @@ -56,8 +56,10 @@ async def catch_errors(self, *args, **kwargs): return await func(self, *args, **kwargs) except Exception as exc: - raise ToolRaisedException(message=f"Intercepted an error from tool , in function=>{func.__name__} on file=>" - f"{func.__module__}", tool_name=tool_name, original_exception=exc) + format_error_message = f"Intercepted an error from tool {tool_name}, in function=" \ + f"{func.__name__}, on file=" \ + f"{func.__module__}" + raise ToolRaisedException(message=format_error_message, tool_name=tool_name, original_exception=exc) return catch_errors return catch_tool_errors_impl From afcf65f2ab8e88458d24fcb5bf89980b90f097e7 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 11 Nov 2025 12:42:45 +0200 Subject: [PATCH 098/286] feat: add caching of Go modules to persistent storage Signed-off-by: Vladimir Belousov --- kustomize/base/exploit_iq_service.yaml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 8d33adf3d..16c17c5d1 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -116,16 +116,20 @@ spec: - name: ENABLE_EXTENDED_JS_PARSERS value: "True" - name: EXPLOIT_IQ_DATA_DIR - value: /exploit-iq-data + value: /exploit-iq-data/ - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: GOMODCACHE + value: /exploit-iq-package-cache/go/pkg/mod volumeMounts: - name: config mountPath: /configs - name: cache mountPath: /exploit-iq-data + - name: package-cache + mountPath: /exploit-iq-package-cache - name: ca-bundle mountPath: /app/certs readOnly: true @@ -136,6 +140,9 @@ spec: - name: cache persistentVolumeClaim: claimName: exploit-iq-data + - name: package-cache + persistentVolumeClaim: + claimName: exploit-iq-package-cache - name: ca-bundle configMap: name: openshift-service-ca.crt @@ -201,3 +208,20 @@ spec: resources: requests: storage: 100Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-package-cache + labels: + purpose: package-cache +spec: + # ReadWriteOncePod is used instead of ReadWriteOnce to avoid long SELinux relabeling + # when volume contains large number of files (tens of thousands or more). + # This prevents pod startup failures and excessive startup times. + # See: https://access.redhat.com/solutions/6221251 and https://access.redhat.com/solutions/7079407 + accessModes: + - ReadWriteOncePod + resources: + requests: + storage: 200Gi From fade1bcb39100b49ce92a116977b43fada332429 Mon Sep 17 00:00:00 2001 From: Roni Hartuv Date: Tue, 18 Nov 2025 10:18:38 +0200 Subject: [PATCH 099/286] feat(kustomize): add Argilla feedback service to base deployment (#144) * chore: update kustomize files to include argille * feat: adding argilla deployment folder to kustomize-base * chore: updating image in deployment file * chore: updating to a temporary image * fix: creating secret to argilla environment variables * chore: add Argilla pull secret instructions to README * fix: update argilla pull secret on README file * fix: update image on deployment file * refactor: move argilla secrets to dedicated base/argilla/feedback_secret.env * docs: exchange the order of level 2 and 3 * fix: revert to resources on kustomization file --- kustomize/README.md | 34 +++++-- kustomize/base/argilla/argilla-route.yaml | 15 +++ kustomize/base/argilla/argilla-service.yaml | 14 +++ kustomize/base/argilla/configmap.yaml | 9 ++ kustomize/base/argilla/deployment.yaml | 106 ++++++++++++++++++++ kustomize/base/argilla/kustomization.yaml | 15 +++ kustomize/base/argilla/rbac.yaml | 21 ++++ kustomize/base/argilla/sa.yaml | 4 + kustomize/base/argilla/service.yaml | 14 +++ kustomize/base/kustomization.yaml | 1 + 10 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 kustomize/base/argilla/argilla-route.yaml create mode 100644 kustomize/base/argilla/argilla-service.yaml create mode 100644 kustomize/base/argilla/configmap.yaml create mode 100644 kustomize/base/argilla/deployment.yaml create mode 100644 kustomize/base/argilla/kustomization.yaml create mode 100644 kustomize/base/argilla/rbac.yaml create mode 100644 kustomize/base/argilla/sa.yaml create mode 100644 kustomize/base/argilla/service.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 6fdcc40d4..3ca9e4b71 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -79,6 +79,7 @@ nvidia_api_key=your_api_key ghsa_api_key=your_api_key registry_redhat_username=your_registry_username registry_redhat_password=your_registry_pass_token + EOF ``` @@ -88,14 +89,29 @@ EOF export YOUR_NAMESPACE_NAME=yourNamespaceNameHere oc new-project $YOUR_NAMESPACE_NAME ``` +3. Create a `base/argilla/feedback_secret.env` file containing the credentials for the Argilla feedback service: + +```shell +cat > base/argilla/feedback_secret.env << EOF +argilla_username=your_argilla_username +argilla_password=your_argilla_password +argilla_api_key=your_argilla_api_key -3. Create an image pull secret to authorize pulling the `ExploitIQ` container image: +EOF +``` +4. Create an image pull secret to authorize pulling the `ExploitIQ` container image: ```shell oc create secret generic exploit-iq-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -4. Edit the `image-registry-credentials.env` and provide image registry credentials required by product scanning to access and pull component images. + Create an image pull secret to authorize pulling the `Argilla` container image: + +```shell +oc create secret generic argilla-user-feedback-ips --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +``` + +5. Edit the `image-registry-credentials.env` and provide image registry credentials required by product scanning to access and pull component images. ```shell # Example: add your desired registries and credentials. "auth" is base64 of ":". @@ -113,7 +129,7 @@ EOF >[!IMPORTANT] >This secret is essential for product scanning to authenticate and pull component images. If you skip this step, kustomize will still deploy, but authenticated pulls will not work until you provide real credentials. -5. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. +6. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret @@ -129,7 +145,7 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -6. Update `ExploitIQ` configuration file with the correct callback URL for the client service. +7. Update `ExploitIQ` configuration file with the correct callback URL for the client service. ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" @@ -137,7 +153,7 @@ find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEH ``` >[!IMPORTANT] You should only run one of 7,8 steps, depends on if you want to run the service with a self hosted LLM or not. -7. To deploy `EXploitIQ` with a self-hosted LLM , run: +8. To deploy `EXploitIQ` with a self-hosted LLM , run: ```shell # Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM @@ -145,14 +161,14 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ``` -8. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: +9. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: ```shell # Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` >[!WARNING] Without completing the following step with the correct secret from step 5, authentication and logging into the UI App will fail! -9. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 5) and generated route +10. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 5) and generated route ```bash oc create -f - < Date: Tue, 18 Nov 2025 11:02:27 +0200 Subject: [PATCH 100/286] chore: retire tag 'nat' for the client UI app Signed-off-by: Zvi Grinberg --- kustomize/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 51640ad40..d5b0837c9 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -74,4 +74,4 @@ images: newTag: nat - name: quay.io/ecosystem-appeng/agent-morpheus-client - newTag: nat + newTag: latest From 338adb79f4471ff4e1b76dff5ef8e1684668582c Mon Sep 17 00:00:00 2001 From: etsien Date: Tue, 18 Nov 2025 10:49:50 -0500 Subject: [PATCH 101/286] APPENG-3801-B - Agent performance fixes - all agent stages (#134) * bugfix in the testing env * update tool descriptions for clarity * refactor tool names to be class constants instead of disparate strings * add initial unit tests * rename tool names to be more consistent and distinct * update unit tests with tool names and tool constants * cleanup startup guide notebook * rework intel source score section * update agent execution stage prompts and make tool descriptions dynamic * add tests for dynamic tool descriptions * revamp the tool description list, as well as the checklist prompt for better clarity and quality * revamp checklist prompt implementation, as well as add in dynamic tool descriptions to both checklist and agent prompts * update tests for tool descriptions * add more detailed agent examples with more useful MRKL-formatted steps * update for summary prompt * update justification prompt with more logic and explanations on how to pick the class * update CVSS prompts and cleanup examples and guidance * bugfix on intel source * bug patch for vdb generation adding in constants during the vdb generation check * bugfix by Tamar * update register_function() and transitive_search() descriptions * bugfix in the testing env * update tool descriptions for clarity * refactor tool names to be class constants instead of disparate strings * add initial unit tests * rename tool names to be more consistent and distinct * update unit tests with tool names and tool constants * cleanup startup guide notebook * rework intel source score section * update agent execution stage prompts and make tool descriptions dynamic * add tests for dynamic tool descriptions * revamp the tool description list, as well as the checklist prompt for better clarity and quality * revamp checklist prompt implementation, as well as add in dynamic tool descriptions to both checklist and agent prompts * update tests for tool descriptions * add more detailed agent examples with more useful MRKL-formatted steps * update for summary prompt * update justification prompt with more logic and explanations on how to pick the class * update CVSS prompts and cleanup examples and guidance * bugfix on intel source * bug patch for vdb generation adding in constants during the vdb generation check * bugfix by Tamar * update register_function() and transitive_search() descriptions * add function locator descriptions * add names to configs * add local output for local testing * bugfix * Update tool_names.py --- kustomize/base/exploit-iq-config.yml | 38 +- kustomize/config-http-openai-local.yml | 66 +- quick_start/quick_start_guide.ipynb | 4372 +---------------- src/vuln_analysis/configs/config-http-nim.yml | 30 +- .../configs/config-http-openai.yml | 38 +- src/vuln_analysis/configs/config-tracing.yml | 47 +- src/vuln_analysis/configs/config.yml | 26 +- .../morpheus:23.11-runtime.json | 1 + src/vuln_analysis/functions/cve_agent.py | 46 +- src/vuln_analysis/functions/cve_checklist.py | 10 + .../functions/cve_generate_cvss.py | 15 +- .../functions/cve_generate_vdbs.py | 5 +- .../tools/container_image_analysis_data.py | 12 +- .../tools/lexical_full_search.py | 11 +- src/vuln_analysis/tools/local_vdb.py | 13 +- src/vuln_analysis/tools/serp.py | 9 +- src/vuln_analysis/tools/tool_names.py | 70 + .../tools/transitive_code_search.py | 125 +- .../utils/checklist_prompt_generator.py | 57 +- src/vuln_analysis/utils/intel_source_score.py | 260 +- .../utils/justification_parser.py | 107 +- src/vuln_analysis/utils/prompting.py | 601 ++- tests/test_base_tool_descriptions.py | 155 + tests/test_tool_filtering.py | 137 + tests/test_tool_names.py | 39 + 25 files changed, 1435 insertions(+), 4855 deletions(-) create mode 100644 src/vuln_analysis/tools/tool_names.py create mode 100644 tests/test_base_tool_descriptions.py create mode 100644 tests/test_tool_filtering.py create mode 100644 tests/test_tool_names.py diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index e0f91dc2a..081505f48 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -54,45 +54,45 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm - Transitive code search tool: + Call Chain Analyzer: _type: transitive_code_search enable_transitive_search: true - Calling Function Name Extractor: + Function Caller Finder: _type: calling_function_name_extractor enable_functions_usage_search: true - Package and Function Locator: + Function Locator: _type: package_and_function_locator - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search - - Transitive code search tool - - Calling Function Name Extractor - - Package and Function Locator + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false @@ -106,10 +106,10 @@ functions: skip: false llm_name: generate_cvss_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Container Image Analysis Data + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data max_concurrency: null max_iterations: 10 prompt_examples: true diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 91fc57fa1..56c366d78 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -55,45 +55,45 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm - Transitive code search tool: + Call Chain Analyzer: _type: transitive_code_search enable_transitive_search: true - Calling Function Name Extractor: + Function Caller Finder: _type: calling_function_name_extractor enable_functions_usage_search: true - Package and Function Locator: + Function Locator: _type: package_and_function_locator - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search - - Transitive code search tool - - Calling Function Name Extractor - - Package and Function Locator + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false @@ -107,10 +107,10 @@ functions: skip: false llm_name: generate_cvss_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Container Image Analysis Data + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data max_concurrency: null max_iterations: 10 prompt_examples: true @@ -124,10 +124,20 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + # cve_file_output: + # _type: cve_file_output + # file_path: .tmp/output.json + # markdown_dir: .tmp/vulnerability_markdown_reports + # overwrite: true cve_http_output: _type: cve_http_output url: http://localhost:8080 endpoint: /reports + cve_file_output: + _type: cve_file_output + file_path: .tmp/output.json + markdown_dir: .tmp/vulnerability_markdown_reports + overwrite: true cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm @@ -138,7 +148,7 @@ functions: llms: checklist_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -146,7 +156,7 @@ llms: top_p: 0.01 code_vdb_retriever_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -154,7 +164,7 @@ llms: top_p: 0.01 doc_vdb_retriever_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -162,7 +172,7 @@ llms: top_p: 0.01 cve_agent_executor_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -170,7 +180,7 @@ llms: top_p: 0.01 generate_cvss_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -178,7 +188,7 @@ llms: top_p: 0.01 summarize_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -186,7 +196,7 @@ llms: top_p: 0.01 justify_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -195,7 +205,7 @@ llms: intel_source_score_llm: _type: openai - api_key: "EMPTY" + api_key: ${OPENAI_API_KEY:-EMPTY} base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 @@ -222,7 +232,7 @@ workflow: cve_generate_cvss_name: cve_generate_cvss cve_summarize_name: cve_summarize cve_justify_name: cve_justify - cve_output_config_name: cve_http_output + cve_output_config_name: cve_file_output eval: general: diff --git a/quick_start/quick_start_guide.ipynb b/quick_start/quick_start_guide.ipynb index 4aae39109..50efc88cb 100644 --- a/quick_start/quick_start_guide.ipynb +++ b/quick_start/quick_start_guide.ipynb @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "tags": [] }, @@ -128,162 +128,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "general:\n", - " use_uvloop: true\n", - "functions:\n", - " cve_generate_vdbs:\n", - " _type: cve_generate_vdbs\n", - " agent_name: cve_agent_executor\n", - " embedder_name: nim_embedder\n", - " base_git_dir: .cache/am_cache/git\n", - " base_vdb_dir: .cache/am_cache/vdb\n", - " base_code_index_dir: .cache/am_cache/code_index\n", - " cve_fetch_intel:\n", - " _type: cve_fetch_intel\n", - " cve_process_sbom:\n", - " _type: cve_process_sbom\n", - " cve_check_vuln_deps:\n", - " _type: cve_check_vuln_deps\n", - " cve_checklist:\n", - " _type: cve_checklist\n", - " llm_name: checklist_llm\n", - " Container Image Code QA System:\n", - " _type: local_vdb_retriever\n", - " embedder_name: nim_embedder\n", - " llm_name: code_vdb_retriever_llm\n", - " vdb_type: code\n", - " return_source_documents: false\n", - " Container Image Developer Guide QA System:\n", - " _type: local_vdb_retriever\n", - " embedder_name: nim_embedder\n", - " llm_name: doc_vdb_retriever_llm\n", - " vdb_type: doc\n", - " return_source_documents: false\n", - " Lexical Search Container Image Code QA System:\n", - " _type: lexical_code_search\n", - " top_k: 5\n", - " Internet Search:\n", - " _type: serp_wrapper\n", - " max_retries: 5\n", - " cve_agent_executor:\n", - " _type: cve_agent_executor\n", - " llm_name: cve_agent_executor_llm\n", - " tool_names:\n", - " - Container Image Code QA System\n", - " - Container Image Developer Guide QA System\n", - " - Internet Search\n", - " max_concurrency: null\n", - " max_iterations: 10\n", - " prompt_examples: false\n", - " replace_exceptions: true\n", - " replace_exceptions_value: I do not have a definitive answer for this checklist\n", - " item.\n", - " return_intermediate_steps: false\n", - " verbose: false\n", - " cve_summarize:\n", - " _type: cve_summarize\n", - " llm_name: summarize_llm\n", - " cve_justify:\n", - " _type: cve_justify\n", - " llm_name: justify_llm\n", - " cve_file_output:\n", - " _type: cve_file_output\n", - " file_path: .tmp/output.json\n", - " markdown_dir: .tmp/vulnerability_markdown_reports\n", - " overwrite: true\n", - "llms:\n", - " checklist_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " code_vdb_retriever_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " doc_vdb_retriever_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " cve_agent_executor_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " summarize_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 1024\n", - " top_p: 0.01\n", - " justify_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 1024\n", - " top_p: 0.01\n", - "embedders:\n", - " nim_embedder:\n", - " _type: nim\n", - " base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5}\n", - " truncate: END\n", - " max_batch_size: 128\n", - "workflow:\n", - " _type: cve_agent\n", - " cve_generate_vdbs_name: cve_generate_vdbs\n", - " cve_fetch_intel_name: cve_fetch_intel\n", - " cve_process_sbom_name: cve_process_sbom\n", - " cve_check_vuln_deps_name: cve_check_vuln_deps\n", - " cve_checklist_name: cve_checklist\n", - " cve_agent_executor_name: cve_agent_executor\n", - " cve_summarize_name: cve_summarize\n", - " cve_justify_name: cve_justify\n", - " cve_output_config_name: cve_file_output\n", - "eval:\n", - " general:\n", - " output_dir: ./.tmp/eval/cve_agent\n", - " dataset:\n", - " _type: json\n", - " file_path: data/eval_datasets/eval_dataset.json\n", - " profiler:\n", - " token_uniqueness_forecast: true\n", - " workflow_runtime_forecast: true\n", - " compute_llm_metrics: true\n", - " csv_exclude_io_text: true\n", - " prompt_caching_prefixes:\n", - " enable: true\n", - " min_frequency: 0.1\n", - " bottleneck_analysis:\n", - " enable_nested_stack: true\n", - " concurrency_spike_analysis:\n", - " enable: true\n", - " spike_threshold: 7\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "with open('configs/config.yml', 'r') as config_in:\n", " workflow_config = yaml.safe_load(config_in)\n", @@ -299,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "tags": [] }, @@ -370,82 +220,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ]\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " },\n", - " \"scan\": {\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-50782\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-36632\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-43804\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-cxfr-5q3r-2rc2\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-554w-xh4j-8w64\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3ww4-gg4f-jr7f\"\n", - " }\n", - " ]\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "with open('data/input_messages/morpheus:23.11-runtime.json', 'r') as input_message_in:\n", " input_message = json.load(input_message_in)\n", @@ -461,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "tags": [] }, @@ -495,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "tags": [] }, @@ -514,70 +294,12 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ]\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " },\n", - " \"scan\": {\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", - " }\n", - " ]\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "print(json.dumps(input_message, indent=2))" ] @@ -591,22 +313,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "requests.post(f\"http://localhost:26466/generate\", json=input_message)" ] @@ -697,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "tags": [] }, @@ -713,3961 +424,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"input\": {\n", - " \"scan\": {\n", - " \"id\": \"4fff5361-9706-41ad-8f0a-15730cc1d300\",\n", - " \"type\": null,\n", - " \"started_at\": \"2025-06-07T19:37:22.118544\",\n", - " \"completed_at\": \"2025-06-07T19:37:27.757216\",\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " }\n", - " ]\n", - " },\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"digest\": null,\n", - " \"platform\": null,\n", - " \"feed_group\": null,\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"eeebf55da604c16e6de3b060cbae6ad3172fdd99\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"eeebf55da604c16e6de3b060cbae6ad3172fdd99\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ],\n", - " \"exclude\": []\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " }\n", - " },\n", - " \"info\": {\n", - " \"vdb\": {\n", - " \"code_vdb_path\": \".cache/am_cache/vdb/code/ccf658f72228c497\",\n", - " \"doc_vdb_path\": \".cache/am_cache/vdb/doc/c89fe6d707036daa\",\n", - " \"code_index_path\": null\n", - " },\n", - " \"intel\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-v4hq-m4wr-6pmj\",\n", - " \"cve_id\": \"CVE-2024-21762\",\n", - " \"summary\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0...\",\n", - " \"description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [],\n", - " \"cvss\": {\n", - " \"score\": 9.8,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-787\",\n", - " \"name\": \"Out-of-bounds Write\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2024-02-09T09:31:31Z\",\n", - " \"updated_at\": \"2024-02-09T09:31:40Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", - " \"type\": \"unreviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-v4hq-m4wr-6pmj\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2024-21762\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2024-21762\",\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\",\n", - " \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\"\n", - " ],\n", - " \"github_reviewed_at\": null,\n", - " \"nvd_published_at\": \"2024-02-09T09:15:08Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 9.8\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": null,\n", - " \"score\": 0.0\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.92448,\n", - " \"percentile\": 0.99718\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2024-21762\",\n", - " \"cve_description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 9.8,\n", - " \"cvss_severity\": \"CRITICAL\",\n", - " \"cwe_name\": \"CWE-787: Out-of-bounds Write (4.17)\",\n", - " \"cwe_description\": \"The product writes data past the end, or before the beginning, of the intended buffer.\",\n", - " \"cwe_extended_description\": null,\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"2.0.14\",\n", - " \"versionStartIncluding\": \"1.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.0.15\",\n", - " \"versionStartIncluding\": \"7.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.2.9\",\n", - " \"versionStartIncluding\": \"7.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.4.3\",\n", - " \"versionStartIncluding\": \"7.4.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.0.18\",\n", - " \"versionStartIncluding\": \"6.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.2.16\",\n", - " \"versionStartIncluding\": \"6.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.4.15\",\n", - " \"versionStartIncluding\": \"6.4.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.0.14\",\n", - " \"versionStartIncluding\": \"7.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.2.7\",\n", - " \"versionStartIncluding\": \"7.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.4.3\",\n", - " \"versionStartIncluding\": \"7.4.0\",\n", - " \"versionEndIncluding\": null\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Fortinet\"\n", - " ],\n", - " \"references\": [\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\",\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2024-02-09T09:15:08.087\",\n", - " \"updated_at\": \"2024-11-29T15:23:32.167\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": null,\n", - " \"id\": null,\n", - " \"url\": null\n", - " },\n", - " \"details\": null,\n", - " \"statement\": null,\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": null\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": null,\n", - " \"notes\": null,\n", - " \"notices\": null,\n", - " \"priority\": null,\n", - " \"ubuntu_description\": null,\n", - " \"impact\": null\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.92448,\n", - " \"percentile\": 0.99719,\n", - " \"date\": \"2025-06-07\",\n", - " \"cve\": \"CVE-2024-21762\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"cve_id\": \"CVE-2023-6975\",\n", - " \"summary\": \"MLFlow Path Traversal Vulnerability\",\n", - " \"description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [\n", - " {\n", - " \"package\": {\n", - " \"ecosystem\": \"pip\",\n", - " \"name\": \"mlflow\"\n", - " },\n", - " \"vulnerable_version_range\": \"< 2.9.2\",\n", - " \"first_patched_version\": \"2.9.2\",\n", - " \"vulnerable_functions\": []\n", - " }\n", - " ],\n", - " \"cvss\": {\n", - " \"score\": 9.8,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-29\",\n", - " \"name\": \"Path Traversal: '..filename'\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2023-12-20T06:30:25Z\",\n", - " \"updated_at\": \"2024-01-02T15:14:39Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", - " \"type\": \"reviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"https://github.com/mlflow/mlflow\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2023-6975\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2023-6975\",\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\",\n", - " \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\"\n", - " ],\n", - " \"github_reviewed_at\": \"2023-12-20T20:32:39Z\",\n", - " \"nvd_published_at\": \"2023-12-20T06:15:45Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 9.8\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": null,\n", - " \"score\": 0.0\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.01542,\n", - " \"percentile\": 0.80461\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2023-6975\",\n", - " \"cve_description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 9.8,\n", - " \"cvss_severity\": \"CRITICAL\",\n", - " \"cwe_name\": \"CWE-29: Path Traversal: '\\\\..\\\\filename' (4.17)\",\n", - " \"cwe_description\": \"The product uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\\\\..\\\\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory.\",\n", - " \"cwe_extended_description\": \"This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.\\nThis is similar to CWE-25, except using \\\"\\\\\\\" instead of \\\"/\\\". Sometimes a program checks for \\\"..\\\\\\\" at the beginning of the input, so a \\\"\\\\..\\\\\\\" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the \\\"/\\\" separator is valid.\",\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"mlflow\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"2.9.2\",\n", - " \"versionStartIncluding\": null,\n", - " \"versionEndIncluding\": null\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Lfprojects\"\n", - " ],\n", - " \"references\": [\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\",\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2023-12-20T06:15:45.553\",\n", - " \"updated_at\": \"2024-11-21T08:44:57.463\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": null,\n", - " \"id\": null,\n", - " \"url\": null\n", - " },\n", - " \"details\": null,\n", - " \"statement\": null,\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": null\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": null,\n", - " \"notes\": null,\n", - " \"notices\": null,\n", - " \"priority\": null,\n", - " \"ubuntu_description\": null,\n", - " \"impact\": null\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.01542,\n", - " \"percentile\": 0.80467,\n", - " \"date\": \"2025-06-07\",\n", - " \"cve\": \"CVE-2023-6975\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"cve_id\": \"CVE-2023-50447\",\n", - " \"summary\": \"Arbitrary Code Execution in Pillow\",\n", - " \"description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [\n", - " {\n", - " \"package\": {\n", - " \"ecosystem\": \"pip\",\n", - " \"name\": \"Pillow\"\n", - " },\n", - " \"vulnerable_version_range\": \"< 10.2.0\",\n", - " \"first_patched_version\": \"10.2.0\",\n", - " \"vulnerable_functions\": []\n", - " }\n", - " ],\n", - " \"cvss\": {\n", - " \"score\": 8.1,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-94\",\n", - " \"name\": \"Improper Control of Generation of Code ('Code Injection')\"\n", - " },\n", - " {\n", - " \"cwe_id\": \"CWE-95\",\n", - " \"name\": \"Improper Neutralization of Directives in Dynamically Evaluated Code ('Eval Injection')\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2024-01-19T21:30:35Z\",\n", - " \"updated_at\": \"2024-11-18T16:26:35Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", - " \"type\": \"reviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"https://github.com/python-pillow/Pillow\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2023-50447\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2023-50447\",\n", - " \"https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", - " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#security\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447\",\n", - " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447\",\n", - " \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\"\n", - " ],\n", - " \"github_reviewed_at\": \"2024-01-22T21:28:18Z\",\n", - " \"nvd_published_at\": \"2024-01-19T20:15:11Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 8.1\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": \"CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N\",\n", - " \"score\": 9.3\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.00408,\n", - " \"percentile\": 0.60132\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2023-50447\",\n", - " \"cve_description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 8.1,\n", - " \"cvss_severity\": \"HIGH\",\n", - " \"cwe_name\": \"CWE-94: Improper Control of Generation of Code ('Code Injection') (4.17)\",\n", - " \"cwe_description\": \"The product constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.\",\n", - " \"cwe_extended_description\": null,\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"pillow\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": null,\n", - " \"versionStartIncluding\": null,\n", - " \"versionEndIncluding\": \"10.1.0\"\n", - " },\n", - " {\n", - " \"package\": \"debian_linux\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": null,\n", - " \"versionStartIncluding\": \"10.0\",\n", - " \"versionEndIncluding\": \"10.0\"\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Debian\",\n", - " \"Python\"\n", - " ],\n", - " \"references\": [\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447/\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447/\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2024-01-19T20:15:11.870\",\n", - " \"updated_at\": \"2024-11-21T08:37:00.967\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": \"pillow: Arbitrary Code Execution via the environment parameter\",\n", - " \"id\": \"2259479\",\n", - " \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=2259479\"\n", - " },\n", - " \"details\": [\n", - " \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"A vulnerability was found in Pillow, a popular Python imaging library. The flaw identified in the PIL.ImageMath.eval function enables arbitrary code execution by manipulating the environment parameter.\"\n", - " ],\n", - " \"statement\": \"The vulnerability in Pillow's PIL.ImageMath.eval function poses a significant threat due to its potential for arbitrary code execution. Pillow's widespread use in diverse domains makes this flaw particularly impactful, as it could lead to unauthorized access, data breaches, and compromise of entire systems. The complex exploitation method involving Python's dunder methods adds sophistication to potential attacks.\",\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": {\n", - " \"cvss3_base_score\": 8.1,\n", - " \"cvss3_scoring_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"status\": \"verified\"\n", - " },\n", - " \"threat_severity\": \"Important\",\n", - " \"public_date\": \"2024-01-19T00:00:00Z\",\n", - " \"cwe\": \"CWE-77\",\n", - " \"affected_release\": [\n", - " {\n", - " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 8\",\n", - " \"release_date\": \"2024-06-10T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:3781\",\n", - " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el8\",\n", - " \"package\": \"python3x-pillow-0:10.3.0-1.el8ap\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 9\",\n", - " \"release_date\": \"2024-06-10T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:3781\",\n", - " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el9\",\n", - " \"package\": \"python-pillow-0:10.3.0-1.el9ap\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 7\",\n", - " \"release_date\": \"2024-02-19T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0857\",\n", - " \"cpe\": \"cpe:/o:redhat:enterprise_linux:7\",\n", - " \"package\": \"python-pillow-0:2.0.0-25.gitd1c6db8.el7_9\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8\",\n", - " \"release_date\": \"2024-02-20T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0893\",\n", - " \"cpe\": \"cpe:/a:redhat:enterprise_linux:8\",\n", - " \"package\": \"python-pillow-0:5.1.1-18.el8_9.1\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.2 Advanced Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1059\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.2\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_2\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Advanced Mission Critical Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Telecommunications Update Service\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_tus:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Update Services for SAP Solutions\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_e4s:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.6 Extended Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1058\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.6\",\n", - " \"package\": \"python-pillow-0:5.1.1-19.el8_6\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.8 Extended Update Support\",\n", - " \"release_date\": \"2024-02-08T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0754\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.8\",\n", - " \"package\": \"python-pillow-0:5.1.1-19.el8_8\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\\nhttps://nvd.nist.gov/vuln/detail/CVE-2023-50447\\nhttp://www.openwall.com/lists/oss-security/2024/01/20/1\\nhttps://devhub.checkmarx.com/cve-details/CVE-2023-50447/\\nhttps://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\\nhttps://github.com/python-pillow/Pillow/releases\"\n", - " ],\n", - " \"name\": \"CVE-2023-50447\",\n", - " \"mitigation\": {\n", - " \"value\": \"Mitigation for this issue is either not available or the currently available options do not meet the Red Hat Product Security criteria comprising ease of use and deployment, applicability to widespread installation base or stability.\",\n", - " \"lang\": \"en:us\"\n", - " },\n", - " \"csaw\": false\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": \"\\nPillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution\\nvia the environment parameter, a different vulnerability than\\nCVE-2022-22817 (which was about the expression parameter).\",\n", - " \"notes\": [],\n", - " \"notices\": [\n", - " {\n", - " \"cves_ids\": [\n", - " \"CVE-2023-44271\",\n", - " \"CVE-2023-50447\"\n", - " ],\n", - " \"description\": \"It was discovered that Pillow incorrectly handled certain long text\\narguments. An attacker could possibly use this issue to cause Pillow to\\nconsume resources, leading to a denial of service. This issue only affected\\nUbuntu 20.04 LTS, and Ubuntu 22.04 LTS. (CVE-2023-44271)\\n\\nDuarte Santos discovered that Pillow incorrectly handled the environment\\nparameter to PIL.ImageMath.eval. An attacker could possibly use this issue\\nto execute arbitrary code. (CVE-2023-50447)\\n\",\n", - " \"id\": \"USN-6618-1\",\n", - " \"instructions\": \"In general, a standard system update will make all the necessary changes.\\n\",\n", - " \"is_hidden\": false,\n", - " \"published\": \"2024-01-30T15:17:54.445907\",\n", - " \"references\": [],\n", - " \"release_packages\": {\n", - " \"focal\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " }\n", - " ],\n", - " \"jammy\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " }\n", - " ],\n", - " \"mantic\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " }\n", - " ]\n", - " },\n", - " \"summary\": \"Several security issues were fixed in Pillow.\\n\",\n", - " \"title\": \"Pillow vulnerabilities\",\n", - " \"type\": \"USN\"\n", - " }\n", - " ],\n", - " \"priority\": \"medium\",\n", - " \"ubuntu_description\": \"\",\n", - " \"impact\": {\n", - " \"baseMetricV3\": {\n", - " \"cvssV3\": {\n", - " \"attackComplexity\": \"HIGH\",\n", - " \"attackVector\": \"NETWORK\",\n", - " \"availabilityImpact\": \"HIGH\",\n", - " \"baseScore\": 8.1,\n", - " \"baseSeverity\": \"HIGH\",\n", - " \"confidentialityImpact\": \"HIGH\",\n", - " \"integrityImpact\": \"HIGH\",\n", - " \"privilegesRequired\": \"NONE\",\n", - " \"scope\": \"UNCHANGED\",\n", - " \"userInteraction\": \"NONE\",\n", - " \"vectorString\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"version\": \"3.1\"\n", - " },\n", - " \"exploitabilityScore\": 2.2,\n", - " \"impactScore\": 5.9\n", - " }\n", - " },\n", - " \"bugs\": [\n", - " \"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1061172\"\n", - " ],\n", - " \"codename\": null,\n", - " \"cvss3\": 8.1,\n", - " \"id\": \"CVE-2023-50447\",\n", - " \"mitigation\": \"\",\n", - " \"notices_ids\": [\n", - " \"USN-6618-1\"\n", - " ],\n", - " \"packages\": [\n", - " {\n", - " \"debian\": \"https://tracker.debian.org/pkg/pillow\",\n", - " \"name\": \"pillow\",\n", - " \"source\": \"https://ubuntu.com/security/cve?package=pillow\",\n", - " \"statuses\": [\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"bionic\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"7.0.0-4ubuntu0.8\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"focal\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"9.0.1-1ubuntu0.2\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"jammy\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"end of life, was needed\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"lunar\",\n", - " \"status\": \"ignored\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.0.0-1ubuntu0.1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"mantic\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"noble\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"oracular\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"plucky\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"trusty\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"upstream\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"xenial\",\n", - " \"status\": \"needs-triage\"\n", - " }\n", - " ],\n", - " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow\"\n", - " },\n", - " {\n", - " \"debian\": \"https://tracker.debian.org/pkg/pillow-python2\",\n", - " \"name\": \"pillow-python2\",\n", - " \"source\": \"https://ubuntu.com/security/cve?package=pillow-python2\",\n", - " \"statuses\": [\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"bionic\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"focal\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"jammy\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"lunar\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"mantic\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"noble\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"oracular\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"plucky\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"trusty\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"upstream\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"xenial\",\n", - " \"status\": \"DNE\"\n", - " }\n", - " ],\n", - " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow-python2\"\n", - " }\n", - " ],\n", - " \"patches\": {\n", - " \"pillow\": [\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/0ca3c33c59927e1c7e0c14dbc1eea1dfb2431a80\",\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/557ba59d13de919d04b3fd4cdef8634f7d4b3348\"\n", - " ],\n", - " \"pillow-python2\": []\n", - " },\n", - " \"published\": \"2024-01-19T20:15:00\",\n", - " \"references\": [\n", - " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\",\n", - " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#imagemath-eval-restricted-environment-keys\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://ubuntu.com/security/notices/USN-6618-1\",\n", - " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\"\n", - " ],\n", - " \"status\": \"active\",\n", - " \"tags\": {},\n", - " \"updated_at\": \"2024-07-24T15:57:39.284958+00:00\"\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.00408,\n", - " \"percentile\": 0.60242,\n", - " \"date\": \"2025-05-30\",\n", - " \"cve\": \"CVE-2023-50447\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " }\n", - " ],\n", - " \"sbom\": {\n", - " \"packages\": [\n", - " {\n", - " \"name\": \"Babel\",\n", - " \"version\": \"2.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Brotli\",\n", - " \"version\": \"1.0.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Brotli\",\n", - " \"version\": \"1.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ConfigArgParse\",\n", - " \"version\": \"1.5.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Flask\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"GitPython\",\n", - " \"version\": \"3.1.40\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Jinja2\",\n", - " \"version\": \"3.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Mako\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Markdown\",\n", - " \"version\": \"3.5.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"MarkupSafe\",\n", - " \"version\": \"2.1.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PyJWT\",\n", - " \"version\": \"2.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PySocks\",\n", - " \"version\": \"1.7.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PyYAML\",\n", - " \"version\": \"6.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Pygments\",\n", - " \"version\": \"2.17.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"SQLAlchemy\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"SciPy\",\n", - " \"version\": \"1.11.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Simple\",\n", - " \"version\": \"Launcher\",\n", - " \"path\": null,\n", - " \"system\": \"1.1.0.14\"\n", - " },\n", - " {\n", - " \"name\": \"Sphinx\",\n", - " \"version\": \"7.2.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Werkzeug\",\n", - " \"version\": \"3.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"absl-py\",\n", - " \"version\": \"1.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"adduser\",\n", - " \"version\": \"3.118ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"alabaster\",\n", - " \"version\": \"0.7.13\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"alembic\",\n", - " \"version\": \"1.13.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"anyio\",\n", - " \"version\": \"3.7.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"appdirs\",\n", - " \"version\": \"1.4.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"apt\",\n", - " \"version\": \"2.4.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"asn1crypto\",\n", - " \"version\": \"1.5.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"attrs\",\n", - " \"version\": \"23.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"base-files\",\n", - " \"version\": \"12ubuntu4.4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"base-passwd\",\n", - " \"version\": \"3.5.52build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bash\",\n", - " \"version\": \"5.1-6ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bc\",\n", - " \"version\": \"1.07.1-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"beautifulsoup4\",\n", - " \"version\": \"4.12.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"betterproto\",\n", - " \"version\": \"1.2.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"binutils\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"binutils-common\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"binutils-x86-64-linux-gnu\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"blinker\",\n", - " \"version\": \"1.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"boa\",\n", - " \"version\": \"0.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"bokeh\",\n", - " \"version\": \"2.4.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"boltons\",\n", - " \"version\": \"23.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"bsdutils\",\n", - " \"version\": \"1:2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"build-essential\",\n", - " \"version\": \"12.9ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bzip2\",\n", - " \"version\": \"1.0.8-5build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates\",\n", - " \"version\": \"20230311ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates-java\",\n", - " \"version\": \"java-archive\",\n", - " \"path\": null,\n", - " \"system\": \"\\u001b[38;2;119;119;119m\\u001b[0m\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates-java\",\n", - " \"version\": \"20190909ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cachetools\",\n", - " \"version\": \"5.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cattrs\",\n", - " \"version\": \"23.2.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"certifi\",\n", - " \"version\": \"2023.11.17\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cffi\",\n", - " \"version\": \"1.15.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cffi\",\n", - " \"version\": \"1.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"chardet\",\n", - " \"version\": \"5.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"charset-normalizer\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"charset-normalizer\",\n", - " \"version\": \"3.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"click\",\n", - " \"version\": \"8.1.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cloudpickle\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"colorama\",\n", - " \"version\": \"0.4.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda\",\n", - " \"version\": \"23.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-build\",\n", - " \"version\": \"3.28.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-libmamba-solver\",\n", - " \"version\": \"23.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-package-handling\",\n", - " \"version\": \"2.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda_index\",\n", - " \"version\": \"0.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda_package_streaming\",\n", - " \"version\": \"0.9.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"configparser\",\n", - " \"version\": \"5.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"confluent-kafka\",\n", - " \"version\": \"1.9.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"contourpy\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"coreutils\",\n", - " \"version\": \"8.32-4.1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cpp\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cpp-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cryptography\",\n", - " \"version\": \"41.0.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cryptography\",\n", - " \"version\": \"41.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cubinlinker\",\n", - " \"version\": \"0.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cccl-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-compat-11-8\",\n", - " \"version\": \"520.61.05-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-compiler-11-8\",\n", - " \"version\": \"11.8.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cudart-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cudart-dev-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cuobjdump-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cupti-11-8\",\n", - " \"version\": \"11.8.87-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cuxxfilt-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-driver-dev-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-gdb-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-keyring\",\n", - " \"version\": \"1.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvcc-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvdisasm-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvprune-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-python\",\n", - " \"version\": \"11.8.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-11-8-config-common\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-11-config-common\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-config-common\",\n", - " \"version\": \"12.3.101-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cudf\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cudf-kafka\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cupy\",\n", - " \"version\": \"12.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"curl\",\n", - " \"version\": \"7.81.0-1ubuntu1.15\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cycler\",\n", - " \"version\": \"0.12.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cytoolz\",\n", - " \"version\": \"0.12.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dash\",\n", - " \"version\": \"0.5.11+git20210903+057cd650a4ed-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dask\",\n", - " \"version\": \"2023.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dask-cuda\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dask-cudf\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"databricks-cli\",\n", - " \"version\": \"0.18.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dataclasses\",\n", - " \"version\": \"0.8\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"datacompy\",\n", - " \"version\": \"0.8.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"debconf\",\n", - " \"version\": \"1.5.79ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"debianutils\",\n", - " \"version\": \"5.5-1ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"diffutils\",\n", - " \"version\": \"1:3.8-0ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dill\",\n", - " \"version\": \"0.3.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dirmngr\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"distributed\",\n", - " \"version\": \"2023.3.2.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docker\",\n", - " \"version\": \"5.0.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docker-pycreds\",\n", - " \"version\": \"0.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docutils\",\n", - " \"version\": \"0.20.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dpkg\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dpkg-dev\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"e2fsprogs\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"entrypoints\",\n", - " \"version\": \"0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"exceptiongroup\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"fastrlock\",\n", - " \"version\": \"0.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"filelock\",\n", - " \"version\": \"3.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"findutils\",\n", - " \"version\": \"4.8.0-1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fontconfig-config\",\n", - " \"version\": \"2.13.1-4.2ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fonts-dejavu-core\",\n", - " \"version\": \"2.37-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fonttools\",\n", - " \"version\": \"4.46.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"fsspec\",\n", - " \"version\": \"2023.12.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"g++\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"g++-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-11-base\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-12-base\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gitdb\",\n", - " \"version\": \"4.0.11\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gmpy2\",\n", - " \"version\": \"2.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg-l10n\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg-utils\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg2\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"googleapis-common-protos\",\n", - " \"version\": \"1.61.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gpg\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-agent\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-wks-client\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-wks-server\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgconf\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgsm\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgv\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"greenlet\",\n", - " \"version\": \"3.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"grep\",\n", - " \"version\": \"3.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"grpcio\",\n", - " \"version\": \"1.54.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"grpclib\",\n", - " \"version\": \"0.4.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gunicorn\",\n", - " \"version\": \"21.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gzip\",\n", - " \"version\": \"1.10-4ubuntu4.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"h2\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"hostname\",\n", - " \"version\": \"3.23ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"hpack\",\n", - " \"version\": \"4.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"hyperframe\",\n", - " \"version\": \"6.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"idna\",\n", - " \"version\": \"3.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"idna\",\n", - " \"version\": \"3.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"imagesize\",\n", - " \"version\": \"1.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"importlib-metadata\",\n", - " \"version\": \"7.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"importlib-resources\",\n", - " \"version\": \"6.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"init-system-helpers\",\n", - " \"version\": \"1.62\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"itsdangerous\",\n", - " \"version\": \"2.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"java-common\",\n", - " \"version\": \"0.72build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"joblib\",\n", - " \"version\": \"1.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jq\",\n", - " \"version\": \"1.6-2.1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"jrt-fs\",\n", - " \"version\": \"11.0.21\",\n", - " \"path\": null,\n", - " \"system\": \"java-archive\"\n", - " },\n", - " {\n", - " \"name\": \"js\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"npm\"\n", - " },\n", - " {\n", - " \"name\": \"json5\",\n", - " \"version\": \"0.9.14\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonpatch\",\n", - " \"version\": \"1.32\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonpointer\",\n", - " \"version\": \"2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonschema\",\n", - " \"version\": \"4.20.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonschema-specifications\",\n", - " \"version\": \"2023.11.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jupyter_client\",\n", - " \"version\": \"8.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jupyter_core\",\n", - " \"version\": \"5.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"kiwisolver\",\n", - " \"version\": \"1.4.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libacl1\",\n", - " \"version\": \"2.3.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libapt-pkg6.0\",\n", - " \"version\": \"2.4.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libarchive-c\",\n", - " \"version\": \"5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libasan6\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libasound2\",\n", - " \"version\": \"1.2.6.1-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libasound2-data\",\n", - " \"version\": \"1.2.6.1-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libassuan0\",\n", - " \"version\": \"2.5.5-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libatomic1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libattr1\",\n", - " \"version\": \"1:2.5.1-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libaudit-common\",\n", - " \"version\": \"1:3.0.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libaudit1\",\n", - " \"version\": \"1:3.0.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-client3\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-common-data\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-common3\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbinutils\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libblkid1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbrotli1\",\n", - " \"version\": \"1.0.9-2build6\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbsd0\",\n", - " \"version\": \"0.11.5-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbz2-1.0\",\n", - " \"version\": \"1.0.8-5build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc-bin\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc-dev-bin\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc6\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc6-dev\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcap-ng0\",\n", - " \"version\": \"0.7.9-2.2build3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcap2\",\n", - " \"version\": \"1:2.44-1ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcbor0.8\",\n", - " \"version\": \"0.8.0-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcc1-0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcom-err2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcrypt-dev\",\n", - " \"version\": \"1:4.4.27-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcrypt1\",\n", - " \"version\": \"1:4.4.27-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libctf-nobfd0\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libctf0\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcublas-11-8\",\n", - " \"version\": \"11.11.3.6-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcufft-11-8\",\n", - " \"version\": \"10.9.0.58-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcups2\",\n", - " \"version\": \"2.4.1op1-1ubuntu4.7\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcurand-11-8\",\n", - " \"version\": \"10.3.0.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcurl4\",\n", - " \"version\": \"7.81.0-1ubuntu1.15\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcusolver-11-8\",\n", - " \"version\": \"11.4.1.48-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdb5.3\",\n", - " \"version\": \"5.3.28+dfsg1-0.8ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdbus-1-3\",\n", - " \"version\": \"1.12.20-2ubuntu4.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdebconfclient0\",\n", - " \"version\": \"0.261ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdpkg-perl\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libedit2\",\n", - " \"version\": \"3.1-20210910-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libexpat1\",\n", - " \"version\": \"2.4.7-1ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libext2fs2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libffi8\",\n", - " \"version\": \"3.4.2-4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfido2-1\",\n", - " \"version\": \"1.10.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfontconfig1\",\n", - " \"version\": \"2.13.1-4.2ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfreetype6\",\n", - " \"version\": \"2.11.1+dfsg-1ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcc-11-dev\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcc-s1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcrypt20\",\n", - " \"version\": \"1.9.4-3ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgdbm-compat4\",\n", - " \"version\": \"1.23-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgdbm6\",\n", - " \"version\": \"1.23-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libglib2.0-0\",\n", - " \"version\": \"2.72.4-0ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgmp10\",\n", - " \"version\": \"2:6.2.1+dfsg-3ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgnutls30\",\n", - " \"version\": \"3.7.3-4ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgomp1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgpg-error0\",\n", - " \"version\": \"1.43-3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgraphite2-3\",\n", - " \"version\": \"1.3.14-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgssapi-krb5-2\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libharfbuzz0b\",\n", - " \"version\": \"2.7.4-1ubuntu3.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libhogweed6\",\n", - " \"version\": \"3.7.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libidn2-0\",\n", - " \"version\": \"2.3.2-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libisl23\",\n", - " \"version\": \"0.24-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libitm1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjpeg-turbo8\",\n", - " \"version\": \"2.1.2-0ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjpeg8\",\n", - " \"version\": \"8c-2ubuntu10\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjq1\",\n", - " \"version\": \"1.6-2.1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libk5crypto3\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkeyutils1\",\n", - " \"version\": \"1.6.1-2ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkrb5-3\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkrb5support0\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libksba8\",\n", - " \"version\": \"1.6.0-2ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblcms2-2\",\n", - " \"version\": \"2.12~rc1-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libldap-2.5-0\",\n", - " \"version\": \"2.5.16+dfsg-0ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblsan0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblz4-1\",\n", - " \"version\": \"1.9.3-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblzma5\",\n", - " \"version\": \"5.2.5-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmambapy\",\n", - " \"version\": \"1.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libmd0\",\n", - " \"version\": \"1.0.4-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmount1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmpc3\",\n", - " \"version\": \"1.2.1-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmpfr6\",\n", - " \"version\": \"4.1.0-3build3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libncurses6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libncursesw6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnettle8\",\n", - " \"version\": \"3.7.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnghttp2-14\",\n", - " \"version\": \"1.43.0-1ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnpth0\",\n", - " \"version\": \"1.6-3build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnsl-dev\",\n", - " \"version\": \"1.3.0-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnsl2\",\n", - " \"version\": \"1.3.0-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnspr4\",\n", - " \"version\": \"2:4.32-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnss3\",\n", - " \"version\": \"2:3.68.2-0ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnuma1\",\n", - " \"version\": \"2.0.14-3ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libonig5\",\n", - " \"version\": \"6.9.7.1-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libp11-kit0\",\n", - " \"version\": \"0.24.0-6build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-modules\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-modules-bin\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-runtime\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam0g\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcre2-8-0\",\n", - " \"version\": \"10.39-3ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcre3\",\n", - " \"version\": \"2:8.39-13ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcsclite1\",\n", - " \"version\": \"1.9.5-3ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libperl5.34\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpng16-16\",\n", - " \"version\": \"1.6.37-3build5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libprocps8\",\n", - " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpsl5\",\n", - " \"version\": \"0.21.0-1.2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libquadmath0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libreadline8\",\n", - " \"version\": \"8.1.2-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"librtmp1\",\n", - " \"version\": \"2.4+20151223.gitfa8646d.1-2build4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsasl2-2\",\n", - " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsasl2-modules-db\",\n", - " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libseccomp2\",\n", - " \"version\": \"2.5.3-2ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libselinux1\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsemanage-common\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsemanage2\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsepol2\",\n", - " \"version\": \"3.3-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsmartcols1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsqlite3-0\",\n", - " \"version\": \"3.37.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libss2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libssh-4\",\n", - " \"version\": \"0.9.6-2ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libssl3\",\n", - " \"version\": \"3.0.2-0ubuntu1.12\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libstdc++-11-dev\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libstdc++6\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsystemd0\",\n", - " \"version\": \"249.11-0ubuntu3.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtasn1-6\",\n", - " \"version\": \"4.18.0-4build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtinfo6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc-common\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc-dev\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc3\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtsan0\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libubsan1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libudev1\",\n", - " \"version\": \"249.11-0ubuntu3.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libunistring2\",\n", - " \"version\": \"1.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libuuid1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libxxhash0\",\n", - " \"version\": \"0.8.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libzstd1\",\n", - " \"version\": \"1.4.8+dfsg-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"linux-libc-dev\",\n", - " \"version\": \"5.15.0-89.99\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"llvmlite\",\n", - " \"version\": \"0.41.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"locket\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"login\",\n", - " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"logsave\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lsb-base\",\n", - " \"version\": \"11.1.0ubuntu4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lto-disabled-list\",\n", - " \"version\": \"24\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lz4\",\n", - " \"version\": \"4.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"make\",\n", - " \"version\": \"4.3-4.1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mamba\",\n", - " \"version\": \"1.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"markdown-it-py\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"matplotlib\",\n", - " \"version\": \"3.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mawk\",\n", - " \"version\": \"1.3.4.20200120-3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mdurl\",\n", - " \"version\": \"0.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"menuinst\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"merlin-core\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"merlin-dataloader\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"more-itertools\",\n", - " \"version\": \"10.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"morpheus\",\n", - " \"version\": \"23.11.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mount\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mpmath\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mrc\",\n", - " \"version\": \"23.11.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"msgpack\",\n", - " \"version\": \"1.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"multidict\",\n", - " \"version\": \"6.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"munkres\",\n", - " \"version\": \"1.1.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nb-conda-kernels\",\n", - " \"version\": \"2.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ncurses-base\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ncurses-bin\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"networkx\",\n", - " \"version\": \"3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"npy-append-array\",\n", - " \"version\": \"0.9.16\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numba\",\n", - " \"version\": \"0.58.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numpy\",\n", - " \"version\": \"1.26.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numpydoc\",\n", - " \"version\": \"1.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nvtabular\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nvtx\",\n", - " \"version\": \"0.2.8\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"oauthlib\",\n", - " \"version\": \"3.2.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"openjdk-11-jre-headless\",\n", - " \"version\": \"11.0.21+9-0ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssh-client\",\n", - " \"version\": \"1:8.9p1-3ubuntu0.4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssl\",\n", - " \"version\": \"3.0.2-0ubuntu1.12\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssl\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"ordered-set\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"packaging\",\n", - " \"version\": \"23.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"packaging\",\n", - " \"version\": \"23.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pandas\",\n", - " \"version\": \"1.3.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"partd\",\n", - " \"version\": \"1.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"passwd\",\n", - " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"patch\",\n", - " \"version\": \"2.7.6-7build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl-base\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl-modules-5.34\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pinentry-curses\",\n", - " \"version\": \"1.1.1-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pip\",\n", - " \"version\": \"23.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pkg-config\",\n", - " \"version\": \"0.29.2-1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pkginfo\",\n", - " \"version\": \"1.9.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pkgutil_resolve_name\",\n", - " \"version\": \"1.3.10\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"platformdirs\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pluggy\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pluggy\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"procps\",\n", - " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"prometheus-client\",\n", - " \"version\": \"0.19.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"prometheus-flask-exporter\",\n", - " \"version\": \"0.23.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"prompt-toolkit\",\n", - " \"version\": \"3.0.41\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"protobuf\",\n", - " \"version\": \"4.21.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"psutil\",\n", - " \"version\": \"5.9.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ptxcompiler\",\n", - " \"version\": \"0.8.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyOpenSSL\",\n", - " \"version\": \"23.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyOpenSSL\",\n", - " \"version\": \"23.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyarrow\",\n", - " \"version\": \"11.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyarrow-hotfix\",\n", - " \"version\": \"0.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pycosat\",\n", - " \"version\": \"0.6.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pycparser\",\n", - " \"version\": \"2.21\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pynvml\",\n", - " \"version\": \"11.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyparsing\",\n", - " \"version\": \"3.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"python\",\n", - " \"version\": \"3.10.12\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"python\",\n", - " \"version\": \"3.10.13\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"python-dateutil\",\n", - " \"version\": \"2.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"python-rapidjson\",\n", - " \"version\": \"1.13\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pytz\",\n", - " \"version\": \"2023.3.post1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyzmq\",\n", - " \"version\": \"25.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"querystring-parser\",\n", - " \"version\": \"1.2.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"readline-common\",\n", - " \"version\": \"8.1.2-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"referencing\",\n", - " \"version\": \"0.31.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"requests\",\n", - " \"version\": \"2.31.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"requests-cache\",\n", - " \"version\": \"1.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rich\",\n", - " \"version\": \"13.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rmm\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rpcsvc-proto\",\n", - " \"version\": \"1.4.2-0ubuntu6\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"rpds-py\",\n", - " \"version\": \"0.13.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ruamel.yaml\",\n", - " \"version\": \"0.17.32\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ruamel.yaml.clib\",\n", - " \"version\": \"0.2.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"scikit-learn\",\n", - " \"version\": \"1.2.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sed\",\n", - " \"version\": \"4.8-1ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"sensible-utils\",\n", - " \"version\": \"0.0.17\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"setuptools\",\n", - " \"version\": \"59.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"setuptools\",\n", - " \"version\": \"68.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"six\",\n", - " \"version\": \"1.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"smmap\",\n", - " \"version\": \"5.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sniffio\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"snowballstemmer\",\n", - " \"version\": \"2.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sortedcontainers\",\n", - " \"version\": \"2.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"soupsieve\",\n", - " \"version\": \"2.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-applehelp\",\n", - " \"version\": \"1.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-devhelp\",\n", - " \"version\": \"1.0.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-htmlhelp\",\n", - " \"version\": \"2.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-jsmath\",\n", - " \"version\": \"1.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-qthelp\",\n", - " \"version\": \"1.0.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-serializinghtml\",\n", - " \"version\": \"1.1.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sqlparse\",\n", - " \"version\": \"0.4.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"stringcase\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sympy\",\n", - " \"version\": \"1.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sysvinit-utils\",\n", - " \"version\": \"3.01-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"tabulate\",\n", - " \"version\": \"0.9.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tar\",\n", - " \"version\": \"1.34+dfsg-1ubuntu0.1.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"tblib\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tensorflow-metadata\",\n", - " \"version\": \"1.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"threadpoolctl\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tomli\",\n", - " \"version\": \"2.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"toolz\",\n", - " \"version\": \"0.12.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"torch\",\n", - " \"version\": \"2.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tornado\",\n", - " \"version\": \"6.3.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tqdm\",\n", - " \"version\": \"4.66.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"traitlets\",\n", - " \"version\": \"5.14.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"triton\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tritonclient\",\n", - " \"version\": \"2.26.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"typing-utils\",\n", - " \"version\": \"0.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"typing_extensions\",\n", - " \"version\": \"4.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ubuntu-keyring\",\n", - " \"version\": \"2021.03.26\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ucf\",\n", - " \"version\": \"3.0043\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ujson\",\n", - " \"version\": \"5.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"unicodedata2\",\n", - " \"version\": \"15.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"unzip\",\n", - " \"version\": \"6.0-26ubuntu3.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"url-normalize\",\n", - " \"version\": \"1.4.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"urllib3\",\n", - " \"version\": \"2.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"urllib3\",\n", - " \"version\": \"2.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"usrmerge\",\n", - " \"version\": \"25ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"util-linux\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"watchdog\",\n", - " \"version\": \"2.1.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"watchgod\",\n", - " \"version\": \"0.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wcwidth\",\n", - " \"version\": \"0.2.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"websocket-client\",\n", - " \"version\": \"1.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"websockets\",\n", - " \"version\": \"12.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wget\",\n", - " \"version\": \"1.21.2-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"wheel\",\n", - " \"version\": \"0.41.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wheel\",\n", - " \"version\": \"0.42.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"xz-utils\",\n", - " \"version\": \"5.2.5-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zict\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"zip\",\n", - " \"version\": \"3.0-12build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zipp\",\n", - " \"version\": \"3.17.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"zlib1g\",\n", - " \"version\": \"1:1.2.11.dfsg-2ubuntu9.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zstandard\",\n", - " \"version\": \"0.19.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " }\n", - " ]\n", - " },\n", - " \"vulnerable_dependencies\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"nvd\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": []\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"ghsa\",\n", - " \"nvd\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": [\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"ghsa\",\n", - " \"nvd\",\n", - " \"ubuntu\",\n", - " \"rhsa\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": [\n", - " {\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - " },\n", - " \"output\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Agent bypassed: no vulnerable packages detected. Checklist not generated.\",\n", - " \"response\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM and so the agent was bypassed.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM.\",\n", - " \"justification\": {\n", - " \"label\": \"false_positive\",\n", - " \"reason\": \"No vulnerable packages or dependencies were detected in the SBOM.\",\n", - " \"status\": \"FALSE\"\n", - " }\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Verify Usage of Vulnerable Path Construction: Review the application code within the container image to check if it constructs pathnames using external input. Specifically, look for instances where user input is used to build file paths, which could be exploited to traverse the file system.\",\n", - " \"response\": \"The application code within the container image does construct pathnames using external input, which could potentially be exploited to traverse the file system. Specifically, the code patterns found in the `path.join()` and string concatenation functions may be vulnerable to path traversal attacks. However, without further information about the container image and its configuration, it is difficult to determine the exact risk and potential impact of these vulnerabilities.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Assess Input Validation and Sanitization: Evaluate the robustness of input validation and sanitization practices within the application, particularly for inputs that are used in constructing file paths. Check if the application properly neutralizes '\\\\..\\\\filename' sequences that could resolve to locations outside the intended directory.\",\n", - " \"response\": \"The application does not properly neutralize '\\\\..\\\\filename' sequences that could resolve to locations outside the intended directory. The custom class `MorpheusRelativePath` and the function `get_package_relative_file` do not validate and sanitize file paths constructed from user input, making the application vulnerable to directory traversal attacks. To fix this, the code should use `os.path.normpath` or `os.path.abspath` to normalize the path and remove any directory traversal sequences, and also validate the resulting path to ensure it's within the intended directory.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Inspect File System Access Controls: Review the file system access controls and permissions within the container to determine if an attacker could exploit the vulnerability to access sensitive files or directories. Consider the potential impact of an attacker gaining access to data and models information.\",\n", - " \"response\": \"The container image likely has restricted access controls and permissions to prevent unauthorized access to sensitive files and directories, but the exact specifics couldn't be determined.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"Based on the provided Checklist and Findings, the CVE is exploitable. Specifically, the application code constructs pathnames using external input, which could be exploited to traverse the file system (Checklist Item 1). Moreover, the application does not properly neutralize directory traversal sequences, making it vulnerable to directory traversal attacks (Checklist Item 2). These findings indicate that the CVE is exploitable, allowing an attacker to potentially access sensitive files or directories.\",\n", - " \"justification\": {\n", - " \"label\": \"vulnerable\",\n", - " \"reason\": \"The analysis indicates that the CVE is exploitable due to the application's construction of pathnames using external input and failure to neutralize directory traversal sequences, making it vulnerable to directory traversal attacks.\",\n", - " \"status\": \"TRUE\"\n", - " }\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Verify Usage of PIL.ImageMath.eval: Check if the application code within the container image uses the PIL.ImageMath.eval function, specifically with the environment parameter. This function is the target of the vulnerability, and its usage with untrusted input could lead to arbitrary code execution.\",\n", - " \"response\": \"Based on the observations from the Container Image Code QA System and the Container Image Developer Guide QA System, it appears that the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, and there is no indication of any dependencies or libraries that might be using it. Therefore, the usage of PIL.ImageMath.eval with untrusted input, which could lead to arbitrary code execution, is unlikely to be a concern for this container image.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Assess Input Handling for PIL.ImageMath.eval: If PIL.ImageMath.eval is used, evaluate how the application handles input data passed to this function, especially focusing on the environment parameter. Ensure that inputs are properly sanitized and validated to prevent potential code injection attacks.\",\n", - " \"response\": \"The container image does not use PIL.ImageMath.eval, but it uses other libraries such as CuPy and Morpheus for mathematical operations. While the provided context does not provide enough information to determine if the inputs are properly sanitized and validated, it is essential to handle input data carefully to prevent potential security risks, such as information leakage across user boundaries. Therefore, it is recommended to implement proper input data sanitization and validation mechanisms when using these libraries.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Review Environment Variable Management: Since the vulnerability involves manipulation of the environment parameter, assess how environment variables are managed within the application and the container. Ensure that sensitive data is not exposed through environment variables and that variables are properly secured to prevent unauthorized access or manipulation.\",\n", - " \"response\": \"The container image's management of environment variables is not clearly documented, and it is unclear if sensitive data is exposed through environment variables or if variables are properly secured to prevent unauthorized access or manipulation. Further investigation is needed to fully assess the security of environment variable management within the container image.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"Based on the provided Checklist and Findings, the CVE is not exploitable. The investigation found that the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, which is the target of the vulnerability. This suggests that the risk of arbitrary code execution through this specific vulnerability is unlikely.\",\n", - " \"justification\": {\n", - " \"label\": \"code_not_reachable\",\n", - " \"reason\": \"The vulnerable code is not executed during runtime because the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, which is the target of the vulnerability.\",\n", - " \"status\": \"FALSE\"\n", - " }\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "print(json.dumps(first_result, indent=2))" ] @@ -4681,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "tags": [] }, @@ -4702,94 +463,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/html": [ - "

\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
vuln_idchecklistsummarylabelreasonaffected
0CVE-2024-21762[{'input': 'Agent bypassed: no vulnerable pack...The VulnerableDependencyChecker did not find a...false_positiveNo vulnerable packages or dependencies were de...FALSE
1GHSA-hh8p-p8mp-gqhm[{'input': 'Verify Usage of Vulnerable Path Co...Based on the provided Checklist and Findings, ...vulnerableThe analysis indicates that the CVE is exploit...TRUE
2GHSA-3f63-hfp8-52jq[{'input': 'Verify Usage of PIL.ImageMath.eval...Based on the provided Checklist and Findings, ...code_not_reachableThe vulnerable code is not executed during run...FALSE
\n", - "
" - ], - "text/plain": [ - " vuln_id checklist \\\n", - "0 CVE-2024-21762 [{'input': 'Agent bypassed: no vulnerable pack... \n", - "1 GHSA-hh8p-p8mp-gqhm [{'input': 'Verify Usage of Vulnerable Path Co... \n", - "2 GHSA-3f63-hfp8-52jq [{'input': 'Verify Usage of PIL.ImageMath.eval... \n", - "\n", - " summary label \\\n", - "0 The VulnerableDependencyChecker did not find a... false_positive \n", - "1 Based on the provided Checklist and Findings, ... vulnerable \n", - "2 Based on the provided Checklist and Findings, ... code_not_reachable \n", - "\n", - " reason affected \n", - "0 No vulnerable packages or dependencies were de... FALSE \n", - "1 The analysis indicates that the CVE is exploit... TRUE \n", - "2 The vulnerable code is not executed during run... FALSE " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "process_results(output_file_path)" ] @@ -4822,7 +500,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.10" + "version": "3.13.7" } }, "nbformat": 4, diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index c95cf05f9..1ff242eb1 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -48,45 +48,45 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm - Transitive code search tool: + Call Chain Analyzer: _type: transitive_code_search enable_transitive_search: true - Calling Function Name Extractor: + Function Caller Finder: _type: calling_function_name_extractor enable_functions_usage_search: true - Package and Function Locator: + Function Locator: _type: package_and_function_locator - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search - - Transitive code search tool - - Calling Function Name Extractor - - Package and Function Locator + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index ccec2190f..8d6377c8a 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -48,45 +48,45 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm - Transitive code search tool: + Call Chain Analyzer: _type: transitive_code_search enable_transitive_search: true - Calling Function Name Extractor: + Function Caller Finder: _type: calling_function_name_extractor enable_functions_usage_search: true - Package and Function Locator: + Function Locator: _type: package_and_function_locator - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search - - Transitive code search tool - - Calling Function Name Extractor - - Package and Function Locator + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false @@ -100,10 +100,10 @@ functions: skip: false llm_name: generate_cvss_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Container Image Analysis Data + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data max_concurrency: null max_iterations: 10 prompt_examples: true diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 689f99629..003c47eff 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -34,44 +34,63 @@ functions: base_git_dir: .cache/am_cache/git base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : _type: cve_check_vuln_deps - skip: false + skip: true cve_checklist: _type: cve_checklist llm_name: checklist_llm - Container Image Code QA System: + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search # Uncomment to enable keyword search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator max_concurrency: null max_iterations: 10 prompt_examples: false @@ -84,10 +103,10 @@ functions: skip: false llm_name: generate_cvss_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Container Image Analysis Data + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data max_concurrency: null max_iterations: 10 prompt_examples: true diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 2094ab65b..9aa0f23b7 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -40,34 +40,34 @@ functions: cve_checklist: _type: cve_checklist llm_name: checklist_llm - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 - Container Image Analysis Data: + Container Analysis Data: _type: container_image_analysis_data cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search + - Code Semantic Search + - Docs Semantic Search + # - Code Keyword Search # Uncomment to enable keyword search + - CVE Web Search max_concurrency: null max_iterations: 10 prompt_examples: false @@ -80,10 +80,10 @@ functions: skip: false llm_name: generate_cvss_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Container Image Analysis Data + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search # Uncomment to enable keyword search + - Container Analysis Data max_concurrency: null max_iterations: 10 prompt_examples: true diff --git a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json index 21ebe640d..9ac547200 100644 --- a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json +++ b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json @@ -2,6 +2,7 @@ "image": { "name": "nvcr.io/nvidia/morpheus/morpheus", "tag": "23.11-runtime", + "analysis_type": "image", "source_info": [ { "type": "code", diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index f86833de1..f1434aa00 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -31,6 +31,7 @@ from langchain_core.prompts import PromptTemplate from pydantic import Field from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.utils.error_handling_decorator import ToolRaisedException from vuln_analysis.utils.prompting import get_agent_prompt from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id @@ -67,26 +68,43 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, state: AgentMorpheusEngineState) -> AgentExecutor: + from vuln_analysis.utils.prompting import build_tool_descriptions + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - prompt = PromptTemplate.from_template(get_agent_prompt(config.prompt, config.prompt_examples)) - - # Filter tools that are not available + + # Filter tools that are not available based on state tools = [ tool for tool in tools - if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or - (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or - (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None) or - (tool.name == "Transitive code search tool" and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) or - (tool.name == "Calling Function Name Extractor" and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) or - (tool.name == "Package and Function Locator" and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) - + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or + (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None) or + (tool.name == ToolNames.CALL_CHAIN_ANALYZER and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) or + (tool.name == ToolNames.FUNCTION_CALLER_FINDER and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) or + (tool.name == ToolNames.FUNCTION_LOCATOR and (not config.transitive_search_tool_enabled or + state.code_index_path is None)) ) ] - + + # Get tool names after filtering for dynamic guidance + enabled_tool_names = [tool.name for tool in tools] + + # Build tool selection guidance with strategic context + tool_descriptions = build_tool_descriptions(enabled_tool_names) + tool_guidance = "\n".join(tool_descriptions) + # Get prompt template + prompt_template_str = get_agent_prompt(config.prompt, config.prompt_examples) + + # Create prompt with tool_selection_strategy as partial variable + prompt = PromptTemplate.from_template( + prompt_template_str, + partial_variables={ + 'tool_selection_strategy': tool_guidance if tool_guidance else "Use available tools as appropriate." + } + ) + agent = create_react_agent(llm=llm, tools=tools, prompt=prompt, diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 99dff2d67..2c367d6ec 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -22,6 +22,7 @@ from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig +from aiq.data_models.component_ref import FunctionRef from pydantic import Field from vuln_analysis.utils import data_utils from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id @@ -34,6 +35,10 @@ class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"): Defines a function that generates tailored, context-sensitive task checklist for impact analysis. """ llm_name: str = Field(description="The LLM model to use") + agent_name: FunctionRef = Field( + default="cve_agent_executor", + description="Name of agent function to get tool configuration from" + ) prompt: str | None = Field( default=None, description= @@ -50,11 +55,16 @@ async def cve_checklist(config: CVEChecklistToolConfig, builder: Builder): llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + # Get agent tool configuration + agent_config = builder.get_function_config(config.agent_name) + agent_tool_names = agent_config.tool_names if hasattr(agent_config, 'tool_names') else None + async def generate_checklist_for_cve(cve_intel): checklist = await generate_checklist(prompt=config.prompt, llm=llm, input_dict=cve_intel, + tool_names=agent_tool_names, enable_llm_list_parsing=False) checklist = await _parse_list([checklist]) diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index b2d198b2b..424c24d22 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -35,6 +35,7 @@ from langchain.agents.mrkl.output_parser import MRKLOutputParser from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.utils.prompting import get_cvss_prompt logger = logging.getLogger(__name__) @@ -186,16 +187,20 @@ async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) is_openai = "openai" in llm.__class__.__module__.lower() - prompt = PromptTemplate.from_template(get_cvss_prompt(config.prompt, config.prompt_examples, is_openai)) - # Filter tools that are not available + # Filter tools that are not available based on state tools = [ tool for tool in tools - if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or - (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or - (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None)) + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or + (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None)) ] + # Get prompt (examples now embedded in template) + prompt = PromptTemplate.from_template( + get_cvss_prompt(config.prompt, config.prompt_examples) + ) + error_handler = _make_parse_error_handler(is_openai) agent = create_react_agent(llm=llm, diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 54aae31ae..53e4ca6fb 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -28,6 +28,7 @@ from vuln_analysis.data_models.common import AnalysisType from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from vuln_analysis.tools.tool_names import ToolNames logger = LoggingFactory.get_agent_logger(__name__) @@ -75,11 +76,11 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): assert isinstance(agent_config, CVEAgentExecutorToolConfig) # Update config based on tools available in agent config - if "Container Image Code QA System" not in agent_config.tool_names: + if ToolNames.CODE_SEMANTIC_SEARCH not in agent_config.tool_names: logger.info("Container Image Code QA System tool is not enabled, setting ignore_code_embedding to True") config.ignore_code_embedding = True - if "Lexical Search Container Image Code QA System" not in agent_config.tool_names: + if ToolNames.CODE_KEYWORD_SEARCH not in agent_config.tool_names: logger.info( "Lexical Search Container Image Code QA System tool is not enabled, setting ignore_code_index to True") config.ignore_code_index = True diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py index 945aca53e..ea0a6d2d9 100644 --- a/src/vuln_analysis/tools/container_image_analysis_data.py +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -65,9 +65,9 @@ async def _arun(query: str) -> list[dict[str, Any]]: yield FunctionInfo.from_fn( _arun, description=( - "Useful when you need to retrieve information from the analysis data of a container image. " - "This tool does not require an input; it uses context to retrieve pre-analyzed data based on a prior scan. " - "This tool is especially useful when your task involves understanding how the container image may be impacted by a reported CVE. " - "The tool provides structured information about the container image's source code on the reported CVE. " - "The tool returns an object with analysis data, which is a list of the container image's source code analysis results, where each item in the list is an object describing a specific finding. " - "Each item in the list includes a 'response' field containing a concise summary of the analysis findings, and an optional 'intermediate_steps' field, which may contain reasoning or supporting details if not null. ")) + "Retrieves pre-analyzed container image source code analysis results for the current CVE. " + "Returns a list of findings with 'response' (summary) and 'intermediate_steps' (reasoning) fields. " + "No input required - uses context automatically." + ) + ) + diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index f584b029e..33609f190 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -57,8 +57,9 @@ async def _arun(query: str) -> list: yield FunctionInfo.from_fn( _arun, - description=("Useful for when you need to check if an application or any dependency " - "within the container image uses a function or a component of a library " - "using keyword search. This tool uses keyword to search code index, and " - "the argument should be a string keyword. You should use this search " - "tool for searching codes before trying other container search related tools.")) + description=( + "Performs keyword search on container source code for exact text matches. " + "Input should be a function name, class name, or code pattern. " + "Use this first before semantic search tools for precise lookups." + ) + ) diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 7a50da9fd..41a79ccd8 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -90,11 +90,16 @@ async def _arun(query: str) -> str | dict: return output_dict["result"] if config.vdb_type == VdbType.CODE: - description = ("Useful for when you need to check if an application or any dependency " - "within the container image uses a function or a component of a library.") + description = ( + "Searches container source code using semantic search. " + "Finds how functions, libraries, or components are used in the codebase. " + "Answers questions about code implementation and dependencies." + ) elif config.vdb_type == VdbType.DOC: - description = ("Useful for when you need to ask questions about the purpose and " - "functionality of the container image.") + description = ( + "Searches container documentation using semantic search. " + "Answers questions about application purpose, architecture, and features." + ) else: raise ValueError(f"Invalid VDB type: {config.vdb_type}. Must be one of {VdbType.CODE} or {VdbType.DOC}.") diff --git a/src/vuln_analysis/tools/serp.py b/src/vuln_analysis/tools/serp.py index f182eb75c..41981e0a7 100644 --- a/src/vuln_analysis/tools/serp.py +++ b/src/vuln_analysis/tools/serp.py @@ -46,5 +46,10 @@ async def _arun(query: str) -> str: result = await search.arun(query) return result - yield FunctionInfo.from_fn(_arun, - description=("Useful for when you need to answer questions about external libraries")) + yield FunctionInfo.from_fn( + _arun, + description=( + "Searches the web for information about CVEs, vulnerabilities, libraries, " + "and security advisories not available in the container." + ) + ) diff --git a/src/vuln_analysis/tools/tool_names.py b/src/vuln_analysis/tools/tool_names.py new file mode 100644 index 000000000..248fec7fd --- /dev/null +++ b/src/vuln_analysis/tools/tool_names.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Tool name constants to avoid hardcoded strings throughout the codebase. + +These constants match the tool names defined in YAML configuration files. +Use these constants instead of string literals when filtering or referencing tools. +""" + + +class ToolNames: + """ + Constants for agent tool names matching YAML configuration keys. + + These names correspond to the function keys in the YAML config files + (e.g., config.yml, config-http-openai-local.yml). + """ + + # Code analysis tools + CODE_SEMANTIC_SEARCH = "Code Semantic Search" + """Searches container source code using semantic vector embeddings""" + + DOCS_SEMANTIC_SEARCH = "Docs Semantic Search" + """Searches container documentation using semantic vector embeddings""" + + CODE_KEYWORD_SEARCH = "Code Keyword Search" + """Searches container source code for exact keyword matches""" + + # Code path analysis + FUNCTION_LOCATOR = "Function Locator" + """Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, provides ecosystem type.""" + + CALL_CHAIN_ANALYZER = "Call Chain Analyzer" + """Checks if a function is reachable from application code""" + + FUNCTION_CALLER_FINDER = "Function Caller Finder" + """Golang only. Finds functions calling specific standard library methods.""" + + # External and cached data sources + CVE_WEB_SEARCH = "CVE Web Search" + """Searches the web for CVE and vulnerability information""" + + CONTAINER_ANALYSIS_DATA = "Container Analysis Data" + """Retrieves pre-analyzed data from earlier container scan steps""" + + +# Export as module-level constants +CODE_SEMANTIC_SEARCH = ToolNames.CODE_SEMANTIC_SEARCH +DOCS_SEMANTIC_SEARCH = ToolNames.DOCS_SEMANTIC_SEARCH +CODE_KEYWORD_SEARCH = ToolNames.CODE_KEYWORD_SEARCH +PACKAGE_FUNCTION_LOCATOR = ToolNames.FUNCTION_LOCATOR +CALL_CHAIN_ANALYZER = ToolNames.CALL_CHAIN_ANALYZER +FUNCTION_CALLER_FINDER = ToolNames.FUNCTION_CALLER_FINDER +CVE_WEB_SEARCH = ToolNames.CVE_WEB_SEARCH +CONTAINER_ANALYSIS_DATA = ToolNames.CONTAINER_ANALYSIS_DATA + + + +__all__ = [ + 'ToolNames', + 'CODE_SEMANTIC_SEARCH', + 'DOCS_SEMANTIC_SEARCH', + 'CODE_KEYWORD_SEARCH', + 'CALL_CHAIN_ANALYZER', + 'FUNCTION_CALLER_FINDER', + 'CVE_WEB_SEARCH', + 'CONTAINER_ANALYSIS_DATA', + 'FUNCTION_LOCATOR' +] diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 0a21c409e..e7f1b9d5a 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -88,9 +88,23 @@ def get_transitive_code_searcher(): return state.transitive_code_searcher +# The below is attached in the prompting section prior to where the tools are introduced, to clarify the tool selection strategy +""" +CODE PATH ANALYSIS WORKFLOW: +When analyzing if vulnerable code is reachable from the application: +1. FIRST: Call 'Package Function Locator' with package_name,function_name to validate names and get ecosystem type +2. THEN: + - If ecosystem is GO and you need to find which functions call a library method, use 'Function Caller Finder' + - Otherwise, use 'Call Chain Analyzer' with the validated names from step 1 +3. Only use validated names that result from the Package Function Locator +""" + @register_function(config_type=TransitiveCodeSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def transitive_search(config: TransitiveCodeSearchToolConfig, builder: Builder): # pylint: disable=unused-argument + """ + Call Chain Analyzer tool used to search source code function reachability. + """ @catch_tool_errors(TRANSITIVE_CODE_SEARCH_TOOL_NAME) async def _arun(query: str) -> tuple: transitive_code_searcher: TransitiveCodeSearcher @@ -100,38 +114,21 @@ async def _arun(query: str) -> tuple: yield FunctionInfo.from_fn( _arun, - description=(""" - - This tool searches the container image's code to determine if a specified function - from a given package is used (directly or transitively) by the application code. - - - Input: package name and function/method name, separated by ','. - - package_name,function_name - package_name,class_name.method_name - - - - CRITICAL: Do not call directly. Mandatory 3-step workflow: - 1. Call 'Package and Function Locator' to get ecosystem type and validate package/function names - 2. Use validated package and function names from locator tool - 3. Select most accurate function name from locator suggestions - Only then call this tool with validated `package_name,function_name`. - WARNING: Guessed names will fail. Workflow is mandatory. - - - Returns a tuple -> (boolean, list_of_documents): - - boolean = True if function is reachable from application code - - list_of_documents = call hierarchy path (only returned if boolean is True) - WARNING: Do not reuse the list_of_documents for further tool calls — - use it only for reporting results to the user. - """)) + description=(""" + Checks if a function from a package is reachable from application code through the call chain. + Input format: 'package_name,function_name'. + Example: 'urllib,parse'. + Returns: (is_reachable: bool, call_hierarchy_path: list). +""")) @register_function(config_type=CallingFunctionNameExtractorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, builder: Builder): # pylint: disable=unused-argument + """ + Function Caller Finder tool used to search source code function specific usages. + Golang only. + """ @catch_tool_errors(FUNCTION_NAME_EXTRACTOR_TOOL_NAME) async def _arun(query: str) -> list: coc_retriever: ChainOfCallsRetriever @@ -144,42 +141,21 @@ async def _arun(query: str) -> list: yield FunctionInfo.from_fn( _arun, - description=(""" - - This tool searches a container image’s code for occurrences - of standard library functions or methods within a package - to list calling functions in that package. Designed for GO only. - - - Input format: 'package_name,function_usage_string' - input_package,sysPackage.functionName(arg, 'textLiteral') - - - CRITICAL: GO ecosystem ONLY. Mandatory workflow: - 1. Call 'Package and Function Locator' first to get ecosystem type - 2. If NOT GO, use other tools. If GO, proceed to step 3 - 3. Use this tool to find functions calling standard library methods - - github.com/golang-jwt/jwt,errors.New("Invalid Key: Key must be PEM encoded") - ["github.com/golang-jwt/jwt,Valid"] - - WARNING: Must verify ecosystem is GO first. - - - Returns a list of all functions containing such usages in the given package. - ['input_package,function1', 'input_package,function2'] - If none found, returns an empty list. - Each returned item can be checked with the 'Transitive code search tool' - to determine if the standard library usage is called from app code. - Note: Even if the result package differs from the expected one, use it anyway. - - + description=(""" + Finds functions in a package that call a specific library function. GO ecosystem only. + Input format: 'package_name,library.function(args_with_literals)'. + Example: 'github.com/golang-jwt/jwt,errors.New(\"Invalid Key: Key must be PEM encoded\")'. + Returns: ['package,caller1', 'package,caller2'] or []. """)) @register_function(config_type=PackageAndFunctionLocatorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def package_and_function_locator(config: PackageAndFunctionLocatorToolConfig, builder: Builder): # pylint: disable=unused-argument + """ + Function Locator tool used to validate package names and find function names using fuzzy matching. + Mandatory first step for code path analysis. + """ @catch_tool_errors(PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME) async def _arun(query: str) -> dict: @@ -202,34 +178,9 @@ async def _arun(query: str) -> dict: yield FunctionInfo.from_fn( _arun, - description=(""" - - FIRST STEP in code analysis. Validates packages, locates functions via fuzzy matching, provides ecosystem type. - - - Input format: 'package_name,function_name' or 'package_name,class_name.method_name' - - libxml2,xmlParseDocument - requests,Session.get - numpy,array.reshape - - - - **MANDATORY FIRST STEP** for ALL code analysis. Use before ANY other tool. - - Returns ecosystem type (GO, PYTHON, JAVASCRIPT, JAVA, C_CPP) - determines which tools to use - - Validates package names, suggests alternatives if missing - - Finds function matches via fuzzy matching - pick the most contextually correct one - - requests,Session.get - ["Session.get", "Session.post", "Session.put", "Session.delete"] - Found class method "Session.get" - - - - Returns dictionary with: - - "ecosystem": ecosystem name (GO, PYTHON, JAVASCRIPT, JAVA, C_CPP) - - "result": suggested function names (if valid package) or error with alternatives (if invalid) - Selected function is input for **'Transitive code search tool' (Step 2)**. - - + description=(""" + Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, and provides ecosystem type (GO/Python/Java/JavaScript/C/C++). + Input format: 'package_name,function_name' or 'package_name,class_name.method_name' + Example: 'libxml2,xmlParseDocument' + Returns: {'ecosystem': str, 'package_msg': str, 'result': [function_names]}. """)) diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 77b6199a8..1822f3c56 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -29,7 +29,10 @@ logger = LoggingFactory.get_agent_logger(__name__) -DEFAULT_CHECKLIST_PROMPT = MOD_FEW_SHOT.format(examples=get_mod_examples()) +# Format MOD_FEW_SHOT with examples, preserving {tool_descriptions} for Jinja2 rendering +# Use double braces for tool_descriptions to escape it during format() +_MOD_FEW_SHOT_ESCAPED = MOD_FEW_SHOT.replace('{tool_descriptions}', '{{tool_descriptions}}') +DEFAULT_CHECKLIST_PROMPT = _MOD_FEW_SHOT_ESCAPED.format(examples=get_mod_examples()) cve_prompt2 = """Parse the following numbered checklist into a python list in the format ["x", "y", "z"], a comma separated list surrounded by square braces: {{template}}""" @@ -112,30 +115,62 @@ async def format_jinja_prompt(template_str, input_dict): async def generate_checklist(prompt: str | None, llm: BaseLanguageModel, input_dict: dict, + tool_names: list[str] | None = None, enable_llm_list_parsing: bool = False) -> str: - + + from vuln_analysis.utils.prompting import build_tool_descriptions + if not prompt: prompt = DEFAULT_CHECKLIST_PROMPT - + + # Build tool descriptions with checklist-specific formatting + if tool_names: + tool_descs = build_tool_descriptions(tool_names) + if tool_descs: + formatted_descs = ["- " + desc for desc in tool_descs] + tool_descriptions = "The following tools can be used to answer checklist questions:\n " + "\n ".join(formatted_descs) + else: + tool_descriptions = "Analysis tools will be used to investigate these questions." + else: + tool_descriptions = "Analysis tools will be used to investigate these questions." + + # Add tool_descriptions to input_dict for Jinja2 rendering + # This treats it as a Jinja2 variable, consistent with all CVE fields + input_dict_with_tools = { + **input_dict, + 'tool_descriptions': tool_descriptions + } + intel = ( additional_intel_prompting + - "\n\nIf a vulnerable function or method is mentioned in the CVE description, ensure the first checklist item verifies whether this function or method is being called from the code or used by the code." - "\nThe vulnerable version of the vulnerable package is already verified to be installed within the container. Check only the other factors that affect exploitability, no need to verify version again." + "\n" + "\n\n" + "\n- If CVE describes a vulnerable function/method, first checklist item MUST " + "check if code calls it" + "\n- Vulnerable package version is already confirmed installed; focus on other " + "exploitability factors" + "\n- Each item must be answerable with available analysis tools (code/doc search, " + "dependency checks)" + "\n- Use specific technical names from CVE details (functions, components, configurations)" + "\n- Maximum 5 checklist items; prioritize most critical exploitability checks" + "\n" + "\n\nGenerate checklist:" ) - + cve_prompt1 = (prompt + intel) try: - format_cve_intel = await format_jinja_prompt(cve_prompt1, input_dict) - + # Jinja2 renders {tool_descriptions} along with all CVE fields + format_cve_intel = await format_jinja_prompt(cve_prompt1, input_dict_with_tools) + gen_checklist = await llm.ainvoke(format_cve_intel) - + if enable_llm_list_parsing: parsing_checklist_template = await format_jinja_prompt(cve_prompt2, {"template": gen_checklist.content}) parsed_checklist = await llm.ainvoke(parsing_checklist_template) return parsed_checklist.content - + except Exception as e: logging.error(f" Error in generating checklist : {e}") raise - + return gen_checklist.content diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index f0715c2d2..8a040a36c 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -49,54 +49,232 @@ async def calculate_intel_score(self, intel: CveIntel) -> CveIntel: return intel def __get_calculate_score_prompt(self, intel: CveIntel) -> str: - return """ - You are an expert in cybersecurity evaluating the quality of a CVE. Calculate a score for the given CVE on a scale from 0 to 100 based on: - - * Technical specificity (20 points) — How precise and in-depth are the technical details? - * Clarity and grammar (10 points) — Is the text clear, well-structured, and free of major grammatical issues? - * Mention of affected component and impact (15 points) — Does it clearly state what is affected and the consequence? - * Reproducibility (15 points) — Would an attacker understand how to exploit the issue from the description? - * Presence of vulnerable function/method (15 points) — Is a specific function, method, or code snippet identified? - * Existence of mitigation (10 points) — Are there any described mitigations, patches, or workarounds? - * Details on environment (10 points) — Is there context about the affected environment (e.g., OS, version)? - * Relevant configuration details (5 points) — Are configuration factors or misconfigurations involved? - - Ensure that you return only JSON response with no markdown symbols: - 1. The individual scores per criterion. - 2. A short justification for each sub-score. - 3. The total score out of 100. - 4. Justification of the total score. + return """ +Evaluate CVE intelligence quality by scoring each criterion independently. + - Ensure the final score does not exceed 100, the weights are respected and the score is returned in total_score tag. - - Use the following 3 examples to calculate the score: - - 1. High Score Example - CVE ID: CVE-2025-30204\n- CVE Description: golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function\'s argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-405: Asymmetric Resource Consumption (Amplification) (4.17)\n- GHSA Details: [{\'first_patched_version\': \'5.2.2\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt/v5\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'>= 5.0.0-rc.1, < 5.2.2\'}, {\'first_patched_version\': \'4.5.2\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt/v4\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 4.5.2\'}, {\'first_patched_version\': None, \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/golang-jwt/jwt\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'>= 3.2.0, <= 3.2.2\'}]\n- CWE Description: The product does not properly control situations in which an adversary can cause the product to consume or produce excessive resources without requiring the adversary to invest equivalent work or otherwise prove authorization, i.e., the adversary\'s influence is "asymmetric."\n- This can lead to poor performance due to "amplification" of resource consumption, typically in a non-linear fashion. This situation is worsened if the product allows malicious users or attackers to consume more resources than their access level permits.\n- RHSA Description: golang-jwt/jwt: jwt-go allows excessive memory allocation during header parsing\n- RHSA Details: ["golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function\'s argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2.", \'A flaw was found in the golang-jwt implementation of JSON Web Tokens (JWT). In affected versions, a malicious request with specially crafted Authorization header data may trigger an excessive consumption of resources on the host system. This issue can cause significant performance degradation or an application crash, leading to a denial of service.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-agent-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-reporter-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:assisted_installer:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhai-tech-preview/assisted-installer-rhel8\', \'product_name\': \'Assisted Installer for Red Hat OpenShift Container Platform\'}, {\'cpe\': \'cpe:/a:redhat:openshift_builds:1\', \'fix_state\': \'Affected\', \'package_name\': \'openshift-builds/openshift-builds-git-cloner-rhel9\', \'product_name\': \'Builds for Red Hat OpenShift\'}, {\'cpe\': \'cpe:/a:redhat:openshift_builds:1\', \'fix_state\': \'Affected\', \'package_name\': \'openshift-builds/openshift-builds-image-bundler-rhel9\', \'product_name\': \'Builds for Red Hat... - Score: 80 - - 2. Medium Score Example - CVE ID: CVE-2022-29810\n- CVE Description: The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter.\n- CVSS Vector: CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N\n- CWE Name: CWE-532: Insertion of Sensitive Information into Log File (4.17)\n- GHSA Details: [{\'first_patched_version\': \'1.5.11\', \'package\': {\'ecosystem\': \'go\', \'name\': \'github.com/hashicorp/go-getter\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 1.5.11\'}]\n- Known Affected Software: [{\'package\': \'go-getter\', \'system\': None, \'versionEndExcluding\': \'1.5.11\', \'versionEndIncluding\': None, \'versionStartExcluding\': None, \'versionStartIncluding\': None}]\n- CWE Description: The product writes sensitive information to a log file.\n- Notable Vulnerable Software Vendors: [\'Hashicorp\']\n- RHSA Description: go-getter: writes SSH credentials into logfile, exposing sensitive credentials to local uses\n- RHSA Details: [\'The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter.\', \'A flaw was found in go-getter, where the go-getter library can write SSH credentials into its log file. This flaw allows a local user with access to read log files to read sensitive credentials, which may lead to privilege escalation or account takeover.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/agent-service-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/clusterlifecycle-state-metrics-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Affected\', \'package_name\': \'rhacm2/managedcluster-import-controller-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/multicloud-manager-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\': \'rhacm2/multiclusterhub-rhel8\', \'product_name\': \'Red Hat Advanced Cluster Management for Kubernetes 2\'}, {\'cpe\': \'cpe:/a:redhat:acm:2\', \'fix_state\': \'Not affected\', \'package_name\':... - Score: 62 - - 3. Low Score Example - CVE ID: CVE-2022-2385\n- CVE Description: A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H\n- CWE Name: [{\'cwe_id\': \'CWE-20\', \'name\': \'Improper Input Validation\'}]\n- GHSA Details: [{\'first_patched_version\': \'0.5.9\', \'package\': {\'ecosystem\': \'go\', \'name\': \'sigs.k8s.io/aws-iam-authenticator\'}, \'vulnerable_functions\': [], \'vulnerable_version_range\': \'< 0.5.9\'}]\n- Known Affected Software: [{\'package\': \'aws-iam-authenticator\', \'system\': \'kubernetes\', \'versionEndExcluding\': \'0.5.9\', \'versionEndIncluding\': None, \'versionStartExcluding\': None, \'versionStartIncluding\': \'0.5.2\'}]\n- Notable Vulnerable Software Vendors: [\'Kubernetes\']\n- RHSA Description: aws-iam-authenticator: AccessKeyID validation bypass\n- RHSA Details: [\'A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges.\', \'A flaw was found in aws-iam-authenticator. This issue occurs when an allow-listed IAM identity may be able to modify their username and escalate privileges.\']\n- RHSA Affected Packages: [{\'cpe\': \'cpe:/a:redhat:openshift:4\', \'fix_state\': \'Affected\', \'package_name\': \'openshift4/ose-hypershift-rhel9\', \'product_name\': \'Red Hat OpenShift Container Platform 4\'}] - Score: 20 - - Given CVE Details:\n - """ + self.__render_template(additional_intel_prompting, intel) + +Provide individual scores for each criterion based on the CVE data below. + +1. technical_specificity (max 20 points) + - How precise and in-depth are the technical details? + - Are specific vulnerable functions, methods, or code paths identified? + +2. clarity (max 10 points) + - Is the text well-structured and grammatically correct? + - Is the description clear and easy to understand? + +3. component_impact (max 15 points) + - Does it clearly state what is affected? + - Are the consequences explicitly described? + +4. reproducibility (max 15 points) + - Could an attacker understand how to exploit this from the description? + - Are attack vectors and preconditions described? + +5. vulnerable_function (max 15 points) + - Is a specific function, method, or code snippet named? + - Are vulnerable code locations identifiable? + +6. mitigation (max 10 points) + - Are patches, workarounds, or mitigations described? + - Is remediation guidance provided? + +7. environment (max 10 points) + - Is there context about the affected environment (OS, version, configuration)? + - Are deployment scenarios mentioned? + +8. configuration (max 5 points) + - Are relevant configuration settings or misconfigurations described? + + + +Return JSON only (no markdown, no code blocks): +{ + "scores": { + "technical_specificity": <0-20>, + "clarity": <0-10>, + "component_impact": <0-15>, + "reproducibility": <0-15>, + "vulnerable_function": <0-15>, + "mitigation": <0-10>, + "environment": <0-10>, + "configuration": <0-5> + }, + "justifications": { + "technical_specificity": "brief reason for score", + "clarity": "brief reason for score", + "component_impact": "brief reason for score", + "reproducibility": "brief reason for score", + "vulnerable_function": "brief reason for score", + "mitigation": "brief reason for score", + "environment": "brief reason for score", + "configuration": "brief reason for score" + } +} + +Do NOT calculate or include a total_score. Only provide the individual criterion scores. + + + + +Example Input: +CVE ID: CVE-2025-30204 +CVE Description: golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function's argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2. +CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H +CWE Name: CWE-405: Asymmetric Resource Consumption (Amplification) +Notable Vendors: Hashicorp + +Example Output (High Quality ~80): +{ + "scores": { + "technical_specificity": 18, + "clarity": 9, + "component_impact": 14, + "reproducibility": 14, + "vulnerable_function": 15, + "mitigation": 9, + "environment": 9, + "configuration": 4 + }, + "justifications": { + "technical_specificity": "Function parse.ParseUnverified identified with O(n) allocation details", + "clarity": "Well-structured with clear impact", + "component_impact": "golang-jwt and DoS impact explicitly stated", + "reproducibility": "Clear attack: malicious Authorization header with periods", + "vulnerable_function": "parse.ParseUnverified explicitly named", + "mitigation": "Patches 5.2.2 and 4.5.2 specified", + "environment": "Versions 3.2.0-5.2.2 listed", + "configuration": "Limited config details" + } +} + +Example Input: +CVE ID: CVE-2022-29810 +CVE Description: The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter. +CVSS Vector: CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N +CWE Name: CWE-532: Insertion of Sensitive Information into Log File +Notable Vendors: Hashicorp + +Example Output (Medium Quality ~62): +{ + "scores": { + "technical_specificity": 12, + "clarity": 8, + "component_impact": 12, + "reproducibility": 10, + "vulnerable_function": 8, + "mitigation": 8, + "environment": 8, + "configuration": 3 + }, + "justifications": { + "technical_specificity": "Moderate detail about SSH key issue", + "clarity": "Clear but brief", + "component_impact": "go-getter and credential exposure stated", + "reproducibility": "Attack path somewhat clear, lacks details", + "vulnerable_function": "General functionality, no specific function", + "mitigation": "Version 1.5.11 patches", + "environment": "Affected versions specified", + "configuration": "Minimal context" + } +} + +Example Input: +CVE ID: CVE-2022-2385 +CVE Description: A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges. +CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +CWE Name: CWE-20: Improper Input Validation +Notable Vendors: Kubernetes + +Example Output (Low Quality ~20): +{ + "scores": { + "technical_specificity": 3, + "clarity": 5, + "component_impact": 5, + "reproducibility": 2, + "vulnerable_function": 0, + "mitigation": 3, + "environment": 4, + "configuration": 0 + }, + "justifications": { + "technical_specificity": "Very vague, no detail", + "clarity": "Brief but understandable", + "component_impact": "General privilege escalation mention", + "reproducibility": "No exploit details", + "vulnerable_function": "No functions identified", + "mitigation": "Patch mentioned", + "environment": "Minimal version info", + "configuration": "None" + } +} + + + + +""" + self.__render_template(additional_intel_prompting, intel) + "\n\n\nProvide your scoring JSON:" def __extract_score(self, text: str) -> int: + """ + Extract individual scores from LLM response and calculate total. + This prevents LLM arithmetic hallucinations by: + 1. Validating each score against its maximum + 2. Calculating total in code (not LLM) + """ text = text.replace("```", "").replace("json", "").strip() if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): - logger.debug("\ntext: %s", str(text)) - data = json.loads(text) + logger.debug("\nRaw LLM response: %s", str(text)) - total_score = data['total_score'] - if isinstance(total_score, dict): - total_score = total_score['score'] + try: + data = json.loads(text) + + # Extract individual scores + scores = data.get('scores', {}) + + # Define maximum values for each criterion + max_values = { + 'technical_specificity': 20, + 'clarity': 10, + 'component_impact': 15, + 'reproducibility': 15, + 'vulnerable_function': 15, + 'mitigation': 10, + 'environment': 10, + 'configuration': 5 + } + + # Validate and cap each score at its maximum + validated_scores = {} + for criterion, max_val in max_values.items(): + score = scores.get(criterion, 0) + # Ensure score is within valid range [0, max] + validated_scores[criterion] = max(0, min(score, max_val)) + + if score > max_val: + logger.warning( + "Score for '%s' (%d) exceeds maximum (%d), capping to max", + criterion, score, max_val + ) + + # Calculate total using code (not LLM) + total_score = sum(validated_scores.values()) + + # Final safety check: total should not exceed 100 + total_score = min(total_score, 100) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nValidated scores: %s", validated_scores) + logger.debug("\nCalculated total: %d", total_score) + + except (json.JSONDecodeError, KeyError, TypeError) as e: + logger.error("Failed to parse scoring response: %s", e) + logger.debug("Problematic text: %s", text) + # Return a default low score on parse failure + return 0 - return total_score + return total_score def __render_template(self, template_str: str, intel: CveIntel) -> str: from jinja2 import Template diff --git a/src/vuln_analysis/utils/justification_parser.py b/src/vuln_analysis/utils/justification_parser.py index 0711ac9f1..78fe26028 100644 --- a/src/vuln_analysis/utils/justification_parser.py +++ b/src/vuln_analysis/utils/justification_parser.py @@ -26,35 +26,84 @@ class JustificationParser: AFFECTED_STATUS_COL_NAME = "affected_status" JUSTIFICATION_PROMPT = dedent(""" - The summary provided below (delimited with XML tags), generated by the software agent named Vulnerability - Analysis for Container Security, evaluates a specific CVE (Common Vulnerabilities and Exposures) against the - backdrop of a software package or environment information. This information may include a Software Bill of - Materials (SBOM), source code, and dependency documentation among others related to environments such as - container images. Vulnerability Analysis for Container Security's role is to analyze this data to ascertain if the CVE - impacts the software environment, determining the necessity for a patch. - - Your task is to review Vulnerability Analysis for Container Security's analysis and classify the situation into - one of the following categories based on the described scenario: - - false_positive: The association between the software package and the CVE is incorrect, either due to an - erroneous package identification or a mismatched CVE. - - code_not_present: The software product is unaffected as it does not contain the code or library that harbors - the vulnerability. - - code_not_reachable: During runtime, the vulnerable code is not executed. - - requires_configuration: The exploitability of this issue depends on whether a specific configuration option is - enabled or disabled. In this case, the configuration is set in a manner that prevents exploitability. - - requires_dependency: Exploitability is contingent upon a missing dependency. - - requires_environment: A specific environment, absent in this case, is required for exploitability. - - compiler_protected: Exploitability hinges on the setting or unsetting of a compiler flag. In this case, the - flag is set in a manner that prevents exploitability. - - runtime_protected: Mechanisms are in place that prevent exploits during runtime. - - perimeter_protected: Protective measures block attacks at the physical, logical, or network perimeters. - - mitigating_control_protected: Implemented controls mitigate the likelihood or impact of the vulnerability. - - uncertain: Not enough information to determine the package's exploitability status. - - vulnerable: the package is actually vulnerable to the CVE and needs to be patched. - - Response only with the category name on the first line, and reasoning on the second line. - {summary} - """).strip("\n") + +Review the CVE exploitability investigation summary and perform two tasks: +1. Classify the exploitability status into one of 12 predefined categories +2. Provide reasoning for your classification, citing specific evidence from the investigation summary + + + +Select the single most appropriate category based on the investigation findings. + +CLASSIFICATION CATEGORIES (in logical precedence order): + +1. false_positive - CVE-to-package association is incorrect (wrong package or mismatched CVE) + +2. code_not_present - Vulnerable code/library is absent from the container + (If code is not present, subsequent factors are irrelevant) + +3. code_not_reachable - Vulnerable code exists but is never executed at runtime + (Only applicable if code IS present but execution path analysis shows no calls) + +4. requires_configuration - Exploitation requires specific configuration that is disabled + (Configuration prevents exploitation) + +5. requires_dependency - Exploitation requires a dependency that is missing + +6. requires_environment - Exploitation requires specific environment that is absent + +7. compiler_protected - Compiler flags prevent exploitation + +8. runtime_protected - Runtime mechanisms (ASLR, DEP, sandboxing) prevent exploitation + +9. perimeter_protected - Network/physical/logical perimeter defenses block exploitation + +10. mitigating_control_protected - Other security controls reduce risk + +11. uncertain - Insufficient information to determine exploitability + +12. vulnerable - Package is actually vulnerable and needs patching + +EXPLOITATION CONDITIONS: +For a CVE to be classified as "vulnerable", ALL of these must be true: +- Vulnerable code is PRESENT in the container +- Vulnerable code is USED/CALLED by the application +- Vulnerable code is REACHABLE from attack surfaces (user input, network, file processing) +- No effective mitigations or protections are in place + +IF EXPLOITATION CONDITIONS ARE NOT MET: +Select the PRIMARY reason for non-exploitability following the logical precedence +order above. For example: +- If code is not present: "code_not_present" (even if other factors would also prevent it) +- If code is not reachable: "code_not_reachable" (not "requires_environment") +- If missing dependency prevents it: "requires_dependency" + +The categories are ordered by logical precedence. Work through the list from top +to bottom and select the first category that applies to the situation. + +Use "uncertain" only as a final fallback when the investigation truly lacks +sufficient information to make any determination. + + + +Provide exactly two lines: + +Line 1: category_name (exact category name from the list above) +Line 2: reasoning (brief explanation citing specific evidence from the summary) + +Do not include labels like "Category:" or "Reasoning:". Just the values on separate lines. + + + +code_not_reachable +The vulnerable PIL.ImageMath.eval function exists in the installed Pillow library, but call chain analysis confirmed it is never invoked from application code. The application only uses PIL.Image.open() and PIL.Image.thumbnail() functions, which do not call ImageMath.eval. + + + +{summary} + + +Provide your justification classification and reasoning on two separate lines:""").strip("\n") RAW_TO_FINAL_JUSTIFICATION_MAP = { "false_positive": "false_positive", diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 46988fdf8..ce36986c6 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -20,232 +20,411 @@ # pylint: disable=line-too-long -SUMMARY_PROMPT = """Summarize the exploitability investigation results of a Common Vulnerabilities and Exposures (CVE) \ -based on the provided Checklist and Findings. Write a concise paragraph focusing only on checklist items with \ -definitive answers. Begin your response by clearly stating whether the CVE is exploitable. Disregard any ambiguous \ -checklist items. -Checklist and Findings: -{response}""" - -# Define a system prompt that sets the context for the language model's task. This prompt positions the assistant -# as a powerful entity capable of investigating CVE impacts on container images. + +def build_tool_descriptions(tool_names: list[str]) -> list[str]: + """ + Build tool descriptions based on enabled tools. + + Returns simple, descriptive tool descriptions that can be formatted + with context-specific text by calling functions. + + Args: + tool_names: List of enabled tool names (after filtering) + + Returns: + List of tool description strings (without bullet points or formatting) + """ + from vuln_analysis.tools.tool_names import ToolNames + + descriptions = [] + + if ToolNames.CODE_SEMANTIC_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CODE_SEMANTIC_SEARCH}: Searches source code using semantic understanding" + ) + + if ToolNames.DOCS_SEMANTIC_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.DOCS_SEMANTIC_SEARCH}: Searches package documentation for architecture/design info" + ) + + if ToolNames.CODE_KEYWORD_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CODE_KEYWORD_SEARCH}: Exact text matching for function names, class names, or imports" + ) + + if ToolNames.FUNCTION_CALLER_FINDER in tool_names and ToolNames.CALL_CHAIN_ANALYZER in tool_names: + descriptions.append( + f"{ToolNames.CALL_CHAIN_ANALYZER}: Checks if functions are reachable from application code\n" + f"{ToolNames.FUNCTION_CALLER_FINDER}: Finds which functions call specific library functions\n" + f"Use '{ToolNames.FUNCTION_CALLER_FINDER}' + '{ToolNames.CALL_CHAIN_ANALYZER}' together to trace function reachability" + ) + elif ToolNames.CALL_CHAIN_ANALYZER in tool_names: + descriptions.append( + f"{ToolNames.CALL_CHAIN_ANALYZER}: Checks if functions are reachable from application code" + ) + elif ToolNames.FUNCTION_CALLER_FINDER in tool_names: + descriptions.append( + f"{ToolNames.FUNCTION_CALLER_FINDER}: Finds which functions call specific library functions" + ) + + if ToolNames.CVE_WEB_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CVE_WEB_SEARCH}: External vulnerability information lookup" + ) + + if ToolNames.CONTAINER_ANALYSIS_DATA in tool_names: + descriptions.append( + f"{ToolNames.CONTAINER_ANALYSIS_DATA}: Retrieves findings from earlier container scan analysis" + ) + + return descriptions + + + + +SUMMARY_PROMPT = """ +Summarize CVE exploitability investigation results into a clear, evidence-based +paragraph. The investigation results consist of checklist items (questions) and +their corresponding conclusions from the security analysis. + + + +Write a 3-5 sentence paragraph following this structure: + +1. VERDICT (sentence 1): Begin with explicit statement + - "The CVE is exploitable" / "The CVE is not exploitable" / "Exploitability is uncertain" + +2. EVIDENCE (sentences 2-4): Support with specific findings + - Cite concrete results: functions found/absent, reachability status, configuration states + - Use technical details: function names, file paths, components + - Connect findings to exploitability conditions + +3. FOCUS: Use only definitive checklist results; ignore inconclusive items + + + +The CVE is not exploitable in this container. Investigation confirmed that while +Python 3.10.0 is installed (vulnerable version), the urllib.parse module is never +imported or called in the application codebase (verified via code search). Additionally, +code analysis revealed that the application does not accept URL inputs from untrusted +sources; all URL handling occurs only with internally generated URLs from configuration +files. The combination of no urllib.parse usage and lack of external URL input eliminates +the attack vector described in the CVE. + + + +{response} + + +Write your summary paragraph:""" + AGENT_SYS_PROMPT = ( - "You are a very powerful assistant who helps investigate the impact of reported Common Vulnerabilities and " - "Exposures (CVE) on container images. Information about the container image under investigation is " - "stored in vector databases available to you via tools.") + "You are an expert security analyst investigating Common Vulnerabilities and " + "Exposures (CVE) in container images. Your role is to methodically answer " + "investigation questions using available tools to determine if vulnerabilities " + "are exploitable in the specific container context. You have access to the " + "container's source code, documentation, and dependency information through " + "specialized search and analysis tools." +) -# Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py -AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. Avoid giving instructions or guidance on how to achieve a task on final answer, rather focus merely on answering the question. You have access to the following tools: +AGENT_PROMPT_TEMPLATE = """ +Answer the investigation question using the available tools. If the input is not a question, formulate it into a question first. A Tool Selection Strategy is provided to help you decide which tools to use. Focus on answering the question. Include your intermediate reasoning in the final answer. + + {tools} + + + +{tool_selection_strategy} + -Use the following format (start each response with one of the following prefixes: [Question, Thought, Action, Action Input, Final Answer]): + +Follow this format exactly (start each line with one of the specified prefixes): Question: the input question you must answer -Thought: you should always think about what to do -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: think about what action to take next +Action: the tool to use, must be one of [{tool_names}] +Action Input: the specific input for that tool +Observation: the result returned by the tool +... (repeat Thought/Action/Action Input/Observation cycle as many times as needed) Thought: I now know the final answer -Final Answer: the final answer to the original input question +Final Answer: provide the answer with supporting evidence from your investigation + -Begin! + + -Question: {input} + +{input} + + +Begin your investigation: Thought:{agent_scratchpad}""" -AGENT_EXAMPLES_FOR_PROMPT = """Example 1: - - Question: 1. Identify the version of Python: Check which version(s) of Python are installed in the container image. The vulnerability affects versions up to and including 3.11.3. - Thought: I should check if Python is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: Python - Observation: 3.10.0 - Thought: Python 3.10.0 is installed, I need to check if the installed version is vulnerable. To do this I'll compare the installed version to the vulnerable version. - Action: container software version comparator - Action Input: 3.10.0, 3.11.3 - Observation: True - Thought: The installed software is vulnerable. I now know the answer. - Final Answer: Python version 3.10.0 is installed and is vulnerable to the CVE. - - Example 2: - - Question: Assess the threat that CVE-20xx-xxxxx poses to the container. - Thought: I should search for more information on CVE-20xx-xxxxx. - Action: Internet Search - Action Input: What is CVE-20xx-xxxxx? - Observation: CVE-20xx-xxxxx causes memory leaks and possible denial of service attack vectors when using urllib.parse - Thought: I should check the code base of the container for instances of urllib.parse - Action: Container Image QA System - Action Input: Is urllib.parse present in the code? - Observation: - Question: Is urllib.parse present in the code? - Helpful answer: No, that function is not called in the code. - Thought: Since the function is not called in the code, the container is not vulnerable. I know the final answer. - Final Answer: The function urllib.parse is not present in the code, so the container is not vulnerable. - - Example 3: - - Question: Check if the container is using Java Runtime Environment (JRE). If it is not using JRE, then it is not vulnerable to CVE-xxxx-xxxxx. - Thought: I should check if JRE is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: JRE - Observation: False - Thought: JRE is not present in the container. I now know the answer. - Final Answer: JRE is not installed in the container. Therefore, it is not vulnerable to CVE-20xx-xxxxx. - - Example 4: - - Question: Check if the container is using Apache. If it is not using Apache, then it is not vulnerable to CVE-xxxx-xxxxx. - Thought: I should check if Apache is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: Apache - Observation: 1.0.1 - Thought: Apache is present in the container. I now know the answer. - Final Answer: Apache is installed in the container. Therefore, it is potentially vulnerable to CVE-20xx-xxxxx. +AGENT_EXAMPLES_FOR_PROMPT = """ +Example 1: Python library function lookup (Code Keyword Search) + +Question: Does the codebase import or call urllib.parse? +Thought: I need to search for exact usage of urllib.parse in the codebase. I'll use keyword search for precise matching. +Action: Code Keyword Search +Action Input: urllib.parse +Observation: Found 3 matches: +- src/utils/url_validator.py:12: from urllib.parse import urlparse +- src/api/endpoints.py:45: parsed = urllib.parse.urlparse(user_url) +- tests/test_urls.py:8: import urllib.parse +Thought: The function is present and actively used in production code. I now know the answer. +Final Answer: Yes, urllib.parse is imported and used in the application. Specifically, url_validator.py imports urlparse, and endpoints.py line 45 calls urllib.parse.urlparse(user_url) to process URLs. + +Example 2: Negative result - function not present + +Question: Does the application call the PIL.ImageMath.eval function? +Thought: I should search for calls to PIL.ImageMath.eval using keyword search for exact matching. +Action: Code Keyword Search +Action Input: PIL.ImageMath.eval +Observation: No matches found +Thought: No direct matches. Let me search for the broader ImageMath module to see if it's imported differently. +Action: Code Keyword Search +Action Input: ImageMath +Observation: No matches found +Thought: The module and function are not present in the codebase. I now know the answer. +Final Answer: No, PIL.ImageMath.eval is not called anywhere in the codebase. The ImageMath module is not imported, so this vulnerability vector does not apply to this container. + +Example 3: C/C++ library with call chain analysis (multiple tools) + +Question: Is the OpenSSL function EVP_EncryptInit_ex2 reachable from application code? +Thought: I need to determine if this OpenSSL function can be reached through the application's call chain. First, let me check for direct calls in the codebase. +Action: Code Keyword Search +Action Input: EVP_EncryptInit_ex2 +Observation: No direct calls found in application code +Thought: No direct calls, but the function might be called indirectly through wrapper functions. Let me find which application functions call this OpenSSL function. +Action: Function Caller Finder +Action Input: openssl,EVP_EncryptInit_ex2() +Observation: ['myapp,encrypt_data', 'myapp,secure_transmit'] +Thought: Two application functions call the OpenSSL function. Now I need to check if these functions are reachable from the application entry points. +Action: Call Chain Analyzer +Action Input: myapp,encrypt_data +Observation: (True, ['main', 'handle_request', 'process_upload', 'encrypt_data']) +Thought: I found a complete call chain from main to the function that uses the vulnerable OpenSSL API. I now know the answer. +Final Answer: Yes, EVP_EncryptInit_ex2 is reachable from application code. The call path is: main → handle_request → process_upload → encrypt_data, where encrypt_data calls the vulnerable OpenSSL function. This means the vulnerability could be exploitable if other conditions are met. + +""" + +AGENT_EXAMPLES_FOR_PROMPT_2 = """ + +Example 4: Understanding application architecture (semantic searches) + +Question: Does the application process images from untrusted sources? +Thought: I need to understand how the application handles images and where they come from. Let me start by checking the documentation for image handling architecture. +Action: Docs Semantic Search +Action Input: How does the application handle image uploads and image processing? +Observation: The application includes an image processing service that accepts image uploads from users via REST API endpoints at /api/upload. Images are validated for file type and size, then resized and stored in cloud storage. +Thought: The documentation indicates user uploads are accepted. Let me examine the code to understand what validation occurs before image processing. +Action: Code Semantic Search +Action Input: image upload validation and PIL image processing +Observation: Found image_processor.py which validates file extensions (.jpg, .png, .gif) and MIME types before passing images to PIL (Pillow library) for resizing. The validate_image() function checks basic properties but processes the image using PIL.Image.open() and PIL.Image.thumbnail(). +Thought: Images come from users via API endpoints and are processed with PIL after basic validation. I now have sufficient information. +Final Answer: Yes, the application processes images from untrusted sources. User-uploaded images are accepted via the /api/upload REST endpoint and processed using the Pillow (PIL) library. While basic validation (file extension and MIME type) is performed, the images originate from untrusted users and are processed before storage, creating potential exploit vectors. + +Example 5: Specific coding pattern detection (keyword + semantic) + +Question: Does the code modify the keylen or ivlen parameters in OSSL_PARAM arrays after calling EVP initialization functions? +Thought: This is a very specific coding pattern in C code. Let me first search for OSSL_PARAM usage in the codebase. +Action: Code Keyword Search +Action Input: OSSL_PARAM +Observation: Found 2 matches: +- src/crypto/cipher_utils.c:45: OSSL_PARAM params[3]; +- src/crypto/cipher_utils.c:78: OSSL_PARAM params[5]; +Thought: Found OSSL_PARAM usage. Now I need to understand if the code modifies keylen or ivlen after EVP initialization. Let me search for this pattern semantically. +Action: Code Semantic Search +Action Input: OSSL_PARAM array modification of keylen or ivlen after EVP_EncryptInit_ex2 EVP_DecryptInit_ex2 EVP_CipherInit_ex2 calls +Observation: The code in cipher_utils.c creates OSSL_PARAM arrays with keylen and ivlen values set during initialization. The arrays are passed to EVP_EncryptInit_ex2() and EVP_DecryptInit_ex2() but are not modified afterward. The parameter values are set once during array creation using OSSL_PARAM_construct_*() functions. +Thought: I have examined the OSSL_PARAM usage pattern. The vulnerable pattern (modifying keylen/ivlen after initialization) is not present. +Final Answer: No, the codebase does not modify keylen or ivlen parameters in OSSL_PARAM arrays after calling EVP initialization functions. The code in cipher_utils.c creates OSSL_PARAM arrays with keylen and ivlen values set during initialization, passes them to EVP initialization functions, and does not subsequently modify these parameter values. The vulnerable coding pattern described in the CVE is not present. + +Example 6: Configuration-specific check (keyword search for config patterns) + +Question: Does the application use any of the affected cipher modes: CCM, GCM, OCB, RC2, RC4, or RC5? +Thought: I need to search for usage of these specific cipher modes in the codebase. Let me start with GCM. +Action: Code Keyword Search +Action Input: EVP_aes_.*_gcm +Observation: Found 1 match: src/crypto/cipher_utils.c:120: cipher = EVP_aes_256_gcm(); +Thought: Found GCM mode usage. Let me check for the other cipher modes mentioned. +Action: Code Keyword Search +Action Input: EVP_aes_.*_ccm +Observation: No matches found +Thought: No CCM. Let me check for OCB, RC2, RC4, RC5 together. +Action: Code Keyword Search +Action Input: RC2|RC4|RC5|OCB +Observation: No matches found +Thought: I found one of the affected cipher modes (GCM) but not the others. I have the answer. +Final Answer: Yes, the application uses GCM (Galois/Counter Mode), which is one of the affected cipher modes. The code uses EVP_aes_256_gcm() in cipher_utils.c at line 120. The other affected cipher modes (CCM, OCB, RC2, RC4, RC5) are not used in the codebase. +""" - """ CVSS_SYS_PROMPT = ( - "You are a very powerful and highly knowledgeable cybersecurity assistant specializing in Common Vulnerabilities and Exposures (CVE) analysis. " - "Your task is to determine the correct CVSS (Common Vulnerability Scoring System) metric value to reflect the impact of a reported CVE on a container image, selecting the most appropriate value from a predefined list." + "You are a cybersecurity expert specializing in Common Vulnerabilities and Exposures (CVE) analysis and CVSS (Common Vulnerability Scoring System) metric evaluation." ) -# Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py -CVSS_PROMPT_TEMPLATE = """You have access to the following tools: +CVSS_PROMPT_TEMPLATE = """ +Determine the correct CVSS metric value for this vulnerability in the context of the analyzed container image. Select the value whose definition most accurately matches evidence from the container analysis. + + {tools} + + + +Use the ReAct format to complete the task, and follow the below rules to determine what next steps to take. + +FORMAT REQUIREMENTS: +- Start your response with "Question:" and "Metric:" (appear ONCE only at top) +- Do NOT repeat "Question:" or "Metric:" after first occurrence +- After every Thought, provide Action + Action Input (unless ready for Final Answer) +- Final Thought must include: Selected: (); Definition Matched: ; Evidence: +- Final Answer must use exact format: : + +TOOL USAGE: +- NEVER output "Action: None" - this is forbidden +- Call "Container Analysis Data" tool ONCE at start; reuse observation throughout +- Never call same tool twice with identical or semantically similar inputs +- Exhaust all relevant tools before making final selection + +VALUE SELECTION: +- Select the value whose definition ACCURATELY matches your evidence +- Never select if definition contradicts your findings +- Only if you've exhausted all tools without sufficient info, select value marked "least_impact: true" +- When using fallback, make ABSOLUTELY sure to pick the value with least_impact: true + + + +Question: the input question about the CVSS metric +Metric: the metric being evaluated in format: () +Thought: your reasoning about what to do next +Action: the tool to use, must be one of [{tool_names}] +Action Input: the input for that tool +Observation: the result from the tool +... (repeat Thought/Action/Action Input/Observation as needed) +Thought: Final reasoning. Selected: (); Definition Matched: ; Evidence: +Final Answer: : + + + + +Example 1: Scope metric with multiple tool calls + +Question: Based on the container analysis, what is the appropriate Scope (S) value for this CVE? +Metric: Scope (S) +Thought: I need to understand if the vulnerability impact extends beyond the vulnerable component. I'll start by retrieving the container analysis data. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable function urllib.parse is present and called. The application processes URLs from external API requests. Container runs with default user privileges but shares network namespace with host system." +Thought: The analysis mentions shared network namespace with host. Let me search for more details about container isolation and scope. +Action: Code Semantic Search +Action Input: container isolation network namespace boundary privilege escalation cross-container +Observation: Found docker-compose.yml configuring network_mode: host, which means the container shares the host's network stack. Documentation indicates the application can access host network interfaces. +Thought: I now know the answer because the container shares the host network namespace, meaning an exploit could affect resources beyond the vulnerable container itself. Selected: Changed (C); Definition Matched: The vulnerability affects resources beyond the security scope of the vulnerable component; Evidence: "network_mode: host", "shares network namespace with host system" +Final Answer: S:C + +Example 2: Attack Vector with definitive answer from initial data + +Question: What is the Attack Vector (AV) for this vulnerability in the container context? +Metric: Attack Vector (AV) +Thought: I need to understand how the vulnerability can be exploited. Let me get the container analysis data first. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "PIL.ImageMath.eval is called in image_processor.py. The function processes user-uploaded images from the /api/upload endpoint which is publicly exposed on port 8080." +Thought: I now know the answer because the vulnerable functionality is accessible via a public network endpoint that accepts user input. Selected: Network (N); Definition Matched: The vulnerable component is bound to the network stack and the attack can be initiated remotely over the internet; Evidence: "publicly exposed on port 8080", "/api/upload endpoint" +Final Answer: AV:N + +Example 3: Privileges Required with additional investigation + +Question: What level of privileges are required to exploit this vulnerability? +Metric: Privileges Required (PR) +Thought: I need to determine if the attacker needs authentication or privileges. Let me check the container analysis first. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable urllib.parse function is called when processing URLs. The application has multiple endpoints." +Thought: The initial analysis doesn't clarify authentication requirements. Let me search for authentication and authorization controls. +Action: Code Semantic Search +Action Input: authentication required API endpoint authorization middleware access control +Observation: Found middleware/auth.py which implements JWT authentication on all /api/* endpoints except /api/health. The vulnerable URL processing occurs in the /api/parse endpoint which requires valid JWT tokens. +Thought: I now know the answer because the vulnerable endpoint requires JWT authentication. Selected: Low (L); Definition Matched: The attacker must have low-level privileges or must be authenticated to exploit the vulnerability; Evidence: "requires valid JWT tokens", "authentication on all /api/* endpoints" +Final Answer: PR:L + +Example 4: Fallback to least impact when uncertain + +Question: What is the Attack Complexity (AC) for this vulnerability? +Metric: Attack Complexity (AC) +Thought: I need to understand the conditions required for successful exploitation. Let me retrieve the analysis data. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable EVP_EncryptInit_ex2 function is reachable through encrypt_data(). The function uses GCM mode." +Thought: The analysis doesn't provide enough detail about exploitation complexity. Let me search for specific conditions or prerequisites. +Action: Code Semantic Search +Action Input: EVP_EncryptInit_ex2 GCM mode keylen ivlen OSSL_PARAM timing race condition +Observation: Found usage of EVP_EncryptInit_ex2 with OSSL_PARAM arrays but no clear indication of whether exploitation requires specific timing, race conditions, or unusual circumstances. +Thought: I now know the answer because after exhausting available tools, I cannot determine if the vulnerability requires special conditions beyond the attacker's control. Due to insufficient information, I must select the least impact value as fallback. Selected: High (H); Definition Matched: Successful attack depends on conditions beyond the attacker's control; Evidence: None +Final Answer: AC:H + + + +Now determine the correct CVSS metric value by using available tools to gather +evidence, then select the value whose definition best matches your findings. + + +{input} + -Use the following format (start each response with one of the following prefixes followed by a colon and a line break: [Question, Metric, Thought, Action, Action Input, Final Answer]): - -Question: the input question you must answer -Metric: the input metric you are evaluating in the following format: () -Thought: you should always think about what to do -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer because . I MUST ALWAYS end my final Thought with exactly: Selected: (); Definition Matched: ; Evidence: <1-2 short quotes>. -Final Answer: the final answer to the original input question in the following format: : - -Begin! - -Input: {input} {agent_scratchpad}""" -CVSS_EXAMPLES_FOR_PROMPT = """Examples: - - Example 1: +def get_agent_prompt(sys_prompt: str | None = None, + prompt_examples: bool = False) -> str: + """ + Get the agent prompt template. - Question: - Metric: Scope (S) - Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE - Action: Container Image Analysis Data - Action Input: - Observation: - Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, I will use the Lexical Search Container Image Code QA System to search for further information - Action: Lexical Search Container Image Code QA System - Action Input: - Observation: - Thought: I now know the answer because , I can now make a confident selection. - Selected: Changed (C), Definition Matched: The impact extends beyond the vulnerable system.; Evidence: 'The scope of the impact has changed'. - Final Answer: S:C - - Example 2: - - Question: - Metric: Attack Vector (AV) - Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE - Action: Container Image Analysis Data - Action Input: - Observation: - Thought: I now know the answer because , I can now make a confident selection. - Selected: Network (N); Definition Matched: Remote attack over the internet.; Evidence: 'The vulnerability can be exploited from anywhere with internet access, posing a significant risk.' - Final Answer: AV:N - - Example 3: - - Question: - Metric: Privilege Required (PR) - Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE - Action: Container Image Analysis Data - Action Input: - Observation: - Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, However, it does not provide enough information to make a confident selection. I need to gather more information, I will use the Lexical Search Container Image Code QA System to search for further information. - Action: Lexical Search Container Image Code QA System - Action Input: - Observation: - Thought: I now know the answer because , but due to insuficient information, it is still unclear what the correct selection should be, therefore, I will select the least impact value as a fallback. - Selected: High (H); Definition Matched: Administrative privileges needed.; Evidence: None. - Final Answer: PR:H - - Example 4: + Tool selection strategy is injected via PromptTemplate partial_variables. - Question: - Metric: User Interaction (UI) - Thought: To answer this question, I will start by calling the Container Image Analysis Data tool to gather information on how the container image may be impacted by a reported CVE - Action: Container Image Analysis Data - Action Input: - Observation: - Thought: The Container Image Analysis Data tool provided some information on how the container image may be impacted by a reported CVE, However, it does not provide enough information to make a confident selection. I need to gather more information, I will use the Lexical Search Container Image Code QA System to search for further information. - Action: Lexical Search Container Image Code QA System - Action Input: - Observation: - Thought: Reconsidering prior Observations, . I now know the answer because , I can now make a confident selection. - Selected: Required (R); Definition Matched: Requires user action.; Evidence: 'User must click a link to trigger the vulnerability'. - Final Answer: UI:R -""" - -def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = False) -> str: - """ - Get the agent prompt based on the system prompt and whether to include examples. + Args: + sys_prompt: Optional system prompt override + prompt_examples: Whether to include few-shot examples + + Returns: + Complete agent prompt template with {tool_selection_strategy} variable """ sys_prompt = sys_prompt or AGENT_SYS_PROMPT - + + # Select template with or without examples if prompt_examples: - prompt_template = AGENT_PROMPT_TEMPLATE.replace("Begin!\n\n", AGENT_EXAMPLES_FOR_PROMPT + "Begin!\n\n") + prompt_template = AGENT_PROMPT_TEMPLATE.replace( + "", + "\n" + AGENT_EXAMPLES_FOR_PROMPT + # AGENT_EXAMPLES_FOR_PROMPT_2 + ) else: prompt_template = AGENT_PROMPT_TEMPLATE + + return f'{sys_prompt}\n\n{prompt_template}' - return f'{sys_prompt} {prompt_template}' - -def get_cvss_prompt(sys_prompt: str | None = None, prompt_examples: bool = True, is_openai: bool = False) -> str: - """ - Get the CVSS prompt based on the system prompt and whether to include examples. +def get_cvss_prompt(sys_prompt: str | None = None, + prompt_examples: bool = True) -> str: """ + Get the CVSS prompt based on the system prompt. + + Args: + sys_prompt: Optional system prompt override + prompt_examples: Whether to include few-shot examples (kept for API compatibility) + Returns: + Complete CVSS prompt with system prompt + """ sys_prompt = sys_prompt or CVSS_SYS_PROMPT - guidance = [ - "Guidance:\n", - "- At the very beginning, start with `Metric:` and `Question:`.\n", - "- PROHIBITED: Do not repeat `Metric:` or `Question:` after the first occurrence. They must appear exactly once at the top.\n", - "- After every Thought, you MUST provide an Action AND Action Input, unless you are ready to provide the Final Answer. Do not stop after a Thought. This rule must always be followed.\n", - "- ABSOLUTE RULE: `Action: None` output is forbidden. Do not output `Action: None` under any circumstances.\n", - "- If you need to reconsider a prior Observation, incorporate that reflection inside your Thought immediately before your next Action or Final Answer. Do not emit a separate Thought for reflection.\n", - "- Exactly one `Thought:` per step is allowed.\n", - "- When you reach the final Thought, always include a brief rationale explaining why the selected value's definition matches your findings and what is the supporting evidence to back this up. You may omit Action/Action Input for this step. Do not call tools in this Thought.\n", - "- REMINDER: In your final Thought, end with exactly: Selected: (); Definition Matched: ; Evidence: <1-2 short quotes or 'None' if no evidence>. Example: Selected: None (N); Definition Matched: No privileges needed; Evidence: 'The container image does not require any privileges to be exploited'. Then output Final Answer: :. Do not call tools in this Thought.\n", - "- If you are ready to provide the Final Answer, you may omit Action/Action Input for the last step.\n", - "- IMPORTANT: Do not include Final Answer with Action or Action Input in the same response.\n", - "- The Final Answer must reuse the exact abbreviation from the Selected line verbatim. Example: Selected: Low (L) → Final Answer: AC:L.\n", - "- VERY IMPORTANT: The Final Answer ABSOLUTELY MUST follow this format: :\n", - "- You MUST start by using the Container Image Analysis Data tool exactly once.\n", - "- After the initial call, you MUST NOT call the Container Image Analysis Data tool again. Reuse the Observation from the first call for all subsequent reasoning and decisions.\n", - "- Use the container image analysis data as your primary reference for this metric to select the most appropriate and accurately descriptive CVSS metric value.\n", - "- If the container image analysis data does not contain sufficient information to make a confident selection, use other tools available to you to gather more information from the container image.\n", - "- You must not call the same tool with the same or semantically similar input more than once. Always try a different input or tool if your prior attempts did not get a definitive answer.\n", - "- ABSOLUTE RULE: From your Observations, you MUST select the value whose provided definition ACCURATELY matches your findings and is supported by the evidence you have collected. Never select a value if its definition contradicts or does not match your evidence.\n", - "- Only if you have thoroughly evaluated all available tools and prior Observations, and still cannot make a confident selection, may you choose the value with least impact marked by `least_impact: true` as a fallback. NEVER fallback to a value marked by `least_impact: false`.\n", - "- IMPORTANT: If using fallback, make ABSOLUTELY sure to pick the value with least_impact: true" - ] + # Examples are now embedded in CVSS_PROMPT_TEMPLATE + # prompt_examples parameter kept for backward compatibility but not used + prompt_template = CVSS_PROMPT_TEMPLATE - if prompt_examples: - prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + CVSS_EXAMPLES_FOR_PROMPT + "\n\n" + "Begin!\n\n") - else: - prompt_template = CVSS_PROMPT_TEMPLATE.replace("Begin!\n\n", "".join(guidance) + "\n\n" + "Begin!\n\n") - return f'{sys_prompt}\n\n{prompt_template}' class PromptBuilder(ABC): @@ -376,11 +555,45 @@ def build_prompt(self) -> str: Given CVE Details: """ + additional_intel_prompting -MOD_FEW_SHOT = """Generate a checklist for a security analyst to use when assessing the exploitability of a specific CVE within a containerized environment. Use the provided examples as a guide to understand how to construct a checklist from a given set of CVE details, then apply this understanding to create a specific checklist for the CVE details provided below. All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes. - +MOD_FEW_SHOT = """ +Generate an investigation checklist for assessing CVE exploitability in a +containerized environment. Your output must be a comma-separated list enclosed +in square brackets, with each item enclosed in quotes. + + + +Create 3-5 checklist items that meet these requirements: + +1. STRUCTURE: Each item must be a clear, answerable question + - Start with interrogative words: Is/Does/Are/Can/Has/etc. + - Be specific and actionable + - Include relevant context from the CVE + +2. CONTENT PRIORITIES: + - If the CVE mentions a specific vulnerable function or method, the FIRST + checklist item must verify whether that function is called or imported + in the codebase + - Focus on exploitability factors (version presence is already confirmed) + - Include specific technical names from the CVE (functions, libraries, + configurations, cipher modes, etc.) + - Consider the attack vector (network exposure, user input, file processing, etc.) + - Address relevant security controls or mitigations + +3. INVESTIGATION TOOLS AVAILABLE: + {tool_descriptions} + + Design questions that can be answered using these analysis capabilities. + +4. COMPLETENESS: + - Cover the vulnerability chain: presence → usage → exploitability + - Each item should independently contribute to understanding exploit risk + + + {examples} + -Given CVE Details: + """ investigation_guideline = """1. Is the flagged component in the product? Determine if the container image includes the flagged package version. Verify the presence of the library or software in question by checking the container's software bill of materials (SBOM). diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py new file mode 100644 index 000000000..9a939aa52 --- /dev/null +++ b/tests/test_base_tool_descriptions.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Tests for the base build_tool_descriptions() function. + +Tests that the consolidated base function provides simple tool descriptions +that can be formatted by specialized functions for different contexts. +""" + +import sys +from pathlib import Path + +# Add src to path +src_path = Path(__file__).parent.parent / "src" +sys.path.insert(0, str(src_path)) + +from vuln_analysis.utils.prompting import build_tool_descriptions +from vuln_analysis.tools.tool_names import ToolNames + + +def test_base_returns_list(): + """Test that base function returns a list, not a string.""" + tool_names = [ToolNames.CODE_SEMANTIC_SEARCH] + + result = build_tool_descriptions(tool_names) + + assert isinstance(result, list) + print("✓ Base function returns list") + + +def test_base_descriptions_format(): + """Test that base descriptions have consistent format.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH + ] + + result = build_tool_descriptions(tool_names) + + # Each description should have format: "Tool Name: Description" + for desc in result: + assert ":" in desc + parts = desc.split(":", 1) + assert len(parts) == 2 + assert len(parts[0].strip()) > 0 # Tool name + assert len(parts[1].strip()) > 0 # Description + + print("✓ Base descriptions have consistent format") + + +def test_base_all_tools(): + """Test that base function includes all available tools.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CONTAINER_ANALYSIS_DATA + ] + + result = build_tool_descriptions(tool_names) + + # Should have 7 descriptions + assert len(result) == 7 + + # Verify all tools are present + all_text = " ".join(result) + assert "Code Semantic Search" in all_text + assert "Docs Semantic Search" in all_text + assert "Code Keyword Search" in all_text + assert "Call Chain Analyzer" in all_text + assert "Function Caller Finder" in all_text + assert "CVE Web Search" in all_text + assert "Container Analysis Data" in all_text + + print("✓ Base function includes all tools") + + +def test_base_empty_list(): + """Test that base function returns empty list when no tools.""" + tool_names = [] + + result = build_tool_descriptions(tool_names) + + assert result == [] + assert isinstance(result, list) + + print("✓ Base function returns empty list for no tools") + + +def test_checklist_formats_descriptions(): + """Test that checklist calling code formats descriptions correctly.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH + ] + + # Simulate what checklist_prompt_generator.py does + tool_descs = build_tool_descriptions(tool_names) + + if tool_descs: + formatted_descs = ["- " + desc for desc in tool_descs] + tool_descriptions = "The following tools can be used to answer checklist questions:\n " + "\n ".join(formatted_descs) + else: + tool_descriptions = "Analysis tools will be used to investigate these questions." + + # Verify checklist formatting + assert "The following tools can be used to answer checklist questions:" in tool_descriptions + assert "Code Semantic Search" in tool_descriptions + assert "Docs Semantic Search" in tool_descriptions + assert " - " in tool_descriptions # Indented bullet points + + print("✓ Checklist formats descriptions correctly") + + +def test_mod_few_shot_structure(): + """Test that MOD_FEW_SHOT has required structure and placeholders.""" + from vuln_analysis.utils.prompting import MOD_FEW_SHOT + + # Check for required section markers + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + + # Check for placeholders + assert "{tool_descriptions}" in MOD_FEW_SHOT + assert "{examples}" in MOD_FEW_SHOT + + # Check for key instructions + assert "3-5 checklist items" in MOD_FEW_SHOT + assert "FIRST" in MOD_FEW_SHOT or "first" in MOD_FEW_SHOT + assert "vulnerable function" in MOD_FEW_SHOT + + print("✓ MOD_FEW_SHOT structure validated") + + +if __name__ == "__main__": + print("Running Base Tool Descriptions tests...\n") + + test_base_returns_list() + test_base_descriptions_format() + test_base_all_tools() + test_base_empty_list() + test_checklist_formats_descriptions() + test_mod_few_shot_structure() + + print("\n✅ All base tool descriptions tests passed!") + diff --git a/tests/test_tool_filtering.py b/tests/test_tool_filtering.py new file mode 100644 index 000000000..f325d9d8c --- /dev/null +++ b/tests/test_tool_filtering.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for tool filtering logic in CVE agent functions. +""" + +import pytest + +from vuln_analysis.tools.tool_names import ToolNames + + +class MockTool: + """Mock tool for testing.""" + def __init__(self, name: str): + self.name = name + + +class MockState: + """Mock state for testing.""" + def __init__( + self, + code_vdb_path: str | None = "/path/to/code", + doc_vdb_path: str | None = "/path/to/docs", + code_index_path: str | None = "/path/to/index" + ): + self.code_vdb_path = code_vdb_path + self.doc_vdb_path = doc_vdb_path + self.code_index_path = code_index_path + + +class TestToolFiltering: + """Test tool filtering logic matches constants correctly.""" + + def create_mock_state( + self, + code_vdb_path: str | None = "/path/to/code", + doc_vdb_path: str | None = "/path/to/docs", + code_index_path: str | None = "/path/to/index" + ) -> MockState: + """Create a mock state for testing.""" + return MockState( + code_vdb_path=code_vdb_path, + doc_vdb_path=doc_vdb_path, + code_index_path=code_index_path + ) + + def test_filter_code_qa_when_path_missing(self): + """Test that Code Semantic Search tool is filtered when code_vdb_path is None.""" + state = self.create_mock_state(code_vdb_path=None) + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + ] + + # Apply filtering logic + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None)) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.DOCS_SEMANTIC_SEARCH + + def test_filter_doc_qa_when_path_missing(self): + """Test that Docs Semantic Search tool is filtered when doc_vdb_path is None.""" + state = self.create_mock_state(doc_vdb_path=None) + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None)) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CODE_SEMANTIC_SEARCH + + def test_filter_lexical_search_when_path_missing(self): + """Test that Code Keyword Search tool is filtered when code_index_path is None.""" + state = self.create_mock_state(code_index_path=None) + tools = [ + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CVE_WEB_SEARCH + + def test_no_filtering_when_all_paths_present(self): + """Test that no tools are filtered when all paths are available.""" + state = self.create_mock_state() + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or + (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None)) + ] + + assert len(filtered_tools) == 3 + + def test_filter_transitive_search_when_disabled(self): + """Test that call chain analysis tools are filtered when disabled.""" + state = self.create_mock_state() + transitive_enabled = False + + tools = [ + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CALL_CHAIN_ANALYZER and + (not transitive_enabled or state.code_index_path is None)) or + (tool.name == ToolNames.FUNCTION_CALLER_FINDER and + (not transitive_enabled or state.code_index_path is None))) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CODE_SEMANTIC_SEARCH diff --git a/tests/test_tool_names.py b/tests/test_tool_names.py new file mode 100644 index 000000000..7d1677717 --- /dev/null +++ b/tests/test_tool_names.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for tool name constants. +""" + +import pytest + +from vuln_analysis.tools.tool_names import ToolNames + + +class TestToolNameConstants: + """Test the tool name constants are properly defined.""" + + def test_all_constants_defined(self): + """Verify all expected tool name constants exist.""" + assert hasattr(ToolNames, 'CODE_SEMANTIC_SEARCH') + assert hasattr(ToolNames, 'DOCS_SEMANTIC_SEARCH') + assert hasattr(ToolNames, 'CODE_KEYWORD_SEARCH') + assert hasattr(ToolNames, 'CALL_CHAIN_ANALYZER') + assert hasattr(ToolNames, 'FUNCTION_CALLER_FINDER') + assert hasattr(ToolNames, 'CVE_WEB_SEARCH') + assert hasattr(ToolNames, 'CONTAINER_ANALYSIS_DATA') + + def test_constants_are_unique(self): + """Verify all tool names are unique.""" + constants = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CONTAINER_ANALYSIS_DATA, + ] + + assert len(constants) == len(set(constants)), "Tool names must be unique" + From 0bc075ad57402b453154aa7066bc9cdbbb0b5c63 Mon Sep 17 00:00:00 2001 From: Roni Hartuv Date: Mon, 24 Nov 2025 13:58:08 +0200 Subject: [PATCH 102/286] chore: add pvc for postgresql and elasticsearch (#149) * chore: add pvc for postgresql and elasticsearch * fix: update pvc settings and names --- .../argilla/argilla-user-feedback-pvc.yaml | 13 +++++++++ kustomize/base/argilla/deployment.yaml | 28 +++++++++++++------ kustomize/base/argilla/kustomization.yaml | 1 + 3 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 kustomize/base/argilla/argilla-user-feedback-pvc.yaml diff --git a/kustomize/base/argilla/argilla-user-feedback-pvc.yaml b/kustomize/base/argilla/argilla-user-feedback-pvc.yaml new file mode 100644 index 000000000..8a730ef7c --- /dev/null +++ b/kustomize/base/argilla/argilla-user-feedback-pvc.yaml @@ -0,0 +1,13 @@ + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: argilla-user-feedback-pvc + labels: + app: morpheus-feedback-api +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/kustomize/base/argilla/deployment.yaml b/kustomize/base/argilla/deployment.yaml index e3ce0c40d..f31b3d2a5 100644 --- a/kustomize/base/argilla/deployment.yaml +++ b/kustomize/base/argilla/deployment.yaml @@ -9,14 +9,19 @@ spec: selector: matchLabels: app: morpheus-feedback-api + strategy: + type: Recreate template: metadata: labels: app: morpheus-feedback-api spec: + restartPolicy: Always imagePullSecrets: - name: argilla-user-feedback-ips serviceAccountName: argilla + securityContext: + fsGroup: 1000 containers: - name: flask-service image: quay.io/exploit-iq/user-feedback-api:latest @@ -33,7 +38,6 @@ spec: value: rh-argilla - name: ARGILLA_API_URL value: 'http://argilla:6900' - # Internal Argilla endpoint - name: argilla-server image: argilla/argilla-server:latest imagePullPolicy: IfNotPresent @@ -76,24 +80,23 @@ spec: - name: elasticsearch image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: user-feedback-storage + subPath: elasticsearch env: - name: node.name value: "elasticsearch" - - name: ES_JAVA_OPTS value: "-Xms512m -Xmx512m" - - name: discovery.type value: "single-node" - - name: cluster.name value: "es-argilla-local" - - name: cluster.routing.allocation.disk.threshold_enabled value: "false" - - name: xpack.security.enabled - value: "false" + value: "false" - name: postgresql image: postgres:13 imagePullPolicy: IfNotPresent @@ -103,4 +106,13 @@ spec: - name: POSTGRES_PASSWORD value: postgres - name: POSTGRES_DB - value: argilla \ No newline at end of file + value: argilla + volumeMounts: + - name: user-feedback-storage + mountPath: /var/lib/postgresql + subPath: postgres + serviceAccount: argilla + volumes: + - name: user-feedback-storage + persistentVolumeClaim: + claimName: argilla-user-feedback-pvc \ No newline at end of file diff --git a/kustomize/base/argilla/kustomization.yaml b/kustomize/base/argilla/kustomization.yaml index 564cbcb34..f95625431 100644 --- a/kustomize/base/argilla/kustomization.yaml +++ b/kustomize/base/argilla/kustomization.yaml @@ -8,6 +8,7 @@ resources: - sa.yaml - service.yaml - argilla-route.yaml + - argilla-user-feedback-pvc.yaml secretGenerator: - name: argilla-feedback-secret From 9f3d9d67fb5d06ae9f1d27cc92e86ff27ecbabde Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Tue, 25 Nov 2025 09:06:27 +0200 Subject: [PATCH 103/286] fix: llm send invalid characters and when get_function in c returns a reserved word (#150) --- src/vuln_analysis/utils/function_name_locator.py | 6 ++++-- .../utils/functions_parsers/c_lang_function_parsers.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 78c2dffc8..2fc509563 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -108,7 +108,9 @@ def common_flow_control(self,input_function: str, package_docs) -> list[str]: package_functions = [] for doc in package_docs: if self.lang_parser: - function_name = self.lang_parser.get_function_name(doc) + function_name = self.lang_parser.get_function_name(doc) + if function_name == "": + continue package_functions.append(f"{function_name}") # Use fuzzy matching to find close matches close_matches = difflib.get_close_matches(input_function, package_functions, n=10, cutoff=0.6) @@ -131,7 +133,7 @@ async def locate_functions(self, query: str) -> list[str]: List of function names in format 'package,function_name' that match the search term, or error message with package suggestions if package is not found """ - query = query.splitlines()[0].replace('"', '').replace("'", "").strip() + query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() parts_input = query.split(",", 1) self.is_std_package = False self.is_package_valid = False diff --git a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py index ff320b295..1a5c7b04e 100644 --- a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py @@ -489,6 +489,7 @@ def is_macro_function_invocation(self,seg: str) -> bool: def get_function_name(self, function: Document) -> str: try: + reservred_words =["break", "case", "continue", "default", "do", "else", "for", "goto", "if", "switch", "while"] if function.metadata.get('func_name') : return function.metadata.get('func_name') @@ -498,6 +499,9 @@ def get_function_name(self, function: Document) -> str: if function_pattern: f_name = function_pattern.group('name') if f_name: + if f_name in reservred_words: + function.metadata['state'] = "invalid" + return "" function.metadata['func_name'] = f_name return f_name function.metadata['state'] = "invalid" From 9c02285352c3cd935904398d390a87f2d43c8087 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 26 Nov 2025 11:22:34 +0200 Subject: [PATCH 104/286] fix: regressive bugs of agent Fixes: https://issues.redhat.com/browse/APPENG-4088 Signed-off-by: Zvi Grinberg --- kustomize/base/exploit-iq-config.yml | 2 +- src/vuln_analysis/configs/config-http-openai.yml | 2 +- src/vuln_analysis/configs/config-tracing.yml | 2 +- src/vuln_analysis/configs/config.yml | 2 +- src/vuln_analysis/utils/prompting.py | 9 +++++---- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 081505f48..09b1427d4 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -103,7 +103,7 @@ functions: verbose: false cve_generate_cvss: _type: cve_generate_cvss - skip: false + skip: true llm_name: generate_cvss_llm tool_names: - Code Semantic Search diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 8d6377c8a..a4983cf08 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -97,7 +97,7 @@ functions: verbose: false cve_generate_cvss: _type: cve_generate_cvss - skip: false + skip: true llm_name: generate_cvss_llm tool_names: - Code Semantic Search diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 003c47eff..bfbe78ddc 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -100,7 +100,7 @@ functions: verbose: false cve_generate_cvss: _type: cve_generate_cvss - skip: false + skip: true llm_name: generate_cvss_llm tool_names: - Code Semantic Search diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 9aa0f23b7..c12c5da4f 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -77,7 +77,7 @@ functions: verbose: false cve_generate_cvss: _type: cve_generate_cvss - skip: false + skip: true llm_name: generate_cvss_llm tool_names: - Code Semantic Search diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index ce36986c6..45837e0c3 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -570,11 +570,12 @@ def build_prompt(self) -> str: - Include relevant context from the CVE 2. CONTENT PRIORITIES: - - If the CVE mentions a specific vulnerable function or method, the FIRST - checklist item must verify whether that function is called or imported - in the codebase + - If the CVE mentions a specific vulnerable function or method in a given package or library, the FIRST + checklist item must verify whether that function in that package or library is called or imported + in the codebase - function should be specified together with the package name, + for example : 'Is the function1 function from the package1 package called in the codebase?' - Focus on exploitability factors (version presence is already confirmed) - - Include specific technical names from the CVE (functions, libraries, + - Include specific technical names from the CVE (functions, libraries, configurations, cipher modes, etc.) - Consider the attack vector (network exposure, user input, file processing, etc.) - Address relevant security controls or mitigations From fe2a7dad828f8ea5a2931a10da7dd19e58140a9d Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:20:01 +0200 Subject: [PATCH 105/286] Java transitive search (#130) * Java transitive search Signed-off-by: Theodor Mihalache --- .tekton/on-pull-request.yaml | 35 + Dockerfile | 21 + .../tests/test_transitive_code_search.py | 74 + .../tools/transitive_code_search.py | 70 +- .../utils/chain_of_calls_retriever.py | 234 +- .../utils/chain_of_calls_retriever_base.py | 111 + .../utils/chain_of_calls_retriever_factory.py | 33 + src/vuln_analysis/utils/data_utils.py | 42 +- src/vuln_analysis/utils/dep_tree.py | 205 +- src/vuln_analysis/utils/document_embedding.py | 45 +- .../utils/function_name_extractor.py | 6 +- .../utils/function_name_locator.py | 6 +- .../c_lang_function_parsers.py | 11 +- .../golang_functions_parsers.py | 158 +- .../java_functions_parsers.py | 2660 +++++++++++++++++ .../lang_functions_parsers.py | 41 +- .../lang_functions_parsers_factory.py | 3 + .../python_functions_parser.py | 61 +- src/vuln_analysis/utils/intel_retriever.py | 1 - .../utils/java_chain_of_calls_retriever.py | 680 +++++ .../utils/java_segmenters_with_methods.py | 2029 +++++++++++++ src/vuln_analysis/utils/java_utils.py | 2440 +++++++++++++++ .../utils/transitive_code_searcher_tool.py | 13 +- 23 files changed, 8566 insertions(+), 413 deletions(-) create mode 100644 src/vuln_analysis/utils/chain_of_calls_retriever_base.py create mode 100644 src/vuln_analysis/utils/chain_of_calls_retriever_factory.py create mode 100644 src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py create mode 100644 src/vuln_analysis/utils/java_chain_of_calls_retriever.py create mode 100644 src/vuln_analysis/utils/java_segmenters_with_methods.py create mode 100644 src/vuln_analysis/utils/java_utils.py diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index afad6c5bc..e8ba883a1 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -178,6 +178,23 @@ spec: print_banner "INSTALLING DEPENDENCIES" uv sync + # Install Java + JAVA_ARCH="x64" + JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_${JAVA_ARCH}_linux_hotspot_22.0.2_9.tar.gz" + JDK_DIR="jdk-22.0.2+9" + + echo ">> Downloading $JDK_URL" + mkdir -p /tekton/home/jdk + curl -fsSL -o /tekton/home/jdk/jdk.tgz "$JDK_URL" + tar -C /tekton/home/jdk -xzf /tekton/home/jdk/jdk.tgz + rm -f /tekton/home/jdk/jdk.tgz + + export JAVA_HOME="/tekton/home/jdk/${JDK_DIR}" + export PATH="$JAVA_HOME/bin:$PATH" + + echo "Java version:" + java -version || true + # Install Go print_banner "Installing Go" @@ -190,6 +207,24 @@ spec: echo "Go version:" go version + # Install Maven + print_banner "Installing Maven" + + MAVEN_VERSION="3.9.11" + ARCHIVE="apache-maven-${MAVEN_VERSION}-bin.tar.gz" + URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${ARCHIVE}" + + curl -s -L -o "${ARCHIVE}" "${URL}" + mkdir -p "$HOME/maven-sdk" + tar -C "$HOME/maven-sdk" -xzf "${ARCHIVE}" + + export MAVEN_HOME="$HOME/maven-sdk/apache-maven-${MAVEN_VERSION}" + export M2_HOME="$MAVEN_HOME" + export PATH="$MAVEN_HOME/bin:$PATH" + + echo "Maven version:" + mvn -v + print_banner "RUNNING LINTER" # Add the current directory to git's safe directories to avoid ownership errors. git config --global --add safe.directory /workspace/source diff --git a/Dockerfile b/Dockerfile index 40054c23c..a2b994916 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,27 @@ RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24. && tar -C /usr/local -xzf /tmp/go1.24.1.linux-amd64.tar.gz \ && rm /tmp/go1.24.1.linux-amd64.tar.gz +# --- Temurin JDK 22 (amd64/x86_64) --- +ARG JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz" +ARG JDK_DIR="jdk-22.0.2+9" +RUN mkdir -p /opt/jdk \ + && curl -fsSL -o /tmp/jdk.tgz "${JDK_URL}" \ + && tar -C /opt/jdk -xzf /tmp/jdk.tgz \ + && rm -f /tmp/jdk.tgz +ENV JAVA_HOME=/opt/jdk/${JDK_DIR} +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# --- Maven 3.9.11 (optional) --- +ARG MVN_VER=3.9.11 +RUN curl -fsSL -o /tmp/maven.tgz \ + "https://archive.apache.org/dist/maven/maven-3/${MVN_VER}/binaries/apache-maven-${MVN_VER}-bin.tar.gz" \ + && tar -C /opt -xzf /tmp/maven.tgz \ + && rm -f /tmp/maven.tgz +ENV PATH="/opt/apache-maven-${MVN_VER}/bin:${PATH}" + +# Verify +RUN java -version && mvn -v + # Set SSL environment variables ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 5622ed206..2c9b2a82f 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -300,3 +300,77 @@ async def test_c_transitive_search_2(): print(f"DEBUG: len(list_path) = {len(list_path)}") assert len(list_path) == 1 assert path_found == False + +@pytest.mark.asyncio +async def test_transitive_search_java_1(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("commons-beanutils:commons-beanutils:1.9.4,org.apache.commons.beanutils.PropertyUtilsBean.getProperty") + (path_found, list_path) = result + print(result) + assert path_found is False + assert len(list_path) is 1 + +@pytest.mark.asyncio +async def test_transitive_search_java_2(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("org.apache.commons:commons-lang3:3.14.0,org.apache.commons.lang3.StringUtils.isBlank") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) is 2 + document = list_path[1] + assert 'src/main/java/io/cryostat' in document.metadata['source'] + assert 'StringUtils.isBlank(' in document.page_content + +# @pytest.mark.asyncio +# async def test_transitive_search_java_3(): +# transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() +# set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", +# git_ref="8f753753379e9381429b476aacbf6890ef101438", +# included_extensions=["**/*.java"], +# excluded_extensions=["target/**/*", +# "build/**/*", +# "*.class", +# ".gradle/**/*", +# ".mvn/**/*", +# ".gitignore", +# "test/**/*", +# "tests/**/*", +# "src/test/**/*", +# "pom.xml", +# "build.gradle"]) +# result = await transitive_code_search_runner_coroutine(",java.net.URLEncoder.encode") +# (path_found, list_path) = result +# print(result) +# assert path_found is True +# assert len(list_path) > 1 +# document = list_path[-1] +# assert 'src/main/java/io/cryostat' in document.metadata['source'] \ No newline at end of file diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index e7f1b9d5a..64bae4005 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os from vuln_analysis.runtime_context import ctx_state @@ -26,14 +25,18 @@ from langchain.docstore.document import Document from vuln_analysis.data_models.state import AgentMorpheusEngineState -from ..utils.chain_of_calls_retriever import ChainOfCallsRetriever -from vuln_analysis.utils.dep_tree import Ecosystem from vuln_analysis.utils.document_embedding import DocumentEmbedding +from ..data_models.input import SourceDocumentsInfo +from ..utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from ..utils.chain_of_calls_retriever_factory import get_chain_of_calls_retriever +from ..utils.dep_tree import Ecosystem from ..utils.error_handling_decorator import catch_pipeline_errors_async, catch_tool_errors from ..utils.function_name_extractor import FunctionNameExtractor from ..utils.function_name_locator import FunctionNameLocator from vuln_analysis.logging.loggers_factory import LoggingFactory +from ..utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever + PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME = "package_and_function_locator" @@ -60,12 +63,13 @@ class PackageAndFunctionLocatorToolConfig(FunctionBaseConfig, name=("%s" % PACKA Package and function locator tool used to validate package names and find function names using fuzzy matching. """ - -def get_call_of_chains_retriever(documents_embedder, si): +def get_call_of_chains_retriever(documents_embedder, si, query: str): documents: list[Document] git_repo = None + code_source_info: SourceDocumentsInfo for source_info in si: if source_info.type == "code": + code_source_info = source_info git_repo = documents_embedder.get_repo_path(source_info) documents = documents_embedder.collect_documents(source_info) if git_repo is None: @@ -73,16 +77,19 @@ def get_call_of_chains_retriever(documents_embedder, si): with open(os.path.join(git_repo, 'ecosystem_data.txt'), 'r', encoding='utf-8') as file: ecosystem = file.read() ecosystem = Ecosystem[ecosystem] - coc_retriever = ChainOfCallsRetriever(documents=documents, ecosystem=ecosystem, manifest_path=git_repo) + coc_retriever = get_chain_of_calls_retriever(ecosystem=ecosystem, + documents=documents, + manifest_path=git_repo, + query=query, + code_source_info=code_source_info) return coc_retriever - -def get_transitive_code_searcher(): +def get_transitive_code_searcher(query: str): state: AgentMorpheusEngineState = ctx_state.get() - if state.transitive_code_searcher is None: + if state.transitive_code_searcher is None or isinstance(state.transitive_code_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): si = state.original_input.input.image.source_info documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query) transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) state.transitive_code_searcher = transitive_code_searcher return state.transitive_code_searcher @@ -108,16 +115,22 @@ async def transitive_search(config: TransitiveCodeSearchToolConfig, @catch_tool_errors(TRANSITIVE_CODE_SEARCH_TOOL_NAME) async def _arun(query: str) -> tuple: transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher() + transitive_code_searcher = get_transitive_code_searcher(query) result = transitive_code_searcher.search(query) return result yield FunctionInfo.from_fn( _arun, description=(""" - Checks if a function from a package is reachable from application code through the call chain. - Input format: 'package_name,function_name'. - Example: 'urllib,parse'. + Checks if a function from a package is reachable from application code through the call chain. + Make sure the input format is matching exactly one of the following formats: + + Input format 1: 'package_name,function_name'. + Example 1: 'urllib,parse'. + + Input format 2(java): 'maven_gav,class_name.function_name'. + Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'. + Returns: (is_reachable: bool, call_hierarchy_path: list). """)) @@ -131,9 +144,9 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, """ @catch_tool_errors(FUNCTION_NAME_EXTRACTOR_TOOL_NAME) async def _arun(query: str) -> list: - coc_retriever: ChainOfCallsRetriever + coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher() + transitive_code_searcher = get_transitive_code_searcher(query) coc_retriever = transitive_code_searcher.chain_of_calls_retriever function_name_extractor = FunctionNameExtractor(coc_retriever) result = function_name_extractor.fetch_list(query) @@ -154,25 +167,24 @@ async def package_and_function_locator(config: PackageAndFunctionLocatorToolConf builder: Builder): # pylint: disable=unused-argument """ Function Locator tool used to validate package names and find function names using fuzzy matching. - Mandatory first step for code path analysis. + Mandatory first step for code path analysis. """ @catch_tool_errors(PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME) async def _arun(query: str) -> dict: - coc_retriever: ChainOfCallsRetriever + coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher() + transitive_code_searcher = get_transitive_code_searcher(query) coc_retriever = transitive_code_searcher.chain_of_calls_retriever locator = FunctionNameLocator(coc_retriever) result = await locator.locate_functions(query) pkg_msg = "Package is valid." - if not locator.is_package_valid and not locator.is_std_package: - pkg_msg = "Package is not valid." - - + if not locator.is_package_valid and not locator.is_std_package: + pkg_msg = "Package is not valid." + return { "ecosystem": coc_retriever.ecosystem.name, - "package_msg": pkg_msg, + "package_msg": pkg_msg, "result": result } @@ -180,7 +192,13 @@ async def _arun(query: str) -> dict: _arun, description=(""" Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, and provides ecosystem type (GO/Python/Java/JavaScript/C/C++). - Input format: 'package_name,function_name' or 'package_name,class_name.method_name' - Example: 'libxml2,xmlParseDocument' + Make sure the input format is matching exactly one of the following formats: + + Input format 1: 'package_name,function_name' or 'package_name,class_name.method_name'. + Example 1: 'libxml2,xmlParseDocument'. + + Input format 2(java): 'maven_gav,class_name.method_name'. + Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'. + Returns: {'ecosystem': str, 'package_msg': str, 'result': [function_names]}. """)) diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index a9957957b..9bc2b1508 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -2,83 +2,21 @@ import time from collections import defaultdict, deque from pathlib import Path -from typing import List, Optional +from typing import List from langchain_core.documents import Document +from .chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, PARENTS_INDEX, \ + calculate_hashable_string_for_function, EXCLUSIONS_INDEX from vuln_analysis.logging.loggers_factory import LoggingFactory from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem -from .functions_parsers.lang_functions_parsers import LanguageFunctionsParser from .functions_parsers.lang_functions_parsers_factory import ( get_language_function_parser, ) from .standard_library_cache import StandardLibraryCache -PARENTS_INDEX = 0 -EXCLUSIONS_INDEX = 1 - logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") - -def calculate_hashable_string_for_function(function_file_name: str, function_name_to_search: str) -> str: - return f"{function_file_name};{function_name_to_search}" - - -def get_extension_of_file(file_path: str): - extension_start = file_path.rfind(".") - return file_path[extension_start:] - - -def get_functions_for_package(package_name: str, documents: list[Document], language_parser: LanguageFunctionsParser, - callee_function_file_name="", function_to_search="", sources_location_packages=True): - # Retrieve documents of packages only ( functions + code) - if sources_location_packages: - for document in documents: - doc_extension = get_extension_of_file(document.metadata.get('source')) - if (not language_parser.is_root_package(document) and - document.metadata.get('content_type') == 'functions_classes' and - language_parser.is_function(document) and - is_function_callable(document, language_parser, callee_function_file_name) and - language_parser.is_supported_file_extensions(doc_extension) and - document_belongs_to_package(language_parser, document, package_name) and - function_called_from_caller_body(document, function_to_search, language_parser)): - yield document - # Retrieve documents of application only ( functions + code) - else: - for document in documents: - doc_extension = get_extension_of_file(document.metadata.get('source')) - if (language_parser.is_root_package(document) - and language_parser.is_supported_file_extensions(doc_extension) - and function_called_from_caller_body(document, function_to_search, language_parser)): - if (language_parser.is_script_language() or - (document.metadata.get('content_type') == 'functions_classes' and - language_parser.is_function(document))): - yield document - - -def is_function_callable(document: Document, language_parser, callee_function_file_name: str) -> bool: - return (language_parser.is_exported_function(document) or - document.metadata['source'].lower() == callee_function_file_name.lower()) - - -def function_called_from_caller_body(document, function_to_search, language_parser: LanguageFunctionsParser) -> bool: - if function_to_search.strip() == "": - return True - function_word = language_parser.get_function_reserved_word() - func_header_template = rf"${function_word} (\(.*\))?\s?${function_to_search}" - return (re.search(pattern=function_to_search, string=document.page_content, - flags=re.IGNORECASE | re.MULTILINE) - # verify caller function or method is not the function - and not re.search(pattern=func_header_template, - string=document.page_content, - flags=re.IGNORECASE | re.MULTILINE)) - - -def document_belongs_to_package(language_parser: LanguageFunctionsParser, document: Document, - package_name: str) -> bool: - return language_parser.is_a_package(package_name, document) - - def is_likely_macro_block(segment: str) -> bool: lines = segment.strip().splitlines() if not lines: @@ -112,25 +50,10 @@ def is_likely_macro_block(segment: str) -> bool: return False -class ChainOfCallsRetriever: +class ChainOfCallsRetriever(ChainOfCallsRetrieverBase): """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being called from the application code base or not. """ - last_visited_parent_package_indexes: Optional[dict] - documents: Optional[List[Document]] - documents_of_full_sources: Optional[dict] - documents_of_types: Optional[list] - documents_of_functions: list | None - language_parser: Optional[LanguageFunctionsParser] - dependency_tree: Optional[DependencyTree] - tree_dict: Optional[dict] - supported_packages: Optional[list[str]] - ecosystem: Optional[Ecosystem] - manifest_path: Optional[Path] - package_name: Optional[str] - found_path: Optional[bool] - types_classes_fields_mapping: Optional[dict[tuple, list[tuple]]] - functions_local_variables_index: Optional[dict[str, dict]] def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_path: Path): """ @@ -181,16 +104,16 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat # filter out full code documents and functions/methods docs, retaining only types/classes docs self.documents_of_types = [doc for doc in filtered_documents if self.language_parser.is_doc_type(doc)] - + # Apply macro filtering only for C/C++ languages - if not self.language_parser.is_language_supports_macro(): + if not self.language_parser.is_language_supports_macro(): no_macro_documents = filtered_documents logger.debug(f"skipping macro check for {self.ecosystem}, using all filtered documents") else: no_macro_documents = [doc for doc in filtered_documents if not is_likely_macro_block(doc.page_content)] - logger.debug(f"no_macro_documents len : {len(no_macro_documents)}") + logger.debug(f"no_macro_documents len : {len(no_macro_documents)}") - if not self.language_parser.is_script_language(): + if not self.language_parser.is_script_language(): # filter out types and full code documents, retaining only functions/methods documents in this attribute. self.documents = [doc for doc in no_macro_documents if self.language_parser.is_function(doc)] self.documents_of_functions = self.documents @@ -198,14 +121,14 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat self.documents = filtered_documents self.documents_of_functions = [doc for doc in self.documents if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] - - logger.debug(f"self.documents len : {len(self.documents)}") + + logger.debug(f"self.documents len : {len(self.documents)}") logger.debug("Chain of Calls Retriever - retaining only types/classes docs " "documents_of_types len %d", len(self.documents_of_types)) # boolean attribute that indicates whether a path was found or not, initially set to False. - self.found_path = False + self.found_path = False logger.debug("Chain of Calls Retriever - after documents_of_full_sources") - + self.last_visited_parent_package_indexes = dict() self.last_visited = dict() # Constructing a map of types and classes to their attributes/members/fields @@ -214,10 +137,10 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat # local variables names mapped to (types, (values or expressions)) self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) logger.debug("Chain of Calls Retriever - after functions_local_variables_index") - + if not self.language_parser.is_search_algo_dfs(): self.sort_docs = self.__group_docs_by_pkg() - + def __group_docs_by_pkg(self) -> dict[str, list[Document]]: """ sort docs by pkg name @@ -266,12 +189,13 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa possible_docs = self.get_possible_docs(function_name_to_search, package, package_exclusions, - sources_location_packages) + sources_location_packages, + frozenset(), + dict()) # Collect all potential caller functions - for doc in get_functions_for_package(package_name=package, + for doc in self.get_functions_for_package(package_name=package, documents=possible_docs, - language_parser=self.language_parser, sources_location_packages=sources_location_packages, function_to_search=function_name_to_search, callee_function_file_name=function_file_name): @@ -291,10 +215,11 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa callee_function_file_name= function_file_name, fields_of_types= - self.types_classes_fields_mapping - , + self.types_classes_fields_mapping, functions_local_variables_index= - self.functions_local_variables_index) + self.functions_local_variables_index, + documents_of_functions= + self.documents_of_functions) # If current document is found to be calling document_function in function_package, then return it as a # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. @@ -328,9 +253,9 @@ def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all # excluded functions, and all function that their body doesn't contain the target function name to search for. def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], - sources_location_packages: bool) \ - -> (list[ - Document], bool): + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): if sources_location_packages: filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') and self.language_parser.is_function(doc) and @@ -348,12 +273,12 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack :param document_function: the document containing the function code and signature :param function_package: the package name containing the function :return: a list of documents of functions that are calling document_function - """ + """ total_start = time.time() direct_parents = list() # gets list of all direct parents of function - + list_of_packages = self.tree_dict.get(function_package) if list_of_packages is not None: direct_parents.extend(list_of_packages[0]) @@ -362,7 +287,7 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) function_file_name = document_function.metadata.get('source') - relevant_docs_to_search_in = list() + relevant_docs_to_search_in = list() # Search for caller functions only at parents according to dependency tree. log_entries = [] loop_start = time.time() @@ -385,13 +310,13 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack func_name = self.language_parser.get_function_name(doc) # check for same doc if (function_name_to_search == func_name) and (file_name == function_file_name): - continue + continue # same function name different files ? # if (function_name_to_search == func_name): - # logger.debug(f"same func name {function_name_to_search}") - - if func_name == "main": - file_path = Path(function_file_name) + # logger.debug(f"same func name {function_name_to_search}") + + if func_name == "main": + file_path = Path(function_file_name) # Get all the parts of the path path_parts = file_path.parts if self.language_parser.dir_name_for_3rd_party_packages() in path_parts: @@ -408,14 +333,16 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack type_documents=self.documents_of_types, callee_function_file_name=function_file_name, fields_of_types=self.types_classes_fields_mapping, - functions_local_variables_index=self.functions_local_variables_index) - + functions_local_variables_index=self.functions_local_variables_index, + documents_of_functions= + self.documents_of_functions) + if found and self.language_parser.is_call_allowed( pkg_docs, doc, document_function): log_entries.append((file_name, func_name, function_name_to_search)) - relevant_docs_to_search_in.append(doc) + relevant_docs_to_search_in.append(doc) except ValueError as ex: logger.error("doc %s / %s", doc, ex) - + loop_elapsed = time.time() - loop_start logger.debug(f"[PROFILE] __find_caller_functions main for-loop took {loop_elapsed:.3f} seconds") # logger.debug table-style summary @@ -430,13 +357,13 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack logger.debug("-" * 100) total_elapsed = time.time() - total_start logger.debug(f"[PROFILE] __find_caller_functions total time took {total_elapsed:.3f} seconds") - + # If didn't find a matching caller function document, returns None. return relevant_docs_to_search_in - - def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, - current_package_name: str) -> tuple[List[Document], bool]: + + def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + current_package_name: str) -> tuple[List[Document], bool]: # main loop. file_counter = 0 q = deque() @@ -451,14 +378,14 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct logger.debug("get_relevant_documents: invalid function %s", target_doc.metadata['source']) continue function_name = self.language_parser.get_function_name(target_doc) - + function_file = target_doc.metadata.get('source') hashed_value = calculate_hashable_string_for_function(function_file, function_name) if hashed_value in self.last_visited: continue self.last_visited[hashed_value] = 1 - logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", + logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", file_counter, target_doc.metadata['source'], function_name, target_pkg, len(q)) file_counter += 1 @@ -470,7 +397,7 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct logger.debug(f"[PROFILE] __find_caller_functions took {sub_elapsed:.3f} seconds") # If found, then add it to path if found_documents is not None: - for doc_candidate in found_documents: + for doc_candidate in found_documents: # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(doc_candidate): matching_documents.append(doc_candidate) @@ -481,24 +408,24 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct # function in input package else: q.append(doc_candidate) - + if self.found_path: break loop_elapsed = time.time() - loop_start logger.debug(f"[PROFILE] Main loop in get_relevant_C_documents took {loop_elapsed:.3f} seconds") # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. + # found or not. logger.debug("get_relevant_documents: result %s", self.found_path) logger.debug("get_relevant_documents: docs %s", matching_documents) return matching_documents, self.found_path - - def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + + def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, current_package_name: str) -> tuple[List[Document], bool]: - """Execute depth-first search with backtracking strategy.""" + """Execute depth-first search with backtracking strategy.""" end_loop = False - - # main loop. - while not end_loop: + + # main loop. + while not end_loop: # Find a caller function and containing package in the dependency tree according to hierarchy found_document = self.__find_caller_function_dfs(document_function=target_function_doc, function_package=current_package_name) @@ -529,7 +456,7 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio target_function_doc = matching_documents[-1] current_package_name = self.__determine_doc_package_name(target_function_doc) # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. + # found or not. return matching_documents, self.found_path # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, @@ -556,20 +483,17 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: package_name = package found_package = True break - # If it's , then create a document for it. if found_package: target_function_doc = self.__find_initial_function(function, package_name=package_name, documents=self.documents, - language_parser=self.language_parser, class_name=class_name) if not target_function_doc and self.language_parser.get_constructor_method_name(): target_function_doc = self.__find_initial_function(function_name=self.language_parser.get_constructor_method_name(), package_name=package_name, documents=self.documents, - language_parser=self.language_parser, class_name=function) - + # If not, there is a chance that the package is some standard library in the ecosystem. else: # Try to create dummy package for ecosystem standard library function, in such case, build a document for @@ -582,18 +506,17 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: "ecosystem": self.ecosystem}) importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, re.escape(package_name)) - root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. - parents = set([self.language_parser.get_package_names(doc)[1] for doc in importing_docs if + parents = list({self.language_parser.get_package_names(doc)[1] for doc in importing_docs if doc.metadata['source'].startswith( prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[1] - in self.tree_dict.keys()]) + in self.tree_dict.keys()}) for doc in importing_docs: if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): - parents.add(root_package[0]) + parents.append(root_package[0]) break self.tree_dict[package_name] = [parents, []] end_loop = False @@ -606,10 +529,10 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # library else: end_loop = True - logger.error("Cannot find initial function=%s, in package=%s", function, package_name) - if end_loop: + logger.error("Cannot find initial function=%s, in package=%s", function, package_name) + if end_loop: return matching_documents, self.found_path - + if self.language_parser.is_search_algo_dfs(): matching_documents, self.found_path = self._depth_first_search( matching_documents, target_function_doc, current_package_name) @@ -627,31 +550,31 @@ def __determine_doc_package_name(self, target_function_doc): if self.tree_dict.get(package_name, None) is not None][0] def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], - language_parser: LanguageFunctionsParser, class_name: str = None) -> Document: - + class_name: str = None) -> Document: + if self.language_parser.is_search_algo_dfs(): pkg_docs = documents else: pkg_docs = self.sort_docs[package_name] - - relevant_docs = language_parser.filter_docs_by_func_pkg_name(function_name, package_name, pkg_docs) + + relevant_docs = self.language_parser.filter_docs_by_func_pkg_name(function_name, package_name, pkg_docs) if class_name: relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] #for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): - for document in get_functions_for_package(package_name, relevant_docs, language_parser): + for document in self.get_functions_for_package(package_name, relevant_docs): # document_function_calls_input_function = True - if function_name.lower() == language_parser.get_function_name(document).lower(): + if function_name.lower() == self.language_parser.get_function_name(document).lower(): # if language_parser.search_for_called_function(document, callee_function=function_name): package_exclusions.append(document) return document - - if language_parser.create_dummy_for_standard_lib(package_name): + + if self.language_parser.create_dummy_for_standard_lib(package_name): dummy_function_doc = Document(page_content=f"func {function_name + '()' + '{}'}", metadata={ - "source": language_parser.dir_name_for_3rd_party_packages() + "/" + package_name + "/dummy.c", + "source": self.language_parser.dir_name_for_3rd_party_packages() + "/" + package_name + "/dummy.c", "ecosystem": self.ecosystem, "func_name": function_name, "pkg_name": package_name}) @@ -683,3 +606,18 @@ def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str] results.append(current_level) return results + + def function_called_from_caller_body(self, document: Document, function_to_search: str) -> bool: + if function_to_search.strip() == "": + return True + function_word = self.language_parser.get_function_reserved_word() + func_header_template = rf"${function_word} (\(.*\))?\s?${function_to_search}" + return (re.search(pattern=function_to_search, string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE) + # verify caller function or method is not the function + and not re.search(pattern=func_header_template, + string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE)) + + def document_belongs_to_package(self, document: Document, package_name: str) -> bool: + return self.language_parser.is_a_package(package_name, document) \ No newline at end of file diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever_base.py b/src/vuln_analysis/utils/chain_of_calls_retriever_base.py new file mode 100644 index 000000000..d9a68deec --- /dev/null +++ b/src/vuln_analysis/utils/chain_of_calls_retriever_base.py @@ -0,0 +1,111 @@ +from abc import abstractmethod, ABC +from pathlib import Path +from typing import List, Optional + +from langchain_core.documents import Document + +from vuln_analysis.utils.dep_tree import DependencyTree, Ecosystem +from vuln_analysis.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +PARENTS_INDEX = 0 + +EXCLUSIONS_INDEX = 1 + +METHOD_EXCLUSIONS_INDEX = 2 + +from vuln_analysis.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +def calculate_hashable_string_for_function(function_file_name: str, function_name_to_search: str) -> str: + return f"{function_file_name};{function_name_to_search}" + +def get_extension_of_file(file_path: str): + extension_start = file_path.rfind(".") + return file_path[extension_start:] + +def is_function_callable(document: Document, language_parser: LanguageFunctionsParser, callee_function_file_name: str) -> bool: + return (language_parser.is_exported_function(document) or + document.metadata['source'].lower() == callee_function_file_name.lower()) + +class ChainOfCallsRetrieverBase(ABC): + last_visited_parent_package_indexes: Optional[dict] + documents: Optional[List[Document]] + documents_of_full_sources: Optional[dict] + documents_of_types: Optional[list] + documents_of_functions: list | None + language_parser: Optional[LanguageFunctionsParser] + dependency_tree: Optional[DependencyTree] + tree_dict: Optional[dict] + supported_packages: Optional[list[str]] + ecosystem: Optional[Ecosystem] + manifest_path: Optional[Path] + package_name: Optional[str] + found_path: Optional[bool] + types_classes_fields_mapping: Optional[dict[tuple, list[tuple]]] + functions_local_variables_index: Optional[dict[str, dict]] + sort_docs: dict[str, list[Document]] + + def get_language_parser(self): + return self.language_parser + + def get_documents(self): + return self.documents + + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all + # excluded functions, and all function that their body doesn't contain the target function name to search for. + @abstractmethod + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + pass + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + @abstractmethod + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + pass + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + @abstractmethod + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) ->list[str]: + pass + + @abstractmethod + def function_called_from_caller_body(self, document, function_to_search) -> bool: + pass + + def get_functions_for_package(self, package_name: str, documents: list[Document], + callee_function_file_name: str = "", function_to_search: str = "", sources_location_packages=True): + # Retrieve documents of packages only ( functions + code) + if sources_location_packages: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (not self.language_parser.is_root_package(document) and + document.metadata.get('content_type') == 'functions_classes' and + self.language_parser.is_function(document) and + is_function_callable(document, self.language_parser, callee_function_file_name) and + self.language_parser.is_supported_file_extensions(doc_extension) and + self.document_belongs_to_package(document, package_name) and + self.function_called_from_caller_body(document, function_to_search)): + yield document + # Retrieve documents of application only ( functions + code) + else: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (self.language_parser.is_root_package(document) + and self.language_parser.is_supported_file_extensions(doc_extension) + and self.function_called_from_caller_body(document, function_to_search)): + if (self.language_parser.is_script_language() or + (document.metadata.get('content_type') == 'functions_classes' and + self.language_parser.is_function(document))): + yield document + + @abstractmethod + def document_belongs_to_package(self, document: Document, + package_name: str) -> bool: + pass \ No newline at end of file diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever_factory.py b/src/vuln_analysis/utils/chain_of_calls_retriever_factory.py new file mode 100644 index 000000000..9c9e4d10a --- /dev/null +++ b/src/vuln_analysis/utils/chain_of_calls_retriever_factory.py @@ -0,0 +1,33 @@ +from pathlib import Path +from typing import List + +from langchain_core.documents import Document + +from vuln_analysis.data_models.input import SourceDocumentsInfo +from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from vuln_analysis.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from vuln_analysis.utils.dep_tree import Ecosystem +from vuln_analysis.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever + +def get_chain_of_calls_retriever(ecosystem: Ecosystem, documents: List[Document], manifest_path: Path, query: str, + code_source_info: SourceDocumentsInfo) -> ChainOfCallsRetrieverBase: + """ + Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated + with the ecosystem. + :param code_source_info: + :param query: + :param manifest_path: + :param documents: + :param ecosystem: the desired programming language parser. + :return: + The right language functions parser associated to the ecosystem, if not exists, return ABC parent that + doesn't do anything + """ + if ecosystem == Ecosystem.JAVA: + return JavaChainOfCallsRetriever(documents, + ecosystem, + manifest_path, + query, + code_source_info) + else: + return ChainOfCallsRetriever(documents, ecosystem, manifest_path) \ No newline at end of file diff --git a/src/vuln_analysis/utils/data_utils.py b/src/vuln_analysis/utils/data_utils.py index 1acab1197..dd74fcb25 100644 --- a/src/vuln_analysis/utils/data_utils.py +++ b/src/vuln_analysis/utils/data_utils.py @@ -14,7 +14,8 @@ # limitations under the License. import json -import logging +import os +import pickle import typing import pandas as pd @@ -25,8 +26,14 @@ from vuln_analysis.data_models.cve_intel import CveIntel from vuln_analysis.logging.loggers_factory import LoggingFactory + logger = LoggingFactory.get_agent_logger(__name__) +PathLike = typing.Union[str, os.PathLike] + +DEFAULT_PICKLE_CACHE_DIRECTORY: PathLike = "./.cache/am_cache/pickle" +DEFAULT_GIT_DIRECTORY: PathLike = "./.cache/am_cache/git" +VDB_DIRECTORY: PathLike = "./.cache/am_cache/vdb" def to_serializable_object(obj: typing.Any): """ @@ -100,3 +107,36 @@ def merge_intel_and_plugin_data_convert_to_dataframe(intel_list: list[CveIntel]) x.parse_plugin_data(x.plugins_intel_data)) ] ], sep="_") + +def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: typing.Any, documents_name: str = ""): + cached_documents_path = \ + (f"{base_pickle_dir}/" + f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" + f"{repo_ref}") + + if documents_name: + cached_documents_path += "-" + documents_name + + logger.debug(f"Saved loaded Documents into Cache => {cached_documents_path}") + with open(cached_documents_path, 'wb') as doc_file: # open a text file + pickle.dump(documents, doc_file) + +def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> tuple: + if not os.path.exists(base_pickle_dir): + os.makedirs(base_pickle_dir) + + cached_documents_path = \ + (f"{base_pickle_dir}/" + f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" + f"{repo_ref}") + + if documents_name: + cached_documents_path += "-" + documents_name + + if os.path.isfile(cached_documents_path): + with open(cached_documents_path, 'rb') as doc_file: + documents = pickle.load(doc_file) # deserialize using load() + logger.debug(f"Cache Hit on Documents, Loaded existing docs at {cached_documents_path}") + return documents, True + + return [], False diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 69c8b2e94..1f9eec046 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -7,15 +7,15 @@ from collections import defaultdict from enum import Enum from pathlib import Path -from typing import Any, Optional +from typing import Any, List, Dict, Tuple, Optional -from langchain_core.documents import Document from packaging.specifiers import SpecifierSet from tqdm import tqdm import logging import ast import json +import zipfile from vuln_analysis.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from contextlib import contextmanager @@ -24,6 +24,7 @@ C_DEP_LIBS_NAME = "rpm_libs" from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager from vuln_analysis.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.java_utils import add_missing_jar_string, is_maven_gav logger = LoggingFactory.get_agent_logger(__name__) @@ -133,7 +134,7 @@ def should_exclude_directory(self, dir_path: str) -> bool: """Check if a directory should be excluded""" path_lower = dir_path.lower() return any(pattern in path_lower for pattern in self.exclude_patterns) - + def relative_matches_end(self, full_path, rel_path): # Normalize to avoid redundant slashes or ../ @@ -443,7 +444,7 @@ def _batch_resolve_header( # Build a sort of "upside down" tree - a dict containing mapping of each # package to a list of all consuming packages def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: - + source_path_obj = Path(manifest_path) for subdir in ["include", "src"]: candidate = source_path_obj / subdir @@ -464,7 +465,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: tree[self.C_STANDARD_LIB] = [] for key in tree: tree[self.C_STANDARD_LIB].append(key) - + return tree # Return only package name, without version @@ -490,7 +491,7 @@ def install_dependencies(self, manifest_path: Path): if container_image == RPMDependencyManager.get_instance().container_image: logger.info(f"Container image {container_image} already downloaded") return - + # Check if container source download is enabled use_container_sources = os.getenv('USE_CONTAINER_SOURCES', 'true').lower() == 'true' if use_container_sources: @@ -505,7 +506,7 @@ def install_dependencies(self, manifest_path: Path): file_data = RPMDependencyManager.get_instance().container_image with open(os.path.join(download_path, "container_image.txt"), "w") as f: f.write(file_data) - + def _install_dependencies_from_container(self, manifest_path: Path, download_path: str): """Install dependencies from container sources using skopeo""" @@ -513,64 +514,64 @@ def _install_dependencies_from_container(self, manifest_path: Path, download_pat container_image = RPMDependencyManager.get_instance().container_image if not container_image: raise ValueError("CONTAINER_IMAGE must be set when USE_CONTAINER_SOURCES=true") - + username = os.getenv('REGISTRY_REDHAT_USERNAME') password = os.getenv('REGISTRY_REDHAT_PASSWORD') if not username or not password: raise ValueError( "REGISTRY_REDHAT_USERNAME and REGISTRY_REDHAT_PASSWORD must be set when USE_CONTAINER_SOURCES=true" ) - + logger.info(f"Downloading container sources for: {container_image}") - + # Setup authentication self._setup_skopeo_auth(username, password) - + # Download container sources container_sources_dir = self._download_container_sources(container_image, manifest_path) - + # Extract and organize RPMs self._extract_and_organize_rpms(container_sources_dir, download_path,container_image) - + # Cleanup temporary files self._cleanup_temp_files(container_sources_dir) def _setup_skopeo_auth(self, username: str, password: str): """Setup skopeo authentication for registry.redhat.io""" logger.info("Setting up skopeo authentication...") - + cmd = [ 'skopeo', 'login', 'registry.redhat.io', '--username', username, '--password', password ] - + result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: raise RuntimeError(f"Skopeo login failed: {result.stderr}") - + logger.info("Skopeo authentication successful") def _download_container_sources(self, container_image: str, manifest_path: Path) -> Path: """Download container source using skopeo with caching support""" # Create source image name (append -source) source_image = f"{container_image}-source" - + # Create clean directory name container_dir = source_image.replace('/', '-').replace(':', '-').replace('@', '-') container_sources_dir = manifest_path / "container_sources" / container_dir - + # Create directory container_sources_dir.mkdir(parents=True, exist_ok=True) - + logger.info(f"Downloading container source: {source_image}") - + cmd = [ - 'skopeo', 'copy', + 'skopeo', 'copy', f'docker://{source_image}', f'dir:{container_sources_dir}' ] - + result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: raise RuntimeError(f"Skopeo copy failed: {result.stderr}") @@ -581,17 +582,17 @@ def _download_container_sources(self, container_image: str, manifest_path: Path) def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: str, container_image: str): """Extract RPMs from container sources and organize them""" logger.info("Extracting RPMs from container sources...") - + metadata_dir = container_sources_dir / "metadata" # Create download path if it doesn't exist os.makedirs(metadata_dir, exist_ok=True) - + # move metadata.json to metadata directory metadata_file = container_sources_dir / "manifest.json" shutil.move(metadata_file, metadata_dir / "manifest.json") version_file = container_sources_dir / "version" shutil.move(version_file, metadata_dir / "version") - + # Extract all tar files in the container sources directory for tar_file in container_sources_dir.glob("*"): if tar_file.is_dir(): @@ -600,30 +601,30 @@ def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: logger.info(f"Extracting: {tar_file.name}") try: result = subprocess.run( - ['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], + ['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], capture_output=True, text=True, check=False ) if result.returncode != 0: logger.warning(f"Failed to extract {tar_file}: {result.stderr}") except Exception as e: logger.warning(f"Error extracting {tar_file}: {e}") - + # Extract app name from container image (same logic as sync_rpms.sh) app_name = self._extract_app_name_from_container_image(container_image) logger.info(f"Extracted app name: {app_name}") - + # Find and copy RPMs to the download path rpm_dir = container_sources_dir / "rpm_dir" if rpm_dir.exists(): logger.info(f"Found RPM directory: {rpm_dir}") - + # Copy RPMs to the standard download path, excluding the container app for rpm_file in rpm_dir.rglob("*.rpm"): # Skip RPMs that belong to the container app if rpm_file.name.startswith(f"{app_name}-"): logger.debug(f"Skipping container app RPM: {rpm_file.name}") continue - + try: subprocess.run(['cp', str(rpm_file), str(download_path)], check=True) logger.debug(f"Copied RPM: {rpm_file.name} -> {download_path}") @@ -645,11 +646,11 @@ def _extract_app_name_from_container_image(self, container_image: str) -> str: def _cleanup_temp_files(self, container_sources_dir: Path): """Clean up temporary container source files (with cache preservation option)""" preserve_cache = os.getenv('PRESERVE_CONTAINER_CACHE', 'false').lower() == 'true' - + if preserve_cache: logger.info("Preserving container source cache (PRESERVE_CONTAINER_CACHE=true)") return - + logger.info("Cleaning up temporary files...") try: # Remove the container sources directory @@ -779,8 +780,8 @@ def determine_go_version(manifest_path): @staticmethod def get_go_mod_graph_tree(manifest_path) -> str: """ - run the go mod graph command to retirive all dependencies from go.mod, - and their hirearchy + run the go mod graph command to retrieve all dependencies from go.mod, + and their hierarchy :param manifest_path: path to the go.mod manifest. :return: a string representation of the go.mod graph, including dependencies and their hierarchy. @@ -833,6 +834,132 @@ def extract_package_name(self, package_name: str) -> str: else: return package_name +class JavaDependencyTreeBuilder(DependencyTreeBuilder): + + def __init__(self, query: str): + self._query = query + + def install_dependencies(self, manifest_path: Path): + source_path = "dependencies-sources" + subprocess.run(["mvn", "dependency:copy-dependencies", "-Dclassifier=sources", + f"-DoutputDirectory={source_path}"], cwd=manifest_path) + + full_source_path = manifest_path / source_path + for jar in full_source_path.glob("*-sources.jar"): + dest = full_source_path / jar.stem # folder named after jar + + if not dest.exists(): + dest.mkdir(exist_ok=True) + with zipfile.ZipFile(jar, "r") as zf: + zf.extractall(dest) + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + dependency_file = manifest_path / "dependency_tree.txt" + package_name = self._query.split(',')[0] + + if is_maven_gav(package_name): + with open(dependency_file, "w") as f: + subprocess.run(["mvn", "dependency:tree", + f"-Dincludes={add_missing_jar_string(package_name)}", + "-Dverbose"], cwd=manifest_path, stdout=f, check=True) + else: + with open(dependency_file, "w") as f: + subprocess.run(["mvn", "dependency:tree", + "-Dverbose"], cwd=manifest_path, stdout=f, check=True) + + with dependency_file.open("r", encoding="utf-8") as f: + lines = f.readlines() + + parent, graph = self.__build_upside_down_dependency_graph(lines) + + # Mark the top level for + graph[parent] = [ROOT_LEVEL_SENTINEL] + + return graph + + def __build_upside_down_dependency_graph( + self, dependency_lines: List[str] + ) -> Tuple[str, Dict[str, List[str]]]: + root: str = "" + stack: List[str] = [] + graph_sets: Dict[str, set] = {} + + for line in dependency_lines: + depth, coord = self.__parse_dependency_line(line) + if depth is None or coord is None: + continue + + if depth == 0: + # start (or restart) a root line + if not root: + root = coord + stack = [coord] + graph_sets.setdefault(coord, set()) + continue + + # trim stack so that parent is at depth-1 + while len(stack) > depth: + stack.pop() + + parent = stack[-1] if stack else None + if parent is not None: + graph_sets.setdefault(coord, set()).add(parent) + graph_sets.setdefault(parent, set()) + else: + graph_sets.setdefault(coord, set()) + + stack.append(coord) + + # no sorting, preserve insertion order of sets -> lists + graph: Dict[str, List[str]] = {k: list(v) for k, v in graph_sets.items()} + return root, graph + + + def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[str]]: + if not line.startswith("[INFO]"): + return None, None + + # Keep indentation blocks; Maven prints exactly one space after "[INFO]" + s = line[6:] + if s.startswith(" "): + s = s[1:] + + # Skip non-tree lines early + if (not s + or s.startswith(("---", "BUILD", "Scanning", "Finished", "Total time")) + or ":" not in s): + return None, None + + # indent blocks ("| " or " ") + optional "+- " or "\- " + rest + m = re.match(r'^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$', s) + if not m: + return None, None + + # Each indent block is 3 chars; add 1 if a branch token is present + depth = (len(m.group('indent')) // 3) + (1 if m.group('branch') else 0) + rest = m.group('rest').strip() + + # First token up to whitespace or ')', optionally starting with '(' + m2 = re.match(r'^\(?([^\s\)]+)\)?', rest) + if not m2: + return None, None + + token = m2.group(1) # e.g., io.foo:bar:jar:1.2.3:compile + parts = token.split(':') + + # Drop trailing Maven scope if present + scopes = {'compile', 'runtime', 'test', 'provided', 'system', 'import'} + if parts and parts[-1] in scopes: + parts = parts[:-1] + + # Expect group:artifact:type:(classifier:)version — return without the type + if len(parts) >= 4: + group, artifact = parts[0], parts[1] + version = parts[-1] + coord = f"{group}:{artifact}:{version}" + return depth, coord + + return None, None class PythonDependencyTreeBuilder(DependencyTreeBuilder): @@ -1003,11 +1130,11 @@ def install_dependency(self, dependency, repo_path): if not res: logger.warning(f'Failed to install dependency {dependency}') - -def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTreeBuilder: +def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "") -> DependencyTreeBuilder: """ A Factory method function that gets an programming language as argument, and returns a DependencyTreeBuilder' instance related to the ecosystem. + :param query: The query containing the package name,function name. :param programming_language: ecosystem to get the dependency tree builder for. :return: DependencyTreeBuilder instance for the ecosystem, If not @@ -1017,6 +1144,8 @@ def get_dependency_tree_builder(programming_language: Ecosystem) -> DependencyTr return GoDependencyTreeBuilder() elif programming_language.value == Ecosystem.PYTHON.value: return PythonDependencyTreeBuilder() + elif programming_language.value == Ecosystem.JAVA.value: + return JavaDependencyTreeBuilder(query=query) elif programming_language.value == Ecosystem.C_CPP.value: return CCppDependencyTreeBuilder() else: @@ -1034,7 +1163,7 @@ class DependencyTree: builder: DependencyTreeBuilder ecosystem: Ecosystem - def __init__(self, ecosystem: Ecosystem): + def __init__(self, ecosystem: Ecosystem, query: str = ""): """ :param ecosystem: the ecosystem of the dependency tree. @@ -1042,7 +1171,7 @@ def __init__(self, ecosystem: Ecosystem): self.ecosystem = ecosystem try: # gets a dependency tree builder object based on ecosystem. - self.builder = get_dependency_tree_builder(ecosystem) + self.builder = get_dependency_tree_builder(ecosystem, query) except ValueError as err: logger.warning("Unable to build dependency tree - %s", str(err)) self.builder = None diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/vuln_analysis/utils/document_embedding.py index e01a600a5..9ccd55d90 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/vuln_analysis/utils/document_embedding.py @@ -15,9 +15,7 @@ import copy import json -import logging import os -import pickle import sys import time import typing @@ -37,12 +35,14 @@ from langchain_core.document_loaders.blob_loaders import Blob from vuln_analysis.data_models.input import SourceDocumentsInfo +from vuln_analysis.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY, DEFAULT_GIT_DIRECTORY, \ + VDB_DIRECTORY, PathLike from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods from vuln_analysis.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods +from vuln_analysis.utils.java_segmenters_with_methods import JavaSegmenterWithMethods from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader from vuln_analysis.utils.git_utils import sanitize_git_url_for_path -from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from vuln_analysis.logging.loggers_factory import LoggingFactory from vuln_analysis.utils.c_segmenter_custom import CSegmenterExtended @@ -50,40 +50,8 @@ if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover -PathLike = typing.Union[str, os.PathLike] - - - logger = LoggingFactory.get_agent_logger(__name__) - -def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref) -> tuple: - if not os.path.exists(base_pickle_dir): - os.makedirs(base_pickle_dir) - - cached_documents_path = \ - (f"{base_pickle_dir}/" - f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" - f"{repo_ref}") - if os.path.isfile(cached_documents_path): - with open(cached_documents_path, 'rb') as doc_file: - documents = pickle.load(doc_file) # deserialize using load() - logger.debug(f"Cache Hit on Documents, Loaded existing docs at {cached_documents_path}") - return documents, True - - return [], False - - -def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: list[Document]): - cached_documents_path = \ - (f"{base_pickle_dir}/" - f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" - f"{repo_ref}") - logger.debug(f"Saved loaded Documents into Cache => {cached_documents_path}") - with open(cached_documents_path, 'wb') as doc_file: # open a text file - pickle.dump(documents, doc_file) - - class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter): """ A version of langchain's RecursiveCharacterTextSplitter that supports multiple languages. @@ -146,6 +114,7 @@ class ExtendedLanguageParser(LanguageParser): } additional_segmenters["go"] = GoSegmenterWithMethods additional_segmenters["python"] = PythonSegmenterWithClassesMethods + additional_segmenters["java"] = JavaSegmenterWithMethods additional_segmenters["c"] = CSegmenterExtended LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { @@ -256,9 +225,9 @@ class DocumentEmbedding: repositories and chunked into smaller pieces for embedding. """ - def __init__(self, *, embedding: "Embeddings", vdb_directory: PathLike = "./.cache/am_cache/vdb", - git_directory: PathLike = "./.cache/am_cache/git", chunk_size: int = 800, chunk_overlap: int = 160, - pickle_cache_directory: PathLike = "./.cache/am_cache/pickle"): + def __init__(self, *, embedding: "Embeddings", vdb_directory: PathLike = VDB_DIRECTORY, + git_directory: PathLike = DEFAULT_GIT_DIRECTORY, chunk_size: int = 800, chunk_overlap: int = 160, + pickle_cache_directory: PathLike = DEFAULT_PICKLE_CACHE_DIRECTORY): """ Create a new DocumentEmbedding instance. diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py index c85ba2c70..0a8dfe109 100644 --- a/src/vuln_analysis/utils/function_name_extractor.py +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -1,8 +1,7 @@ import os import re -from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever -from vuln_analysis.utils.dep_tree import Ecosystem +from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetrieverBase from vuln_analysis.logging.loggers_factory import LoggingFactory @@ -66,7 +65,7 @@ def handle_argument(param: str) -> str: class FunctionNameExtractor: - def __init__(self, coc_retriever: ChainOfCallsRetriever): + def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.lang_parser = coc_retriever.language_parser self.docs = coc_retriever.documents self.ecosystem = coc_retriever.ecosystem @@ -102,7 +101,6 @@ def fetch_list(self, query: str) -> list[str]: self.lang_parser.get_package_name(doc, package) or infer_if_short_package_name_match(package, self.lang_parser.get_package_names(doc), - final_package)] the_final_package = list(final_package) # If short package name was inferred , use it also in searching all the way diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 2fc509563..9f28caa2d 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -16,8 +16,8 @@ import difflib -from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever from vuln_analysis.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase from vuln_analysis.utils.dep_tree import Ecosystem from vuln_analysis.utils.standard_library_cache import StandardLibraryCache from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper @@ -31,7 +31,7 @@ class FunctionNameLocator: Initially designed for C/C++ but can be extended for other languages. """ - def __init__(self, coc_retriever: ChainOfCallsRetriever): + def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.lang_parser = coc_retriever.language_parser self.coc_retriever = coc_retriever self.is_std_package = False @@ -224,6 +224,8 @@ async def locate_functions(self, query: str) -> list[str]: return self.common_flow_control(function, package_docs) case Ecosystem.GO.value: return self.common_flow_control(function, package_docs) + case Ecosystem.JAVA.value: + return self.common_flow_control(function.rsplit(".")[-1], package_docs) case _: return self.common_flow_control(function, package_docs) diff --git a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py index 1a5c7b04e..992ed6cfe 100644 --- a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py @@ -1,3 +1,5 @@ +from typing import Tuple, List + from langchain_core.documents import Document from .lang_functions_parsers import LanguageFunctionsParser import re @@ -462,7 +464,7 @@ def create_map_of_local_vars(self, functions_methods_documents: list[Document]) # This method returns (for the given ecosystem) , a mapping of all types/classes in all packages to a dict # containing mapping of all fields names in the type/class to their defining types. # a list of - def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: types_mapping = dict() for _type in types: #parse the document @@ -521,7 +523,9 @@ def search_for_called_function( type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict] + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document]=None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None ) -> bool: """ Returns True if caller_function calls callee_function (directly or via struct/function pointer). @@ -703,9 +707,6 @@ def create_dummy_for_standard_lib(self,package_name: str)-> bool: def get_dummy_function(self, function_name): return f"void {function_name}() {{}}" - def is_same_package(self, package_name_from_input, package_name_from_tree): - return package_name_from_input.lower() == package_name_from_tree.lower() - def is_call_allowed(self, pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: caller_pkg = self.get_package_names(caller_function)[0] callee_pkg = self.get_package_names(callee_function)[0] diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 94a677b08..423bad8b2 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -1,11 +1,10 @@ import os import re +from typing import Tuple, List from langchain_core.documents import Document from .lang_functions_parsers import LanguageFunctionsParser -from ..dep_tree import Ecosystem -from ..standard_library_cache import StandardLibraryCache EMBEDDED_TYPE = "embedded_type" @@ -22,8 +21,6 @@ SLICE_TYPES_REGEX = r"\[\][a-zA-Z0-9]+ " -# def parse_type_struct(fieldName: str, ) -> : dict[str,str] - def get_package_name_file(function: Document): content = function.page_content index_of_package = content.find("package") @@ -32,91 +29,11 @@ def get_package_name_file(function: Document): return content[index_of_package + 8:index_of_end_of_line].strip() -def check_types_from_callee_package(params: list[tuple], type_documents: list[Document], callee_package: str, - code_documents: dict[str, Document], parameter: str, - callee_function_file_name: str) -> bool: - callee_function_doc = code_documents.get(callee_function_file_name) - callee_function_file_package_name = get_package_name_file(callee_function_doc) - for param in params: - param_name, param_type = param - if param_name == parameter: - param_type_stripped = str(param_type).replace("*", "").replace("&", "") - parts = param_type_stripped.split(".") - # Only type without package - if len(parts) == 1: - for the_type in type_documents: - if the_type.page_content.startwith(f"type {parts[0]}"): - code_with_type_file = code_documents.get(the_type.metadata['source']) - type_file_package_name = get_package_name_file(code_with_type_file) - if type_file_package_name == callee_function_file_package_name: - return True - # type with package qualifier - else: - for the_type in type_documents: - if the_type.page_content.startwith(f"type {parts[1]}"): - code_with_type_file = code_documents.get(the_type.metadata['source']) - package_match = handle_imports(code_with_type_file, parts[0], callee_package) - return package_match - else: - return False - - -def handle_imports(code_content: str, identifier: str, callee_package: str) -> bool: - start_search_for_import = max(code_content.rfind("//import"), code_content.rfind("// import")) - if start_search_for_import > -1: - after_last_import_comment_index = start_search_for_import + 4 - else: - after_last_import_comment_index = 0 - after_last_import_comment = code_content[after_last_import_comment_index:] - first_import = after_last_import_comment.find("import") - last_import = after_last_import_comment.rfind("import") - # return after_last_import_comment, first_import, last_import - if first_import == last_import: - block_of_import = after_last_import_comment[first_import + 6:] - after_left_bracket = block_of_import.find("(") - before_right_bracket = block_of_import.find(")") - inside_block = block_of_import[after_left_bracket + 1: before_right_bracket] - position_of_identifier_in_import_block = inside_block.find(identifier) - if position_of_identifier_in_import_block != -1: - previous_end_of_line = inside_block[:position_of_identifier_in_import_block].rfind( - os.linesep) - alias_package_extended = inside_block[previous_end_of_line:].strip() - index_of_end_of_line = alias_package_extended.find(os.linesep) - row_of_identifier = alias_package_extended[:index_of_end_of_line] - if len(row_of_identifier.split(" ")) > 1: - package_name_of_alias = row_of_identifier.split(" ")[1].strip('"') - else: - package_name_of_alias = row_of_identifier.split(" ")[0].strip('"') - - if (package_name_of_alias.strip().lower() == callee_package.strip().lower() - or callee_package.strip().lower() in package_name_of_alias.strip().lower()): - return True - - # Checks if there is a dedicated import with alias and imported package name - else: - identifier_import_position = code_content.find(f"import {identifier}") - if identifier_import_position != -1: - start_of_package_name = code_content[identifier_import_position + len("import ") - + len(identifier):] - index_of_end_of_line = start_of_package_name.find(os.linesep) - package_name_to_check = start_of_package_name[:index_of_end_of_line] - if package_name_to_check.strip().lower() == callee_package.strip().lower(): - return True - # import without alias, in this case maybe package name contain alias - else: - # re.search(regex, caller_function_body, re.MULTILINE) - matching = re.search(rf"import [\'\"].*{identifier}[\'\"]", code_content) - if matching and matching.group(0): - import_line = code_content[matching.start():] - import_package_line = import_line[:import_line.find(os.linesep)].strip() - package_name = import_package_line.split(r"\s")[1] - if package_name.strip().lower() == callee_package.strip().lower(): - return True - return False - - class GoLanguageFunctionsParser(LanguageFunctionsParser): + def is_same_package(self, package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() in package_name_from_tree.lower() + def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}() {{}}" @@ -309,7 +226,7 @@ def get_type_name(self, the_type: Document) -> str: parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content return parts[1] - def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: types_mapping = dict() for the_type in types: # multiple types in a single block declaration @@ -413,8 +330,8 @@ def is_searchable_file_name(self, function: Document) -> bool: file_path = str(function.metadata['source']) return "test" not in file_path[file_path.rfind("/") + 1:].split(".")[0].lower() - def is_function(self, function: Document) -> bool: - return function.page_content.startswith("func") + def is_function(self, doc: Document) -> bool: + return doc.page_content.startswith(self.get_function_reserved_word()) def supported_files_extensions(self) -> list[str]: return [".go"] @@ -466,7 +383,9 @@ def search_for_called_function(self, caller_function: Document, callee_function_ callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict]) -> bool: + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document]=None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> bool: index_of_function_opening = caller_function.page_content.index("{") index_of_function_closing = caller_function.page_content.rfind("}") caller_function_body = str( @@ -540,7 +459,7 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum if matching and matching.group(0): return True - identifier_is_imported = handle_imports(code_content, identifier, callee_package) + identifier_is_imported = self.is_package_imported(code_content, identifier, callee_package) if identifier_is_imported: return True @@ -608,4 +527,57 @@ def is_root_package(self, function: Document) -> bool: return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) def is_comment_line(self, line: str) -> bool: - return line.strip().startswith("//") + return line.strip().startswith("//") + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + start_search_for_import = max(code_content.rfind("//import"), code_content.rfind("// import")) + if start_search_for_import > -1: + after_last_import_comment_index = start_search_for_import + 4 + else: + after_last_import_comment_index = 0 + after_last_import_comment = code_content[after_last_import_comment_index:] + first_import = after_last_import_comment.find("import") + last_import = after_last_import_comment.rfind("import") + # return after_last_import_comment, first_import, last_import + if first_import == last_import: + block_of_import = after_last_import_comment[first_import + 6:] + after_left_bracket = block_of_import.find("(") + before_right_bracket = block_of_import.find(")") + inside_block = block_of_import[after_left_bracket + 1: before_right_bracket] + position_of_identifier_in_import_block = inside_block.find(identifier) + if position_of_identifier_in_import_block != -1: + previous_end_of_line = inside_block[:position_of_identifier_in_import_block].rfind( + os.linesep) + alias_package_extended = inside_block[previous_end_of_line:].strip() + index_of_end_of_line = alias_package_extended.find(os.linesep) + row_of_identifier = alias_package_extended[:index_of_end_of_line] + if len(row_of_identifier.split(" ")) > 1: + package_name_of_alias = row_of_identifier.split(" ")[1].strip('"') + else: + package_name_of_alias = row_of_identifier.split(" ")[0].strip('"') + + if (package_name_of_alias.strip().lower() == callee_package.strip().lower() + or callee_package.strip().lower() in package_name_of_alias.strip().lower()): + return True + + # Checks if there is a dedicated import with alias and imported package name + else: + identifier_import_position = code_content.find(f"import {identifier}") + if identifier_import_position != -1: + start_of_package_name = code_content[identifier_import_position + len("import ") + + len(identifier):] + index_of_end_of_line = start_of_package_name.find(os.linesep) + package_name_to_check = start_of_package_name[:index_of_end_of_line] + if package_name_to_check.strip().lower() == callee_package.strip().lower(): + return True + # import without alias, in this case maybe package name contain alias + else: + # re.search(regex, caller_function_body, re.MULTILINE) + matching = re.search(rf"import [\'\"].*{identifier}[\'\"]", code_content) + if matching and matching.group(0): + import_line = code_content[matching.start():] + import_package_line = import_line[:import_line.find(os.linesep)].strip() + package_name = import_package_line.split(r"\s")[1] + if package_name.strip().lower() == callee_package.strip().lower(): + return True + return False \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py new file mode 100644 index 000000000..294fc0061 --- /dev/null +++ b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py @@ -0,0 +1,2660 @@ +import hashlib +import os +import re +from typing import Dict, Tuple, Any, List, Optional +from tqdm import tqdm + +from langchain_core.documents import Document + +from .lang_functions_parsers import LanguageFunctionsParser +from ..java_utils import extract_jar_name, PRIMITIVE_TYPES, collect_fields_from_types, get_type_name, \ + is_java_type, is_java_method, extract_method_name_with_params, find_function, get_target_class_names, \ + strip_java_generics, JAVA_ANNOTATION_SYMBOL +from ...logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +PARAMETER = "parameter" + +LOCAL_VAR_USAGE = "local_var_usage" +LOCAL_IMPLICIT = "local_implicit" +LOCAL_INDIRECT_TYPES_INDICATIONS = [LOCAL_VAR_USAGE, LOCAL_IMPLICIT] +MODIFIERS = { + "public","protected","private","static","final","abstract","synchronized", + "native","strictfp","default","transient","volatile","sealed","non-sealed" +} + +TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') + +class JavaLanguageFunctionsParser(LanguageFunctionsParser): + def parse_all_type_struct_class_to_fields(self, types: list[Document], inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]]) -> dict[tuple, list[tuple[str, str]]]: + """ + Returns (TypeSimpleName, doc.metadata['source']) -> [(memberName, memberType), ...] + - Enums are ignored entirely. + - Interfaces: returns ONLY fields (constants). No default/static methods. + - Classes/records: returns fields. + """ + results: Dict[Tuple[str, Any], List[Tuple[str, str]]] = {} + + for doc in tqdm(types, total=len(types), desc="Parsing class/interface/enum/record documents for members..."): + src = doc.page_content + source = doc.metadata['source'] + if not src: + continue + + # One pass per document — returns {type_name: [(field, type), ...]} + per_file = collect_fields_from_types(src, include_nested=True, include_anonymous=True) + + for class_name, fields in per_file.items(): + results[(class_name, source)] = fields + + # # TODO add inherited fields support + # for (result_class_name, result_source), fields in results.items(): + # # Get inherited classes + # try: + # list_of_types = inheritance_map[(result_class_name, result_source)] + # except KeyError: + # i = 1 + # + # for (inherited_class_name, inherited_source) in list_of_types: + # if inherited_class_name != result_class_name and inherited_source != result_source: + # inherited_fields = results[(inherited_class_name, inherited_source)] + # fields.extend(inherited_fields) + + return results + + def get_dummy_function(self, function_name): + return f"public void {function_name + '()' + '{}'}" + + def get_class_name_from_class_function(self, func: Document): + """ + Extract the final path segment and strip its last extension. + Works with / or \ path separators. If there's no dot, returns the segment as-is. + """ + source = func.metadata.get('source') + + # get last non-empty segment + tail = re.split(r"[\\/]", source.rstrip("/\\"))[-1] + # drop final extension if present + return tail.rsplit(".", 1)[0] + + def get_function_reserved_word(self) -> str: + return "" + + def get_type_reserved_word(self) -> str: + pass + + def is_searchable_file_name(self, function: Document) -> bool: + pass + + def is_function(self, function: Document) -> bool: + return is_java_method(function.page_content) + + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + def supported_files_extensions(self) -> list[str]: + return [".java"] + + def get_comment_line_notation(self) -> str: + return "//" + + def dir_name_for_3rd_party_packages(self) -> str: + return "dependencies-sources" + + def is_exported_function(self, function: Document) -> bool: + return True + + def get_function_name(self, function: Document) -> str: + # --- constants & tiny char helpers (scoped) --- + JAVA_CTRL = {"if", "for", "while", "switch", "catch", "synchronized", "try"} + # Disallow these as the *first* token of a "type" to avoid false positives (e.g., record header) + DISALLOW_TYPE_HEAD = {"class","interface","enum","record","new","return"} + + WS = " \t\r\n\f\v" + DIG = "0123456789" + IDF_EXTRA = "_$" + + def is_id_start(ch: str) -> bool: + return ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or (ch in IDF_EXTRA) + + def is_id_part(ch: str) -> bool: + return is_id_start(ch) or (ch in DIG) + + def prev_non_ws(s: str, idx: int): + i = idx + while i >= 0 and s[i] in WS: + i -= 1 + return i + + # --- pass 1: strip //, /* */, and string/char literals (keeps layout lightweight) --- + s = function.page_content + n = len(s) + out = [] + i = 0 + while i < n: + c = s[i] + if c == '/' and i + 1 < n and s[i+1] == '/': + i += 2 + while i < n and s[i] != '\n': + i += 1 + out.append('\n' if i < n else '') + if i < n: i += 1 + elif c == '/' and i + 1 < n and s[i+1] == '*': + i += 2 + while i + 1 < n and not (s[i] == '*' and s[i+1] == '/'): + i += 1 + i = min(i + 2, n) + out.append(' ') + elif c in ("'", '"'): + q = c + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == q: + i += 1 + break + else: + i += 1 + out.append(' ') + else: + out.append(c) + i += 1 + s = ''.join(out) + n = len(s) + + # Quick lambda presence flag (after stripping strings/comments) + saw_lambda_arrow = '->' in s + + # --- tiny scanners on cleaned source --- + def skip_ws(idx: int) -> int: + while idx < n and s[idx] in WS: + idx += 1 + return idx + + def skip_ws_back(idx: int) -> int: + while idx >= 0 and s[idx] in WS: + idx -= 1 + return idx + + def match_paren_close(start: int) -> int: + """start at '(' and find matching ')' with nesting inside the parameter list.""" + depth, i = 0, start + while i < n: + ch = s[i] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + def skip_angle(i: int) -> int: + """Skip a balanced <...> block (handles nesting); return index just after the matching '>'.""" + depth, j = 0, i + while j < n: + ch = s[j] + if ch == '<': + depth += 1 + elif ch == '>': + depth -= 1 + if depth == 0: + return j + 1 + j += 1 + return i # malformed + + def skip_annotation(i: int) -> int: + """Skip @Anno and optional (args).""" + j = i + 1 + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + j = skip_ws(j) + if j < n and s[j] == '(': + end = match_paren_close(j) + if end == -1: + return i + j = end + 1 + return j + + def parse_return_type(hstart: int, name_start: int) -> int: + """ + Try to parse a return type in [hstart, name_start). + Return index right after the type (including any array suffixes), + or -1 if no valid type. Ensures there is at least one char (ws or []), + and *only* ws/[] between type end and name (guards against "int x = foo("). + """ + i = skip_ws(hstart) + # annotations + modifiers + while True: + i = skip_ws(i) + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i); continue + j = i + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + if s[i:j] in MODIFIERS: + i = j + continue + break + + i = skip_ws(i) + # leading method type-parameters: + if i < n and s[i] == '<': + new_i = skip_angle(i) + if new_i == i: # malformed + return -1 + i = new_i + i = skip_ws(i) + + # type-use annotations before the return type + while i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i) + i = skip_ws(i) + + # parse 'void' or qualified identifier (with optional generic args) + j = i + if j + 4 <= n and s[j:j+4] == 'void' and (j + 4 >= n or not is_id_part(s[j+4])): + j += 4 + else: + if j < n and is_id_start(s[j]): + t0 = j + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + # The first identifier cannot be one of these keywords + if s[t0:j] in DISALLOW_TYPE_HEAD: + return -1 + # dotted qualifiers + while True: + jj = skip_ws(j) + if jj < n and s[jj] == '.': + j = jj + 1 + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + continue + return -1 + break + else: + return -1 + # optional generic type arguments on the return type + jj = skip_ws(j) + if jj < n and s[jj] == '<': + nx = skip_angle(jj) + if nx == jj: + return -1 + j = nx + + # Optional array suffixes on return type, with arbitrary spacing inside []. + k = j + while True: + kk = skip_ws(k) + if kk + 1 < n and s[kk] == '[': + kk += 1 + kk = skip_ws(kk) + if kk < n and s[kk] == ']': + kk += 1 + k = kk + continue + break + + # Must have a (non-empty) gap between type end and name (constructor guard) + if k >= name_start: + return -1 + + # Gap must contain only whitespace or [] decorations + scan = k + while scan < name_start: + if s[scan] in WS: + scan += 1 + elif s[scan] == '[': + scan += 1 + scan = skip_ws(scan) + if scan < n and s[scan] == ']': + scan += 1 + else: + return -1 + else: + return -1 + return k + + # --- main scan: seek '(' and validate the candidate as a declaration --- + i = 0 + while True: + open_idx = s.find('(', i) + if open_idx == -1: + break # no more candidates + + # candidate name is the identifier immediately before '(' + j = skip_ws_back(open_idx - 1) + if j < 0 or not is_id_part(s[j]): + i = open_idx + 1 + continue + + end = j + 1 + start = j + while start - 1 >= 0 and is_id_part(s[start - 1]): + start -= 1 + name = s[start:end] + + # skip control keywords (if(), for(), etc.) + if name in JAVA_CTRL: + i = open_idx + 1 + continue + + # skip chained calls / annotations like obj.m( or @Anno( + prev = prev_non_ws(s, start - 1) + if prev is not None and s[prev] in ('.', JAVA_ANNOTATION_SYMBOL): + i = open_idx + 1 + continue + + # find matching ')' + close_idx = match_paren_close(open_idx) + if close_idx == -1: + i = open_idx + 1 + continue + + # after ')': optional ws + optional 'throws ...' + then '{' or ';' + p = skip_ws(close_idx + 1) + # lambda guard: "() ->" + if p < n and s[p] == '-' and p + 1 < n and s[p + 1] == '>': + # it's a lambda parameter list, not a method decl + i = open_idx + 1 + continue + + if p + 6 <= n and s[p:p+6] == 'throws' and (p + 6 >= n or s[p+6] in WS): + p = p + 6 + while p < n: + p = skip_ws(p) + ch = s[p] if p < n else '' + if ch in '{;': + break + if ch == '<': + p = skip_angle(p) + elif is_id_start(ch): + # qualified exception with optional generic args and commas + p += 1 + while p < n and is_id_part(s[p]): + p += 1 + while True: + q = skip_ws(p) + if q < n and s[q] == '.': + p = q + 1 + if p < n and is_id_start(s[p]): + p += 1 + while p < n and is_id_part(s[p]): + p += 1 + continue + break + q = skip_ws(p) + if q < n and s[q] == '<': + p = skip_angle(q) + else: + p = q + q = skip_ws(p) + if q < n and s[q] == ',': + p = q + 1 + else: + p = q + else: + p += 1 + p = skip_ws(p) + + if p >= n or s[p] not in '{;': + # likely a call or lambda, not a declaration + i = open_idx + 1 + continue + + # Find header start (after previous ';' or '{' or '}' before the name) + h = start - 1 + while h >= 0 and s[h] not in ';{}': + h -= 1 + h_start = h + 1 + + # Require a valid return type and no extra identifier between type and name + if parse_return_type(h_start, start) == -1: + i = open_idx + 1 + continue + + return name + + # --- fallback: if we saw a lambda arrow anywhere, treat this as a lambda-only snippet --- + if saw_lambda_arrow: + logger.info("get_function_name is called on a lambda function") + return "lambda" # e.g., "o -> o.getParameterTypes().length" TODO handle lambda correctly + + return "" + + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, + callee_function_package: str, code_documents: list[Document], type_documents: list[Document], + callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> bool: + + def _find_matching_paren(s: str, open_idx: int) -> int: + """Find matching ')' for the '(' at open_idx. Ignores strings/char literals.""" + depth = 0 + i = open_idx + n = len(s) + in_str = in_chr = False + esc = False + while i < n: + ch = s[i] + if in_str: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + def _next_token_after(s: str, idx: int) -> str: + """Return the next non-space token after idx: '{', 'throws', or first char/empty.""" + n = len(s) + i = idx + 1 + while i < n and s[i].isspace(): + i += 1 + if s.startswith('throws', i) and (i + 6 == n or not s[i + 6].isalnum() and s[i + 6] != '_'): + return 'throws' + return s[i] if i < n else '' + + # Precise left-scan to the start of the expression that owns this call. + # Stops at the first top-level boundary (not inside (),[],{} or strings): + # ; , = + - * / % ! ~ ? : & | ^ < > { } \n + # We allow dots, identifiers, casts, and 'new ...' as part of the expression. + BOUNDARY_CHARS = set(';,=+-*/%!?::&|^<>{}\n') + + def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: + i = pos - 1 + in_str = in_chr = False + esc = False + # depths for (), [], {} + dp = db = dbr = 0 + limit = max(0, pos - max_back) + while i >= limit: + ch = s[i] + if in_str: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == '"': in_str = False + i -= 1; continue + if in_chr: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == "'": in_chr = False + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + + # Track bracket depths (we treat scanning left; depths just indicate "inside something") + if ch == ')': dp += 1; i -= 1; continue + if ch == '(': dp = max(0, dp - 1); i -= 1; continue + if ch == ']': db += 1; i -= 1; continue + if ch == '[': db = max(0, db - 1); i -= 1; continue + if ch == '}': dbr += 1; i -= 1; continue + if ch == '{': dbr = max(0, dbr - 1); i -= 1; continue + + # Only consider boundaries at top level + if dp == db == dbr == 0 and ch in BOUNDARY_CHARS: + return i + 1 # start right after the boundary + + i -= 1 + return limit # we hit the scan cap; good enough + + import re + + # Extract body once + src = caller_function.page_content + try: + lo = src.index("{") + hi = src.rfind("}") + caller_function_body = src[lo + 1:hi] + except ValueError: + caller_function_body = src + + # Fast regex: name followed by '(' with some receiver context allowed before it + pattern = re.compile( + rf'(?:\breturn\b\s*\(?[^;]*?)?(?:[\w$()\[\].]*?\.?)?\b{re.escape(callee_function_name)}\s*\(', + re.MULTILINE + ) + + # Dedup by the (start,end) slice of the exact call expression we extracted + seen_slices = set() + + # Target class names for the resolver + target_class_names: frozenset[str] + class_name = self.get_class_name_from_class_function(callee_function) + key = (class_name, callee_function.metadata['source']) + if "dummy" not in callee_function_file_name: + target_class_names = get_target_class_names(type_inheritance[key]) + else: + target_class_names = frozenset([class_name]) + + for m in pattern.finditer(caller_function_body): + call = m.group(0) + if not m or not call: + continue + + open_paren_pos = m.end(0) - 1 # the '(' + # Balance to closing ')' + close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) + if close_paren_pos == -1: + continue + + # This filters out method declarations (headers) + nxt = _next_token_after(caller_function_body, close_paren_pos) + if nxt == '{' or nxt == 'throws': + continue + + # Precisely slice from the start of the owning expression to the close paren + start_ctx = _expr_start_left(caller_function_body, m.start(), max_back=512) + end_ctx = close_paren_pos + 1 + slice_key = (start_ctx, end_ctx) + if slice_key in seen_slices: + continue + seen_slices.add(slice_key) + + # Robust resolver (handles casts, chains, static roots, helper returns, etc.) + if self.__check_identifier_resolved_to_callee_function_package( + function=caller_function, + identifier_function=call, + callee_package=callee_function_package, + code_documents=code_documents, + type_documents=type_documents, + callee_function_file_name=callee_function_file_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + target_class_names=target_class_names, + documents_of_functions=documents_of_functions, + callee_function_name=callee_function_name, + type_inheritance=type_inheritance + ): + logger.debug("__check_identifier_resolved_to_callee_function_package resolved successfully - callee_function_name = " + + callee_function_name + ", identifier_function = " + + call + ", target_class_names = " + str(target_class_names) + ", \ncaller_function_source = " + caller_function.metadata['source'] + + ", \ncaller_function = " + caller_function.page_content) + return True + + logger.debug("__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - callee_function_name = " + + callee_function_name + ", identifier_function = " + + call + ", target_class_names = " + str(target_class_names) + ", \ncaller_function_source = " + caller_function.metadata['source'] + + ", \ncaller_function = " + caller_function.page_content) + return False + + def __check_identifier_resolved_to_callee_function_package( + self, + function: Document, + identifier_function: str, + callee_package: str, + code_documents: dict[str, Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + target_class_names: frozenset[str], + documents_of_functions: list[Document], + callee_function_name: str, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] + ) -> bool: + """ + Decide if the found call expression (`identifier_function`) in `function` actually targets + the method in `callee_package`, supporting: + - Unqualified calls inside same package. + - Static calls: ClassName.method(…) or pkg.ClassName.method(…) + - 'new' receivers: new Type(...).method(…) + - Arbitrary-depth chains: A.staticB(...).c().d(...).targetMethod(…) + - Receiver that is a method call: foo().bar(…) → resolve foo()'s return type. + - Unqualified helper call in same class: helper(…) → resolve helper() return type. + - Receiver that is a cast: ((Type) x).target(…) → use Type as a strong hint. + - SUPER handling uses `target_class_names` (parents/children set) — no file header parsing. + """ + import re, hashlib + + def _strip_return_and_ws(s: str) -> str: + s = s.strip() + if s.startswith("return"): + s = s[6:].strip() + return s + + def _strip_leading_this(s: str) -> str: + return s.replace("this.", "", 1) if s.startswith("this.") else s + + def _is_upper_camel(token: str) -> bool: + return bool(token) and token[0].isalpha() and token[0].upper() == token[0] + + def _split_top_level_dots(expr: str) -> list[str]: + """Split by dots ignoring dots inside (), [], {}, <>, and strings/chars.""" + parts, buf = [], [] + dp = db = dbr = da = 0 + in_str = in_chr = False + i, n = 0, len(expr) + while i < n: + ch = expr[i] + if in_str: + buf.append(ch) + if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + buf.append(ch) + if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 + elif ch == "'": in_chr = False + i += 1; continue + + if ch == '"': in_str = True; buf.append(ch); i += 1; continue + if ch == "'": in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': dp += 1; buf.append(ch); i += 1; continue + if ch == ')': dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': db += 1; buf.append(ch); i += 1; continue + if ch == ']': db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': da += 1; buf.append(ch); i += 1; continue + if ch == '>': + if da > 0: da -= 1 + buf.append(ch); i += 1; continue + + if ch == '.' and dp == db == dbr == da == 0: + parts.append(''.join(buf).strip()); buf = [] + i += 1; continue + + buf.append(ch); i += 1 + parts.append(''.join(buf).strip()) + return [p for p in parts if p] + + def _strip_call_parens(s: str) -> str: + """Remove a single trailing '(...)' from the end if balanced.""" + s2 = s.rstrip() + if not s2.endswith(')'): + return s + depth = 0 + in_str = in_chr = False + i = len(s2) - 1 + while i >= 0: + ch = s2[i] + if in_str: + if ch == '\\': i -= 2; continue + if ch == '"': in_str = False; i -= 1; continue + i -= 1; continue + if in_chr: + if ch == '\\': i -= 2; continue + if ch == "'": in_chr = False; i -= 1; continue + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == ')': depth += 1; i -= 1; continue + if ch == '(': + depth -= 1 + if depth == 0: + return s2[:i].rstrip() + i -= 1; continue + i -= 1 + return s + + def _is_fqcn_like(token: str) -> bool: + t = token.replace('$', '.') + if '.' in t: + pkg, _, cls = t.rpartition('.') + return bool(pkg) and _is_upper_camel(cls[:1] + cls[1:]) + return _is_upper_camel(token) + + # --- resolve the return type of a method declared in the same source file --- + def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: + sig = re.compile( + rf""" + (?\[\]?]*?) + \s+{re.escape(method_name)}\s*\( + """, + re.VERBOSE, + ) + m = sig.search(source_text) + if not m: + return None + ret = m.group(1).strip() + ret = re.sub(r'<[^<>]*>', '', ret) + ret = re.sub(r'\s*(\[\])+$', '', ret) + return ret or None + + # peel a leading Java cast "((Type) expr)" → (cast_type | None, remainder_expr) + _CAST_RE = re.compile( + r'^\(\s*([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:<[^()<>]*>)?(?:\s*(?:\[\]))*)\s*\)\s*(.*)\Z' + ) + def _peel_leading_cast(expr: str) -> tuple[str | None, str]: + s = expr.strip() + # unwrap outermost parens only if they wrap the whole expr (not a call) + changed = True + while changed and s.startswith('(') and s.endswith(')'): + changed = False + depth = 0 + for i, ch in enumerate(s): + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0 and i != len(s) - 1: + break + else: + s = s[1:-1].strip() + changed = True + # peel one or more casts + last_type = None + rest = s + while True: + m = _CAST_RE.match(rest) + if not m: + break + last_type = m.group(1) + rest = m.group(2).strip() + return last_type, rest + + # reduce snippet to the actual callee + its immediate receiver + def _extract_recv_and_method(snippet: str) -> tuple[str | None, str | None]: + s = snippet.strip() + m = re.search(r'([A-Za-z_$][\w$]*)\s*\($', s) or \ + (list(re.finditer(r'([A-Za-z_$][\w$]*)\s*\(', s))[-1] if re.search(r'\(', s) else None) + if not m: + return None, None + method = m.group(1) + method_start = m.start(1) + + # immediate '.' before the method token? + i = method_start - 1 + while i >= 0 and s[i].isspace(): i -= 1 + if i < 0 or s[i] != '.': + return None, method + + # walk left to get the *smallest* receiver just before the dot + dot_idx = i + + def _match_paren_reverse(text: str, close_idx: int) -> int: + depth = 1 + i2 = close_idx - 1 + in_str = in_chr = False + esc = False + while i2 >= 0: + ch = text[i2] + if in_str: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == '"': in_str = False + i2 -= 1; continue + if in_chr: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == "'": in_chr = False + i2 -= 1; continue + if ch == '"': in_str = True; i2 -= 1; continue + if ch == "'": in_chr = True; i2 -= 1; continue + if ch == ')': depth += 1 + elif ch == '(': + depth -= 1 + if depth == 0: + return i2 + i2 -= 1 + return -1 + + j = dot_idx - 1 + while j >= 0 and s[j].isspace(): j -= 1 + + if j >= 0 and s[j] == ')': + # get the minimal '( ... )' group immediately before the dot + open_idx = _match_paren_reverse(s, j) + if open_idx == -1: + start = j + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + return (recv if recv else None), method + + # If the '(' belongs to a *call* (e.g. foo(...)) — include the callee token to its left. + k = open_idx - 1 + while k >= 0 and s[k].isspace(): k -= 1 + if k >= 0 and (s[k].isalnum() or s[k] in '_$.'): + start = k + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + else: + # parenthesized group/cast → keep minimal group + recv = s[open_idx:dot_idx].strip() + else: + start = j + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + + return (recv if recv else None), method + + caller_src = function.metadata.get('source') + caller_doc = code_documents.get(caller_src) + caller_text = caller_doc.page_content if caller_doc else "" + caller_class_name = self.get_class_name_from_class_function(caller_doc) if caller_doc else "" + + # Does a type token (simple or qualified) belong to the *callee type(s)/package*? + def _type_token_matches_callee(type_token: str) -> bool: + if not type_token: + return False + tt = re.sub(r'<[^<>]*>', '', type_token).strip() + tt = re.sub(r'\s*(\[\])+$', '', tt) + simple_tt = tt.rsplit('.', 1)[-1] + if target_class_names and strip_java_generics(simple_tt) not in target_class_names: + return False + code_text = caller_text + if '.' in tt and _is_fqcn_like(tt): + if self.is_package_imported(code_text, tt): + return True + matches = self.__get_type_docs_matched_with_callee_type( + callee_package, tt, type_documents, target_class_names + ) + return len(matches) > 0 + + def _caller_key() -> str: + content = function.page_content + md5hex = hashlib.md5(content.strip().encode("utf-8")).hexdigest() + return f"{extract_method_name_with_params(content)}@{md5hex}@{caller_src}" + + # ---- normalize the found call snippet ---- + expr = _strip_return_and_ws(_strip_leading_this(identifier_function)) + expr = re.sub(r'\s+', ' ', expr).strip() + + recv_norm, method_norm = _extract_recv_and_method(expr) + if method_norm: + expr = (f"{recv_norm}.{method_norm}(" if recv_norm else f"{method_norm}(").strip() + + # decide qualified vs unqualified using recv_norm + is_unqualified = (recv_norm is None) + + # ======== SUPER / THIS handling via target_class_names ======== + if recv_norm is not None: + recv_trim = recv_norm.strip() + + # Interface.super.method(…) + if recv_trim.endswith('.super'): + iface_type = recv_trim[:-6].strip() # drop ".super" + if _type_token_matches_callee(iface_type): + return True + + # plain super.method(…) + if recv_trim == 'super': + caller_inheritance_list = get_target_class_names(type_inheritance[(caller_class_name, caller_src)]) + # Use target_class_names (parents/children set). Prefer excluding current class name if present. + for cand in caller_inheritance_list: + if cand == strip_java_generics(caller_class_name): + continue + if _type_token_matches_callee(cand): + return True + + # explicit this.method(…) + if recv_trim == 'this': + if caller_class_name and _type_token_matches_callee(caller_class_name): + return True + + if is_unqualified: + # helper method in same class? use its return type + base_before_paren = method_norm + if base_before_paren and callee_function_name != base_before_paren: + fn = find_function(caller_src, base_before_paren, documents_of_functions) + if fn: + ret_type = _find_method_return_type_in_file(fn.page_content, base_before_paren) + if ret_type and _type_token_matches_callee(ret_type): + return True + + # fallback: same-package unqualified call rule + try: + callee_doc = code_documents[callee_function_file_name] + callee_pkg_decl = (self.get_package_name_file(callee_doc) or "").strip() + except KeyError: + callee_pkg_decl = callee_function_file_name # fallback + + caller_pkg_decl = (self.get_package_name_file(caller_doc) or "").strip() if caller_doc else "" + jar_name = extract_jar_name(caller_src or "") + return ( + callee_package.__contains__(jar_name) + and callee_pkg_decl + and callee_pkg_decl == caller_pkg_decl + and strip_java_generics(caller_class_name) in target_class_names + ) + + # ---- dotted/chained expression handling ---- + chain = _split_top_level_dots(expr) + if not chain: + return False + + recv_raw = chain[-2] if len(chain) >= 2 else "" + recv_raw = recv_raw.strip() + + # Casts first: '((Type) x).method(' + cast_type, post_cast_expr = _peel_leading_cast(recv_raw) + if cast_type: + if _type_token_matches_callee(cast_type): + return True + m_id_cast = re.match(r'\s*([A-Za-z_$][\w$]*)', post_cast_expr) + if m_id_cast: + caller_key = _caller_key() + traced_cast = self.__trace_down_package( + expression=m_id_cast.group(1), + type_documents=type_documents, + callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_key, + target_class_names=target_class_names, + function=function, + code_documents=code_documents + ) + if traced_cast: + return True + + # Also allow Interface.super receiver in a chain + if recv_raw.endswith('.super'): + iface_type = recv_raw[:-6].strip() + if _type_token_matches_callee(iface_type): + return True + + # Explicit super/this in chained receiver + if recv_raw == 'super': + caller_inheritance_list = get_target_class_names(type_inheritance[(caller_class_name, caller_src)]) + for cand in caller_inheritance_list: + if cand == strip_java_generics(caller_class_name): + continue + if _type_token_matches_callee(cand): + return True + elif recv_raw == 'this': + if caller_class_name and _type_token_matches_callee(caller_class_name): + return True + + # Case A: any static-style class root in the chain + for seg in chain[:-1]: + seg_core = _strip_call_parens(seg).strip() + if seg_core.startswith("new "): + typ = _strip_call_parens(seg_core[4:].strip()) + if _type_token_matches_callee(typ): + return True + continue + if _is_fqcn_like(seg_core) and _type_token_matches_callee(seg_core): + return True + + # Case B: identifier/expression → dataflow OR rightmost receiver-call return type + recv_for_ident = _strip_call_parens(recv_raw).strip() if not cast_type else post_cast_expr.strip() + if recv_for_ident: + caller_key = _caller_key() + + m_id = re.match(r'\s*([A-Za-z_$][\w$]*)', recv_for_ident) + left_ident = m_id.group(1) if m_id else None + if left_ident and left_ident not in ('this', 'super'): + traced = self.__trace_down_package( + expression=left_ident.strip(), + type_documents=type_documents, + callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_key, + target_class_names=target_class_names, + function=function, + code_documents=code_documents + ) + if traced: + return True + + if recv_for_ident.endswith(')'): + segs = _split_top_level_dots(recv_for_ident) + recv_call_name = None + for seg in reversed(segs): + seg = seg.strip() + if seg.endswith(')'): + mtok = re.match(r'\s*([A-Za-z_$][\w$]*)\s*\(', seg) + if mtok: + recv_call_name = mtok.group(1) + break + if recv_call_name: + # Use local method return type (cheap heuristic) + ret_type = _find_method_return_type_in_file(caller_text, recv_call_name) + if ret_type and _type_token_matches_callee(ret_type): + return True + + # Case C: final receiver is a direct 'new Type(...)' + if recv_raw.startswith("new "): + typ = _strip_call_parens(recv_raw[4:].strip()) + if _type_token_matches_callee(typ): + return True + + # Case D: immediate receiver is a (pkg.)Class token + if _is_fqcn_like(recv_raw) and _type_token_matches_callee(recv_raw): + return True + + return False + + def __trace_down_package(self, expression: str, type_documents: list[Document], + callee_package: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str, + target_class_names: frozenset[str], + function: Document, + code_documents: dict[str, Document]) -> bool: + + variables_mappings = functions_local_variables_index[caller_function_index] + parts = expression.split(".") + result = False + + (resolved_type, struct_initializer_expression, + value_list, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-1) + + if (var_properties is not None + and (struct_initializer_expression or + resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or PARAMETER in value_list)): + result = self.__lookup_package(callee_package, resolved_type, struct_initializer_expression, + type_documents, value_list, target_class_names) + + # Property/member is not in function, check if it's member/property of a type + elif var_properties is None and (expression[0].islower() or expression[0] == '_'): + class_name = self.get_class_name_from_class_function(function) + possible_types = {key: value for (key, value) in fields_of_types.items() + if key == (class_name, function.metadata['source'])} + for mappings in possible_types.values(): + for mapping in mappings: + if expression in mapping: + returned_matched_types = self.__get_type_docs_matched_with_callee_type(callee_package, + mapping[1], + type_documents, + target_class_names) + if len(returned_matched_types) > 0: + result = True + + elif var_properties is not None: + value = var_properties.get("value", None) + if value is not None: + inferred = self._infer_type_from_var_initializer( + initializer=value_list[-1], + variables_mappings=variables_mappings, + type_documents=type_documents, + caller_src=function.metadata['source'], + code_documents=code_documents + ) + if inferred: + returned_matched_types = self.__get_type_docs_matched_with_callee_type(callee_package, + inferred, + type_documents, + target_class_names) + if len(returned_matched_types) > 0: + result = True + return result + + def document_imports_package(self, documents: list[Document], package_name: str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if self.is_package_imported(value.page_content, package_name)] + return importing_docs + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + """ + returns Match if `java_src` contains: + - import ; + - import .*; + Static imports are excluded. + `identifier` is a fully qualified class name (may include nested classes, e.g., java.util.Map.Entry). + """ + # Derive the base *package* (up to the first Type segment, usually starting with uppercase) + parts = identifier.split('.') + pkg_end = None + for i, seg in enumerate(parts): + if seg and seg[0].isalpha() and seg[0].upper() == seg[0]: # heuristically treat Uppercase as type segment + pkg_end = i + break + if pkg_end is None: + # Fallback: assume last segment is the type + pkg_end = max(0, len(parts) - 1) + + base_pkg = '.'.join(parts[:pkg_end]) # e.g., "java.util" for "java.util.Map.Entry" + + # Build the regex: match non-static imports only + if base_pkg: + pattern = rf'(?m)^\s*import\s+(?!static\s)(?:{re.escape(identifier)}|{re.escape(base_pkg)}\.\*)\s*;' + else: + # No sensible package (edge case) -> only match the direct import + pattern = rf'(?m)^\s*import\s+(?!static\s){re.escape(identifier)}\s*;' + + return re.search(pattern, code_content, flags=re.MULTILINE) is not None + + def get_package_name_file(self, function: Document): + content = function.page_content + index_of_package = content.find("package") + index_of_end_of_line = index_of_package + content[index_of_package:].find(os.linesep) + if index_of_package > -1: + return content[index_of_package + 8:index_of_end_of_line].strip() + + def get_package_names(self, function: Document) -> list[str]: + result, acc = [], [] + full_doc_path = str(function.metadata['source']) + parts = full_doc_path.split("/") + + if parts[0].startswith(self.dir_name_for_3rd_party_packages()): + for piece in parts[2:-1]: + acc.append(piece) + result.append('.'.join(acc)) + else: + for piece in parts[0:-1]: + acc.append(piece) + result.append('.'.join(acc)) + + return result + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + pass + + def is_doc_type(self, doc: Document) -> bool: + return is_java_type(doc.page_content) + + def create_map_of_local_vars(self, functions_methods_documents: List[Document]) -> Dict[str, Dict]: + """ + Parse each Document (containing exactly one Java method OR one lambda) and return: + "(sig)@@" for methods, or "lambda@@" for lambdas -> { + : {"value": , "type": }, + return_types: [] # empty for void/constructor/lambda + } + + Notes: + - Catch parameters (e.g., `catch (IOException ex)`) are returned as parameters. + - Catch parameters are **not** included in the signature part of the key. + - For locals, "value" is a list of all assignment RHS expressions to that local in the body + (declaration initializers + later reassignments, including augmented ops like +=, <<=, etc.). + """ + + # --- constants for output shape --- + RETURN_TYPES = "return_types" + + CONTROL = { + "if", "else", "while", "do", "for", "switch", "case", "default", "return", + "break", "continue", "throw", "throws", "try", "catch", "finally", "new", + "assert", "yield" + } + + NON_TYPE_STARTS = { + "return", "throw", "break", "continue", "assert", "yield", + "if", "else", "for", "while", "switch", "case", "default", + "try", "catch", "finally", "synchronized", "new", + "this", "super" + } + + # --- hot-path compiled/aliased helpers for perf --- + ANN_RE = re.compile(r'@\w+(?:\s*\([^()]*\))?') + WS_RE = re.compile(r'\s+') + VAR_END_RE = re.compile(r'([A-Za-z_$][\w$]*)\s*((?:\[\s*\])*)\s*$') + + ANN_SUB = ANN_RE.sub + WS_SUB = WS_RE.sub + RE_SEARCH = re.search + + # ------------------------- + # low-level scanners + # ------------------------- + def strip_comments_keep_strings(s: str) -> str: + if "/*" not in s and "//" not in s: + return s + out = [] + i, n = 0, len(s) + sl = ml = False + in_str = in_chr = False + esc = False + while i < n: + ch = s[i] + nx = s[i+1] if i+1 < n else '' + if sl: + if ch == '\n': + sl = False; out.append('\n') + else: + out.append(' ') + i += 1; continue + if ml: + if ch == '*' and nx == '/': + ml = False; out.extend((' ', ' ')); i += 2 + else: + out.append('\n' if ch == '\n' else ' '); i += 1 + continue + if in_str: + out.append(ch) + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + out.append(ch) + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i += 1; continue + + if ch == '/' and nx == '/': + sl = True; out.extend((' ', ' ')); i += 2; continue + if ch == '/' and nx == '*': + ml = True; out.extend((' ', ' ')); i += 2; continue + if ch == '"': + in_str = True; out.append(ch); i += 1; continue + if ch == "'": + in_chr = True; out.append(ch); i += 1; continue + + out.append(ch); i += 1 + return ''.join(out) + + def prev_non_ws(s: str, i: int) -> int: + while i >= 0 and s[i].isspace(): i -= 1 + return i + + def scan_back_to_boundary(s: str, pos: int) -> int: + i = pos - 1 + while i >= 0 and s[i] not in "{;\n": + i -= 1 + return i + 1 + + def balanced_region(s: str, open_pos: int, open_ch: str, close_ch: str) -> Tuple[Optional[str], Optional[int]]: + if open_pos < 0 or open_pos >= len(s) or s[open_pos] != open_ch: + return None, None + i = open_pos + depth = 0 + in_str = in_chr = False + esc = False + n = len(s) + while i < n: + ch = s[i] + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == open_ch: depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + return s[open_pos+1:i], i+1 + i += 1 + return None, None + + def balanced_region_reverse(s: str, close_pos: int, close_ch: str, open_ch: str) -> Tuple[Optional[str], Optional[int]]: + if close_pos < 0 or close_pos >= len(s) or s[close_pos] != close_ch: + return None, None + i = close_pos + depth = 0 + in_str = in_chr = False + esc = False + while i >= 0: + ch = s[i] + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i -= 1; continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == close_ch: depth += 1 + elif ch == open_ch: + depth -= 1 + if depth == 0: + return s[i+1:close_pos], i + i -= 1 + return None, None + + def split_top_level(s: str, delim: str) -> List[str]: + out = [] + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + start = 0 + for idx, ch in enumerate(s): + if in_str: + if ch == '\\': + continue + elif ch == '"': + in_str = False + continue + if in_chr: + if ch == '\\': + continue + elif ch == "'": + in_chr = False + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '(': depth_p += 1 + elif ch == ')': depth_p -= 1 + elif ch == '[': depth_b += 1 + elif ch == ']': depth_b -= 1 + elif ch == '{': depth_br += 1 + elif ch == '}': depth_br -= 1 + elif ch == '<': depth_a += 1 + elif ch == '>' and depth_a > 0: depth_a -= 1 + if ch == delim and depth_p == 0 and depth_b == 0 and depth_br == 0 and depth_a == 0 and not (in_str or in_chr): + out.append(s[start:idx]); start = idx + 1 + out.append(s[start:]); return out + + def remove_modifiers(s: str) -> str: + tokens = WS_RE.split(s.strip()) + tokens = [t for t in tokens if t and t not in MODIFIERS] + return ' '.join(tokens).strip() + + def canon_ws(s: Optional[str]) -> str: + return WS_SUB(' ', s or '').strip() + + def count_array_pairs(text: str) -> int: + i = 0; n = len(text); cnt = 0 + while i < n: + if text[i] == '[': + j = i + 1 + while j < n and text[j].isspace(): j += 1 + if j < n and text[j] == ']': + cnt += 1 + i = j + 1 + continue + i += 1 + return cnt + + def scan_identifier(s: str, i: int): + n = len(s) + if i >= n or not (s[i] == '_' or s[i] == '$' or s[i].isalpha()): + return None, i + j = i + 1 + while j < n and (s[j] == '_' or s[j] == '$' or s[j].isalnum()): + j += 1 + return s[i:j], j + + def skip_ws(s: str, i: int) -> int: + n = len(s) + while i < n and s[i].isspace(): i += 1 + return i + + def scan_type_prefix(s: str): + i = skip_ws(s, 0) + if i >= len(s): + return None, None + if s[i:i+3] == 'var' and (i+3 == len(s) or not (s[i+3] == '_' or s[i+3].isalnum() or s[i+3] == '$')): + j = skip_ws(s, i+3) + k = j + while True: + kk = skip_ws(s, k) + if kk < len(s) and s[kk] == '[': + kk = skip_ws(s, kk+1) + if kk < len(s) and s[kk] == ']': + k = kk + 1; continue + break + return s[i:k], k + + start = i + name, j = scan_identifier(s, i) + if not name: return None, None + while True: + jj = skip_ws(s, j) + if jj < len(s) and s[jj] == '.': + jj = skip_ws(s, jj+1) + name2, j2 = scan_identifier(s, jj) + if not name2: break + j = j2 + else: + break + + j = skip_ws(s, j) + if j < len(s) and s[j] == '<': + depth = 0; k = j; n = len(s) + while k < n: + ch = s[k] + if ch == '<': depth += 1 + elif ch == '>': + depth -= 1 + if depth == 0: + k += 1; break + elif ch == '"': + _, end = balanced_region(s, k, '"', '"') + k = (end or (k+1)); continue + k += 1 + j = k + + k = j + while True: + kk = skip_ws(s, k) + if kk < len(s) and s[kk] == '[': + kk = skip_ws(s, kk+1) + if kk < len(s) and s[kk] == ']': + k = kk + 1; continue + break + return s[start:k], k + + def count_and_strip_trailing_array_dims(type_part: str) -> tuple[str, int, bool]: + """ + Count only trailing [] pairs at the END of the type, strip them, and + return (type_without_trailing_arrays, trailing_dims_count, space_before_arrays). + + Examples: + "Object[][]" -> ("Object", 2, False) + "List" -> ("List", 0, False) + "Map[]" + -> ("Map", 1, False) + "String []" -> ("String", 1, True) + "String [][][]" -> ("String", 3, True) # any whitespace before the first '[' counts + """ + # Matches trailing sequences like: " []", "[][]", " [] [] ", etc. + # It's ok if your code already defines TRAILING_ARR_RE; otherwise: + # TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') + m = TRAILING_ARR_RE.search(type_part) + if not m: + return type_part, 0, False + + suffix = m.group(0) + dims = suffix.count('[') + + # If suffix starts with whitespace, then there was a space before the first '[' + space_before = bool(suffix) and suffix[0].isspace() + + # Strip the whole suffix from the type + return type_part[:m.start()].rstrip(), dims, space_before + + def parse_param_list(text: str) -> list: + t = (text or "").strip() + if not t: return [] + parts = split_top_level(t, ',') + res = [] + for raw in parts: + item = ANN_SUB(' ', raw.strip()) + item = remove_modifiers(item) + item = WS_SUB(' ', item).strip() + if not item: continue + + varargs = item.endswith('...') + if varargs: item = item[:-3].rstrip() + + m = VAR_END_RE.search(item) + if not m: + # untyped parameter (e.g., lambda param) + res.append({"name": item, "type": None, "varargs": varargs, "array_dims": 0}) + continue + + name = m.group(1) + # [] after the variable name + arr_after = count_array_pairs(m.group(2) or "") + + # Type part may have generics, and may END with [] (which belong to this param) + type_part = (item[:m.start()].strip()) + + # ✅ Only count/strip [] at the END of the type, not inside generics + type_part, trailing_dims, space_before = count_and_strip_trailing_array_dims(type_part) + + type_clean = WS_SUB(' ', type_part).strip() if type_part else None + res.append({ + "name": name, + "type": type_clean if type_clean else None, + "varargs": varargs, + "array_dims": arr_after + trailing_dims, + # Preserve whether there was whitespace before type-trailing []. + # Only meaningful if trailing_dims > 0 (i.e., [] were part of the type). + "space_before_arrays": bool(space_before and trailing_dims > 0) + }) + return res + + def render_param_type(p: dict) -> str: + """ + Render a parameter type for the method signature. + + Rules: + - If varargs: "T..." (arrays ignored if present). + - If array_dims > 0: preserve whether there was a space before the type-trailing []. + * If the original had "String []": render "String []" + * If it was "String[]": render "String[]" + * [] moved from variable name (e.g., "String a[]") are aggregated into the type; we render with no extra space by default, + unless type-trailing [] originally had whitespace (captured in 'space_before_arrays'). + - If type is missing (e.g., lambda), use "var". + """ + t = canon_ws(p.get("type") or "") + array_dims = int(p.get("array_dims") or 0) + + if p.get("varargs"): + base = t or "var" + return base + "..." + + if array_dims > 0: + base = t or "var" + # Honor preserved whitespace only when the original trailing [] were part of the type + sep = " " if p.get("space_before_arrays") else "" + return base + sep + "[]" * array_dims + + return t + + def render_param_decl(p: dict) -> str: + t = render_param_type(p) + return (f"{t} {p['name']}".strip() if t else p["name"]) + + def find_top_eq(s: str): + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + for i, ch in enumerate(s): + if in_str: + if ch == '\\': continue + elif ch == '"': in_str = False; continue + continue + if in_chr: + if ch == '\\': continue + elif ch == "'": in_chr = False; continue + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '(': depth_p += 1 + elif ch == ')': depth_p -= 1 + elif ch == '[': depth_b += 1 + elif ch == ']': depth_b -= 1 + elif ch == '{': depth_br += 1 + elif ch == '}': depth_br -= 1 + elif ch == '<': depth_a += 1 + elif ch == '>' and depth_a > 0: depth_a -= 1 + elif ch == '=' and depth_p == 0 and depth_b == 0 and depth_br == 0 and depth_a == 0: + return i + return None + + def _has_comp_ops_before_assign(text: str) -> bool: + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + i = 0; n = len(text) + while i < n: + ch = text[i] + if in_str: + if ch == '\\': i += 2; continue + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == '\\': i += 2; continue + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == '(': depth_p += 1; i += 1; continue + if ch == ')': depth_p -= 1; i += 1; continue + if ch == '[': depth_b += 1; i += 1; continue + if ch == ']': depth_b -= 1; i += 1; continue + if ch == '{': depth_br += 1; i += 1; continue + if ch == '}': depth_br -= 1; i += 1; continue + if ch == '<': depth_a += 1; i += 1; continue + if ch == '>' and depth_a > 0: depth_a -= 1; i += 1; continue + + if depth_p==depth_b==depth_br==depth_a==0: + if i+1 < n and text[i:i+2] in ('==','!=','&&','||','<=','>='): + return True + if ch == '?': + return True + if ch in '=#,': + return False + i += 1 + return False + + def parse_decl_with_inits(stmt: str): + s = stmt.strip().rstrip(';').strip() + if not s: return None + s1 = s + if JAVA_ANNOTATION_SYMBOL in s1: + while True: + m = ANN_RE.match(s1) + if not m: break + s1 = s1[m.end():].lstrip() + s1 = re.sub( + r'^(?:(?:public|protected|private|static|final|abstract|synchronized|native|strictfp|transient|volatile)\s+)+', + '', s1).lstrip() + + if _has_comp_ops_before_assign(s1): + return None + i0 = skip_ws(s1, 0) + tok0, _ = scan_identifier(s1, i0) + if tok0 and tok0 in NON_TYPE_STARTS: + return None + + type_str, end_idx = scan_type_prefix(s1) + if not type_str: + return None + rest = s1[end_idx:].lstrip() + if not rest: + return None + + id0, _ = scan_identifier(rest, 0) + if not id0: + return None + + decls = split_top_level(rest, ',') + out = [] + for dec in decls: + d = dec.strip() + if not d: continue + eq = find_top_eq(d) + left = d[:eq].rstrip() if eq is not None else d + init = d[eq+1:].strip() if eq is not None else "" + m2 = VAR_END_RE.match(left) + if not m2: continue + name = m2.group(1) + dims = count_array_pairs(m2.group(2) or "") + out.append({"name": name, "dims": dims, "init": init}) + if not out: + return None + return (canon_ws(type_str), out) + + def strip_anon_and_local_class_bodies(s: str) -> str: + """ + Remove bodies of *real* class/interface/enum/record declarations and 'new T(...) { ... }' + anonymous bodies, but DO NOT touch '.class' literals (e.g., 'Foo.class'). + Single pass, preserves string/char literals, keeps statement layout with spaces. + """ + if ('class' not in s and 'interface' not in s and 'enum' not in s and 'record' not in s + and ' new ' not in s and 'new ' not in s): + return s + + out = [] + i = 0 + n = len(s) + in_str = in_chr = False + + def skip_balanced_at(open_pos: int) -> int: + content, endp = balanced_region(s, open_pos, '{', '}') + if endp is None: + # unmatched '{' – be conservative + out.extend(' ') + return open_pos + 1 + # blank out the whole balanced region + out.extend(' ' * (endp - open_pos)) + return endp + + def scan_type_from_index(idx: int): + type_str, end_idx = scan_type_prefix(s[idx:]) + if not type_str: + return None, idx + return type_str, idx + end_idx + + def prev_non_ws_idx(k: int) -> int: + k -= 1 + while k >= 0 and s[k].isspace(): + k -= 1 + return k + + while i < n: + ch = s[i] + + # string/char handling + if in_str: + out.append(ch) + if ch == '"': + in_str = False + i += 1 + continue + if in_chr: + out.append(ch) + if ch == "'": + in_chr = False + i += 1 + continue + if ch == '"': + in_str = True + out.append(ch) + i += 1 + continue + if ch == "'": + in_chr = True + out.append(ch) + i += 1 + continue + + # identifiers + if ch.isalpha() or ch == '_': + j = i + 1 + while j < n and (s[j].isalnum() or s[j] == '_'): + j += 1 + word = s[i:j] + + # ---- real type declarations (NOT '.class' literals) ---- + if word in ("class", "interface", "enum", "record"): + # If previous non-space is '.', this is likely a '.class' literal → keep it. + p = prev_non_ws_idx(i) + if p >= 0 and s[p] == '.': + out.append(s[i:j]) # copy the literal 'class' + i = j + continue + + # Otherwise, treat as a declaration keyword and blank until its body '{...}' + k = j + # blank everything up to the next '{' (header/implements/extends) + while k < n and s[k] != '{': + out.append(' ') + k += 1 + if k < n and s[k] == '{': + i = skip_balanced_at(k) # blanks the body + continue + # No '{' found (malformed). Just copy the token and continue. + out.append(s[i:j]) + i = j + continue + + # ---- 'new T(...) { ... }' anonymous body stripping ---- + if word == "new": + k = j + while k < n and s[k].isspace(): + k += 1 + _t, k2 = scan_type_from_index(k) + if k2 != k: + k = k2 + while k < n and s[k].isspace(): + k += 1 + if k < n and s[k] == '(': + _, endp = balanced_region(s, k, '(', ')') + if endp: + k = endp + while k < n and s[k].isspace(): + k += 1 + # If a '{' follows, it's an anonymous body → blank it. + if k < n and s[k] == '{': + i = skip_balanced_at(k) + continue + + # default: just copy the identifier + out.append(s[i:j]) + i = j + continue + + # default: copy char + out.append(ch) + i += 1 + + return ''.join(out) + + # Canonicalize initializer/assignment RHS expressions + def canon_init(expr: str) -> str: + if not expr: + return "" + e = expr.strip() + e = re.sub(r'\s*\.\s*', '.', e) + e = re.sub(r'\s*\(\s*', '(', e) + e = re.sub(r'\s*\)\s*', ')', e) + e = re.sub(r'\s*,\s*', ', ', e) + e = re.sub(r'\s*<\s*', '<', e) + e = re.sub(r'\s*>\s*', '>', e) + e = re.sub(r'\s+\)', ')', e) + e = re.sub(r'\(\s+', '(', e) + e = re.sub(r'\s+', ' ', e) + return e + + # --------- collect catch parameters from body ---------- + def collect_catch_params(body: str) -> List[dict]: + if not body or "catch" not in body: + return [] + s = body + params = [] + i = 0 + n = len(s) + depth_p = depth_b = 0 + in_str = in_chr = False + while i < n: + ch = s[i] + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + + if depth_p == 0 and depth_b == 0 and (ch.isalpha() or ch == '_'): + j = i + 1 + while j < n and (s[j].isalnum() or s[j] == '_'): j += 1 + token = s[i:j] + k = j + while k < n and s[k].isspace(): k += 1 + if token == "catch" and k < n and s[k] == '(': + content, endp = balanced_region(s, k, '(', ')') + if endp is None: + i = k + 1 + continue + c = ANN_SUB(' ', content or "") + c = remove_modifiers(c) + c = WS_SUB(' ', c).strip() + name_match = RE_SEARCH(r'([A-Za-z_$][\w$]*)\s*$', c) + if name_match: + name = name_match.group(1) + type_part = c[:name_match.start()].strip() + params.append({"name": name, "type": type_part or None, "varargs": False, "array_dims": 0, "is_catch": True}) + i = endp + continue + i = j + continue + i += 1 + return params + + # --- assignment parsing helpers --- + def _split_top_level_assignment(stmt: str) -> Optional[Tuple[str, str, str]]: + """ + Return (lhs, op, rhs) for the first top-level assignment in `stmt`, or None. + Supports =, +=, -=, *=, /=, &=, |=, ^=, %=, <<=, >>=, >>>=. + Ignores ==, !=, <=, >=, &&, || and anything inside (), [], {}, <> or strings/chars. + """ + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + i = 0 + n = len(stmt) + + while i < n: + ch = stmt[i] + + if in_str: + if ch == '\\': i += 2; continue + if ch == '"': in_str = False; i += 1; continue + i += 1; continue + + if in_chr: + if ch == '\\': i += 2; continue + if ch == "'": in_chr = False; i += 1; continue + i += 1; continue + + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': depth_p += 1; i += 1; continue + if ch == ')': depth_p -= 1; i += 1; continue + if ch == '[': depth_b += 1; i += 1; continue + if ch == ']': depth_b -= 1; i += 1; continue + if ch == '{': depth_br += 1; i += 1; continue + if ch == '}': depth_br -= 1; i += 1; continue + if ch == '<': depth_a += 1; i += 1; continue + if ch == '>' and depth_a > 0: depth_a -= 1; i += 1; continue + + if ch == '=' and depth_p == depth_b == depth_br == depth_a == 0: + # detect assignment operator + op_start = i + op = '=' + if i >= 3 and stmt[i-3:i+1] == '>>>=': + op_start = i-3; op = '>>>=' + elif i >= 2 and stmt[i-2:i+1] in ('>>=', '<<='): + op_start = i-2; op = stmt[i-2:i+1] + elif i >= 1 and stmt[i-1] in '+-*/&|^%': + op_start = i-1; op = stmt[i-1] + '=' + else: + prevc = stmt[i-1] if i-1 >= 0 else '' + if prevc in ('=', '!', '<', '>'): + i += 1; continue # not an assignment (==, !=, <=, >=) + lhs = stmt[:op_start].strip() + rhs = stmt[i+1:].strip() + return lhs, op, rhs + i += 1 + return None + + def collect_locals_single_pass(body: str) -> list: + if not body: + return [] + if (';' not in body) and ('for' not in body) and ('catch' not in body) and ('try(' not in body) and ('try (' not in body): + return [] + + scrubbed = strip_anon_and_local_class_bodies(body) + + locals_out = [] + i = 0 + n = len(scrubbed) + depth_p = depth_b = depth_br = 0 + in_str = in_chr = False + stmt_start = 0 + + while i < n: + ch = scrubbed[i] + + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + + if ch == '{' and depth_p == 0 and depth_b == 0: + depth_br += 1 + stmt_start = i + 1 + i += 1 + continue + if ch == '}' and depth_p == 0 and depth_b == 0: + depth_br = max(0, depth_br - 1) + stmt_start = i + 1 + i += 1 + continue + + if ch == ';' and depth_p == 0 and depth_b == 0: + stmt = scrubbed[stmt_start:i+1].strip() + stmt_start = i + 1 + if stmt: + st0 = stmt.lstrip() + t0, _ = scan_identifier(st0, 0) + if not (t0 and t0 in NON_TYPE_STARTS): + decl = parse_decl_with_inits(stmt) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i += 1 + continue + + if depth_p == 0 and depth_b == 0 and (ch.isalpha() or ch == '_'): + j = i + 1 + while j < n and (scrubbed[j].isalnum() or scrubbed[j] == '_'): j += 1 + token = scrubbed[i:j] + k = j + while k < n and scrubbed[k].isspace(): k += 1 + if token == "for" and k < n and scrubbed[k] == '(': + content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + c = (content or "").strip() + depth_lp = depth_lb = depth_lbr = depth_la = 0 + colon_idx = None + for idx2, ch2 in enumerate(c): + if ch2 == '(': + depth_lp += 1 + elif ch2 == ')': + depth_lp -= 1 + elif ch2 == '[': + depth_lb += 1 + elif ch2 == ']': + depth_lb -= 1 + elif ch2 == '{': + depth_lbr += 1 + elif ch2 == '}': + depth_lbr -= 1 + elif ch2 == '<': + depth_la += 1 + elif ch2 == '>' and depth_la > 0: + depth_la -= 1 + elif ch2 == ':' and depth_lp == 0 and depth_lb == 0 and depth_lbr == 0 and depth_la == 0: + colon_idx = idx2; break + if colon_idx is not None: + left = c[:colon_idx].strip() + right = c[colon_idx+1:].strip() + left = ANN_SUB(' ', left) + left = remove_modifiers(left) + left = WS_SUB(' ', left).strip() + m = VAR_END_RE.search(left) + if m: + name = m.group(1) + dims = count_array_pairs(m.group(2) or "") + type_part = left[:m.start()].strip() + locals_out.append({ + "type": type_part or None, + "name": name, + "array_dims": dims, + "init": canon_init(right) + }) + else: + segs = split_top_level(c, ';') + if segs: + init_seg = segs[0].strip() + decl = parse_decl_with_inits(init_seg) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i = endp + continue + + elif token == "catch" and k < n and scrubbed[k] == '(': + _content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + i = endp + continue + + elif token == "try" and k < n and scrubbed[k] == '(': + content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + for part in split_top_level((content or "").strip(), ';'): + decl = parse_decl_with_inits(part.strip()) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i = endp + continue + + i = j + continue + + i += 1 + + return locals_out + + def collect_all_assignment_values(body: str, declared_locals: List[dict]) -> Dict[str, List[str]]: + """ + For each declared local variable, collect ALL RHS expressions assigned to it within the method body: + - Declaration initializers + - Later simple and augmented assignments (e.g., x = ..., x += ..., x <<= ...) + Returns: { localName: [canon_rhs1, canon_rhs2, ...] } + """ + out: Dict[str, List[str]] = {} + if not body: + return out + + locals_set = {d["name"] for d in declared_locals} + seen: Dict[str, set] = {} + + scrubbed = strip_anon_and_local_class_bodies(body) + + # 1) declaration initializers (always included) + for d in declared_locals: + nm = d["name"] + init = d.get("init", "") + if init: + val = canon_init(init) + out.setdefault(nm, []).append(val) + seen.setdefault(nm, set()).add(val) + + # 2) later reassignments (simple and augmented) + i = 0 + n = len(scrubbed) + depth_p = depth_b = depth_br = 0 + in_str = in_chr = False + stmt_start = 0 + + while i < n: + ch = scrubbed[i] + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + if ch == '{' and depth_p == 0 and depth_b == 0: + depth_br += 1; stmt_start = i + 1; i += 1; continue + if ch == '}' and depth_p == 0 and depth_b == 0: + depth_br = max(0, depth_br - 1); stmt_start = i + 1; i += 1; continue + + if ch == ';' and depth_p == 0 and depth_b == 0: + stmt = scrubbed[stmt_start:i].strip() + stmt_start = i + 1 + + # skip declarations (we already captured their initializers) + decl = None if _has_comp_ops_before_assign(stmt) else parse_decl_with_inits(stmt + ';') + + if not decl: + parsed = _split_top_level_assignment(stmt) + if parsed: + lhs, _op, rhs = parsed + # Only record locals that are plain identifiers on the LHS + m = RE_SEARCH(r'^[A-Za-z_$][\w$]*$', lhs) + if m: + nm = m.group(0) + if nm in locals_set: + val = canon_init(rhs) + s = seen.setdefault(nm, set()) + if val not in s: + out.setdefault(nm, []).append(val) + s.add(val) + i += 1 + continue + + i += 1 + + return out + + # ---------- helpers for anchoring ---------- + def first_open_brace_outside_strings(s: str) -> int: + in_str = in_chr = False + esc = False + for idx, ch in enumerate(s): + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '{': + return idx + return -1 + + def prev_identifier_word(s: str, start: int) -> Optional[str]: + i = start - 1 + while i >= 0 and s[i].isspace(): i -= 1 + if i >= 0 and s[i] == '.': + return '.' + while i >= 0 and s[i] in '.=+-/*%&|^!~?:,([{<': + if s[i] == '.': + return '.' + i -= 1 + if i < 0 or not (s[i].isalnum() or s[i] in '_$'): + return None + j = i + while j >= 0 and (s[j].isalnum() or s[j] in '_$'): + j -= 1 + return s[j+1:i+1] + + # ---------- robust: anchor at the real method body, then walk backward ---------- + def parse_single_method(s: str) -> dict: + brace_idx = first_open_brace_outside_strings(s) + if brace_idx == -1: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + close_paren = s.rfind(')', 0, brace_idx) + if close_paren == -1: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + param_text, open_paren = balanced_region_reverse(s, close_paren, ')', '(') + if open_paren is None: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + name_tok_start = prev_non_ws(s, open_paren - 1) + name_tok = None + if name_tok_start is not None: + j = name_tok_start + while j >= 0 and (s[j].isalnum() or s[j] in '_$'): + j -= 1 + name_tok = s[j+1:name_tok_start+1] if j < name_tok_start else None + name_start = j + 1 + else: + name_start = None + + if not name_tok or name_tok in CONTROL: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + prev_word = prev_identifier_word(s, name_start) + if prev_word == '.' or (prev_word in NON_TYPE_STARTS or prev_word in CONTROL): + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + header_start = scan_back_to_boundary(s, open_paren) + pre_name_raw = s[header_start:name_start].rstrip() + pre_clean = remove_modifiers(ANN_SUB(' ', pre_name_raw)).strip() + ret_type = pre_clean or None + + header_params = parse_param_list(param_text or "") + + body_text, _ = balanced_region(s, brace_idx, '{', '}') + body_text = body_text or "" + + locals_ = collect_locals_single_pass(body_text) if body_text else [] + + catch_params = collect_catch_params(body_text) + all_params = list(header_params) + (catch_params or []) + + return { + "name": name_tok, + "return_type": ret_type, + "parameters": all_params, # used for map (includes catch) + "header_params": header_params, # used for key (excludes catch) + "locals": locals_, + "body": body_text + } + + def parse_single_lambda(s: str) -> dict: + arrow = s.find("->") + j = arrow - 1 + while j >= 0 and s[j].isspace(): j -= 1 + if j >= 0 and s[j] == ')': + ptext, _start = balanced_region_reverse(s, j, ')', '(') + params_text = ptext or "" + else: + m = RE_SEARCH(r'[A-Za-z_$][\w$]*$', s[:arrow]) + params_text = m.group(0) if m else "" + + i = arrow + 2 + n = len(s) + while i < n and s[i].isspace(): i += 1 + is_block = False + if i < n and s[i] == '{': + body_text, _ = balanced_region(s, i, '{', '}') + body_text = body_text or "" + is_block = True + else: + body_text = "" + + p = (params_text or "").strip() + if p.startswith('(') and p.endswith(')'): + p = p[1:-1] + header_params = [] + if p: + for raw in split_top_level(p, ','): + item = ANN_SUB(' ', raw.strip()) + item = remove_modifiers(item) + item = WS_SUB(' ', item).strip() + if not item: continue + m = VAR_END_RE.search(item) + if not m: + header_params.append({"name": item, "type": None, "varargs": False, "array_dims": 0}) + continue + name = m.group(1) + arr_after = count_array_pairs(m.group(2) or "") + left = item[:m.start()].rstrip() + varargs = left.endswith('...') + if varargs: left = left[:-3].rstrip() + type_str = left if left else None + header_params.append({"name": name, "type": type_str, "varargs": varargs, "array_dims": arr_after}) + + locals_ = collect_locals_single_pass(body_text) if is_block else [] + + catch_params = collect_catch_params(body_text) if is_block else [] + all_params = list(header_params) + (catch_params or []) + + return { + "kind": "lambda", + "return_type": None, + "parameters": all_params, # used for map (includes catch) + "header_params": header_params, # not used in key for lambda anyway + "locals": locals_, + "body": body_text + } + + # ------------------------- + # Main loop over documents + # ------------------------- + mappings: Dict[str, Dict] = {} + docs = functions_methods_documents + + for doc in tqdm(docs, total=len(docs), desc="Parsing methods/lambda documents for signature and local variables..."): + raw = doc.page_content + source = str(doc.metadata.get("source", "")) + s = strip_comments_keep_strings(raw) + + md5 = hashlib.md5(s.strip().encode("utf-8")).hexdigest() + + # Prefer method parsing even if the body contains lambdas. + m_entry = parse_single_method(s) + is_valid_method = ( + m_entry.get("name") not in (None, "") + and (m_entry.get("body") or m_entry.get("header_params")) + ) + + if is_valid_method: + entry = m_entry + # Use ONLY header parameters for the signature part (exclude catch params) + header_params = entry.get("header_params") or [] + params_sig = ", ".join(render_param_decl(p) for p in header_params) + func_key = f"{entry.get('name','')}({params_sig})@{md5}@{source}" + else: + # Fallback to lambda-only documents + entry = parse_single_lambda(s) + func_key = f"lambda@{md5}@{source}" + + all_vars: Dict[str, Dict[str, object]] = {} + + # parameters (method/lambda + catch) + for p in entry.get("parameters", []): + ptype = render_param_type(p) + all_vars[p["name"]] = {"value": PARAMETER, "type": ptype} + + # return type + ret = canon_ws(entry.get("return_type")) + all_vars[RETURN_TYPES] = ([] if (not ret or ret == "void") else [ret]) + + # collect ALL assignments for locals + locals_list = entry.get("locals", []) or [] + assigns = collect_all_assignment_values(entry.get("body", "") or "", locals_list) + + # locals (do not override parameters) + for loc in locals_list: + name = loc["name"] + if name in all_vars and all_vars[name].get("value") == PARAMETER: + continue + dtype = canon_ws(loc.get("type") or "") + if loc.get("array_dims", 0) > 0: + dtype = (dtype or "var") + "[]" * loc["array_dims"] + dtype_out = LOCAL_IMPLICIT if dtype == "var" else dtype + values_list = assigns.get(name, []) + all_vars[name] = {"value": values_list, "type": dtype_out} + + mappings[func_key] = all_vars + + return mappings + + def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): + struct_initializer_expression_list: Dict[str, str] = {} # TODO not sure its needed at all + var_properties = variables_mappings.get(parts[the_part], None) + if var_properties is not None: + resolved_type = var_properties.get("type") + + value_list = var_properties.get("value") + for val in value_list: + struct_initializer_expression = re.search(r"""\bnew\s+(?:<[^>]*>\s*)?(?:[A-Za-z_$][\w$]*\.)*[A-Za-z_$][\w$]* + (?:\s*<[^<>]*(?:<[^<>]*>[^<>]*)*>)?\s* + (?: + (?:\[\s*\])+\s*\{ # array initializer + | + \((?:[^()"']+|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\( + (?:[^()"']+|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')* + \))*\) # constructor args (approx) + (?:\s*\{)? # optional anon-class brace + )""", val) + + if struct_initializer_expression is not None: + struct_initializer_expression_list[val] = struct_initializer_expression.group(0) + resolved_type = str(resolved_type) + return resolved_type, struct_initializer_expression_list, value_list, var_properties + else: + return None, None, None, None + + def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, + value_list, target_class_names: frozenset[str]) -> bool: + result = False + if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + docs = self.__get_type_docs_matched_with_callee_type(callee_package, resolved_type, type_documents, target_class_names) + + if len(docs) > 0: + result = True + elif PARAMETER in value_list: + result = False + + elif struct_initializer_expression: + struct_type = (struct_initializer_expression.group(0)) # TODO struct_initializer_expression is a list of expressions + docs = self.__get_type_docs_matched_with_callee_type(callee_package, struct_type, type_documents, target_class_names) + if len(docs) > 0: + result = True + return result + + def __get_type_docs_matched_with_callee_type(self, callee_package, checked_type, type_documents, target_class_names: frozenset[str]) -> list[ + Document]:# TODO Make sure Fix works - the target classes can be in other jars + if checked_type and strip_java_generics(checked_type) not in target_class_names: + return [] + + return [a_type for a_type in type_documents if self.is_same_package(extract_jar_name(a_type.metadata['source']), callee_package.partition(":")[2]) + and strip_java_generics(get_type_name(a_type)) in target_class_names] + + def _find_method_return_type_in_type_docs( + self, + receiver_type: str, # simple or qualified type name + method_name: str, + type_documents: list[Document], + ) -> str | None: + """ + Find the declared return type of `receiver_type#method_name(...)` by scanning + the matching type document in `type_documents`. Returns a normalized type + (generics/arrays stripped) or None if not found. + """ + # Normalize the receiver simple name (e.g. "org.quartz.JobDetail" -> "JobDetail") + simple = re.sub(r'\s+', '', receiver_type).rsplit('.', 1)[-1] + simple = re.sub(r'<[^<>]*>', '', simple) + simple = re.sub(r'\s*(\[\])+$', '', simple) + + # A light signature regex (like the one you already use) + sig = re.compile( + rf""" + (?[A-Za-z_.$?][\w$.<>\[\]?]*?) # return type + \s+{re.escape(method_name)}\s*\( # method name + '(' + """, + re.VERBOSE, + ) + + # Pick the first doc whose header declares this simple type + header = re.compile(rf'\b(class|interface|enum|record)\b\s+{re.escape(simple)}\b') + for doc in type_documents: + src = doc.page_content + if not src: + continue + if not header.search(src): + continue + m = sig.search(src) + if not m: + # Could be overloaded; try a looser scan (multiple matches) + matches = list(sig.finditer(src)) + if not matches: + continue + m = matches[0] + + ret = m.group('ret').strip() + ret = re.sub(r'<[^<>]*>', '', ret) # strip generics + ret = re.sub(r'\s*(\[\])+$', '', ret) # strip array suffixes + return ret or None + + return None + + def _infer_type_from_var_initializer( + self, + initializer: str, # e.g. "getDescriptorById(conn, recording.remoteId)" + variables_mappings: dict, # per-function locals/params map + type_documents: list[Document], + caller_src: str, # function.metadata['source'] + code_documents: dict[str, Document], + ) -> str | None: + def _split_top_level_dots(expr: str) -> list[str]: + parts, buf = [], [] + dp = db = dbr = da = 0 + in_str = in_chr = False + i, n = 0, len(expr) + while i < n: + ch = expr[i] + if in_str: + buf.append(ch) + if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + buf.append(ch) + if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 + elif ch == "'": in_chr = False + i += 1; continue + + if ch == '"': in_str = True; buf.append(ch); i += 1; continue + if ch == "'": in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': dp += 1; buf.append(ch); i += 1; continue + if ch == ')': dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': db += 1; buf.append(ch); i += 1; continue + if ch == ']': db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': da += 1; buf.append(ch); i += 1; continue + if ch == '>': + if da > 0: da -= 1 + buf.append(ch); i += 1; continue + + if ch == '.' and dp == db == dbr == da == 0: + parts.append(''.join(buf).strip()); buf = []; i += 1; continue + buf.append(ch); i += 1 + parts.append(''.join(buf).strip()) + return [p for p in parts if p] + + def _strip_call_parens(s: str) -> str: + s2 = s.rstrip() + if not s2.endswith(')'): + return s + depth = 0 + in_str = in_chr = False + i = len(s2) - 1 + while i >= 0: + ch = s2[i] + if in_str: + if ch == '\\': i -= 2; continue + if ch == '"': in_str = False; i -= 1; continue + i -= 1; continue + if in_chr: + if ch == '\\': i -= 2; continue + if ch == "'": in_chr = False; i -= 1; continue + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == ')': depth += 1; i -= 1; continue + if ch == '(': + depth -= 1 + if depth == 0: return s2[:i].rstrip() + i -= 1; continue + i -= 1 + return s + + def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: + sig = re.compile( + rf"""(?[A-Za-z_.$?][\w$.<>\[\]?]*?)\s+{re.escape(method_name)}\s*\( + """, re.VERBOSE) + m = sig.search(source_text) + if not m: + return None + ret = m.group('ret').strip() + ret = re.sub(r'<[^<>]*>', '', ret) + ret = re.sub(r'\s*(\[\])+$', '', ret) + return ret or None + + expr = initializer.strip().rstrip(';') + chain = _split_top_level_dots(expr) + if not chain: + return None + + # ----- CASE 1: starts with a variable identifier (old behavior) ----- + start = chain[0].strip() + ident_only = re.match(r'^[A-Za-z_$][\w$]*$', start) + if ident_only and (len(chain) == 1 or not start.endswith(')')): + start_entry = variables_mappings.get(start, {}) + start_type = (start_entry.get('type') if isinstance(start_entry, dict) else None) + if not start_type: + return None + cur_type = re.sub(r'<[^<>]*>', '', start_type) + cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + + else: + # Examine only the "head" before '(' to decide qualification + head = _strip_call_parens(start).strip() + + # ----- CASE 2: unqualified method call in same file: foo(a,b) ----- + if start.endswith(')') and '.' not in head: + mname = re.match(r'^([A-Za-z_$][\w$]*)$', head) + if not mname: + return None + method_name = mname.group(1) + caller_doc = code_documents.get(caller_src) + caller_text = caller_doc.page_content if caller_doc else "" + ret = _find_method_return_type_in_file(caller_text, method_name) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + + # ----- CASE 3: static-style or qualified head: ClassName.staticMethod(...) or pkg.Class.m(...) + elif start.endswith(')') and '.' in head: + # Split head into type-like left and method right + left, _, right = head.rpartition('.') + # Left should be a type token (simple or qualified). We don't verify imports here. + base_type = re.sub(r'<[^<>]*>', '', left) + base_type = re.sub(r'\s*(\[\])+$', '', base_type) + method_name = right + if not re.match(r'^[A-Za-z_$][\w$]*$', method_name): + return None + # Return type of static method on 'base_type' + ret = self._find_method_return_type_in_type_docs(base_type, method_name, type_documents) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + + else: + return None + + # ----- Walk remaining segments; method calls change the type ----- + for seg in chain[1:]: + seg_core = _strip_call_parens(seg).strip() + if not seg.endswith(')'): # field access — keep type + continue + mname = re.match(r'^([A-Za-z_$][\w$]*)\s*$', seg_core) + if not mname: + continue + method_name = mname.group(1) + ret = self._find_method_return_type_in_type_docs(cur_type, method_name, type_documents) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + + return cur_type or None + + def get_package_name(self, function: Document, package_name: str) -> str: + return package_name if extract_jar_name(function.metadata['source']) in package_name else '' \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py index c6da4ccaf..c816ebfac 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py @@ -1,10 +1,10 @@ from abc import ABC, abstractmethod +from typing import Tuple, List +from warnings import warn import re from langchain_core.documents import Document - - class LanguageFunctionsParser(ABC): # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing @@ -19,7 +19,7 @@ def create_map_of_local_vars(self, functions_methods_documents: list[Document]) # containing mapping of all fields names in the type/class to their defining types. # a list of @abstractmethod - def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> dict[tuple, list[tuple]]: pass @abstractmethod @@ -32,7 +32,9 @@ def get_function_name(self, function: Document) -> str: def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, callee_function_package: str, code_documents: list[Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict]) -> bool: + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> bool: pass # This method gets a function document, and return a list with the containing package name. if @@ -74,10 +76,8 @@ def is_exported_function(self, function: Document) -> bool: # This method get a source document, and returns True only if the document type # is a function ( not type/class/full_source) for the ecosystem. - def is_function(self, function: Document) -> bool: - return function.page_content.startswith(self.get_function_reserved_word()) - + pass # This method returns the name of the directory containing the 3rd party packages/libs source code files for the # ecosystem @@ -105,12 +105,14 @@ def is_searchable_file_name(self, function: Document) -> bool: # This method returns the function/method reserved word in code for the ecosystem/programming language. @abstractmethod def get_function_reserved_word(self) -> str: - pass + warn("This function is deprecated. Use is_function() instead.", DeprecationWarning, stacklevel=2) + return "" # This function returns the type reserved word in code for the ecosystem/programming language. @abstractmethod def get_type_reserved_word(self) -> str: - pass + warn("This function is deprecated. Use is_type() instead.", DeprecationWarning, stacklevel=2) + return "" @abstractmethod def get_dummy_function(self, function_name) -> str: @@ -121,7 +123,7 @@ def is_script_language(): return False def is_same_package(self, package_name_from_input, package_name_from_tree): - return package_name_from_input.lower() in package_name_from_tree.lower() + return package_name_from_input.lower() == package_name_from_tree.lower() @staticmethod def get_constructor_method_name(): @@ -133,6 +135,9 @@ def get_constructor_method_name(): def get_class_name_from_class_function(self, func: Document): return None + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + pass + # This method gets a list of documents and a package name as arguments, and returns a list of # documents that contain import statements for the specified package. It searches for both # single import statements and grouped import statements that include the package name. @@ -145,37 +150,37 @@ def document_imports_package(self, documents: list[Document], package_name: str) def is_doc_type(self, doc: Document) -> bool: return doc.page_content.startswith(self.get_type_reserved_word()) - + def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: relevant_docs = [ - doc for doc in documents + doc for doc in documents if doc.metadata.get('source').__contains__(package_name) and doc.page_content.__contains__(function_name) ] return relevant_docs - + def is_a_package(self, package_name: str, doc: Document) -> bool: b_3party = doc.metadata.get('source').startswith(self.dir_name_for_3rd_party_packages()) - b_package = (not self.is_root_package(doc) and (self.get_package_name(function=doc, package_name=package_name) != "")) + b_package = (not self.is_root_package(doc) and (self.get_package_name(function=doc, package_name=package_name) != "")) return b_3party and b_package - + # This method determines if the search algorithm uses Depth-First Search (DFS) for traversing # function call graphs and dependency trees. Returns True if DFS is used, False for other algorithms. def is_search_algo_dfs(self) -> bool: return True - + # This method checks if the programming language supports macro definitions/preprocessor # capabilities that can affect function parsing and analysis. Returns True for languages # with macros (like C/C++), False otherwise. def is_language_supports_macro(self) -> bool: return False - + # This method determines if a dummy function should be created for standard library packages # that may not have complete source code available for analysis. Returns True if a dummy # function should be created for the given package, False if complete source is available. def create_dummy_for_standard_lib(self, package_name: str) -> bool: return False - + # is this call allowed by the language? def is_call_allowed(self,pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: return True \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py index f6252bd2d..4f1178c68 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py @@ -1,6 +1,7 @@ from ..dep_tree import Ecosystem, DependencyTree from .golang_functions_parsers import GoLanguageFunctionsParser from .python_functions_parser import PythonLanguageFunctionsParser +from .java_functions_parsers import JavaLanguageFunctionsParser from .lang_functions_parsers import LanguageFunctionsParser from .c_lang_function_parsers import CLanguageFunctionsParser @@ -23,5 +24,7 @@ def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | No if tree is not None: parser_obj.init_CParser(tree) return parser_obj + elif ecosystem == Ecosystem.JAVA: + return JavaLanguageFunctionsParser() else: return LanguageFunctionsParser() diff --git a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py index 0b8500fef..5007e8624 100644 --- a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py +++ b/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py @@ -1,6 +1,7 @@ import ast import os.path import re +from typing import Tuple, List from langchain_core.documents import Document @@ -27,10 +28,6 @@ class PythonLanguageFunctionsParser(LanguageFunctionsParser): - - def is_same_package(self, package_name_from_input, package_name_from_tree): - return package_name_from_input.lower() == package_name_from_tree.lower() - def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}(): pass" @@ -87,38 +84,19 @@ def create_map_of_local_vars(self, functions_methods_documents: list[Document]) return mappings - @staticmethod - def get_class_name_from_class_function(func: Document): + def get_class_name_from_class_function(self, func: Document): pattern = r'#\(class:\s*([\w_]+)\)$' match = re.search(pattern, func.page_content) class_name = match.group(1) if match else None return class_name - def is_package_imported(self, code_content, callee_package, identifier=None) -> bool: - for line in code_content.page_content.split(os.linesep): - if not self.is_comment_line(line): - if 'import' in line and callee_package in line: - if identifier: - if identifier == callee_package: - if all(word in line for word in ['import', callee_package]): - return True - else: - if all(word in line for word in ['import', callee_package, 'as', identifier]) \ - or all(word in line for word in ['from', callee_package, 'import', identifier]): - return True - else: - if all(word in line for word in ['from', callee_package, 'import']): - return True - return False - def get_type_reserved_word(self) -> str: return "class" def get_function_reserved_word(self) -> str: return "def" - @staticmethod - def is_searchable_file_name(function: Document) -> bool: + def is_searchable_file_name(self, function: Document) -> bool: file_path = function.metadata['source'] file_name = os.path.basename(file_path) return re.search(r"^test_|.*_test.py", file_name) is None @@ -155,7 +133,9 @@ def search_for_called_function(self, caller_function: Document, callee_function_ callee_function_package: str, code_documents: dict[str, Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict]) -> bool: + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document]=None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> bool: calls = self._get_function_calls(caller_function, callee_function_name, code_documents) if not calls: @@ -175,8 +155,8 @@ def search_for_called_function(self, caller_function: Document, callee_function_ caller_function_package = self.get_package_names(caller_function)[0] if callee_function_package == caller_function_package: return True - return (self.is_package_imported(caller_document, callee_function_package, identifier) or - self.is_package_imported(caller_function, caller_function_package, identifier)) + return (self.is_package_imported(caller_document.page_content, identifier, callee_function_package) or + self.is_package_imported(caller_function.page_content, identifier, caller_function_package)) # There is a qualifier identifier. else: identifier = parts[-2] @@ -187,7 +167,7 @@ def search_for_called_function(self, caller_function: Document, callee_function_ # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , # if it's the same as callee package name - if self.is_package_imported(caller_document, callee_function_package, identifier): + if self.is_package_imported(caller_document.page_content, identifier, callee_function_package): return True ## otherwise, the identifier is in the caller function body or in signature/receiver function argument. @@ -384,7 +364,7 @@ def __get_variable_data(variable_declaration:str) -> tuple[str, str, str]: return variable_name, variable_type, variable_value - def parse_all_type_struct_class_to_fields(self, types: list[Document]) -> dict[tuple, list[tuple]]: + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: types_mapping = dict() for the_type in types: type_key = the_type.page_content.split('class')[1].split('(')[0].strip() @@ -403,7 +383,24 @@ def is_script_language(): @staticmethod def get_constructor_method_name(): return '__init__' - + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + for line in code_content.split(os.linesep): + if not self.is_comment_line(line): + if 'import' in line and callee_package in line: + if identifier: + if identifier == callee_package: + if all(word in line for word in ['import', callee_package]): + return True + else: + if all(word in line for word in ['import', callee_package, 'as', identifier]) \ + or all(word in line for word in ['from', callee_package, 'import', identifier]): + return True + else: + if all(word in line for word in ['from', callee_package, 'import']): + return True + return False + def is_a_package(self, package_name: str, doc: Document) -> bool: return (not self.is_root_package(doc) and - self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file + self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index d782c3a0c..ece0d3df2 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import os from contextlib import asynccontextmanager diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py new file mode 100644 index 000000000..47ffefec8 --- /dev/null +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -0,0 +1,680 @@ +import re +from pathlib import Path +from typing import List +from tqdm import tqdm + +from langchain_core.documents import Document + +from .chain_of_calls_retriever_base import PARENTS_INDEX, calculate_hashable_string_for_function, EXCLUSIONS_INDEX, \ + ChainOfCallsRetrieverBase, METHOD_EXCLUSIONS_INDEX +from .data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY +from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from .functions_parsers.lang_functions_parsers_factory import ( + get_language_function_parser, +) + +from vuln_analysis.logging.loggers_factory import LoggingFactory +from .java_utils import convert_from_maven_artifact, extract_jar_name, is_maven_gav, extract_method_name_with_params, \ + create_inheritance_map, get_target_class_names, dummy_package_name +from ..data_models.input import SourceDocumentsInfo + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +# Lowercase package segments; class segments start with uppercase; allow dots or $ for inners +_FQCN_STRICT_RE = re.compile( + r""" + ^(?=.+\.) # must contain at least one dot (have a package) + (?:[a-z_][a-z0-9_]*\.)+ # package segments (conventional lowercase) + [A-Z][A-Za-z0-9_$]* # top-level class (starts uppercase by convention) + (?:[.$][A-Z][A-Za-z0-9_$]*)* # optional inner classes via '.' or '$' + $""", + re.X, +) + +# Matches "--sources/" in the path +_GAV_DIR_RE = re.compile(r'([^/\\]+)-([0-9][^/\\]*)-sources(?:/|\\)') + +class JavaChainOfCallsRetriever(ChainOfCallsRetrieverBase): + """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being + called from the application code base or not. + """ + def __init__(self, documents: List[Document], + ecosystem: Ecosystem, + manifest_path: Path, + query: str, + code_source_info: SourceDocumentsInfo): + """ + :param documents: + List of documents containing the functions/methods and classes/types of the application code + + application dependencies' code + :param ecosystem: Ecosystem + The programming language of the list of documents + :param manifest_path: str + The manifest of the application, defining all the direct and transitive dependencies of the application, being + used here to build the dependency tree for a more efficient lookup and search. + """ + + logger.debug("Creating Chain of Calls Retriever") + logger.debug("Starting building Chain of Calls Retriever") + self.ecosystem = ecosystem + logger.debug("Chain of Calls Retriever - creating dependency tree") + # Build dependency tree based on the parameter programming language/package manager. + self.dependency_tree = DependencyTree(ecosystem=ecosystem, query=query) + logger.debug("Chain of Calls Retriever - get language parser") + self.language_parser = get_language_function_parser(ecosystem, self.dependency_tree) + self.manifest_path = manifest_path + # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. + if self.dependency_tree.builder is None: + raise RuntimeError("Couldn't continue as dependency tree weren't generated") + + self.tree_dict = dict() + + # Build a dependency tree using the dependency tree builder logic. + tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) + for package, parents in tree.items(): + parents.extend([package]) + self.tree_dict[package] = list() + self.tree_dict[package].append(parents) + self.tree_dict[package].append([]) # For package exclusions + self.tree_dict[package].append({}) # For methods exclusions + self.supported_packages = list(self.tree_dict.keys()) + logger.debug("Chain of Calls Retriever - populating functions documents") + allowed_files_extensions = self.language_parser.supported_files_extensions() + + # filter out unsupported files extensions. + self.documents = [doc for doc in documents + if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) + .endswith(ext)])] + + documents_of_functions, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_functions") + # filter out types and full code documents, retaining only functions/methods documents in this attribute. + if not exist: + self.documents_of_functions = [doc for doc in tqdm(self.documents, total=len(self.documents), desc="Filtering documents, leaving functions/methods ...") + if self.language_parser.is_function(doc)] + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_functions, "documents_of_functions") + else: + self.documents_of_functions = documents_of_functions + + documents_of_types, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_types") + # filter out full code documents and functions/methods docs, retaining only types/classes docs + if not exist: + self.documents_of_types = [doc for doc in tqdm(self.documents, total=len(self.documents) , desc="Filtering documents, leaving types/classes ...") + if self.language_parser.is_doc_type(doc)] + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_types, "documents_of_types") + else: + self.documents_of_types = documents_of_types + + # boolean attribute that indicates whether a path was found or not, initially set to False. + self.found_path = False + # Filter out all documents but full source docs. + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in self.documents + if doc.metadata.get('content_type') == 'simplified_code'} + if not self.language_parser.is_script_language(): + self.documents = self.documents_of_functions + logger.debug("Chain of Calls Retriever - after documents_of_full_sources") + self.last_visited_parent_package_indexes = dict() + + type_inheritance, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "type_inheritance") + if not exist: + self.type_inheritance = create_inheritance_map([fsrc for fsrc in self.documents_of_full_sources.values()]) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.type_inheritance, "type_inheritance") + else: + self.type_inheritance = type_inheritance + + types_classes_fields_mapping, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "types_classes_fields_mapping") + # Constructing a map of types and classes to their attributes/members/fields + if not exist: + self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields( + self.documents_of_types, self.type_inheritance) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.types_classes_fields_mapping, "types_classes_fields_mapping") + else: + self.types_classes_fields_mapping = types_classes_fields_mapping + + functions_local_variables_index, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "functions_local_variables_index") + # Create a data structure containing dict of key=(function_name@source_file),value = dict of + # local variables names mapped to (types, (values or expressions)) + if not exist: + self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.functions_local_variables_index, "functions_local_variables_index") + else: + self.functions_local_variables_index = functions_local_variables_index + + logger.debug("Chain of Calls Retriever - after functions_local_variables_index") + + def __find_caller_function(self, document_function: Document, function_package: str) -> Document: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :return: a document of a function that is calling document_function + """ + # Create a copy of the document types list + documents_of_types: list[Document] = self.documents_of_types[:] + direct_parents = list() + # gets list of all direct parents of function + + list_of_packages = self.tree_dict.get(function_package) + if list_of_packages: + direct_parents.extend(list_of_packages[PARENTS_INDEX]) + # gets list of documents to search in only from parents of function' package. + function_name_to_search = self.language_parser.get_function_name(document_function) + + function_file_name = document_function.metadata.get('source') + relevant_docs_to_search_in = list() + last_visited_package_index = (self.last_visited_parent_package_indexes + .get(calculate_hashable_string_for_function(function_file_name, + function_name_to_search), 0)) + func_pack_from_tree = self.tree_dict.get(function_package) + package_exclusions = func_pack_from_tree[EXCLUSIONS_INDEX] + method_exclusions = func_pack_from_tree[METHOD_EXCLUSIONS_INDEX] + + target_class_names: frozenset[str] + class_name = self.language_parser.get_class_name_from_class_function(document_function) + key = (class_name, document_function.metadata['source']) + if "dummy" not in function_file_name: + target_class_names = get_target_class_names(self.type_inheritance[key]) + else: + target_class_names = frozenset([class_name]) + target_type_doc = Document(page_content="public class " + class_name + "{}", + metadata={"source": self.language_parser.dir_name_for_3rd_party_packages() + + "/" + function_package.partition(':')[-1] + + "/" + class_name + ".java", + "ecosystem": self.ecosystem}) + documents_of_types.append(target_type_doc) + + # Search for caller functions only at parents according to dependency tree. + for package in direct_parents[last_visited_package_index:]: + sources_location_packages = True + if self.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: + sources_location_packages = False + + artifact_id = package.split(":", 2)[1] + possible_docs = self.get_possible_docs(function_name_to_search, + artifact_id, + package_exclusions, + sources_location_packages, + target_class_names, + method_exclusions) + + jar_name = convert_from_maven_artifact(package) + # Collect all potential caller functions + for doc in self.get_functions_for_package(package_name=jar_name, + documents=possible_docs, + sources_location_packages=sources_location_packages, + function_to_search=function_name_to_search, + callee_function_file_name=function_file_name): + relevant_docs_to_search_in.append(doc) + # Perform the search only on the subset of potential caller functions + for doc in relevant_docs_to_search_in: + function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, + callee_function_name= + function_name_to_search, + callee_function=document_function, + callee_function_package= + function_package, + code_documents= + self.documents_of_full_sources, + type_documents= + documents_of_types, + callee_function_file_name= + function_file_name, + fields_of_types= + self.types_classes_fields_mapping, + functions_local_variables_index= + self.functions_local_variables_index, + documents_of_functions= + self.documents_of_functions, + type_inheritance=self.type_inheritance) + + # If current document is found to be calling document_function in function_package, then return it as a + # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. + if function_is_being_called: + package_exclusions.append(doc) + return doc + else: + method_name = extract_method_name_with_params(doc.page_content) + key = (doc.metadata['source'], function_name_to_search, target_class_names, method_name) + method_exclusions[key] = True + + # If didn't find a matching caller function document, returns None. + return None + + def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: + """ + Checks if a document is in the exclusions list based on its + function name, function body and source metadata. + """ + doc_function_content = doc.page_content.strip() + doc_source = doc.metadata.get('source').strip() + + for exclusion_doc in exclusions: + exclusion_function_content = exclusion_doc.page_content.strip() + exclusion_source = exclusion_doc.metadata.get('source').strip() + + if doc_function_content == exclusion_function_content and doc_source == exclusion_source: + return True + return False + + def _is_method_excluded(self, function_name_to_search: str, target_class_names: frozenset[str], doc: Document, method_exclusions: dict) -> bool: + method_source = doc.metadata['source'] + method_name = extract_method_name_with_params(doc.page_content) + + key = (method_source, function_name_to_search, target_class_names, method_name) + + return key in method_exclusions + + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all + # excluded functions, and all function that their body doesn't contain the target function name to search for. + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + if sources_location_packages: + filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') + and self.language_parser.is_function(doc) and + not self._is_doc_excluded(doc, exclusions) and + not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] + else: + filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and + (self.language_parser.is_function(doc) or self.language_parser.is_script_language()) and + not self._is_doc_excluded(doc, exclusions) and + not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] + + return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + """Sync implementations for retriever.""" + self.found_path = False + + class_name, method_name, package_name = self.extract_from_query(query) + + end_loop = False + found_package = False + matching_documents = [] + # Check if input package is in dependency tree + for package in self.tree_dict: + if self.language_parser.is_same_package(package_name, package): + package_name = package + found_package = True + break + # If it's , then create a document for it. + if found_package: + target_function_doc = self.__find_initial_function(class_name=class_name, + method_name=method_name, + package_name=package_name, + documents=self.documents) + + if target_function_doc: + matching_documents.append(target_function_doc) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # Otherwise, don't even start the process as the target function is not in dependencies tree or not a standard + # library + else: + end_loop = True + logger.error(f"Cannot find initial function=${method_name}, in package=${package_name}") + # If not, there is a chance that the package is some standard library in the ecosystem. + else: + # Try to create dummy package for ecosystem standard library function, in such case, build a document for + # vulnerable function in standard lib package of language. + package_name = dummy_package_name + + page_content = self.language_parser.get_dummy_function(method_name) + target_function_doc = Document(page_content=page_content + , metadata={"source": package_name + "/" + class_name.replace(".", "/") + ".java", + "ecosystem": self.ecosystem}) + matching_documents.append(target_function_doc) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + + importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, class_name) + + root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] + prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() + # find all parents ( all importing packages) of the input package so we'll have candidate pkgs to search in. + parents = list({ + next( + (key for key in self.tree_dict.keys() if extract_jar_name(doc.metadata['source']) in key), + None # Default value if no key is found (optional) + ) + for doc in importing_docs + if doc.metadata['source'].startswith(prefix_of_3rd_parties_libs) + }) + + for doc in importing_docs: + if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): + parents.append(root_package[0]) + break + self.tree_dict[package_name] = [parents, [], {}] + + current_package_name = package_name + # main loop. + while not end_loop: + # Find a caller function and containing package in the dependency tree according to hierarchy + found_document = self.__find_caller_function(document_function=target_function_doc, + function_package=current_package_name) + # If found, then add it to path + if found_document: + matching_documents.append(found_document) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(found_document): + end_loop = True + self.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + target_function_doc = found_document + # extract package name from function document + current_package_name = self.__determine_doc_package_name(target_function_doc) + else: + # end loop because didn't find a caller for initial function + if len(matching_documents) == 1: + end_loop = True + # Backtrack - means that we're going back because current function has no callers anywhere, so we + # need to remove it, and continue with other possible potential called functions from its caller + else: + dead_end_node = matching_documents.pop() + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # Excludes dead end function node from future searches, as it led to nowhere. + self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + target_function_doc = matching_documents[-1] + current_package_name = self.__determine_doc_package_name(target_function_doc) + + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, self.found_path + + def extract_from_query(self, query: str) -> tuple[str, str, str]: + (package_name, function) = tuple( query.splitlines()[0].strip('"\'').replace("#", ".").split(",")) + + class_name = function.rpartition('.')[0] + method_name = function.rpartition('.')[2] + + if not class_name and self.is_java_fqcn(package_name): + class_name = package_name + + package_name, class_name = self.infer_class_name_and_package_name(method_name, class_name) + + return class_name, method_name, package_name + + def __get_parents(self, importing_docs: list[Document], prefix_of_3rd_parties_libs: str) -> set[str]: + pkg_names = { + names[0] + for doc in importing_docs + if doc.metadata['source'].startswith(prefix_of_3rd_parties_libs) + for names in [self.language_parser.get_package_names(doc)] + if names + } + + # 2) Keep only k's where the package name appears in them (substring logic preserved) + parents = {k for k in self.tree_dict if any(pkg in k for pkg in pkg_names)} + + return parents + + def __determine_doc_package_name(self, target_function_doc): + """ + Determine the logical package/dependency identifier for a given document. + + This method inspects `target_function_doc.metadata['source']` and tries to + resolve it to a "package name" or dependency key based on `self.tree_dict` + and the language parser's 3rd-party directory convention. + + Behaviour: + - If the source path contains the directory name returned by + `self.language_parser.dir_name_for_3rd_party_packages()`, the document + is treated as coming from a 3rd-party JAR: + * Extract a `jar_name` from the source path via `extract_jar_name`. + * Iterate over all values in `self.tree_dict`; for each entry look at + `dep_vals[0]` (a collection of dependency identifiers). + * Return the first dependency string (`dep_val`) that contains + `jar_name` as a substring. + + - Otherwise, the document is treated as non-3rd-party (e.g. first-party): + * Extract a `jar_name` from `"/" + src`. + * If `jar_name` exists as a key in `self.tree_dict` and its value is + truthy, return `jar_name`. + + - If no matching dependency/package can be found, return an empty string. + + Parameters + ---------- + target_function_doc : + A document object whose `metadata['source']` field contains the source + path used to infer the package/dependency. + + Returns + ------- + str + The resolved package/dependency identifier, or an empty string if no + match could be determined. + """ + src = target_function_doc.metadata['source'] + if self.language_parser.dir_name_for_3rd_party_packages() in src: + jar_name = extract_jar_name(src) + for dep_vals in self.tree_dict.values(): + for dep_val in dep_vals[0]: + if jar_name in dep_val: + return dep_val + else: + jar_name = extract_jar_name('/' + src) + if self.tree_dict[jar_name]: + return jar_name + + return "" + + def __find_initial_function(self, class_name: str, method_name: str, package_name: str ,documents: list[Document]) -> Document | None: + jar_name = convert_from_maven_artifact(package_name) + + # TODO check if its correct to handle none 3rd party code here as well + relevant_docs = [doc for doc in documents + if (doc.metadata.get('source').__contains__(self.language_parser.dir_name_for_3rd_party_packages()) + and doc.metadata.get('source').__contains__(jar_name) or + not doc.metadata.get('source').__contains__(self.language_parser.dir_name_for_3rd_party_packages())) and + self.language_parser.get_class_name_from_class_function(doc) == class_name.rpartition('.')[2] and + self.language_parser.get_function_name(doc) == method_name] + package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] + # TODO handle method overloading + for document in relevant_docs: + package_exclusions.append(document) + return document + + return None + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str]: + results = [] + for i, package_function in enumerate(reversed(call_hierarchy_list)): + class_name = self.language_parser.get_class_name_from_class_function(package_function) + function_name = self.language_parser.get_function_name(package_function) + jar_name = extract_jar_name(package_function.metadata['source']) + current_level = f"(jar_name={jar_name}, class_name={class_name}, function={function_name},depth={i})" + + results.append(current_level) + + return results + + def function_called_from_caller_body(self, document: Document, function_to_search: str) -> bool: + """ + Return True iff the Java method body (document.page_content) contains a call + to `function_to_search`. Assumptions: the source is a single method (no + comments/annotations/javadocs before it). + """ + import re + + name_raw = (function_to_search or "").strip() + if not name_raw: + return True # preserve original behavior + + # Strip any "(...)" suffix if given and extract optional qualifier + base = re.sub(r"\s*\(.*\)\s*$", "", name_raw) + m = re.match(r"^(?:(?P[\w$.]+)\.)?(?P[A-Za-z_$][\w$]*)$", base) + if m: + target_qual = m.group("qual") + target_method = m.group("mname") + else: + target_qual, target_method = None, re.sub(r"[^\w$]", "", base) + + src = document.page_content or "" + + # Search only in the method *body* to avoid matching the header. + body_start = src.find("{") + body = src[body_start + 1:] if body_start != -1 else src + + # Lightly mask string/char literals to avoid false positives inside quotes. + def _strip_strings(s: str) -> str: + out = [] + i, n = 0, len(s) + while i < n: + c = s[i] + if c == '"': + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == '"': + i += 1 + break + else: + i += 1 + out.append(' ') + elif c == "'": + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == "'": + i += 1 + break + else: + i += 1 + out.append(' ') + else: + out.append(c) + i += 1 + return ''.join(out) + + body = _strip_strings(body) + + name_esc = re.escape(target_method) + parts = [ + rf"\.\s*{name_esc}\s*\(", # chained call: ... .getProperty( + rf"(? bool: + return (not self.language_parser.is_root_package(document) and + document.metadata.get('source').__contains__(package_name)) # package_name = jar_name + + def is_java_fqcn(self, s: str) -> bool: + """ + Return True iff `s` looks like a fully qualified Java class name. + + Strict mode (default): + - package: one or more lowercase segments like 'com.example.' + - class & inners: segments start uppercase (e.g., 'Map.Entry' or 'Map$Entry') + Permissive mode: + - allows uppercase/underscore in package segments and lowercase class names + + Rejects arrays/generics/whitespace/etc. Examples: + True: 'java.lang.String', 'java.util.Map.Entry', 'java.util.Map$Entry' + False: 'String', 'java.util', 'java.lang.String[]', 'com..example.Class', + 'com.example.Class' + """ + s = s.strip() + if not s: + return False + + return _FQCN_STRICT_RE.match(s) is not None + + def infer_class_name_and_package_name(self, method_name: str, class_name: str) -> (str, str): + simple_class_name = class_name + parts = class_name.split(".") + # if fqcn, take the simple class name + if len(parts) > 1: + simple_class_name = parts[-1] + + for doc in self.documents_of_functions: + if method_name in doc.page_content and method_name == self.language_parser.get_function_name(doc): + if self.language_parser.get_class_name_from_class_function(doc) == simple_class_name: + source = doc.metadata['source'] + + artifact_name = self.extract_maven_artifact(source) + + for val in self.tree_dict: + if artifact_name in val: + return val, self.extract_fqcn(source) + + return "", class_name + + def extract_fqcn(self, text: str) -> str: + """ + Convert a source file path to an FQCN. + Works both when a '-sources/' prefix exists and when it doesn't. + + Examples: + 'dependencies-sources/hibernate-core-6.6.13.Final-sources/org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + 'org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + """ + + p = text.replace("\\", "/") + + # If there's a '-sources/' prefix, drop everything up to and including the last occurrence. + marker = "-sources/" + cut = p.rfind(marker) + tail = p[cut + len(marker):] if cut != -1 else p + + # Strip leading slash if any + if tail.startswith("/"): + tail = tail[1:] + + # Drop the .java suffix (case-sensitive per your example; change to lower() if needed) + if tail.endswith(".java"): + tail = tail[:-5] + + # Convert path separators to dots to form the FQCN + return tail.replace("/", ".") + + def extract_maven_artifact(self, path: str) -> str: + """ + Extract "artifactId:version" from a path like: + '.../hibernate-core-6.6.13.Final-sources/org/hibernate/type/.../ArrayJavaType.java' + -> 'org.hibernate:hibernate-core:6.6.13.Final' + + If the input is only 'org/hibernate/.../ArrayJavaType.java' (no artifact/version in the path), + returns "". + + """ + p = path.replace("\\", "/") + m = _GAV_DIR_RE.search(p) + if not m: + return "" # no artifact/version info in the path + + artifact, version = m.group(1), m.group(2) + + # Tail after the "-sources/" marker; use the first 2 dirs for groupId (e.g., "org.hibernate") + tail = p[m.end():] + pkg_dirs = tail.split("/") + # drop the filename + if pkg_dirs and pkg_dirs[-1].endswith(".java"): + pkg_dirs = pkg_dirs[:-1] + + if len(pkg_dirs) < 2: + return "" + + return f"{artifact}:{version}" diff --git a/src/vuln_analysis/utils/java_segmenters_with_methods.py b/src/vuln_analysis/utils/java_segmenters_with_methods.py new file mode 100644 index 000000000..19e9bf4cc --- /dev/null +++ b/src/vuln_analysis/utils/java_segmenters_with_methods.py @@ -0,0 +1,2029 @@ +import bisect +import os + +from typing import List, Tuple, Optional, Dict, Set + +from langchain_community.document_loaders.parsers.language.java import JavaSegmenter + + +class JavaSegmenterWithMethods(JavaSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + def extract_functions_classes(self) -> List[str]: + """ + This method extracts all the golang methods and anonymous functions from a source code, and appends them to + the list of all golang regular functions calculated in parent class' method. + : + :return: a list of all golang functions/methods/anonymous functions ( each one with a complete implementation, + signature + body) + """ + function_classes = super().extract_functions_classes() + methods = extract_methods(self.code) # TODO Add constructors to the method list + inner_classes = extract_inner_classes(self.code) # TODO Add anonymous inner classes support + function_classes.extend(methods) + function_classes.extend(inner_classes) + + return function_classes + + def simplify_code(self) -> str: + opt_java_embedding = os.environ.get('OPTIMIZE_JAVA_FOR_EMBEDDING') + if opt_java_embedding: + return self.__original_simplify_code() + + return self.code + + def __original_simplify_code(self) -> str: + """ + Replace each top-level or nested type declaration body with a single + comment line of the form: + // Code for:
+ while preserving package/imports and any non-declaration code. + + Performance notes: + - Uses byte offsets from tree-sitter to avoid per-line per-node loops. + - Finds header via a single regex on the node slice (bytes). + - Records removal intervals (line ranges) and merges them once. + - Produces output with a single linear pass. + """ + import re + from bisect import bisect_right + + language = self.get_language() + query = language.query(self.get_chunk_query()) + + parser = self.get_parser() + # Work with bytes once + code_str = self.code + code_bytes = code_str.encode("utf-8", "surrogatepass") + tree = parser.parse(code_bytes) + + # Precompute newline byte indices for fast byte->line mapping + # lines: 0..n-1; newline_indices holds byte offsets of '\n' + newline_indices = [] + for i, ch in enumerate(code_bytes): + if ch == 10: # b'\n' + newline_indices.append(i) + n_lines = len(self.source_lines) + + def byte_to_line(b: int) -> int: + # Returns 0-based line number for byte offset b + # Count how many '\n' are before b + return bisect_right(newline_indices, b - 1) + + # Header matcher on BYTES (handles @interface and the keywords) + header_re = re.compile(rb"\b(class|interface|enum|record|module)\b|@interface") + + # We'll collect: + # - header_map: line -> replacement string (comment line) + # - intervals_to_remove: list of (start_line, end_line) inclusive, not touching header_line + header_map = {} + intervals_to_remove = [] + + # Iterate captures once; sort by start byte to favor outer declarations first + captures = query.captures(tree.root_node) + captures.sort(key=lambda cap: cap[0].start_byte) + + for node, _ in captures: + sb, eb = node.start_byte, node.end_byte + + # Search header keyword within the node range (first small window, then full if needed) + # Small window optimized to catch typical cases quickly + SMALL = 4096 # 4KB window is enough for very long headers + slice1 = code_bytes[sb:min(eb, sb + SMALL)] + m = header_re.search(slice1) + if not m and eb - sb > SMALL: + # Fallback: search the rest + m = header_re.search(code_bytes[sb + SMALL:eb]) + offset = SMALL if m else 0 + else: + offset = 0 + + if not m: + # Shouldn't happen; fall back to node.start_point line + start_line = node.start_point[0] + end_line = node.end_point[0] + header_line = start_line + else: + header_byte = sb + offset + m.start() + header_line = byte_to_line(header_byte) + start_line = node.start_point[0] + end_line = node.end_point[0] + + # Determine the header line text to echo in the comment + # Prefer the exact header line; fallback to the node's first line if empty + if 0 <= header_line < n_lines: + header_text = self.source_lines[header_line].rstrip() + else: + header_text = "" + + if not header_text and 0 <= start_line < n_lines: + header_text = self.source_lines[start_line].rstrip() + + header_map.setdefault( + header_line, self.make_line_comment(f"Code for: {header_text}") + ) + + # Add removal intervals for everything EXCEPT the header line + if start_line <= end_line: + if start_line <= header_line - 1: + intervals_to_remove.append((start_line, header_line - 1)) + if header_line + 1 <= end_line: + intervals_to_remove.append((header_line + 1, end_line)) + + # Merge intervals to minimize membership checks + if intervals_to_remove: + intervals_to_remove.sort() + merged = [] + cs, ce = intervals_to_remove[0] + for s, e in intervals_to_remove[1:]: + if s <= ce + 1: + ce = max(ce, e) + else: + merged.append((cs, ce)) + cs, ce = s, e + merged.append((cs, ce)) + intervals_to_remove = merged + + # Build the output in a single pass over lines: + # skip any line that falls inside a removal interval; replace header lines from header_map + out_lines = [] + it = iter(intervals_to_remove) + try: + cur = next(it) + except StopIteration: + cur = None + + for ln in range(n_lines): + # Advance intervals until current could include ln + while cur is not None and cur[1] < ln: + try: + cur = next(it) + except StopIteration: + cur = None + break + + # If ln is in a removal interval -> skip + if cur is not None and cur[0] <= ln <= cur[1]: + continue + + # Replace header line if present; else original line + repl = header_map.get(ln) + if repl is not None: + out_lines.append(repl) + else: + out_lines.append(self.source_lines[ln]) + + return "\n".join(out_lines) + + def get_chunk_query(self) -> str: + return CHUNK_QUERY + +CHUNK_QUERY = """ + [ + (class_declaration) @class + (interface_declaration) @interface + (enum_declaration) @enum + (record_declaration) @record + (annotation_type_declaration) @ann_interface + (module_declaration) @module + ] +""".strip() + +_NON_METHOD_TOKENS = { + "if", "for", "while", "switch", "catch", "try", "synchronized", + "assert", "return", "throw", "this", "super", "new", "case", "default" +} + +def _is_ident_start(ch: str) -> bool: + return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') + +def _is_ident_part(ch: str) -> bool: + return _is_ident_start(ch) or ('0' <= ch <= '9') + +# ------------------------- whitespace & comment skipper ----------------------- + +def _strip_body_comments_and_annotations(src: str, lbrace: int, rbrace: int) -> str: + """ + Return the body substring src[lbrace:rbrace+1] with // and /*...*/ comments + and Java annotations (e.g., @SuppressWarnings(...), @Nullable, @A.B) removed. + Strings/chars/text blocks are preserved verbatim. Keeps '@interface' intact. + Inserts a single space when removal would glue tokens together. + """ + out: List[str] = [] + i = lbrace + n = min(rbrace + 1, len(src)) + + def last_char() -> str: + if not out: + return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + # --- comments --- + if ch == '/' and i + 1 < n and src[i + 1] == '/': + # line comment: drop but preserve newline if present + i += 2 + j = src.find('\n', i, n) + i = n if j == -1 else j + if i < n and src[i] == '\n': + out.append('\n') + continue + if ch == '/' and i + 1 < n and src[i + 1] == '*': + # block / Javadoc comment + j = src.find('*/', i + 2, n) + if j == -1: + break + prev = last_char() + nxt = src[j + 2] if j + 2 < n else '' + if (prev and (prev.isalnum() or prev in '$_)[]>') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + i = j + 2 + continue + + # --- literals: copy verbatim --- + if src.startswith('"""', i) or ch in ('"', "'"): + j = _skip_java_string_like(src, i) + out.append(src[i:j]) + i = j + continue + + # --- annotations --- + if ch == '@': + # Keep annotation type declarations: '@interface' + if src.startswith('@interface', i): + out.append('@interface') + i += len('@interface') + continue + + prev = last_char() + i += 1 # skip '@' + + # Qualified annotation name: ident ('.' ident)* + if i < n and _is_ident_start(src[i]): + i += 1 + while i < n and _is_ident_part(src[i]): i += 1 + while i < n and src[i] == '.': + j = i + 1 + if j < n and _is_ident_start(src[j]): + i = j + 1 + while i < n and _is_ident_part(src[i]): i += 1 + else: + break + + # optional ws/comments then optional "( ... )" + i = _skip_ws_comments(src, i) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + + # spacer to avoid token gluing + nxt = src[i] if i < n else '' + if (prev and (prev.isalnum() or prev in '$_)>]') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + continue + + # default: copy + out.append(ch) + i += 1 + + return ''.join(out) + +def _strip_annotations_in_range(src: str, start: int, end: int) -> str: + """ + Return src[start:end] with Java annotations removed: + - '@Qualified.Name' + - '@Name(...)' (skips balanced parentheses; strings inside respected) + - multiple/dotted names (e.g., @a.b.C) + Works on any header slice (e.g., from the first modifier to the '{'). + Inserts a single space when removal would glue tokens together. + """ + out: List[str] = [] + i = start + n = min(end, len(src)) + + def last_char() -> str: + if not out: + return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + if ch == '@': + # Remember the previous emitted char to decide if we need a spacer + prev = last_char() + + i += 1 # skip '@' + + # Qualified annotation name: ident ('.' ident)* + if i < n and (_is_ident_start(src[i])): + i += 1 + while i < n and _is_ident_part(src[i]): + i += 1 + while i < n and src[i] == '.': + j = i + 1 + if j < n and _is_ident_start(src[j]): + i = j + 1 + while i < n and _is_ident_part(src[i]): + i += 1 + else: + break + + # Optional whitespace/comments before arguments + i = _skip_ws_comments(src, i) + + # Optional "( ... )" with balanced parens (may contain strings/braces) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + + # If removing glued two tokens together (e.g., 'T,@A U' -> 'T U'), pad one space + nxt = src[i] if i < n else '' + if (prev and (prev.isalnum() or prev in '$_)>]') and + (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + continue + + # Copy through normally (comments/strings can remain here; we strip comments later) + out.append(ch) + i += 1 + + return ''.join(out) + +def _strip_comments_keep_strings_range(src: str, start: int, end: int) -> str: + """ + Return src[start:end] with // and /*...*/ comments removed, preserving + string/char literals and text blocks. Inserts a single space if removing + a block comment would glue two tokens together. + """ + out: List[str] = [] + i = start + n = min(end, len(src)) + + def last_char() -> str: + if not out: return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + # line comment + if ch == '/' and i + 1 < n and src[i + 1] == '/': + i += 2 + j = src.find('\n', i, n) + i = n if j == -1 else j # keep newline (added below if present) + if i < n and src[i] == '\n': + out.append('\n') + continue + + # block/Javadoc comment + if ch == '/' and i + 1 < n and src[i + 1] == '*': + j = src.find('*/', i + 2, n) + if j == -1: + # unterminated: drop rest defensively + break + prev = last_char() + nxt = src[j + 2] if j + 2 < n else '' + if (prev and (prev.isalnum() or prev in '$_)[]>') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + i = j + 2 + continue + + # literals: copy verbatim + if src.startswith('"""', i) or ch in ('"', "'"): + j = _skip_java_string_like(src, i) + out.append(src[i:j]) + i = j + continue + + out.append(ch) + i += 1 + + return ''.join(out) + +def _skip_ws_comments(src: str, i: int) -> int: + """Skip ASCII whitespace and //... or /*...*/ comments.""" + n = len(src) + while i < n: + ch = src[i] + # whitespace + if ch in (' ', '\t', '\r', '\n', '\f'): + i += 1 + continue + # line comment + if i + 1 < n and ch == '/' and src[i + 1] == '/': + i += 2 + nl = src.find('\n', i) + i = n if nl == -1 else nl + 1 + continue + # block comment (incl. Javadoc) + if i + 1 < n and ch == '/' and src[i + 1] == '*': + j = src.find('*/', i + 2) + i = n if j == -1 else j + 2 + continue + break + return i + +# --------------------------- string / char / text blocks ---------------------- + +def _skip_java_text_block(src: str, i: int) -> int: + # src[i:i+3] == '"""' (Java text block) + n = len(src) + i += 3 + while i < n: + j = src.find('"""', i) + if j == -1: + return n + # if escaped \"\"\", keep scanning + k = j - 1 + backslashes = 0 + while k >= 0 and src[k] == '\\': + backslashes += 1 + k -= 1 + if (backslashes % 2) == 1: + i = j + 1 + continue + return j + 3 + return n + +def _skip_java_string(src: str, i: int) -> int: + # src[i] == '"' + n = len(src) + i += 1 + while i < n: + ch = src[i] + if ch == '\\': + i += 2 + elif ch == '"': + return i + 1 + else: + i += 1 + return n + +def _skip_java_char(src: str, i: int) -> int: + # src[i] == '\'' + n = len(src) + i += 1 + while i < n: + ch = src[i] + if ch == '\\': + i += 2 + elif ch == "'": + return i + 1 + else: + i += 1 + return n + +def _skip_java_string_like(src: str, i: int) -> int: + if src.startswith('"""', i): + return _skip_java_text_block(src, i) + if src[i] == '"': + return _skip_java_string(src, i) + if src[i] == "'": + return _skip_java_char(src, i) + return i + +# --------------- delimiter & ignored-span index (one-pass pre-scan) ---------- + +_PAR_O2C: Dict[int, int] = {} +_PAR_C2O: Dict[int, int] = {} +_BR_O2C: Dict[int, int] = {} +_BR_C2O: Dict[int, int] = {} + +# All spans to ignore for scanning (comments, strings, text blocks) +_IGN_SPANS: List[Tuple[int, int]] = [] # [(start, end)], end exclusive +_IGN_STARTS: List[int] = [] +_IGN_ENDS: List[int] = [] + +_DELIM_SRC_ID: Optional[int] = None + +def _ignored_span_at(pos: int) -> Optional[Tuple[int, int]]: + """Return (start,end) ignored span containing pos, or None.""" + if not _IGN_SPANS: + return None + idx = bisect.bisect_right(_IGN_STARTS, pos) - 1 + if idx >= 0 and _IGN_STARTS[idx] <= pos < _IGN_ENDS[idx]: + return _IGN_SPANS[idx] + return None + +def _build_delim_index(source: str) -> Tuple[ + Dict[int, int], Dict[int, int], Dict[int, int], Dict[int, int], List[Tuple[int, int]] +]: + """ + Scan `source` once to build delimiter maps and a list of ignored spans. + + Returns + ------- + ( + paren_open_to_close : Dict[int, int], + paren_close_to_open : Dict[int, int], + brace_open_to_close : Dict[int, int], + brace_close_to_open : Dict[int, int], + ignored_spans : List[Tuple[int, int]], + ) + + Where: + - paren_open_to_close maps index of '(' to the index of its matching ')'. + - paren_close_to_open maps index of ')' to the index of its matching '('. + - brace_open_to_close maps index of '{' to the index of its matching '}'. + - brace_close_to_open maps index of '}' to the index of its matching '{'. + - ignored_spans is a list of (start, end) slices for comments and literals + (end is exclusive) covering: + * // line comments + * /* ... */ block/Javadoc comments + * "..." strings, '...' chars, and \"\"\"...\"\"\" text blocks + + Notes + ----- + - Delimiter pairs inside ignored spans are not tracked (they're skipped). + - Runs in O(len(source)) time; memory proportional to number of delimiters/spans. + """ + paren_open_to_close: Dict[int, int] = {} + paren_close_to_open: Dict[int, int] = {} + brace_open_to_close: Dict[int, int] = {} + brace_close_to_open: Dict[int, int] = {} + + ignored_spans: List[Tuple[int, int]] = [] + paren_stack: List[int] = [] + brace_stack: List[int] = [] + + pos, length = 0, len(source) + while pos < length: + ch = source[pos] + + # Whitespace: skip quickly + if ch in (' ', '\t', '\r', '\n', '\f'): + pos += 1 + continue + + # // line comment + if pos + 1 < length and ch == '/' and source[pos + 1] == '/': + span_start = pos + pos += 2 + newline_idx = source.find('\n', pos) + pos = length if newline_idx == -1 else newline_idx + 1 + ignored_spans.append((span_start, pos)) + continue + + # /* block/Javadoc */ comment + if pos + 1 < length and ch == '/' and source[pos + 1] == '*': + span_start = pos + close_idx = source.find('*/', pos + 2) + pos = length if close_idx == -1 else close_idx + 2 + ignored_spans.append((span_start, pos)) + continue + + # Strings / chars / text blocks + if source.startswith('"""', pos) or ch in ('"', "'"): + span_start = pos + pos = _skip_java_string_like(source, pos) + ignored_spans.append((span_start, pos)) + continue + + # Delimiters outside ignored spans + if ch == '(': + paren_stack.append(pos) + elif ch == ')' and paren_stack: + open_idx = paren_stack.pop() + paren_open_to_close[open_idx] = pos + paren_close_to_open[pos] = open_idx + elif ch == '{': + brace_stack.append(pos) + elif ch == '}' and brace_stack: + open_idx = brace_stack.pop() + brace_open_to_close[open_idx] = pos + brace_close_to_open[pos] = open_idx + + pos += 1 + + ignored_spans.sort(key=lambda span: span[0]) + return ( + paren_open_to_close, + paren_close_to_open, + brace_open_to_close, + brace_close_to_open, + ignored_spans, + ) + +def _populate_delim_index_cache(java_source: str): + """ + Build and cache delimiter/ignored-span indexes for a given Java source string. + + Parameters + ---------- + java_source : str + The entire Java source text to index. + + Side Effects + ------------ + Populates (overwrites) the following global caches for O(1) delimiter lookups: + - _PAR_O2C : Dict[int, int] # map of '(' byte index -> matching ')' + - _PAR_C2O : Dict[int, int] # map of ')' byte index -> matching '(' + - _BR_O2C : Dict[int, int] # map of '{' byte index -> matching '}' + - _BR_C2O : Dict[int, int] # map of '}' byte index -> matching '{' + - _IGN_SPANS : List[Tuple[int, int]] # ignored spans (comments/strings/text blocks), end-exclusive + - _IGN_STARTS : List[int] # start positions of _IGN_SPANS (for bisect) + - _IGN_ENDS : List[int] # end positions of _IGN_SPANS (for bisect) + - _DELIM_SRC_ID : Optional[int] # identity of the currently indexed source + + Complexity + ---------- + O(len(src)) time and O(k) memory, where k is the number of delimiters and + ignored spans discovered. + + Notes + ----- + The cache is keyed by the *identity* of `src` (via id(src)), not its value. + Rebuild the index if you switch to a different source object. + """ + global _PAR_O2C, _PAR_C2O, _BR_O2C, _BR_C2O, _DELIM_SRC_ID, _IGN_SPANS, _IGN_STARTS, _IGN_ENDS + + # Local typed temps (ok to annotate) + par_o2c: Dict[int, int] + par_c2o: Dict[int, int] + br_o2c: Dict[int, int] + br_c2o: Dict[int, int] + spans: List[Tuple[int, int]] + + par_o2c, par_c2o, br_o2c, br_c2o, spans = _build_delim_index(java_source) + + # Assign to globals (no annotations here) + _PAR_O2C = par_o2c + _PAR_C2O = par_c2o + _BR_O2C = br_o2c + _BR_C2O = br_c2o + + _IGN_SPANS = spans + _IGN_STARTS = [a for (a, _) in spans] + _IGN_ENDS = [b for (_, b) in spans] + + _DELIM_SRC_ID = id(java_source) + +def _clear_delim_index_cache(): + global _PAR_O2C, _PAR_C2O, _BR_O2C, _BR_C2O, _DELIM_SRC_ID, _IGN_SPANS, _IGN_STARTS, _IGN_ENDS + _PAR_O2C.clear(); _PAR_C2O.clear(); _BR_O2C.clear(); _BR_C2O.clear() + _IGN_SPANS.clear(); _IGN_STARTS.clear(); _IGN_ENDS.clear() + _DELIM_SRC_ID = None + +# -------------------------- forward balanced matchers ------------------------- + +def _match_balanced_angles(source: str, open_angle_index: int) -> int: + """ + Return the index of the '>' that matches the '<' at `open_angle_index`. + + Scans forward from just after the opening '<', tracking nested angle pairs + (e.g., `Map>`). Whitespace and comments are skipped via + `_skip_ws_comments`, and string/char/text-block literals are copied over with + `_skip_java_string_like` so any '<' or '>' inside them are ignored. + + Parameters + ---------- + source : str + The Java source code to scan. + open_angle_index : int + Index in `source` where the opening '<' resides. The caller should ensure + `source[open_angle_index] == '<'`. + + Returns + ------- + int + The index of the matching closing '>' if found; otherwise `len(source) - 1` + (a defensive fallback for unterminated input). + + Notes + ----- + - This routine treats angle brackets as generic-type delimiters. It does not try + to disambiguate comparison operators vs generics; the caller should invoke it + only in contexts where a type parameter list is expected. + - Runs in O(N) from the opening bracket to the end (or until the match is found). + """ + length = len(source) + nesting = 1 # we've consumed the first '<' + i = open_angle_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so '<' or '>' inside them don't affect nesting. + if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + i = _skip_java_string_like(source, i) + continue + + char = source[i] + if char == '<': + nesting += 1 + elif char == '>': + nesting -= 1 + if nesting == 0: + return i + i += 1 + + # Fallback: unterminated angle list + return length - 1 + +def _match_balanced_parens(source: str, open_paren_index: int) -> int: + """ + Return the index of the ')' that matches the '(' at `open_paren_index`. + + Fast path: if a delimiter index was precomputed for this exact `source` + (see `_set_delim_index`), use the cached mapping for O(1) lookup. + + Otherwise, scan forward from just after the opening '(' while: + - Skipping whitespace and comments via `_skip_ws_comments`. + - Skipping over string/char/text-block literals via `_skip_java_string_like`. + - Tracking nested parentheses depth. + + Parameters + ---------- + source : str + The Java source to scan. + open_paren_index : int + Index in `source` where the opening '(' resides. Caller must ensure + `source[open_paren_index] == '('`. + + Returns + ------- + int + Index of the matching ')' if found; otherwise `len(source) - 1` + as a defensive fallback for unterminated input. + + Notes + ----- + - Time complexity is O(K) where K is the distance from `open_paren_index` + to the matching ')', or to the end if unterminated. + - The cache is used only when `_DELIM_SRC_ID == id(source)`. + """ + # Cached O(1) lookup when indices were prebuilt for this exact source object. + if _DELIM_SRC_ID == id(source): + cached_close = _PAR_O2C.get(open_paren_index) + if cached_close is not None: + return cached_close + + # Fallback: linear scan with nesting. + length = len(source) + nesting = 1 # we've consumed the first '(' + i = open_paren_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so any '(' or ')' inside them don't affect nesting. + if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + i = _skip_java_string_like(source, i) + continue + + ch = source[i] + if ch == '(': + nesting += 1 + elif ch == ')': + nesting -= 1 + if nesting == 0: + return i + i += 1 + + # Unterminated parentheses list – return last valid index defensively. + return length - 1 + +def _match_balanced_braces(source: str, open_brace_index: int) -> int: + """ + Return the index of the '}' that matches the '{' at `open_brace_index`. + + Fast path: + If delimiter indexes were precomputed for this exact `source` + via `_set_delim_index`, use the cached brace map for O(1) lookup. + + Fallback scan: + Starting just after the opening '{', scan forward while: + - Skipping whitespace and comments with `_skip_ws_comments`. + - Skipping over string/char/text-block literals with `_skip_java_string_like`. + - Tracking nested braces `{ ... }` with a depth counter. + - When encountering '(', skip the entire parenthesized section with + `_match_balanced_parens` to avoid confusing braces that may appear inside. + + Parameters + ---------- + source : str + The Java source text to scan. + open_brace_index : int + Index in `source` where the opening '{' resides. + Caller must ensure `source[open_brace_index] == '{'`. + + Returns + ------- + int + Index of the matching '}' if found; otherwise `len(source) - 1` + as a defensive fallback for unterminated input. + + Notes + ----- + - This function does not treat '<' specially inside bodies (to avoid + misinterpreting comparisons as generics). + - Time complexity is O(K), where K is the distance from the opening brace + to its match (or to the end for unterminated input). + """ + # Cached O(1) close-brace lookup if indexes were built for this exact source object. + if _DELIM_SRC_ID == id(source): + cached_close = _BR_O2C.get(open_brace_index) + if cached_close is not None: + return cached_close + + # Fallback: linear scan with nesting depth. + length = len(source) + nesting = 1 # we've consumed the first '{' + i = open_brace_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so braces inside them don't affect nesting. + if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + i = _skip_java_string_like(source, i) + continue + + ch = source[i] + if ch == '{': + nesting += 1 + elif ch == '}': + nesting -= 1 + if nesting == 0: + return i + elif ch == '(': + # Skip an entire (...) block to avoid miscounting braces seen within. + i = _match_balanced_parens(source, i) + + # NOTE: do not treat '<' specially inside bodies (comparisons). + i += 1 + + # Unterminated block – return last valid index defensively. + return length - 1 + +# ------------------------------ backward helpers ------------------------------ + +def _prev_word(src: str, pos: int) -> str: + j = pos - 1 + while j >= 0 and src[j] in ' \t\r\n\f': + j -= 1 + if j < 0: + return "" + end = j + 1 + while j >= 0 and _is_ident_part(src[j]): + j -= 1 + return src[j+1:end] + +def _skip_leading_annotations(src: str, i: int) -> int: + """ + From position i (roughly at the beginning of a signature), skip any number of + annotations and comments so that method-level annotations do NOT appear in the + returned snippet. Returns first non-annotation token index. + """ + n = len(src) + i = _skip_ws_comments(src, i) + while i < n: + if src[i] == '@': + i += 1 + while i < n and (src[i].isalnum() or src[i] in ('_', '$', '.')): + i += 1 + i = _skip_ws_comments(src, i) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + i = _skip_ws_comments(src, i) + continue + j = _skip_ws_comments(src, i) + if j != i: + i = j + continue + break + return i + +def _back_ident(src: str, i: int) -> Tuple[int, Optional[str]]: + """ + From position i (typically the '(' of a parameter list), scan left to find the + simple identifier just before it. Returns (start_index, name) or (i, None). + """ + j = i - 1 + # skip ws + while j >= 0 and src[j] in ' \t\r\n\f': + j -= 1 + if j < 0: + return i, None + end = j + 1 + while j >= 0 and _is_ident_part(src[j]): + j -= 1 + start = j + 1 + name = src[start:end] + if name and _is_ident_start(name[0]): + return start, name + return i, None + +def _find_left_boundary(src: str, before: int) -> int: + """ + Return index just after the nearest real boundary (';', '{', '}') occurring before `before`, + ignoring comments/strings and jumping over balanced () for speed. + IMPORTANT: Treat '}', '{', ';' as immediate boundaries BEFORE any jump, so we don't leap + across the previous method body and accidentally glue multiple methods together. + """ + if _DELIM_SRC_ID != id(src): + # simple forward fallback + i = 0 + last = 0 + before = max(0, min(before, len(src))) + while i < before: + j = _skip_ws_comments(src, i) + if j != i: + i = j; continue + if i < before and (src.startswith('"""', i) or src[i] in ('"', "'")): + i = _skip_java_string_like(src, i); continue + if src[i] in (';', '{', '}'): + last = i + 1 + i += 1 + return last + + # fast backward using indexes + i = min(before - 1, len(src) - 1) + while i >= 0: + # hop out of ignored span + span = _ignored_span_at(i) + if span: + i = span[0] - 1 + continue + + ch = src[i] + + # FIRST: boundary check + if ch in (';', '{', '}'): + return i + 1 + + # jump across groups for speed (do NOT jump across '}' – handled above) + if ch == ')': + o = _PAR_C2O.get(i) if _DELIM_SRC_ID == id(src) else None + i = (o - 1) if o is not None else (i - 1) + continue + + i -= 1 + return 0 + +def _match_paren_backwards(source: str, close_paren_index: int) -> int: + """ + Return the index of the '(' that matches the ')' at `close_paren_index`. + + Fast path + --------- + If delimiter indexes were precomputed for this exact `source` (see + `_set_delim_index`), use the cached map `_PAR_C2O` for O(1) lookup. + + Fallback scan + ------------- + Walk left from just before `close_paren_index` while: + - Tracking nested parentheses depth. + - Skipping over string and char literals (so any parentheses inside them + don't affect nesting). + + Parameters + ---------- + source : str + The Java source text to scan. + close_paren_index : int + Index in `source` where the closing ')' resides. Caller must ensure + `source[close_paren_index] == ')'`. + + Returns + ------- + int + Index of the matching '(' if found; otherwise 0 as a defensive fallback. + + Notes + ----- + - Time complexity is O(K), where K is the distance from the closing paren + to its match (or to the start if unterminated). + - The cache is used only when `_DELIM_SRC_ID == id(source)`. + """ + # Cached O(1) lookup when indices were built for this exact source object. + if _DELIM_SRC_ID == id(source): + cached_open = _PAR_C2O.get(close_paren_index) + if cached_open is not None: + return cached_open + + # Fallback: linear backward scan with nesting. + nesting = 1 # we've seen one ')' + i = close_paren_index - 1 + + while i >= 0: + ch = source[i] + + if ch == ')': + nesting += 1 + elif ch == '(': + nesting -= 1 + if nesting == 0: + return i + elif ch in ('"', "'"): + # Skip over a string/char literal going left, handling escapes. + quote = ch + i -= 1 + while i >= 0: + if source[i] == '\\': + i -= 2 # skip the escape and the escaped char + continue + if source[i] == quote: + i -= 1 + break + i -= 1 + continue + + i -= 1 + + # Unterminated or not found – safest fallback. + return 0 + +# ------------------------------- throws skipper ------------------------------- + +def _skip_throws_clause(src: str, i: int) -> int: + n = len(src) + i = _skip_ws_comments(src, i) + if not src.startswith('throws', i): + return i + i += len('throws') + while i < n: + i = _skip_ws_comments(src, i) + if i >= n: + break + if src.startswith('"""', i) or src[i] in ('"', "'"): + i = _skip_java_string_like(src, i); continue + ch = src[i] + if ch == '<': + i = _match_balanced_angles(src, i) + 1; continue + if ch == '(': + i = _match_balanced_parens(src, i) + 1; continue + if ch == '[': + # zero or more [] after a type + while i < n and src[i] == '[': + rb = src.find(']', i + 1) + if rb == -1: return n + i = rb + 1 + continue + if ch == '@': + # annotation on thrown type + i += 1 + while i < n and (src[i].isalnum() or src[i] in ('_', '$', '.')): + i += 1 + i = _skip_ws_comments(src, i) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + continue + if ch == ',': + i += 1; continue + if ch in ('{', ';'): + break + i += 1 + return _skip_ws_comments(src, i) + +# ----------------------- type discovery for constructor filter ---------------- + +class _TypeRegion: + __slots__ = ("name", "start", "end") # '{' index .. matching '}' index + def __init__(self, name: str, start: int, end: int) -> None: + self.name = name + self.start = start + self.end = end + +def _find_all_named_types(src: str) -> List[_TypeRegion]: + """ + Find all class/interface/enum/record bodies and names, including nested types. + """ + n = len(src) + i = 0 + types: List[_TypeRegion] = [] + while i < n: + i = _skip_ws_comments(src, i) + if i >= n: break + + # skip literals + if src.startswith('"""', i) or src[i] in ('"', "'"): + i = _skip_java_string_like(src, i); continue + + # token + if _is_ident_start(src[i]) or src[i] == '@': + tok_start = i + (1 if src[i] == '@' else 0) + j = tok_start + while j < n and _is_ident_part(src[j]): j += 1 + tok = src[tok_start:j] + + if tok in ('class', 'interface', 'enum', 'record'): + # name + k = _skip_ws_comments(src, j) + name_start = k + while k < n and _is_ident_part(src[k]): k += 1 + if k == name_start: # '.class' or broken + i = j; continue + name = src[name_start:k] + + # type params / record header + k = _skip_ws_comments(src, k) + if k < n and src[k] == '<': + k = _match_balanced_angles(src, k) + 1 + k = _skip_ws_comments(src, k) + if tok == 'record' and k < n and src[k] == '(': + k = _match_balanced_parens(src, k) + 1 + k = _skip_ws_comments(src, k) + + # to opening '{' + while k < n and src[k] != '{': + if src.startswith('"""', k) or src[k] in ('"', "'"): + k = _skip_java_string_like(src, k) + else: + k += 1 + k = _skip_ws_comments(src, k) + + if k < n and src[k] == '{': + end = _match_balanced_braces(src, k) + if name: + types.append(_TypeRegion(name=name, start=k, end=end)) + i = k + 1 + continue + + i = j + continue + + i += 1 + + return types + +def _enclosing_type_name(types: List[_TypeRegion], pos: int) -> Optional[str]: + best: Optional[_TypeRegion] = None + for t in types: + if t.start <= pos <= t.end and (best is None or t.start >= best.start): + best = t + return best.name if best else None + +# --------------------------------- lambdas ------------------------------------ + +def _capture_lambda_at_arrow(src: str, arrow: int) -> Optional[Tuple[int, int]]: + """ + Given '->' at src[arrow:arrow+2], return (start, end) slice indices of the + lambda expression WITHOUT trailing delimiters. + """ + n = len(src) + # body start + j = _skip_ws_comments(src, arrow + 2) + # find parameter start + p = arrow - 1 + while p >= 0 and src[p] in ' \t\r\n\f': + p -= 1 + if p < 0: + return None + if src[p] == ')': + param_start = _match_paren_backwards(src, p) + else: + # single identifier parameter + end = p + 1 + start = p + while start >= 0 and _is_ident_part(src[start]): + start -= 1 + start += 1 + if start >= end or not _is_ident_start(src[start]): + return None + param_start = start + + # BODY + if j < n and src[j] == '{': + end = _match_balanced_braces(src, j) + 1 + return (param_start, end) + + # expression-bodied: consume until delimiter at depth 0 + depth_p = depth_b = depth_c = 0 + k = j + while k < n: + k2 = _skip_ws_comments(src, k) + if k2 != k: + k = k2; continue + if k < n and (src.startswith('"""', k) or src[k] in ('"', "'")): + k = _skip_java_string_like(src, k); continue + ch = src[k] + if ch == '(': + depth_p += 1 + elif ch == ')': + if depth_p > 0: + depth_p -= 1 + else: + break + elif ch == '[': + depth_b += 1 + elif ch == ']': + if depth_b > 0: + depth_b -= 1 + elif ch == '{': + depth_c += 1 + elif ch == '}': + if depth_c == 0 and depth_p == 0 and depth_b == 0: + break + if depth_c > 0: + depth_c -= 1 + elif ch in (';', ','): + if depth_p == 0 and depth_b == 0 and depth_c == 0: + break + k += 1 + end = k + return (param_start, end) + +def _extract_lambdas_in_range(src: str, start: int, end: int) -> List[str]: + out: List[str] = [] + i = start + n = len(src) + end = min(end, n) + while i < end: + j = _skip_ws_comments(src, i) + if j != i: + i = j; continue + if i < end and (src.startswith('"""', i) or src[i] in ('"', "'")): + i = _skip_java_string_like(src, i); continue + if i + 1 < end and src[i] == '-' and src[i + 1] == '>': + span = _capture_lambda_at_arrow(src, i) + if span: + s, e = span + if s >= start and e <= end: + clean = _strip_comments_keep_strings_range(src, s, e) + out.append(clean) + i = e + continue + i += 2 + continue + i += 1 + return out + +# ------------------ small helpers for method detection ------------------------ + +def _header_closes_at_brace(src: str, close_p: int, brace_idx: int) -> bool: + """Return True if src[close_p] == ')' and after optional ws/comments + throws we land exactly on '{'.""" + k = _skip_ws_comments(src, close_p + 1) + k = _skip_throws_clause(src, k) + k = _skip_ws_comments(src, k) + return k == brace_idx + +# ------------------------------- core extractor ------------------------------- + +def _extract_methods_anywhere( + src: str, + start: int = 0, + end: Optional[int] = None, + types: Optional[List[_TypeRegion]] = None, +) -> List[str]: + """ + Extract Java method definitions (with bodies) anywhere within [start, end), + excluding constructors. Anonymous/inner/local class bodies are recursively scanned. + Method-level annotations preceding a signature are excluded from the returned slice. + Lambdas are NOT returned here (they are gathered in a separate pass). + """ + if end is None: + end = len(src) + if types is None: + types = _find_all_named_types(src) + + # Enclosing named-type lookup (innermost) + types_sorted = sorted(types, key=lambda t: t.start) + type_starts = [t.start for t in types_sorted] + + def _enclosing_region(pos: int) -> Optional[_TypeRegion]: + """ + Return the innermost named type region containing pos. + IMPORTANT: keep scanning left until we either find an enclosing region + or run out of candidates. Do NOT early-return None if the nearest + left type ended before pos; an outer type may still enclose pos. + """ + idx = bisect.bisect_right(type_starts, pos) - 1 + while idx >= 0: + t = types_sorted[idx] + if t.start <= pos <= t.end: + return t + # If this candidate doesn't enclose pos, step left and keep looking. + idx -= 1 + return None + + # quick enum-constants section end cache + enum_term_cache: Dict[int, int] = {} + + def _region_kind(reg: _TypeRegion) -> str: + if not reg: + return "" + i2 = _find_left_boundary(src, reg.start) + while i2 < reg.start: + j2 = _skip_ws_comments(src, i2); i2 = j2 + if i2 >= reg.start: break + if src.startswith('"""', i2) or src[i2] in ('"', "'"): + i2 = _skip_java_string_like(src, i2); continue + if _is_ident_start(src[i2]) or src[i2] == '@': + tok_start = i2 + (1 if src[i2] == '@' else 0) + j2 = tok_start + while j2 < reg.start and _is_ident_part(src[j2]): j2 += 1 + tok = src[tok_start:j2] + if tok in ('class', 'interface', 'enum', 'record'): + return tok + i2 = j2; continue + i2 += 1 + return "" + + def _enum_constants_terminator(type_region: _TypeRegion) -> int: + """ + Find the position of the semicolon that terminates the enum-constants list + inside an enum body, or return -1 if no explicit terminator exists. + + Parameters + ---------- + type_region : _TypeRegion + The region describing the enum body. `type_region.start` is the index + of the opening '{' and `type_region.end` is the index of the matching '}'. + + Returns + ------- + int + The index of the terminating ';' that appears at *brace depth 1* inside + the enum body. If the constants section runs directly into the closing + brace (i.e., no explicit ';'), returns -1. Also returns -1 for + unterminated/degenerate cases. + + Caching + ------- + Results are cached in `enum_term_cache` keyed by `type_region.start`. + """ + # Cache fast-path + cached_pos = enum_term_cache.get(type_region.start) + if cached_pos is not None: + return cached_pos + + scan_pos: int = type_region.start + 1 # first character after '{' + body_end: int = type_region.end # index of matching '}' + brace_depth: int = 1 # already inside the enum body + + while scan_pos < body_end: + next_pos = _skip_ws_comments(src, scan_pos) + if next_pos != scan_pos: + scan_pos = next_pos + continue + + # Skip over literals so braces/semicolons inside them are ignored. + if src.startswith('"""', scan_pos) or src[scan_pos] in ('"', "'"): + scan_pos = _skip_java_string_like(src, scan_pos) + continue + + ch = src[scan_pos] + + if ch == '{': + brace_depth += 1 + elif ch == '}': + brace_depth -= 1 + if brace_depth == 0: + # Reached end of enum body without seeing a terminator. + enum_term_cache[type_region.start] = -1 + return -1 + elif ch == ';' and brace_depth == 1: + # The terminator separating constants from members. + enum_term_cache[type_region.start] = scan_pos + return scan_pos + elif ch == '(': + # Skip (...) blocks to avoid miscounting nested tokens. + scan_pos = _match_balanced_parens(src, scan_pos) + 1 + continue + elif ch == '<': + # Skip generic type argument lists inside declarations. + scan_pos = _match_balanced_angles(src, scan_pos) + 1 + continue + + scan_pos += 1 + + # No explicit terminator found. + enum_term_cache[type_region.start] = -1 + return -1 + + def _in_enum_constants_section(name_pos: int) -> bool: + reg = _enclosing_region(name_pos) + if not reg or _region_kind(reg) != 'enum': + return False + term = _enum_constants_terminator(reg) + return term == -1 or name_pos < term + + def _scan_paren_internals_for_anonymous_classes(open_idx: int, close_idx: int, out_accum: List[str]): + # Look for "new Type(...){ ... }" inside argument lists; recurse into their bodies + k2 = open_idx + 1 + while k2 < close_idx: + kk = _skip_ws_comments(src, k2) + if kk != k2: + k2 = kk; continue + if k2 < close_idx and (src.startswith('"""', k2) or src[k2] in ('"', "'")): + k2 = _skip_java_string_like(src, k2); continue + + pos = src.find('new', k2, close_idx) + if pos == -1: + break + k2 = pos + + # skip false positives inside ignored spans + sp = _ignored_span_at(k2) if _DELIM_SRC_ID == id(src) else None + if sp and sp[0] <= k2 < sp[1]: + k2 = sp[1]; continue + + pre = src[k2 - 1] if k2 > open_idx + 1 else '' + post = src[k2 + 3] if k2 + 3 < close_idx else '' + if (k2 == open_idx + 1 or not _is_ident_part(pre)) and (k2 + 3 >= close_idx or not _is_ident_part(post)): + t = _skip_ws_comments(src, k2 + 3) + # qualified type with optional type params + while t < close_idx: + if _is_ident_start(src[t]): + t += 1 + while t < close_idx and _is_ident_part(src[t]): t += 1 + if t < close_idx and src[t] == '.': + t += 1; continue + elif t < close_idx and src[t] == '<': + t = _match_balanced_angles(src, t) + 1 + else: + break + t = _skip_ws_comments(src, t) + + if t < close_idx and src[t] == '(': + ctor_close = _match_balanced_parens(src, t) + u = _skip_ws_comments(src, ctor_close + 1) + if u < close_idx and src[u] == '{': + body_end = _match_balanced_braces(src, u) + if u + 1 <= body_end: + out_accum.extend(_extract_methods_anywhere(src, start=u + 1, end=body_end, types=types)) + k2 = body_end + 1; continue + k2 = ctor_close + 1; continue + + k2 = k2 + 3 # move past this "new" + + def _body_might_have_local_types(beg: int, end_: int) -> bool: + s = src + return (s.find('class', beg, end_) != -1 or + s.find('interface', beg, end_) != -1 or + s.find('enum', beg, end_) != -1 or + s.find('record', beg, end_) != -1) + + out: List[str] = [] + i = start + n = end + seen_spans: Set[Tuple[int, int]] = set() + + while i < n: + j = _skip_ws_comments(src, i) + if j != i: + i = j; continue + if i >= n: break + + if src.startswith('"""', i) or src[i] in ('"', "'"): + i = _skip_java_string_like(src, i); continue + + if src[i] == '(': + name_start, name = _back_ident(src, i) + close_p = _match_balanced_parens(src, i) + + if not name: + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + i = close_p + 1 + continue + + if _prev_word(src, name_start) == 'record': + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + i = close_p + 1 + continue + + if name in _NON_METHOD_TOKENS: + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + i = close_p + 1 + continue + + left_bound = _find_left_boundary(src, name_start) + + # "new" between boundary and name means likely a constructor call or anonymous class literal + def _has_new_token_between(s: str, left: int, right: int) -> bool: + left = max(0, left); right = min(len(s), right) + i2 = left + while True: + pos = s.find('new', i2, right) + if pos == -1: return False + if _DELIM_SRC_ID == id(s): + sp = _ignored_span_at(pos) + if sp and sp[0] <= pos < sp[1]: + i2 = pos + 3; continue + pre = s[pos - 1] if pos > left else '' + post = s[pos + 3] if pos + 3 < right else '' + if (pos == left or not _is_ident_part(pre)) and (pos + 3 >= right or not _is_ident_part(post)): + return True + i2 = pos + 3 + + if _has_new_token_between(src, left_bound, name_start): + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + kk = _skip_ws_comments(src, close_p + 1) + if kk < n and src[kk] == '{': + body_end = _match_balanced_braces(src, kk) + if kk + 1 <= body_end: + out.extend(_extract_methods_anywhere(src, start=kk + 1, end=body_end, types=types)) + i = body_end + 1 + else: + i = close_p + 1 + continue + + k = _skip_ws_comments(src, close_p + 1) + if k < n and src[k] == '{' and _in_enum_constants_section(name_start): + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + body_end = _match_balanced_braces(src, k) + if k + 1 <= body_end: + out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) + i = body_end + 1 + continue + + # must have method body "{" + k = _skip_throws_clause(src, k) + if k >= n or src[k] != '{': + if close_p < n: + _scan_paren_internals_for_anonymous_classes(i, close_p, out) + i = close_p + 1 + continue + + sig_guess = _find_left_boundary(src, name_start) + sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) + + # constructor? compare with enclosing type simple name + encl_reg = _enclosing_region(name_start) + if encl_reg and name == encl_reg.name: + # do not emit constructor, but scan its body + body_end = _match_balanced_braces(src, k) + if k + 1 <= body_end and _body_might_have_local_types(k + 1, body_end): + out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) + i = body_end + 1 + continue + + body_end = _match_balanced_braces(src, k) + sig_start = _skip_ws_comments(src, sig_start) # final guard + + header_no_ann = _strip_annotations_in_range(src, sig_start, k) + clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) + + clean_body = _strip_body_comments_and_annotations(src, k, body_end) + + method_text = clean_header + clean_body + + span = (sig_start, body_end) + if span not in seen_spans: + seen_spans.add(span) + out.append(method_text) + + if k + 1 <= body_end and _body_might_have_local_types(k + 1, body_end): + out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) + + i = body_end + 1 + continue + + # Secondary pattern: brace-first line (signature ended previous line) + if src[i] == '{': + p = src.rfind(')', start, i) + if p != -1: + open_p = _match_paren_backwards(src, p) + if open_p > 0 and _header_closes_at_brace(src, p, i): + name_start, name = _back_ident(src, open_p) + if name and name not in _NON_METHOD_TOKENS: + left_bound = _find_left_boundary(src, name_start) + # filter out anonymous class/ctor and constructors + def _has_new_between(s, a, b): + a = max(0, a); b = min(len(s), b) + pos = s.find('new', a, b) + while pos != -1: + sp = _ignored_span_at(pos) if _DELIM_SRC_ID == id(s) else None + if not sp or not (sp[0] <= pos < sp[1]): + pre = s[pos - 1] if pos > a else '' + post = s[pos + 3] if pos + 3 < b else '' + if (pos == a or not _is_ident_part(pre)) and (pos + 3 >= b or not _is_ident_part(post)): + return True + pos = s.find('new', pos + 3, b) + return False + if not _has_new_between(src, left_bound, name_start): + encl_reg = _enclosing_region(name_start) + if not (encl_reg and name == encl_reg.name) and _prev_word(src, name_start) != 'record': + body_end = _match_balanced_braces(src, i) + sig_guess = _find_left_boundary(src, name_start) + sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) + sig_start = _skip_ws_comments(src, sig_start) + + header_no_ann = _strip_annotations_in_range(src, sig_start, i) + clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) + + clean_body = _strip_body_comments_and_annotations(src, i, body_end) + + method_text = clean_header + clean_body + + span = (sig_start, body_end) + if span not in seen_spans: + seen_spans.add(span) + out.append(method_text) + + if i + 1 <= body_end and _body_might_have_local_types(i + 1, body_end): + out.extend(_extract_methods_anywhere(src, start=i + 1, end=body_end, types=types)) + i = body_end + 1 + continue + i += 1 + + return out + +# ============================== public API ==================================== + +def extract_methods(java_source: str) -> List[str]: + """ + Returns a list of Java method bodies/signatures (with bodies) plus all arrow-lambda + forms found in the file. Excludes constructors, comments/Javadoc, and strips + any method-level annotations that start before the signature token. + Order: methods first, then lambdas. Optimized for large files by pre-indexing + comments/strings and delimiter pairs (O(1) matching for (), {}). + """ + _populate_delim_index_cache(java_source) + try: + # Extract + methods = _extract_methods_anywhere(java_source, 0, len(java_source)) + # Lambdas in a separate pass across the whole file (so we don't double-count) + lambdas = _extract_lambdas_in_range(java_source, 0, len(java_source)) + return methods + lambdas + finally: + _clear_delim_index_cache() + +def extract_inner_classes(src: str) -> List[str]: + """ + Return a list of source slices for *inner* named types (class/interface/enum/record), + including nested inner types. Each returned slice is prefixed with the file's + top-level import statements (including 'import static ...;'). + """ + + # ----------------------- + # Tiny local "utilities" + # ----------------------- + def _is_ident_char(ch): + return ch.isalnum() or ch in ('_', '$') + + def _skip_string(s, i): + i += 1 + n = len(s) + while i < n: + c = s[i] + if c == '\\': + i += 2 + elif c == '"': + return i + 1 + else: + i += 1 + return n + + def _skip_char(s, i): + i += 1 + n = len(s) + while i < n: + c = s[i] + if c == '\\': + i += 2 + elif c == "'": + return i + 1 + else: + i += 1 + return n + + def _skip_ws_comments(s, i): + """Skip whitespace and //... or /*...*/ comments. Always makes progress or returns.""" + n = len(s) + while i < n: + start = i + # whitespace + while i < n and s[i].isspace(): + i += 1 + # // line comment + if i + 1 < n and s[i] == '/' and s[i + 1] == '/': + i += 2 + while i < n and s[i] != '\n': + i += 1 + continue + # /* block comment */ + if i + 1 < n and s[i] == '/' and s[i + 1] == '*': + i += 2 + end = s.find('*/', i) + if end == -1: + return n + i = end + 2 + continue + if i == start: + break + return i + + def _read_ident(s, i): + j, n = i, len(s) + while j < n and _is_ident_char(s[j]): + j += 1 + return s[i:j], j + + def _word_at(s, i, word): + n = len(s) + end = i + len(word) + if end > n or s[i:end] != word: + return False + pre = s[i - 1] if i > 0 else '' + post = s[end] if end < n else '' + return (not _is_ident_char(pre)) and (not _is_ident_char(post)) + + def _skip_parenthesized(s, i): + n = len(s) + if i >= n or s[i] != '(': + return i + depth = 1 + i += 1 + while i < n and depth > 0: + i = _skip_ws_comments(s, i) + if i >= n: + break + c = s[i] + if c == '"': + i = _skip_string(s, i); continue + if c == "'": + i = _skip_char(s, i); continue + if c == '(': + depth += 1; i += 1; continue + if c == ')': + depth -= 1; i += 1; continue + i += 1 + return i + + def _skip_leading_annotations(s, i): + """Skip @Anno, @pkg.Anno, @Anno(...), stacked.""" + n = len(s) + while True: + i = _skip_ws_comments(s, i) + if i >= n or s[i] != '@': + return i + i += 1 + # qualified name + while i < n and (_is_ident_char(s[i]) or s[i] == '.'): + i += 1 + i = _skip_ws_comments(s, i) + if i < n and s[i] == '(': + i = _skip_parenthesized(s, i) + # loop to allow many annotations + + def _match_brace(s, i_open): + n = len(s) + if i_open >= n or s[i_open] != '{': + return None + depth, i = 1, i_open + 1 + steps, limit = 0, 10_000_000 + while i < n and depth > 0: + steps += 1 + if steps > limit: + return None + i = _skip_ws_comments(s, i) + if i >= n: + break + c = s[i] + if c == '"': + i = _skip_string(s, i); continue + if c == "'": + i = _skip_char(s, i); continue + if c == '{': + depth += 1; i += 1; continue + if c == '}': + depth -= 1 + if depth == 0: + return i + i += 1; continue + i += 1 + return None + + def _find_left_boundary(s, brace_pos): + """ + Find the start position of the type declaration (keyword position). + Backward bounded search. + """ + window_size = 4000 + base = max(0, brace_pos - window_size) + window = s[base:brace_pos] + best = -1 + for kw in ('class', 'interface', 'enum', 'record'): + j = window.rfind(kw) + while j != -1: + pre = window[j - 1] if j > 0 else '' + post = window[j + len(kw)] if j + len(kw) < len(window) else '' + if not _is_ident_char(pre) and not _is_ident_char(post): + best = max(best, j) + break + j = window.rfind(kw, 0, j) + return base + best if best != -1 else max(0, brace_pos - 1) + + # ------------------------------- + # Collect top-level import block + # ------------------------------- + def _collect_top_level_imports(s): + imports = [] + NORMAL, LINE, BLOCK, STRING, CHAR = 0, 1, 2, 3, 4 + state = NORMAL + depth = 0 + i, n = 0, len(s) + + def _word_at_local(pos, word): + end = pos + len(word) + if end > n or s[pos:end] != word: + return False + pre = s[pos - 1] if pos > 0 else '' + post = s[end] if end < n else '' + return (not _is_ident_char(pre)) and (not _is_ident_char(post)) + + saw_top_level_type = False + + while i < n and not saw_top_level_type: + ch = s[i] + if state == NORMAL: + if ch == '/': + if i + 1 < n and s[i + 1] == '/': + state = LINE; i += 2; continue + if i + 1 < n and s[i + 1] == '*': + state = BLOCK; i += 2; continue + if ch == '"': state = STRING; i += 1; continue + if ch == "'": state = CHAR; i += 1; continue + + if ch == '{': + depth += 1 + if depth == 1: + saw_top_level_type = True + i += 1 + continue + if ch == '}': + depth = max(0, depth - 1); i += 1; continue + + if depth == 0 and ch == 'i' and _word_at_local(i, 'import'): + j = i + 6 + while j < n and s[j] != ';': + j += 1 + if j < n and s[j] == ';': + imports.append(s[i:j + 1].strip()) + i = j + 1 + continue + + i += 1 + continue + + elif state == LINE: + if ch == '\n': state = NORMAL + i += 1; continue + + elif state == BLOCK: + if ch == '*' and i + 1 < n and s[i + 1] == '/': + state = NORMAL; i += 2 + else: + i += 1 + continue + + elif state == STRING: + if ch == '\\': i += 2 + else: + if ch == '"': state = NORMAL + i += 1 + continue + + else: # CHAR + if ch == '\\': i += 2 + else: + if ch == "'": state = NORMAL + i += 1 + continue + + return ("\n".join(imports) + ("\n" if imports else "")) + + # ----------------------------------- + # Scan: find all named type regions + # ----------------------------------- + def _find_all_named_types(s): + kinds = ('class', 'interface', 'enum', 'record') + regions = [] # tuples: (name, kind, start, end) + i, n = 0, len(s) + steps, limit = 0, 10_000_000 + + while i < n: + steps += 1 + if steps > limit: + break + + i = _skip_ws_comments(s, i) + if i >= n: + break + + c = s[i] + if c == '"': + i = _skip_string(s, i); continue + if c == "'": + i = _skip_char(s, i); continue + + i = _skip_leading_annotations(s, i) + if i >= n: + break + + matched_kind = None + for kw in kinds: + if _word_at(s, i, kw): + matched_kind = kw + break + if not matched_kind: + i += 1 + continue + + # consume keyword + i += len(matched_kind) + i = _skip_ws_comments(s, i) + + # read type name + name, i2 = _read_ident(s, i) + if not name: + i += 1 + continue + i = i2 + + # consume header up to '{' + while i < n: + i = _skip_ws_comments(s, i) + if i >= n or s[i] == '{': + break + if s[i] == '"': + i = _skip_string(s, i); continue + if s[i] == "'": + i = _skip_char(s, i); continue + if _is_ident_char(s[i]): + _, i = _read_ident(s, i); continue + if s[i] == '<': + depth = 1; i += 1 + while i < n and depth > 0: + i = _skip_ws_comments(s, i) + if i >= n: break + ch = s[i] + if ch == '"': i = _skip_string(s, i); continue + if ch == "'": i = _skip_char(s, i); continue + if ch == '<': depth += 1; i += 1; continue + if ch == '>': depth -= 1; i += 1; continue + i += 1 + continue + if s[i] == '(': + i = _skip_parenthesized(s, i); continue + i += 1 + + if i >= n or s[i] != '{': + continue + + body_open = i + body_close = _match_brace(s, body_open) + if body_close is None: + break + + regions.append((name, matched_kind, body_open, body_close)) + i = body_close + 1 + + return regions + + # ----------------------------------- + # Assemble results for inner classes + # ----------------------------------- + imports_block = _collect_top_level_imports(src) + imports_prefix = imports_block if (not imports_block or imports_block.endswith("\n")) else imports_block + "\n" + + regions = _find_all_named_types(src) # (name, kind, start, end) + regions.sort(key=lambda t: (t[2], -(t[3] - t[2]))) # by start, longer first + + results = [] + stack = [] # elements: (end, sig_start) + + for name, kind, start, end in regions: + # pop completed outers + while stack and start >= stack[-1][0]: + stack.pop() + + # compute header/signature start + hdr = _find_left_boundary(src, start) + hdr = _skip_ws_comments(src, hdr) + sig_start = _skip_leading_annotations(src, hdr) + sig_start = _skip_ws_comments(src, sig_start) + + is_inner = bool(stack) + if is_inner and end >= sig_start: + type_src = src[sig_start:end + 1] + results.append(imports_prefix + type_src) + + stack.append((end, sig_start)) + + return results \ No newline at end of file diff --git a/src/vuln_analysis/utils/java_utils.py b/src/vuln_analysis/utils/java_utils.py new file mode 100644 index 000000000..9b687ae6d --- /dev/null +++ b/src/vuln_analysis/utils/java_utils.py @@ -0,0 +1,2440 @@ +import hashlib +import re +from dataclasses import dataclass +from typing import Dict, List, Tuple, Optional, Any +from langchain_core.documents import Document +from tqdm import tqdm + +JAVA_ANNOTATION_SYMBOL = '@' +PRIMITIVE_TYPES = {"byte","short","int","long","float","double","boolean","char","void"} +dummy_package_name = "dummy:dummy:1.0.0" + +# Accepts: +# - groupId: dot-separated segments; each segment alnum, may contain '-' or '_' inside +# - artifactId: alnum, may contain '.', '_' or '-' inside +# - version: alnum start, may contain '.', '_' or '-' (e.g., 1.2.3, 1.0.0-SNAPSHOT) +_GAV_RE = re.compile( + r""" + ^(?P + (?:[A-Za-z0-9]+(?:[A-Za-z0-9_-]*[A-Za-z0-9])?) + (?:\.(?:[A-Za-z0-9]+(?:[A-Za-z0-9_-]*[A-Za-z0-9])?))* + ) + : + (?P[A-Za-z0-9]+(?:[A-Za-z0-9._-]*[A-Za-z0-9])?) + : + (?P[A-Za-z0-9][A-Za-z0-9._-]*) + $ + """, + re.X, +) + +@dataclass +class ParseResult: + kind: str # "class" | "interface" | "enum" | "record" | "method" | "unknown" + name: Optional[str] = None # simple name if found (class or method) + meta: Optional[Dict[str, Any]] = None + +def add_missing_jar_string(s: str) -> str: + i = s.rfind(':') + if i == -1: + return s + return s[:i] + ':jar' + s[i:] + + +def convert_from_maven_artifact(package_name: str) -> str: + s = package_name + + # 1) Remove substring up to and including the first ':' + first = s.find(":") + if first != -1: + s = s[first + 1:] + + # 2) Replace the last ':' (if any) with '-' + last = s.rfind(":") + if last != -1: + s = s[:last] + "-" + s[last + 1:] + + return s + +def _convert_to_maven_artifact(package_name: str) -> str: + last_index = package_name.rfind("-") + + if last_index != -1: # Check if : exists and it should + # Construct the new string + new_package_name = ( + package_name[:last_index] + + ":" + + package_name[last_index + len("-"):] + ) + return new_package_name + + return package_name + +def extract_jar_name(source: str) -> str: + parts = source.split('/') + + if len(parts) > 1: + return _convert_to_maven_artifact(parts[1].rstrip("-sources")) + + return "" + +def is_maven_gav(text: str) -> bool: + """ + Return True iff `s` is a Maven GAV in the form GroupId:ArtifactId:Version. + Examples of valid inputs: + - "commons-beanutils:commons-beanutils:1.9.4" + - "org.apache.commons:commons-lang3:3.14.0" + - "com.example:my-lib:1.0.0-SNAPSHOT" + """ + text = text.strip() + if ":" not in text: # quick reject + return False + return _GAV_RE.match(text) is not None + +def collect_fields_from_types( + src: str, + include_nested: bool = True, + include_anonymous: bool = True, +) -> Dict[str, List[Tuple[str, str]]]: + """ + Faster single-pass walker: + - Masks comments and string/char/text-blocks once. + - Precomputes global () and {} match maps (O(1) lookups afterward). + - Finds named types and (optionally) anonymous classes. + - For each non-enum type, collects ONLY field declarations (non-primitive, non-wrapper). + - Recurse into nested/local types without re-scanning the whole file. + Returns: { class_simple_name_or_@MD5 : [(fieldName, fieldType), ...] } + """ + + s = src + n = len(s) + out: Dict[str, List[Tuple[str, str]]] = {} + + startswith = s.startswith + find = s.find + + WRAPS = {"Boolean","Byte","Character","Short","Integer","Long","Float","Double","Void"} + MODS = { + "public","protected","private","abstract","static","final", + "transient","volatile","strictfp","synchronized","default","native" + } + MRK_SET = frozenset("(){}=;") + + # ---------- 0) Mask comments & strings with spaces (same length) ---------- + def mask_code(src: str) -> str: + m = list(src) + i = 0; N = len(src) + + def zap(a: int, b: int): + if a < b: m[a:b] = ' ' * (b - a) + + while i < N: + ch = src[i] + # line comment + if ch == '/' and i + 1 < N and src[i+1] == '/': + j = find('\n', i + 2) + if j == -1: j = N + zap(i, j); i = j; continue + # block comment + if ch == '/' and i + 1 < N and src[i+1] == '*': + j = find('*/', i + 2) + j = N if j == -1 else j + 2 + zap(i, j); i = j; continue + # text block + if startswith('"""', i): + j = i + 3 + while True: + k = find('"""', j) + if k == -1: + zap(i, N); i = N; break + # check escapes before """ + t = k - 1; back = 0 + while t >= 0 and src[t] == '\\': + back += 1; t -= 1 + j = k + 3 + if back % 2 == 0: + zap(i, j); i = j; break + continue + # string literal + if ch == '"': + j = i + 1 + while j < N: + c = src[j]; j += 1 + if c == '\\': j += 1 + elif c == '"': break + zap(i, j); i = j; continue + # char literal + if ch == "'": + j = i + 1 + while j < N: + c = src[j]; j += 1 + if c == '\\': j += 1 + elif c == "'": break + zap(i, j); i = j; continue + i += 1 + return ''.join(m) + + m = mask_code(s) + + # ---------- 1) Precompute () and {} match maps (O(n)) ---------- + # This removes repeated O(k) walks for every match_brace/paren call. + paren_match: Dict[int, int] = {} + brace_match: Dict[int, int] = {} + ps = [] + bs = [] + for idx, ch in enumerate(m): + if ch == '(': + ps.append(idx) + elif ch == ')': + if ps: + j = ps.pop() + paren_match[j] = idx + elif ch == '{': + bs.append(idx) + elif ch == '}': + if bs: + j = bs.pop() + brace_match[j] = idx + + # ---------- tiny lexing helpers (ASCII fast-path) ---------- + def is_id_start(ch: str) -> bool: + o = ord(ch) + if o < 128: + return ch == '_' or ch == '$' or (65 <= o <= 90) or (97 <= o <= 122) + return ch.isalpha() + + def is_id_part(ch: str) -> bool: + o = ord(ch) + if o < 128: + return (ch == '_' or ch == '$' or (48 <= o <= 57) or + (65 <= o <= 90) or (97 <= o <= 122)) + return ch.isalnum() + + def skip_ws(i: int) -> int: + while i < n and m[i] <= ' ': + i += 1 + return i + + # O(1) using precomputed maps; fall back (very rare) if unmatched. + def match_paren(i: int) -> int: + j = paren_match.get(i) + if j is not None: return j + # fallback (degenerate/patched sources) + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + i += 1 + return i - 1 + + def match_brace(i: int) -> int: + j = brace_match.get(i) + if j is not None: return j + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '{': depth += 1 + elif ch == '}': depth -= 1 + i += 1 + return i - 1 + + def match_angles(i: int) -> int: + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '<': depth += 1 + elif ch == '>': depth -= 1 + i += 1 + return i - 1 + + def read_ident(i: int, end: int): + if i < end and is_id_start(s[i]): + j = i + 1 + while j < end and is_id_part(s[j]): j += 1 + return s[i:j], j + return None, i + + def collapse_ws(t: str) -> str: + return " ".join(t.split()) + + def base_type_of(typ: str) -> str: + if not typ: return "" + depth = 0; out = [] + for ch in typ: + if ch == '<': depth += 1 + elif ch == '>': depth = depth-1 if depth else 0 + elif depth == 0: out.append(ch) + t = "".join(out).strip() + if t.endswith("..."): t = t[:-3].rstrip() + while t.endswith("[]"): t = t[:-2].rstrip() + return collapse_ws(t) + + def keep_type(typ: str) -> bool: + if not typ: return False + base = base_type_of(typ) + if not base: return False + simple = base.split(".")[-1] + if simple.lower() in PRIMITIVE_TYPES: return False + if simple in WRAPS: return False + if base.startswith("java.lang.") and simple in WRAPS: return False + return True + + def parse_field_header(header: str) -> List[Tuple[str, str]]: + i, L = 0, len(header) + + def ws(k): + while k < L and header[k].isspace(): k += 1 + return k + + def rid(k): + if k < L and is_id_start(header[k]): + j = k + 1 + while j < L and is_id_part(header[j]): j += 1 + return header[k:j], j + return None, k + + def paren(k): + depth = 1; k += 1 + while k < L and depth > 0: + c = header[k] + if c == '(': + depth += 1 + elif c == ')': + depth -= 1 + elif c in ('"', "'"): + k += 1 + while k < L: + d = header[k]; k += 1 + if d == '\\': k += 1 + elif d == '"': break + k += 1 + return k - 1 + + def angles(k): + depth = 1; k += 1 + while k < L and depth > 0: + c = header[k] + if c == '<': depth += 1 + elif c == '>': depth -= 1 + k += 1 + return k - 1 + + # drop leading annotations on type + i = ws(i) + while i < L and header[i] == JAVA_ANNOTATION_SYMBOL: + i += 1 + _, i = rid(i); i = ws(i) + while i < L and header[i] == '.': + _, i = rid(i + 1); i = ws(i) + if i < L and header[i] == '(': + i = paren(i) + 1 + i = ws(i) + + # read base type + generics + []/... + type_start = i + _, j = rid(i) + if j == i: + return [] + i = ws(j) + if i < L and header[i] == '<': + i = angles(i) + 1; i = ws(i) + while i < L and header[i] == '.': + _, i = rid(i + 1); i = ws(i) + if i < L and header[i] == '<': + i = angles(i) + 1; i = ws(i) + while True: + if header.startswith('[]', i): i += 2; i = ws(i); continue + if header.startswith('...', i): i += 3; i = ws(i); continue + break + base = header[type_start:i].strip() + + out_fields: List[Tuple[str, str]] = [] + + def var_and_suffix(k): + k = ws(k) + while k < L and header[k] == JAVA_ANNOTATION_SYMBOL: + k += 1 + _, k = rid(k); k = ws(k) + while k < L and header[k] == '.': + _, k = rid(k + 1); k = ws(k) + if k < L and header[k] == '(': + k = paren(k) + 1 + k = ws(k) + name, j2 = rid(k) + if not name: return None, "", k + k = ws(j2) + suf = "" + while header.startswith('[]', k): + suf += "[]"; k += 2; k = ws(k) + return name, suf, k + + i = ws(i) + v, suf, i2 = var_and_suffix(i) + if not v: return [] + out_fields.append((v, (base + suf).strip())) + i = i2 + while i < L and header[i] == ',': + i += 1 + v, suf, i2 = var_and_suffix(i) + if not v: break + out_fields.append((v, (base + suf).strip())) + i = i2 + return out_fields + + def next_marker(i: int, end: int, ms=MRK_SET): + while i < end: + i = skip_ws(i) + if i >= end: break + c = m[i] + if c in ms: return i, c + i += 1 + return -1, '' + + def scan_to_toplevel_semicolon(p: int, end: int) -> int: + par=brc=brk=0 + while p < end: + ch = m[p] + if ch == '(': + par += 1 + elif ch == ')': + if par: par -= 1 + elif ch == '{': + brc += 1 + elif ch == '}': + if brc: brc -= 1 + elif ch == '[': + brk += 1 + elif ch == ']': + if brk: brk -= 1 + elif ch == ';' and par == 0 and brc == 0 and brk == 0: + return p + p += 1 + return -1 + + # -------- scan anonymous classes inside a range -------- + def scan_anonymous_range(a: int, b: int): + if not include_anonymous: + return + i = a + while i < b: + i = skip_ws(i) + if i >= b: break + # fast token check for 'new' (masked buffer, so no comments/strings) + if m[i] != 'n': + i += 1; continue + if i + 3 <= b and m[i:i+3] == 'new' and (i == a or not is_id_part(s[i-1])): + j = i + 3 + if j < b and is_id_part(s[j]): # avoid 'newX' + i += 1; continue + + anon_start = i + k = skip_ws(j) + # type-use annotations + while k < b and s[k] == JAVA_ANNOTATION_SYMBOL: + k += 1 + nm, k = read_ident(k, b) + while k < b and s[k] == '.': + nm2, k = read_ident(k + 1, b) + if not nm2: break + k = skip_ws(k) + if k < b and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + + nm, k2 = read_ident(k, b) + if not nm: + i += 1; continue + k = k2 + while True: + k = skip_ws(k) + if k < b and m[k] == '<': + k = match_angles(k) + 1; continue + if k < b and m[k] == '.': + nm2, k2 = read_ident(k + 1, b) + if not nm2: break + k = k2; continue + break + k = skip_ws(k) + if k < b and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + if k < b and m[k] == '{': + r = match_brace(k) + segment = s[anon_start:r + 1] + md5_hex = hashlib.md5(segment.encode("utf-8")).hexdigest() + anon_name = f"@{md5_hex}" + collect_type('class', anon_name, k, r) + i = r + 1 + continue + i += 1 + + # ---------------- core: scan whole file, process types inline ---------------- + def collect_type(kind: str, name: str, lb: int, rb: int): + if kind == 'enum': + return + fields: List[Tuple[str, str]] = [] + i, end = lb + 1, rb + + while i < end: + i = skip_ws(i) + if i >= end: break + + ch = m[i] + + # initializer blocks + if ch == '{': + rblk = match_brace(i) + scan_anonymous_range(i, rblk) + i = rblk + 1 + continue + + # static { ... } + if is_id_start(s[i]): + j = i + 1 + while j < end and is_id_part(s[j]): j += 1 + if s[i:j] == 'static': + j2 = skip_ws(j) + if j2 < end and m[j2] == '{': + rblk = match_brace(j2) + scan_anonymous_range(j2, rblk) + i = rblk + 1 + continue + + # normalize: skip annotations & modifiers + start = i + tmp = start + while True: + t2 = skip_ws(tmp) + if t2 != tmp: tmp = t2; continue + if tmp < end and s[tmp] == JAVA_ANNOTATION_SYMBOL: + j = tmp + 1 + nm, j = read_ident(j, end) + while j < end and s[j] == '.': + nm2, j = read_ident(j + 1, end) + if not nm2: break + j = skip_ws(j) + if j < end and m[j] == '(': + j = match_paren(j) + 1 + tmp = j; continue + if tmp < end and is_id_start(s[tmp]): + t3 = tmp + 1 + while t3 < end and is_id_part(s[t3]): t3 += 1 + if s[tmp:t3] in MODS: + tmp = t3; continue + break + start = tmp + + # nested named type at normalized start (e.g., 'public class X { ... }') + if include_nested: + j = start + if j < end and is_id_start(s[j]): + j2 = j + 1 + while j2 < end and is_id_part(s[j2]): j2 += 1 + tok2 = s[j:j2] + if tok2 in ('class','interface','enum','record'): + m2 = skip_ws(j2) + name_start = m2 + while m2 < end and is_id_part(s[m2]): m2 += 1 + if m2 > name_start: + tname = s[name_start:m2] + m2 = skip_ws(m2) + if m2 < end and m[m2] == '<': m2 = match_angles(m2) + 1; m2 = skip_ws(m2) + if tok2 == 'record' and m2 < end and m[m2] == '(': + m2 = match_paren(m2) + 1; m2 = skip_ws(m2) + # use .find to jump to body quickly + body_open = m.find('{', m2, end) + if body_open != -1: + r2 = match_brace(body_open) + collect_type(tok2, tname, body_open, r2) + i = r2 + 1 + continue + + # decide by first structural marker + pos, mrk = next_marker(start, end, MRK_SET) + if pos == -1: break + + if mrk == '(': + # method/ctor: scan body for anonymous, then skip body + pclose = match_paren(pos) + q = skip_ws(pclose + 1) + # optional throws … + if q + 6 <= end and s[q:q+6] == 'throws' and (q + 6 == end or not is_id_part(s[q+6])): + j = skip_ws(q + 6) + while j < end and m[j] not in '{;': + if m[j] == '<': j = match_angles(j) + 1 + elif m[j] == '[' and j + 1 < end and m[j+1] == ']': j += 2 + elif is_id_start(s[j]): + k = j + 1 + while k < end and is_id_part(s[k]): k += 1 + j = k + elif m[j] in '.,': j += 1 + else: break + j = skip_ws(j) + q = skip_ws(j) + if q < end and m[q] == '{': + rbody = match_brace(q) + scan_anonymous_range(q + 1, rbody) + i = rbody + 1 + elif q < end and m[q] == ';': + i = q + 1 + else: + semi = scan_to_toplevel_semicolon(q, end) + i = semi + 1 if semi != -1 else end + continue + + if mrk == '{': + rblk = match_brace(pos) + scan_anonymous_range(pos, rblk) + i = rblk + 1 + continue + + if mrk in ('=', ';'): + if mrk == '=': + semi = scan_to_toplevel_semicolon(pos + 1, end) + if semi == -1: semi = end - 1 + scan_anonymous_range(pos + 1, semi) # anon in initializer + header = s[start:pos].strip() + for (vn, vt) in parse_field_header(header): + if keep_type(vt): + fields.append((vn, vt)) + i = semi + 1 + continue + else: # ';' + header = s[start:pos].strip() + for (vn, vt) in parse_field_header(header): + if keep_type(vt): + fields.append((vn, vt)) + i = pos + 1 + continue + + i = pos + 1 + + out[name] = fields + + # ---- top-level scan ---- + i = 0 + if ("class" not in s and "interface" not in s and "enum" not in s and "record" not in s and "new" not in s): + return out + + while i < n: + i = skip_ws(i) + if i >= n: break + if not is_id_start(s[i]): + i += 1; continue + + j = i + 1 + while j < n and is_id_part(s[j]): j += 1 + tok = s[i:j] + + if tok in ('class','interface','enum','record'): + k = skip_ws(j) + name_start = k + while k < n and is_id_part(s[k]): k += 1 + if k == name_start: i = j; continue + name = s[name_start:k] + k = skip_ws(k) + if k < n and m[k] == '<': k = match_angles(k) + 1; k = skip_ws(k) + if tok == 'record' and k < n and m[k] == '(': + k = match_paren(k) + 1; k = skip_ws(k) + body_open = m.find('{', k, n) + if body_open != -1: + rb = match_brace(body_open) + collect_type(tok, name, body_open, rb) + i = rb + 1 + continue + + if include_anonymous and tok == 'new': + anon_start = i + k = skip_ws(j) + # type-use annotations + while k < n and s[k] == JAVA_ANNOTATION_SYMBOL: + nm, k = read_ident(k + 1, n) + while k < n and s[k] == '.': + nm2, k = read_ident(k + 1, n) + if not nm2: break + k = skip_ws(k) + if k < n and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + + nm, k2 = read_ident(k, n) + if nm: + k = k2 + while True: + k = skip_ws(k) + if k < n and m[k] == '<': + k = match_angles(k) + 1; continue + if k < n and m[k] == '.': + nm2, k2 = read_ident(k + 1, n) + if not nm2: break + k = k2; continue + break + k = skip_ws(k) + if k < n and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + if k < n and m[k] == '{': + rb = match_brace(k) + segment = s[anon_start:rb + 1] + md5_hex = hashlib.md5(segment.encode("utf-8")).hexdigest() + anon_name = f"@{md5_hex}" + collect_type('class', anon_name, k, rb) + i = rb + 1 + continue + + i += 1 + + return out + +def get_type_name(the_type: Document) -> str: + """ + Return the simple name of the first class/interface/enum/record declared + anywhere in the given Java source (top-level or nested). Makes no assumptions + about leading comments, javadocs, annotations, package/imports, etc. + + - Skips // line comments, /* ... */ (incl. Javadoc) block comments, strings "..." + and character literals '...'. + - Handles 'record' headers (e.g., `record Point(int x, int y) { ... }`). + - Matches annotation type declarations too (e.g., `public @interface Foo {}`), + because the token `interface` is still present. + - Returns the *simple* identifier (e.g., 'Point'). + """ + src = the_type.page_content + n = len(src) + i = 0 + + # States + NORMAL, LINE, BLOCK, STRING, CHAR = 0, 1, 2, 3, 4 + state = NORMAL + + KW = ("class", "interface", "enum", "record") + + def is_ws(ch: str) -> bool: + return ch <= " " and ch in " \t\r\n\f\v" + + def is_id_start(ch: str) -> bool: + # Java allows '_' and '$'; also allow unicode letters + return ch == "_" or ch == "$" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) + + def is_id_part(ch: str) -> bool: + return is_id_start(ch) or ("0" <= ch <= "9") + + def word_at(pos: int, word: str) -> bool: + """Return True if 'word' starts at src[pos] with Java-style word boundaries.""" + end = pos + len(word) + if end > n or src[pos:end] != word: + return False + # left boundary: start of file or not an identifier part + if pos > 0 and (src[pos-1].isalnum() or src[pos-1] in "_$"): + return False + # right boundary: end of file or not an identifier part + if end < n and (src[end].isalnum() or src[end] in "_$"): + return False + return True + + while i < n: + ch = src[i] + + if state == NORMAL: + # comment/string/char entry + if ch == "/": + if i + 1 < n and src[i + 1] == "/": + state = LINE + i += 2 + continue + if i + 1 < n and src[i + 1] == "*": + state = BLOCK + i += 2 + continue + elif ch == '"': + state = STRING + i += 1 + continue + elif ch == "'": + state = CHAR + i += 1 + continue + + # Try to match a type keyword at this position + for kw in KW: + if word_at(i, kw): + j = i + len(kw) + # skip whitespace between keyword and identifier + while j < n and is_ws(src[j]): + j += 1 + # Next must be a valid Java identifier start (type name) + if j < n and is_id_start(src[j]): + k = j + 1 + while k < n and is_id_part(src[k]): + k += 1 + return src[j:k] + # If not a valid identifier here, keep scanning + # (defensive: malformed code or rare construct) + i += 1 + + elif state == LINE: + if ch == "\n": + state = NORMAL + i += 1 + + elif state == BLOCK: + if ch == "*" and i + 1 < n and src[i + 1] == "/": + state = NORMAL + i += 2 + else: + i += 1 + + elif state == STRING: + if ch == "\\" and i + 1 < n: + i += 2 # skip escaped + continue + if ch == '"': + state = NORMAL + i += 1 + + elif state == CHAR: + if ch == "\\" and i + 1 < n: + i += 2 + continue + if ch == "'": + state = NORMAL + i += 1 + + return "" + +def is_java_method(source: str) -> bool: + """ + Return True iff the given Java source snippet is a *method* declaration, + or (fallback) contains a lambda ('->') outside of comments/strings. + Constructors are *not* considered methods here (no return type). + """ + s = source + n = len(s) + i = 0 + + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + brace = 0 + saw_lambda = False + + def is_ws(ch: str) -> bool: + return ch <= " " and ch in " \t\r\n\f\v" + + def is_id_start(ch: str) -> bool: + return ch == "_" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) + + def is_id_part(ch: str) -> bool: + return ( + ch == "_" or ch == "$" or + ("A" <= ch <= "Z") or ("a" <= ch <= "z") or ("0" <= ch <= "9") or + (ch >= "\u0080" and ch.isalnum()) + ) + + CLASS_KINDS = ("class", "interface", "enum", "record") + CLASS_MODS = {"public","protected","private","abstract","final","static","sealed","strictfp"} + CTRL_WORDS = {"if","for","while","switch","catch","synchronized","return","new","case"} + METH_MODS = {"public","protected","private","abstract","static","final","synchronized", + "native","strictfp","default","transient"} + PRIM_TYPES = {"void","int","long","double","float","short","byte","boolean","char"} + + def skip_paren(j: int) -> int: + depth = 1 + st = NORMAL + while j < n and depth > 0: + ch = s[j] + if st == NORMAL: + if ch == "/": + if j + 1 < n: + c2 = s[j + 1] + if c2 == "/": st = LINE; j += 2; continue + if c2 == "*": st = BLOCK; j += 2; continue + elif ch == '"': + if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': + st = TEXT; j += 3; continue + st = STRING; j += 1; continue + elif ch == "'": + st = CHAR; j += 1; continue + elif ch == "(": + depth += 1; j += 1; continue + elif ch == ")": + depth -= 1; j += 1; continue + else: + j += 1; continue + elif st == LINE: + nl = s.find("\n", j) + j = n if nl == -1 else nl + 1 + st = NORMAL + elif st == BLOCK: + end = s.find("*/", j) + j = n if end == -1 else end + 2 + st = NORMAL + elif st == STRING: + while j < n: + c2 = s[j]; j += 1 + if c2 == "\\": j += 1 + elif c2 == '"': break + st = NORMAL + elif st == CHAR: + while j < n: + c2 = s[j]; j += 1 + if c2 == "\\": j += 1 + elif c2 == "'": break + st = NORMAL + else: # TEXT + while True: + end = s.find('"""', j) + if end == -1: + j = n + break + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1; k -= 1 + j = end + 3 + if back % 2 == 0: + break + st = NORMAL + return j + + def read_ident(j: int) -> tuple[Optional[str], int]: + if j >= n or not is_id_start(s[j]): return None, j + k = j + 1 + while k < n and is_id_part(s[k]): k += 1 + return s[j:k], k + + def read_qual_ident(j: int) -> tuple[Optional[str], int]: + name, k = read_ident(j) + if not name: return None, j + while k < n and s[k] == ".": + nm2, k2 = read_ident(k + 1) + if not nm2: break + k = k2 + return name, k + + def skip_ws_comments(j: int) -> int: + while j < n: + ch = s[j] + if is_ws(ch): + j += 1; continue + if ch == "/" and j + 1 < n: + c2 = s[j + 1] + if c2 == "/": + j = (s.find("\n", j + 2) + 1) or n; continue + if c2 == "*": + end = s.find("*/", j + 2) + j = n if end == -1 else end + 2; continue + break + return j + + def has_dot_between(j: int, k: int) -> bool: + j = skip_ws_comments(j) + while j < k: + ch = s[j] + if ch == ".": return True + if is_ws(ch): + j += 1; continue + if ch == "/" and j + 1 < n: + if s[j+1] == "/": + j = (s.find("\n", j + 2) + 1) or n; continue + if s[j+1] == "*": + end = s.find("*/", j + 2) + j = n if end == -1 else end + 2; continue + j += 1 + return False + + def plausible_method_decl(name_start: int) -> bool: + """ + Must see a real return type token before the name (excludes constructors). + """ + j = name_start + j0 = max(s.rfind("\n", 0, j), s.rfind("{", 0, j), s.rfind(";", 0, j)) + 1 + j = skip_ws_comments(j0) + + while j < name_start: + # annotations + if s[j] == JAVA_ANNOTATION_SYMBOL: + _, j = read_qual_ident(j + 1) + j = skip_ws_comments(j) + if j < n and s[j] == "(": + j = skip_paren(j + 1) + j = skip_ws_comments(j) + continue + + # type params + if s[j] == "<": + depth = 1; j += 1 + while j < n and depth > 0: + if s[j] == "<": depth += 1 + elif s[j] == ">": depth -= 1 + j += 1 + j = skip_ws_comments(j) + continue + + tok, k = read_ident(j) + if tok: + if tok in PRIM_TYPES: + return True # primitive (incl. void) return + + if tok in METH_MODS: + j = skip_ws_comments(k) + continue + + # reference type start + j = k + j = skip_ws_comments(j) + + # optional generics + if j < n and s[j] == "<": + depth = 1; j += 1 + while j < n and depth > 0: + if s[j] == "<": depth += 1 + elif s[j] == ">": depth -= 1 + j += 1 + j = skip_ws_comments(j) + + # optional arrays + while j + 1 < n and s[j] == "[" and s[j + 1] == "]": + j += 2 + j = skip_ws_comments(j) + + # optional intersections: A & B & ... + tcount = 0 + while tcount < 4: + j = skip_ws_comments(j) + if j < n and s[j] == "&": + j = skip_ws_comments(j + 1) + nm2, k2 = read_ident(j) + if not nm2: break + j = k2 + j = skip_ws_comments(j) + if j < n and s[j] == "<": + depth = 1; j += 1 + while j < n and depth > 0: + if s[j] == "<": depth += 1 + elif s[j] == ">": depth -= 1 + j += 1 + j = skip_ws_comments(j) + while j + 1 < n and s[j] == "[" and s[j + 1] == "]": + j += 2 + j = skip_ws_comments(j) + tcount += 1 + else: + break + + return True # saw a bona fide reference return type + + break + return False + + while i < n: + ch = s[i] + + if state == NORMAL: + if ch == "-" and i + 1 < n and s[i + 1] == ">": + saw_lambda = True + i += 2 + continue + + if ch == "/": + if i + 1 < n: + c2 = s[i + 1] + if c2 == "/": state = LINE; i += 2; continue + if c2 == "*": state = BLOCK; i += 2; continue + elif ch == '"': + if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + state = TEXT; i += 3; continue + state = STRING; i += 1; continue + elif ch == "'": + state = CHAR; i += 1; continue + elif ch == "{": + brace += 1; i += 1; continue + elif ch == "}": + brace = max(0, brace - 1); i += 1; continue + + if brace == 0: + if is_ws(ch): + i += 1; continue + + # skip annotations at top level + if ch == JAVA_ANNOTATION_SYMBOL: + _, i = read_qual_ident(i + 1) + i = skip_ws_comments(i) + if i < n and s[i] == "(": + i = skip_paren(i + 1) + continue + + # skip modifiers & detect type headers early (we only care to *not* misclassify) + if is_id_start(ch): + ident, j = read_ident(i) + if ident in CLASS_MODS: + i = skip_ws_comments(j); continue + if ident == "non": + k = skip_ws_comments(j) + if k < n and s[k] == "-": + k2 = skip_ws_comments(k + 1) + id2, j2 = read_ident(k2) + if id2 == "sealed": + i = skip_ws_comments(j2); continue + if ident in CLASS_KINDS: + i = skip_ws_comments(j) + name, j2 = read_ident(i) + if name: + return False + i = j + continue + + # potential method header: look for '(' with a plain name just before it + if ch == "(": + j = i - 1 + while j >= 0 and is_ws(s[j]): j -= 1 + end = j + 1 + while j >= 0 and is_id_part(s[j]): j -= 1 + name = s[j+1:end] if end > (j + 1) else None + + if name and (name not in CTRL_WORDS) and not has_dot_between(end, i): + k = skip_paren(i + 1) + q = skip_ws_comments(k) + + # optional 'throws ...' clause + if s.startswith("throws", q) and (q+6 == n or not is_id_part(s[q+6])): + q = skip_ws_comments(q + 6) + while q < n and s[q] not in "{;": + if is_ws(s[q]) or s[q] in ".,": q += 1; continue + if s[q] == "<": + depth = 1; q += 1 + while q < n and depth > 0: + if s[q] == "<": depth += 1 + elif s[q] == ">": depth -= 1 + q += 1 + continue + if is_id_start(s[q]): + _, q = read_ident(q); continue + if s[q] == "[" and q + 1 < n and s[q+1] == "]": + q += 2; continue + break + q = skip_ws_comments(q) + + # must end with '{' or ';' to be a decl, then validate return type + if q < n and s[q] in "{;": + if plausible_method_decl(j + 1): + return True # it's a method + + i += 1 + continue + + i += 1 + continue + + elif state == LINE: + nl = s.find("\n", i) + i = n if nl == -1 else nl + 1 + state = NORMAL + continue + + elif state == BLOCK: + end = s.find("*/", i) + i = n if end == -1 else end + 2 + state = NORMAL + continue + + elif state == STRING: + while i < n: + c2 = s[i]; i += 1 + if c2 == "\\": i += 1 + elif c2 == '"': break + state = NORMAL + continue + + elif state == CHAR: + while i < n: + c2 = s[i]; i += 1 + if c2 == "\\": i += 1 + elif c2 == "'": break + state = NORMAL + continue + + else: # TEXT + while True: + end = s.find('"""', i) + if end == -1: + i = n + break + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1; k -= 1 + i = end + 3 + if back % 2 == 0: + break + state = NORMAL + continue + + # lambda fallback → treat snippet as "method-like" + if saw_lambda: + return True + + return False + +def is_java_type(source: str) -> bool: + """ + Extremely fast: single pass, early exit. + Detects top-level class/interface/enum/record only. + Avoids misclassifying method snippets. + """ + s = source + n = len(s) + i = 0 + + if ("class" not in s and "interface" not in s and + "enum" not in s and "record" not in s and "(" not in s): + return False + + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + brace = 0 + + ws = " \t\r\n\f\v" + class_kinds = {"class", "interface", "enum", "record"} + class_mods = {"public","protected","private","abstract","final","static","sealed","strictfp"} + meth_ctrl = {"if","for","while","switch","catch","synchronized","return","new","case"} + + def is_ws(ch): return ch in ws + def is_id_start(ch): return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or (ch >= '\u0080' and ch.isalpha()) + def is_id_part(ch): return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or ('0' <= ch <= '9') or (ch >= '\u0080' and ch.isalnum()) + + def read_ident(j: int): + if j >= n or not is_id_start(s[j]): return None, j + k = j + 1 + while k < n and is_id_part(s[k]): k += 1 + return s[j:k], k + + def skip_ws_comments(j: int) -> int: + while j < n: + ch = s[j] + if is_ws(ch): j += 1; continue + if ch == '/' and j + 1 < n: + c2 = s[j + 1] + if c2 == '/': + j = (s.find('\n', j + 2) + 1) or n; continue + if c2 == '*': + end = s.find("*/", j + 2) + j = n if end == -1 else end + 2; continue + break + return j + + def skip_paren_balanced(j: int) -> int: + depth = 1; st = NORMAL + while j < n and depth > 0: + ch = s[j] + if st == NORMAL: + if ch == '/': + if j + 1 < n and s[j+1] == '/': + j = (s.find('\n', j + 2) + 1) or n; continue + if j + 1 < n and s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2; continue + elif ch == '"': + if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': + st = TEXT; j += 3; continue + st = STRING; j += 1; continue + elif ch == "'": + st = CHAR; j += 1; continue + elif ch == '(': + depth += 1; j += 1; continue + elif ch == ')': + depth -= 1; j += 1; continue + else: + j += 1; continue + elif st == STRING: + while j < n: + c2 = s[j]; j += 1 + if c2 == '\\': j += 1 + elif c2 == '"': break + st = NORMAL + elif st == CHAR: + while j < n: + c2 = s[j]; j += 1 + if c2 == '\\': j += 1 + elif c2 == "'": break + st = NORMAL + else: # TEXT + while True: + end = s.find('"""', j) + if end == -1: j = n; break + k = end - 1; back = 0 + while k >= 0 and s[k] == '\\': back += 1; k -= 1 + j = end + 3 + if back % 2 == 0: break + st = NORMAL + return j + + def skip_angles_balanced(j: int) -> int: + depth = 1; j += 1 + while j < n and depth > 0: + ch = s[j] + if ch == '<': depth += 1 + elif ch == '>': depth -= 1 + elif ch == '/': + if j + 1 < n and s[j+1] == '/': + j = (s.find('\n', j + 2) + 1) or n; continue + if j + 1 < n and s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2; continue + j += 1 + return j + + def has_dot_between(a: int, b: int) -> bool: + j = a + while j < b: + ch = s[j] + if ch == '.': return True + if is_ws(ch): + j += 1; continue + if ch == '/' and j + 1 < n: + if s[j+1] == '/': + j = (s.find('\n', j + 2) + 1) or n; continue + if s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2; continue + j += 1 + return False + + # NEW: skip a single qualified type (with optional annotations, generics, array suffix) + def skip_one_type(p: int) -> int: + p = skip_ws_comments(p) + # annotations on the type + while p < n and s[p] == JAVA_ANNOTATION_SYMBOL: + _, p = read_ident(p + 1) + while p < n and s[p] == '.': + _, p = read_ident(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p) + # qualified name + optional generics per segment + nm, p2 = read_ident(p) + if not nm: return p + p = p2; p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p); p = skip_ws_comments(p) + while p < n and s[p] == '.': + nm2, p2 = read_ident(p + 1) + if not nm2: break + p = p2; p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p); p = skip_ws_comments(p) + # array suffix(es) + while p + 1 < n and s[p] == '[' and s[p+1] == ']': + p += 2; p = skip_ws_comments(p) + return p + + # NEW: skip comma-separated type list after extends/implements/permits + def skip_type_list(p: int) -> int: + p = skip_ws_comments(p) + tcount = 0 + while p < n: + p2 = skip_one_type(p) + if p2 == p: break + p = skip_ws_comments(p2) + tcount += 1 + if p < n and s[p] == ',': + p = skip_ws_comments(p + 1) + continue + break + return p + + while i < n: + ch = s[i] + + if state == NORMAL: + if ch == '/': + if i + 1 < n and s[i+1] == '/': + state = LINE; i += 2; continue + if i + 1 < n and s[i+1] == '*': + state = BLOCK; i += 2; continue + elif ch == '"': + if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + state = TEXT; i += 3; continue + state = STRING; i += 1; continue + elif ch == "'": + state = CHAR; i += 1; continue + elif ch == '{': + brace += 1; i += 1; continue + elif ch == '}': + brace = max(0, brace - 1); i += 1; continue + + if brace != 0: + i += 1; continue + if is_ws(ch): + i += 1; continue + + if ch == JAVA_ANNOTATION_SYMBOL: + i += 1 + _, i = read_ident(i) + while i < n and s[i] == '.': + _, i = read_ident(i + 1) + i = skip_ws_comments(i) + if i < n and s[i] == '(': + i = skip_paren_balanced(i + 1) + continue + + if is_id_start(ch): + tok, j = read_ident(i) + + # early: method/ctor at top level => not a class + k = skip_ws_comments(j) + if k < n and s[k] == '(' and tok not in meth_ctrl and not has_dot_between(j, k): + r = skip_paren_balanced(k + 1) + q = skip_ws_comments(r) + if q + 6 <= n and s[q:q+6] == 'throws' and (q + 6 == n or not is_id_part(s[q+6])): + q = skip_ws_comments(q + 6) + while q < n and s[q] not in '{;': + if is_ws(s[q]) or s[q] in '.,': q += 1; continue + if s[q] == '<': + q = skip_angles_balanced(q); q = skip_ws_comments(q); continue + if is_id_start(s[q]): + _, q = read_ident(q); q = skip_ws_comments(q); continue + if s[q] == '[' and q + 1 < n and s[q+1] == ']': + q += 2; continue + break + q = skip_ws_comments(q) + if q < n and s[q] in '{;': + return False + + # class modifiers (incl. "non-sealed") + if tok in class_mods: + i = skip_ws_comments(j) + if tok == "non" and i < n and s[i] == '-': + i = skip_ws_comments(i + 1) + t2, j2 = read_ident(i) + if t2 == "sealed": + i = skip_ws_comments(j2) + continue + + if tok in class_kinds: + p = skip_ws_comments(j) + name, p2 = read_ident(p) + if not name: + i = j; continue + p = skip_ws_comments(p2) + if tok == 'record' and p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p) + p = skip_ws_comments(p) + + # allow extends/implements/permits with full type lists + while p < n: + if s.startswith("extends", p) and (p+7 == n or not is_id_part(s[p+7])): + p = skip_type_list(skip_ws_comments(p + 7)); continue + if s.startswith("implements", p) and (p+10 == n or not is_id_part(s[p+10])): + p = skip_type_list(skip_ws_comments(p + 10)); continue + if s.startswith("permits", p) and (p+7 == n or not is_id_part(s[p+7])): + p = skip_type_list(skip_ws_comments(p + 7)); continue + # rare: stray type annotation chunk before the brace + if s[p] == JAVA_ANNOTATION_SYMBOL: + p = skip_ws_comments(p + 1) + _, p = read_ident(p) + while p < n and s[p] == '.': + _, p = read_ident(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p); continue + if s[p] == '<': # stray generics chunk + p = skip_angles_balanced(p); p = skip_ws_comments(p); continue + break + + if p < n and s[p] == '{': + return True + + i = j + continue + + i = j + continue + + i += 1 + continue + + elif state == LINE: + nl = s.find('\n', i) + i = n if nl == -1 else nl + 1 + state = NORMAL + continue + + elif state == BLOCK: + end = s.find('*/', i) + i = n if end == -1 else end + 2 + state = NORMAL + continue + + elif state == STRING: + while i < n: + c2 = s[i]; i += 1 + if c2 == '\\': i += 1 + elif c2 == '"': break + state = NORMAL + continue + + elif state == CHAR: + while i < n: + c2 = s[i]; i += 1 + if c2 == '\\': i += 1 + elif c2 == "'": break + state = NORMAL + continue + + else: # TEXT + while True: + end = s.find('"""', i) + if end == -1: i = n; break + k = end - 1; back = 0 + while k >= 0 and s[k] == '\\': back += 1; k -= 1 + i = end + 3 + if back % 2 == 0: break + state = NORMAL + continue + + return False + +def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: + """ + Returns: + { + (SimpleTypeName, SourcePath): [ + ("Self", "SourcePath"), + ("Parent", "SourcePath-or-empty"), + ("GrandParent", "SourcePath-or-empty"), + ("Iface1", "SourcePath-or-empty"), + ... + ("ImplOrSubclass1", "SourcePath-or-empty"), + ("ImplOrSubclass2", "SourcePath-or-empty"), + ... + ] + } + + - Key is (type simple name, doc.metadata['source'] or 'path' or '') + - Handles 'Code for:' stubs and full files with comments/imports + - Order-preserving, no duplicates + - Explicit-imports-first resolution; star imports allowed + - Only DFS into interfaces present in the provided set (externals are leaves) + - Includes self, ancestors, implemented interfaces, and transitive implementers/subclasses + """ + + # ---------------- helpers ---------------- + def strip_comments_preserving_newlines(s: str) -> str: + """Remove /* ... */ and // ... comments, preserving newlines for positions.""" + def _blk(m): + t = m.group(0) + return re.sub(r"[^\n]", " ", t) + s2 = re.sub(r"/\*.*?\*/", _blk, s, flags=re.S) + s2 = re.sub(r"//[^\n]*", "", s2) + return s2 + + def find_header_slice_no_comments(src: str) -> str | None: + """ + Strip comments; find first real type header; return text from that point + up to the first '{' that follows it. + """ + s = strip_comments_preserving_newlines(src) + NAME_KIND_RE = re.compile( + r"(?m)^\s*" + r"(?:@[A-Za-z_$][\w$]*(?:\s*\([^()]*\))?\s*)*" # annotations (line-wise) + r"(?:(?:public|protected|private|abstract|final|static|sealed|non-sealed|strictfp)\s+)*" + r"\b(class|interface|enum|record)\b" + ) + m = NAME_KIND_RE.search(s) + if not m: + return None + brace_pos = s.find("{", m.end()) + return s[m.start():].strip() if brace_pos == -1 else s[m.start():brace_pos].strip() + + def before_brace(s: str) -> str: + i = s.find("{") + return s if i == -1 else s[:i] + + def drop_leading_annotations(s: str) -> str: + i = 0 + n = len(s) + while True: + while i < n and s[i].isspace(): + i += 1 + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i += 1 + while i < n and (s[i].isalnum() or s[i] in ('_', '$', '.')): + i += 1 + if i < n and s[i] == '(': + depth = 1 + i += 1 + while i < n and depth > 0: + ch = s[i] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + i += 1 + continue + break + return s[i:] + + def base_token(t: str) -> str: + """Strip generics and [] to get a core type token.""" + t = (t or "").strip() + out, depth = [], 0 + for ch in t: + if ch == '<': + depth += 1 + elif ch == '>': + depth = max(0, depth - 1) + elif depth == 0: + out.append(ch) + return ''.join(out).replace("[]", "").strip() + + def simple_name_from_fqcn(fq: str) -> str: + return fq.split('.')[-1] if fq else fq + + def dedup_keep_order(items: List[str]) -> List[str]: + """ + Remove duplicates from a list of strings while preserving the order of their + first appearance. Falsy values (e.g., "", None) are ignored. + + Parameters + ---------- + items : List[str] + The input sequence of strings. Elements are considered duplicates if + their string values are equal. Whitespace differences matter—that is, + "Foo" and " Foo" are different. + + Returns + ------- + List[str] + A new list containing the first occurrence of each non-falsy, unique + string from `items`, in original order. + + Examples + -------- + >>> dedup_keep_order(["A", "B", "A", "C"]) + ['A', 'B', 'C'] + + >>> dedup_keep_order(["", "A", "", "A", "B"]) + ['A', 'B'] + + # Whitespace differences are preserved and treated as distinct + >>> dedup_keep_order(["Foo", " Foo", "Foo "]) + ['Foo', ' Foo', 'Foo '] + + >>> dedup_keep_order([]) + [] + """ + seen: set[str] = set() + ordered: List[str] = [] + for item in items: + if item and item not in seen: + seen.add(item) + ordered.append(item) + return ordered + + def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: + """ + Remove duplicate (name, source) pairs while preserving the order + of their first appearance. + + Parameters + ---------- + pairs : List[Tuple[str, str]] + A list of (simple_type_name, source_path) tuples. Two entries are + considered duplicates only if BOTH the name and source are identical. + + Returns + ------- + List[Tuple[str, str]] + A new list containing the first occurrence of each unique pair, in the + same relative order as `pairs`. + + Examples + -------- + >>> tup_dedup_keep_order([("A", "/src/A.java"), ("B", "/src/B.java"), ("A", "/src/A.java")]) + [('A', '/src/A.java'), ('B', '/src/B.java')] + + # Different source_path → not a duplicate + >>> tup_dedup_keep_order([("A", "/src/A.java"), ("A", "/other/A.java")]) + [('A', '/src/A.java'), ('A', '/other/A.java')] + + # Empty input + >>> tup_dedup_keep_order([]) + [] + """ + seen: set[Tuple[str, str]] = set() + deduped: List[Tuple[str, str]] = [] + for pair in pairs: + if pair and pair not in seen: + seen.add(pair) + deduped.append(pair) + return deduped + + def meta_source(doc) -> str: + try: + meta = getattr(doc, "metadata", None) or {} + return meta.get("source") or meta.get("path") or "" + except Exception: + return "" + + def simple_name_from_filename(doc) -> str | None: + src = meta_source(doc) + m = re.search(r"([A-Za-z_$][\w$]*)\.java$", src) + return m.group(1) if m else None + + def split_type_list_strict(segment: str) -> List[str]: + """Split by commas outside <...> and (...).""" + if not segment: + return [] + items, cur = [], [] + depth_ang = depth_par = 0 + for ch in segment: + if ch == '<': depth_ang += 1 + elif ch == '>': depth_ang = max(0, depth_ang - 1) + elif ch == '(': depth_par += 1 + elif ch == ')': depth_par = max(0, depth_par - 1) + if ch == ',' and depth_ang == 0 and depth_par == 0: + tok = ''.join(cur).strip() + if tok: items.append(tok) + cur = [] + else: + cur.append(ch) + last = ''.join(cur).strip() + if last: items.append(last) + return items + + # Clause extractor that starts scanning AFTER the 'kind name' match. + name_kind_re = re.compile(r"""\b(class|interface|enum|record)\b\s+([A-Za-z_$][\w$]*)""", re.X) + + def extract_clauses_after_kindname(header: str) -> Tuple[str | None, str | None]: + m = name_kind_re.search(header) + if not m: + return None, None + scan_start = m.end() # start right after 'class|interface|enum|record ' + + def is_word_at(pos: int, word: str) -> bool: + if pos < 0 or pos + len(word) > len(header): return False + if header[pos:pos+len(word)] != word: return False + left_ok = pos == 0 or not (header[pos-1].isalnum() or header[pos-1] == '_') + right_ok = pos+len(word) == len(header) or not (header[pos+len(word)].isalnum() or header[pos+len(word)] == '_') + return left_ok and right_ok + + depth_ang = depth_par = 0 + ext_start = impl_start = None + i = scan_start + while i < len(header): + ch = header[i] + if ch == '<': depth_ang += 1 + elif ch == '>': depth_ang = max(0, depth_ang - 1) + elif ch == '(': depth_par += 1 + elif ch == ')': depth_par = max(0, depth_par - 1) + elif depth_ang == 0 and depth_par == 0: + if ext_start is None and is_word_at(i, "extends"): + j = i + len("extends") + while j < len(header) and header[j].isspace(): j += 1 + ext_start = j + i = j + continue + if impl_start is None and is_word_at(i, "implements"): + j = i + len("implements") + while j < len(header) and header[j].isspace(): j += 1 + impl_start = j + i = j + continue + i += 1 + + def segment_until_next(start: int) -> str | None: + if start is None: return None + depth_a = depth_p = 0 + k = start + while k < len(header): + c = header[k] + if c == '<': depth_a += 1 + elif c == '>': depth_a = max(0, depth_a - 1) + elif c == '(': depth_p += 1 + elif c == ')': depth_p = max(0, depth_p - 1) + elif depth_a == 0 and depth_p == 0: + if is_word_at(k, "extends") or is_word_at(k, "implements"): + break + k += 1 + seg = header[start:k].strip().rstrip(",").strip() + return seg or None + + ext_seg = segment_until_next(ext_start) + impl_seg = segment_until_next(impl_start) + + # Fallbacks: broad but safe, used only when structured scan yields nothing. + if ext_seg is None and "extends" in header: + m_ext = re.search(r"\bextends\b\s+([^{};]+?)(?:\bimplements\b|$)", header) + if m_ext: + ext_seg = m_ext.group(1).strip() + if impl_seg is None and "implements" in header: + m_impl = re.search(r"\bimplements\b\s+([^{};]+)$", header) + if m_impl: + impl_seg = m_impl.group(1).strip() + + return ext_seg, impl_seg + + # Inline-friendly (work on full source) + pkg_re = re.compile(r"\bpackage\s+([A-Za-z_][\w.]*)\s*;") + import_re = re.compile(r"\bimport\s+(?!static\b)\s*([A-Za-z_][\w.]*?(?:\.\*|))\s*;") + + # ---------------- first pass: parse & collect ---------------- + known_fqcns: set[str] = set() + parsed: List[dict] = [] + + for doc in tqdm(java_types, total=len(java_types), desc="Building types inheritance map..."): + src_all = doc.page_content or "" + + # Prefer 'Code for:' header if present + idx = src_all.lower().find("code for:") + if idx != -1: + line_end = src_all.find('\n', idx) + if line_end == -1: line_end = len(src_all) + header_line = src_all[idx:line_end] + header_text_raw = re.sub(r".*?Code\s*for\s*:\s*", "", header_line, flags=re.I).strip() + else: + header_text_raw = find_header_slice_no_comments(src_all) or "" + + if not header_text_raw: + header_text_raw = drop_leading_annotations(before_brace(src_all)) + + header_text = before_brace(header_text_raw) + header_text = re.sub(r"\s+", " ", header_text).strip() + + m_name = name_kind_re.search(header_text) + if not m_name: + # fallback to filename + file_simple = simple_name_from_filename(doc) + if not file_simple: + continue + pkg = (pkg_re.search(src_all) or [None, ""])[1] + imps = import_re.findall(src_all) + fqcn = f"{pkg}.{file_simple}" if pkg else file_simple + parsed.append({ + "fqcn": fqcn, + "simple": file_simple, + "pkg": pkg, + "imports": imps, + "kind": "class", + "extends_list": [], + "implements_list": [], + "source": meta_source(doc), + }) + known_fqcns.add(fqcn) + continue + + kind, parsed_name = m_name.group(1), m_name.group(2) + + # Depth-aware, start-after-name extractor (robust for 'extends' after name) + ext_seg, impl_seg = extract_clauses_after_kindname(header_text) + + # ---------- LAST-RESORT: scan the de-commented full source for "class ... extends ..." ---------- + if (not ext_seg) and ("extends" in src_all): + s_nc = strip_comments_preserving_newlines(src_all) + # anchor on the same class name we already parsed + m_ext2 = re.search( + rf"\bclass\s+{re.escape(parsed_name)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", + s_nc + ) + if m_ext2: + ext_seg = m_ext2.group(1).strip() + # ---------- END LAST-RESORT ---------- + + extends_names = split_type_list_strict(ext_seg) if ext_seg else [] + implements_names = split_type_list_strict(impl_seg) if impl_seg else [] + + file_simple = simple_name_from_filename(doc) + simple_name = file_simple or parsed_name + + pkg = (pkg_re.search(src_all) or [None, ""])[1] + imps = import_re.findall(src_all) + + fqcn = f"{pkg}.{simple_name}" if pkg else simple_name + parsed.append({ + "fqcn": fqcn, + "simple": simple_name, + "pkg": pkg, + "imports": imps, + "kind": kind, + "extends_list": extends_names, + "implements_list": implements_names, + "source": meta_source(doc), + }) + known_fqcns.add(fqcn) + + # simple -> known FQCNs + by_simple: Dict[str, List[str]] = {} + for fq in known_fqcns: + by_simple.setdefault(simple_name_from_fqcn(fq), []).append(fq) + + # ---------------- resolver ---------------- + def make_resolver(pkg: str, imports: List[str]): + """ + Build a simple type resolver for a compilation unit. + + Purpose + ------- + Given the package declaration (`pkg`) and the list of non-static import + statements (`imports`) from a single Java source file, this factory + returns a closure `resolve(token: str) -> str` that maps a *type token* + (possibly with generics/arrays and possibly unqualified) to a best-effort + fully qualified class name (FQCN) **string**, when it can be determined + unambiguously from what is already known. + + Process (high level) + -------------------- + 1) Precompute two import helpers: + - `explicit`: map of simple class name -> explicit import (e.g. "List" -> "java.util.List"). + - `stars`: list of star-imported packages (e.g. "java.util" for "java.util.*"). + + 2) The returned `resolve` closure: + - Normalizes the input token by stripping generics and `[]` to a base type. + - If the token is already qualified (contains a '.'), return it unchanged. + - If it matches an explicit import, return that FQCN. + - Otherwise try `.` if such a type is known in `known_fqcns`. + - Otherwise check star imports: if exactly one candidate in star packages exists, return it. + - Otherwise, if the simple name is globally unique among `known_fqcns`, return that FQCN. + - If none of the above rules yield an unambiguous answer, return the **simple name** unchanged. + + Return value of the closure + --------------------------- + The inner `resolve(token)` returns a **string**: + * either a fully qualified class name (e.g., "java.util.List") + * or, if resolution is ambiguous/unknown, the original *simple* type name + (e.g., "List"). It never returns `None`. + + Notes + ----- + - `known_fqcns` and `by_simple` are expected to be available in the enclosing scope: + * `known_fqcns: set[str]` — all FQCNs discovered across parsed units + * `by_simple: Dict[str, List[str]]` — simple name -> list of FQCNs having that simple name + - `base_token()` and `simple_name_from_fqcn()` are helpers also available in the enclosing scope. + + Parameters + ---------- + pkg : str + The package name declared in the current Java file (e.g., "org.example.util"). + imports : List[str] + A list of non-static import strings as they appear in the file + (e.g., ["java.util.List", "org.example.model.*"]). + + Returns + ------- + str + A resolver function that takes a type token and returns an FQCN string + or the original simple name if resolution is not unique. + """ + explicit: Dict[str, str] = {} + stars: List[str] = [] + for imp in imports: + if imp.endswith(".*"): + stars.append(imp[:-2]) # package of a star import + else: + explicit[simple_name_from_fqcn(imp)] = imp # simple -> explicit FQCN + + def resolve(token: str) -> str: + """ + Resolve a type token to an FQCN when possible. + + Purpose + ------- + Convert a possibly decorated and/or unqualified Java type reference + (e.g., "List", "Foo[]", "Foo") into its canonical fully + qualified class name when it can be determined unambiguously using: + - explicit imports, + - the current package, + - star imports, + - or global uniqueness among known types. + + Process (concise) + ----------------- + 1) Normalize `token` to its base (strip generics and array suffixes). + 2) If already qualified, return as-is. + 3) Check explicit imports. + 4) Check current package (`pkg`). + 5) Check star-import packages, but only if it yields exactly one match. + 6) If the simple name is globally unique among `known_fqcns`, return that FQCN. + 7) Otherwise, return the **simple name** unchanged (ambiguous/unknown). + + Returns + ------- + str + Either a fully qualified class name (e.g., "java.util.Map") or, + if the type cannot be resolved uniquely, the original simple name + (e.g., "Map"). Never returns None. + """ + tok = base_token(token) + if not tok: + return "" # empty/invalid token → empty string for caller logic + + # Already qualified + if '.' in tok: + return tok + + # Explicit import wins + if tok in explicit: + return explicit[tok] + + # Try same package + if pkg: + candidate = f"{pkg}.{tok}" + if candidate in known_fqcns: + return candidate + + # Star imports – accept only a unique hit + candidates: List[str] = [] + for fq in by_simple.get(tok, []): + if any(fq.startswith(base + ".") for base in stars): + candidates.append(fq) + if len(candidates) == 1: + return candidates[0] + + # Globally unique simple name across all known types + all_candidates = by_simple.get(tok, []) + if len(all_candidates) == 1: + return all_candidates[0] + + # Unknown or ambiguous → keep simple name + return tok + + return resolve + + # ---------------- build graph ---------------- + info: Dict[str, Dict] = {} + for h in parsed: + resolve = make_resolver(h["pkg"], h["imports"]) + + if h["kind"] == "record": + parents = ["java.lang.Record"] + elif h["kind"] == "class": + parents = [resolve(x) for x in (h.get("extends_list") or [])][:1] + elif h["kind"] == "enum": + parents = [] + else: # interface + parents = [resolve(x) for x in (h.get("extends_list") or [])] + + impls = [resolve(x) for x in (h.get("implements_list") or [])] + + info[h["fqcn"]] = { + "kind": h["kind"], + "parents": parents, + "impls": impls, # interfaces that this type implements + "simple": h["simple"], + "source": h.get("source", ""), + } + + def is_iface(fq: str) -> bool: + return fq in info and info[fq]["kind"] == "interface" + + # ---------------- ancestry (upwards) ---------------- + def extends_chain(fq: str) -> List[str]: + if fq not in info: + return [] + k = info[fq]["kind"] + chain: List[str] = [] + + if k == "interface": + seen = set() + def dfs_iface(x: str): + for p in info.get(x, {}).get("parents", []): + if p and p not in seen: + seen.add(p) + if is_iface(p): + dfs_iface(p) + chain.append(p) + dfs_iface(fq) + return dedup_keep_order(chain) + + # class/record/enum: linear chain + cur, visited = fq, set() + while True: + ps = info.get(cur, {}).get("parents", []) + if not ps: break + p = ps[0] + if not p or p in visited: break + visited.add(p) + chain.append(p) + if p in info and not is_iface(p): + cur = p + else: + break + return dedup_keep_order(chain) + + def all_interfaces(fq: str) -> List[str]: + """Interfaces implemented by a type (directly, via ancestors, and via super-interfaces).""" + ordered: List[str] = [] + added, visiting = set(), set() + + def add(i: str): + if i and i not in added: + added.add(i) + ordered.append(i) + + def dfs_ifaces(i: str): + if not i or i in visiting or i in added: + return + if i not in info or not is_iface(i): + add(i) + return + visiting.add(i) + for p in info.get(i, {}).get("parents", []): + dfs_ifaces(p) + visiting.remove(i) + add(i) + + for i in info.get(fq, {}).get("impls", []): + dfs_ifaces(i) + + for anc in extends_chain(fq): + if anc in info and not is_iface(anc): + for i in info.get(anc, {}).get("impls", []): + dfs_ifaces(i) + + if is_iface(fq): + for i in info.get(fq, {}).get("parents", []): + dfs_ifaces(i) + + return ordered + + # ---------------- reverse indices (downwards) ---------------- + rev_extends: Dict[str, List[str]] = {} # class -> direct subclasses + rev_iface_extends: Dict[str, List[str]] = {} # interface -> direct sub-interfaces + direct_impl: Dict[str, List[str]] = {} # interface -> classes that directly list it + + for fq, rec in info.items(): + for p in rec.get("parents", []): + if not p: continue + if is_iface(fq) and is_iface(p): + rev_iface_extends.setdefault(p, []).append(fq) + else: + rev_extends.setdefault(p, []).append(fq) + for i in rec.get("impls", []): + if i: + direct_impl.setdefault(i, []).append(fq) + + # Precompute all interfaces per-type (for fast reverse-implementers) + all_ifaces_map: Dict[str, List[str]] = { fq: all_interfaces(fq) for fq in info.keys() } + + # Interface -> all implementers (classes), including via superclass inheritance + rev_impl_all: Dict[str, List[str]] = {} + for c, ifaces in all_ifaces_map.items(): + if is_iface(c): # only classes + continue + for i in ifaces: + rev_impl_all.setdefault(i, []).append(c) + + def all_subclasses(fq: str) -> List[str]: + out: List[str] = [] + seen = set() + q = list(rev_extends.get(fq, [])) + while q: + x = q.pop(0) + if x in seen: continue + seen.add(x) + out.append(x) + q.extend(rev_extends.get(x, []) or []) + return out + + def all_subinterfaces(fq: str) -> List[str]: + out: List[str] = [] + seen = set() + q = list(rev_iface_extends.get(fq, [])) + while q: + x = q.pop(0) + if x in seen: continue + seen.add(x) + out.append(x) + q.extend(rev_iface_extends.get(x, []) or []) + return out + + def all_implementers_of_interface(i_fq: str) -> List[str]: + impls: List[str] = [] + impls.extend(rev_impl_all.get(i_fq, [])) + for sub_i in all_subinterfaces(i_fq): + impls.extend(rev_impl_all.get(sub_i, [])) + # stable dedup + seen = set() + out: List[str] = [] + for x in impls: + if x and x not in seen: + seen.add(x) + out.append(x) + return out + + # ---------------- final map ---------------- + out: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + for fq, rec in info.items(): + self_name = rec["simple"] + source = rec.get("source", "") + + up_chain = [ + (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") + for x in extends_chain(fq) + ] + iface_list = [ + (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") + for x in all_ifaces_map[fq] + ] + + down_list: List[Tuple[str, str]] = [] + if is_iface(fq): + impls = all_implementers_of_interface(fq) + down_list.extend( + (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") + for x in impls + ) + else: + subs = all_subclasses(fq) + down_list.extend( + (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") + for x in subs + ) + + vals = tup_dedup_keep_order([(self_name, source)] + up_chain + iface_list + down_list) + out[(self_name, source)] = vals + + return out + +def extract_method_name_with_params(java_src: str) -> str | None: + """ + Return 'methodName()' from a Java method header string. + Return 'lambda' ONLY if the snippet itself starts with a lambda expression like: + - (a, b) -> ... + - x -> ... + Assumptions: + - The string begins at the actual method header OR a top-level lambda. + - No comments/javadocs/annotations are present at the very beginning (before a lambda or header). + """ + import re + + s = java_src.lstrip() + n = len(s) + + # ---------- helpers ---------- + def is_id_start(ch: str) -> bool: + return ch.isalpha() or ch in "_$" + + def is_id_part(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + def skip_ws(i: int) -> int: + while i < n and s[i].isspace(): + i += 1 + return i + + def match_paren_close(start: int) -> int: + """Given s[start] == '(', return index of its matching ')', else -1.""" + depth, i2 = 0, start + while i2 < n: + ch = s[i2] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i2 + i2 += 1 + return -1 + + # ---------- detect lambda ONLY if it is at the beginning ---------- + i0 = skip_ws(0) + if i0 < n: + if s[i0] == '(': # (params) -> ... + end = match_paren_close(i0) + if end != -1: + j = skip_ws(end + 1) + if j + 1 < n and s[j] == '-' and s[j + 1] == '>': + return "lambda" + elif is_id_start(s[i0]): # x -> ... + j = i0 + 1 + while j < n and is_id_part(s[j]): + j += 1 + k = skip_ws(j) + if k + 1 < n and s[k] == '-' and s[k + 1] == '>': + return "lambda" + + # ---------- regular method header parsing ---------- + open_idx = s.find('(', i0) + if open_idx == -1: + return None + + # Backtrack to method name just before '(' + i = open_idx - 1 + while i >= 0 and s[i].isspace(): + i -= 1 + if i < 0 or not is_id_part(s[i]): + return None + start = i + while start - 1 >= 0 and is_id_part(s[start - 1]): + start -= 1 + name = s[start:i + 1] + + # Find matching ')' + close_idx = match_paren_close(open_idx) + if close_idx == -1: + return None + + raw_params = s[open_idx + 1:close_idx] + + # ---- split ONLY on top-level commas between parameters ---- + parts = [] + depth_p = depth_b = depth_a = 0 # () [] <> + in_str = in_chr = False + token_start = 0 + for idx, ch in enumerate(raw_params): + if in_str: + if ch == '\\': + continue + elif ch == '"': + in_str = False + continue + if in_chr: + if ch == '\\': + continue + elif ch == "'": + in_chr = False + continue + + if ch == '"': + in_str = True + continue + if ch == "'": + in_chr = True + continue + + if ch == '(': + depth_p += 1 + elif ch == ')': + depth_p -= 1 + elif ch == '[': + depth_b += 1 + elif ch == ']': + depth_b -= 1 + elif ch == '<': + depth_a += 1 + elif ch == '>' and depth_a > 0: + depth_a -= 1 + elif ch == ',' and depth_p == 0 and depth_b == 0 and depth_a == 0: + parts.append(raw_params[token_start:idx]) + token_start = idx + 1 + parts.append(raw_params[token_start:]) + + # ---- per-parameter cleanup (preserve intra-type spacing; normalize name-side arrays) ---- + cleaned = [] + for p in parts: + # Trim edges; preserve interior spacing + p = p.strip() + if not p: + continue + + # Remove annotations like @Foo or @Foo(bar) anywhere in the parameter + # (consume following whitespace so we don't leave double spaces). + p = re.sub(r'@\w+(?:\s*\([^()]*\))?\s*', '', p) + + # Remove standalone 'final' tokens (do not collapse other spacing) + p = re.sub(r'(?.*?)(?P\s+)(?P[A-Za-z_$][\w$]*)(?P(?:\s*\[\s*\])*)\s*$', + p, + flags=re.DOTALL, + ) + + if not m: + # Fallback: ensure there's at least one space before the trailing identifier + mname = re.search(r'([A-Za-z_$][\w$]*)\s*$', p) + if mname: + pname = mname.group(1) + tpart = p[:mname.start()] + if len(tpart) == 0 or not tpart[-1].isspace(): + p = f"{tpart} {pname}" + else: + p = f"{tpart}{pname}" + cleaned.append(p) + continue + + tpart = m.group('t') + pname = m.group('name') + name_arr = m.group('name_arr') or '' + + if name_arr.strip(): + # Move name-side brackets to the type side, canonicalized as '[]' per dim. + dims = len(re.findall(r'\[\s*\]', name_arr)) + t_clean = tpart.rstrip() # drop trailing space before appending + t_with_arr = t_clean + '[]' * dims + p = f"{t_with_arr} {pname}" + else: + # Keep original spacing around tpart; ensure at least one space before name + if len(tpart) == 0 or not tpart[-1].isspace(): + p = f"{tpart} {pname}" + else: + p = f"{tpart}{pname}" + + cleaned.append(p) + + params_norm = ', '.join(filter(None, cleaned)) + return f"{name}({params_norm})" + +def find_function(caller_src:str, method_name:str, documents_of_functions:list[Document]) -> Document | None: + # TODO make it smarter, also use the method parameters to check if this is the actual method + for doc in documents_of_functions: + if caller_src == doc.metadata["source"]: + extracted_method_name = extract_method_name_with_params(doc.page_content) + + if extracted_method_name and method_name in extracted_method_name: + return doc + +def get_target_class_names(inheritance_list: List[Tuple[str, str]]) -> frozenset[str]: + return frozenset([class_name for (class_name, source) in inheritance_list]) + +def strip_java_generics(type_str: str) -> str: + """ + Remove all balanced <...> generic sections from a Java type string. + + Examples: + "List" -> "List" + "Map>" -> "Map" + "Map>[]" -> "Map[]" + "Map.Entry" -> "Map.Entry" + "List..." -> "List..." + "java.util.Map " -> "java.util.Map" + "@Nonnull List<@A ? super T>" -> "@Nonnull List" + """ + s = type_str.strip() + out = [] + depth = 0 + for ch in s: + if ch == '<': + depth += 1 + continue + if ch == '>': + if depth > 0: + depth -= 1 + # ignore stray '>' if any + continue + if depth == 0: + out.append(ch) + + # Join and normalize spaces around punctuation commonly found in types + res = ''.join(out) + + # Collapse multiple spaces to one + res = re.sub(r'\s+', ' ', res).strip() + + # Remove spaces around punctuation like dots, brackets, commas, and varargs + res = re.sub(r'\s*([\[\]\(\),.])\s*', r'\1', res) # e.g., "Map []" -> "Map[]", "Map . Entry" -> "Map.Entry" + res = re.sub(r'\s*(\.\.\.)', r'\1', res) # e.g., "List ..." -> "List..." + + return res \ No newline at end of file diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/vuln_analysis/utils/transitive_code_searcher_tool.py index bf65ac3d7..7c5053f08 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/vuln_analysis/utils/transitive_code_searcher_tool.py @@ -12,13 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import os from pathlib import Path from langchain.docstore.document import Document -from .chain_of_calls_retriever import ChainOfCallsRetriever +from .chain_of_calls_retriever_base import ChainOfCallsRetrieverBase from .dep_tree import ( GOLANG_MANIFEST, JAVA_MANIFEST, @@ -44,12 +43,12 @@ class TransitiveCodeSearcher: Parameters ---------- - chain_of_calls_retriever : ChainOfCallsRetriever + chain_of_calls_retriever : ChainOfCallsRetrieverBase Chain Of Calls Retriever object that knows how to search in code """ - def __init__(self, chain_of_calls_retriever: ChainOfCallsRetriever): + def __init__(self, chain_of_calls_retriever: ChainOfCallsRetrieverBase): self.chain_of_calls_retriever = chain_of_calls_retriever @staticmethod @@ -84,14 +83,14 @@ def download_dependencies(git_repo_path: Path) -> bool: C_CPLUSPLUS_MANIFEST_4) ] found = [p for p in candidates if p.is_file()] - if found: + if found: ecosystem = MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] else: logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") return False try: - logger.info(f"Started installing packages for {ecosystem}") + logger.info(f"Started installing packages for {ecosystem}") tree_builder = get_dependency_tree_builder(ecosystem) tree_builder.install_dependencies(git_repo_path) with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: @@ -141,4 +140,4 @@ def search(self, query: str) -> tuple[bool, list[Document]]: f"-------------------------------------------\n{function_method.page_content}\n" logger.debug(content_of_files_in_path) logger.debug(content_of_files_in_path, extra=MULTI_LINE_MESSAGE_TRUE) - return found_path, call_hierarchy_list + return found_path, call_hierarchy_list \ No newline at end of file From f85e7114c72ccbcbd8b26f5e4b302ea9171735c4 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 1 Dec 2025 18:01:27 +0200 Subject: [PATCH 106/286] chore: add /health endpoint to expose agent status Signed-off-by: Vladimir Belousov --- .../configs/config-http-openai.yml | 9 ++++ .../functions/health_endpoint.py | 50 +++++++++++++++++++ src/vuln_analysis/register.py | 1 + 3 files changed, 60 insertions(+) create mode 100644 src/vuln_analysis/functions/health_endpoint.py diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index a4983cf08..fa3cdd7e4 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -14,6 +14,13 @@ # limitations under the License. general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check use_uvloop: true telemetry: tracing: @@ -127,6 +134,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + health_check: + _type: health_check llms: checklist_llm: diff --git a/src/vuln_analysis/functions/health_endpoint.py b/src/vuln_analysis/functions/health_endpoint.py new file mode 100644 index 000000000..6697354bd --- /dev/null +++ b/src/vuln_analysis/functions/health_endpoint.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel + +from aiq.builder.builder import Builder +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig + + +class HealthCheckTool(FunctionBaseConfig, name="health_check"): + pass + + +class HealthStatusResponse(BaseModel): + status: str + + +@register_function(config_type=HealthCheckTool) +async def health_check(config: HealthCheckTool, builder: Builder): + """ + Registers a health-check function used to verify that the agent runtime + is operational and capable of handling requests. + """ + + async def _health(request: None = None) -> HealthStatusResponse: + """ + Returns operational status data for the agent runtime. + + Returns: + HealthStatusResponse: Structured health-status payload. + """ + return HealthStatusResponse(status="ok") + + yield FunctionInfo.from_fn( + _health, description="Health-status endpoint returning operational state." + ) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 179e70db6..520cdb9ab 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -40,6 +40,7 @@ from vuln_analysis.functions import cve_process_sbom from vuln_analysis.functions import cve_summarize from vuln_analysis.functions import cve_generate_cvss +from vuln_analysis.functions import health_endpoint from vuln_analysis.tools import lexical_full_search # This is actually registers the tool in the type registry of NAT! from vuln_analysis.tools import transitive_code_search From 3077965322931352fa83fb426b00f5c1e3c3b1cc Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 2 Dec 2025 08:45:29 +0200 Subject: [PATCH 107/286] chore: add health_check function to tracing and base config Signed-off-by: Vladimir Belousov --- kustomize/base/exploit-iq-config.yml | 11 +++++++++-- src/vuln_analysis/configs/config-tracing.yml | 9 +++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 09b1427d4..f39fe866b 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -14,8 +14,14 @@ # limitations under the License. general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check use_uvloop: true - telemetry: tracing: phoenix: @@ -23,7 +29,6 @@ general: endpoint: http://localhost:6006/v1/traces project: cve_agent - functions: cve_generate_vdbs: _type: cve_generate_vdbs @@ -136,6 +141,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + health_check: + _type: health_check llms: checklist_llm: diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index bfbe78ddc..ff11867da 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -14,6 +14,13 @@ # limitations under the License. general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check use_uvloop: true telemetry: logging: @@ -131,6 +138,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + health_check: + _type: health_check llms: checklist_llm: From 622e25a140ae27cce6112bfcecfec491de975031 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache Date: Tue, 2 Dec 2025 20:10:11 +0200 Subject: [PATCH 108/286] Java transitive search performance improvements Signed-off-by: Theodor Mihalache --- .../java_functions_parsers.py | 48 +++- .../utils/java_chain_of_calls_retriever.py | 45 ++-- src/vuln_analysis/utils/java_utils.py | 212 +++++++++++------- 3 files changed, 209 insertions(+), 96 deletions(-) diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py index 294fc0061..c0cb77062 100644 --- a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py @@ -7,7 +7,7 @@ from langchain_core.documents import Document from .lang_functions_parsers import LanguageFunctionsParser -from ..java_utils import extract_jar_name, PRIMITIVE_TYPES, collect_fields_from_types, get_type_name, \ +from ..java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, get_type_name, \ is_java_type, is_java_method, extract_method_name_with_params, find_function, get_target_class_names, \ strip_java_generics, JAVA_ANNOTATION_SYMBOL from ...logging.loggers_factory import LoggingFactory @@ -2422,7 +2422,7 @@ def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, value_list, target_class_names: frozenset[str]) -> bool: result = False - if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + if not struct_initializer_expression and resolved_type not in JAVA_METHOD_PRIM_TYPES: docs = self.__get_type_docs_matched_with_callee_type(callee_package, resolved_type, type_documents, target_class_names) if len(docs) > 0: @@ -2439,11 +2439,47 @@ def __lookup_package(self, callee_package, resolved_type, struct_initializer_exp def __get_type_docs_matched_with_callee_type(self, callee_package, checked_type, type_documents, target_class_names: frozenset[str]) -> list[ Document]:# TODO Make sure Fix works - the target classes can be in other jars - if checked_type and strip_java_generics(checked_type) not in target_class_names: - return [] + """ + Return all type_documents whose jar "package" matches callee_package and whose + (generic-stripped) type name is in target_class_names. + + Performance notes: + - Short-circuits early if checked_type cannot possibly match. + - Normalizes callee_package once. + - Reuses cached extract_jar_name, get_type_name, and strip_java_generics. + """ + # Early exit if the checked_type is present and does not match any target class + if checked_type: + stripped_checked = strip_java_generics(checked_type) + if stripped_checked not in target_class_names: + return [] + + # Normalize the "package" part of callee_package once (after first ':') + _, _, callee_pkg_tail = callee_package.partition(":") + callee_pkg_tail_lower = callee_pkg_tail.lower() + + result: List["Document"] = [] + + for a_type in type_documents: + src = a_type.metadata['source'] - return [a_type for a_type in type_documents if self.is_same_package(extract_jar_name(a_type.metadata['source']), callee_package.partition(":")[2]) - and strip_java_generics(get_type_name(a_type)) in target_class_names] + # extract_jar_name is cached and fairly cheap after first call + jar_name = extract_jar_name(src) + + # Same logic as is_same_package(extract_jar_name(...), callee_package.partition(":")[2]) + if jar_name.lower() != callee_pkg_tail_lower: + continue + + # get_type_name is cached per source text + type_name = get_type_name(a_type.page_content) + if not type_name: + continue + + # Preserve original logic: strip generics before membership check + if strip_java_generics(type_name) in target_class_names: + result.append(a_type) + + return result def _find_method_return_type_in_type_docs( self, diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py index 47ffefec8..c4642f8a4 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -82,14 +82,14 @@ def __init__(self, documents: List[Document], allowed_files_extensions = self.language_parser.supported_files_extensions() # filter out unsupported files extensions. - self.documents = [doc for doc in documents + documents = [doc for doc in documents if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) .endswith(ext)])] documents_of_functions, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_functions") # filter out types and full code documents, retaining only functions/methods documents in this attribute. if not exist: - self.documents_of_functions = [doc for doc in tqdm(self.documents, total=len(self.documents), desc="Filtering documents, leaving functions/methods ...") + self.documents_of_functions = [doc for doc in tqdm(documents, total=len(documents), desc="Filtering documents, leaving functions/methods ...") if self.language_parser.is_function(doc)] save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_functions, "documents_of_functions") else: @@ -98,7 +98,7 @@ def __init__(self, documents: List[Document], documents_of_types, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_types") # filter out full code documents and functions/methods docs, retaining only types/classes docs if not exist: - self.documents_of_types = [doc for doc in tqdm(self.documents, total=len(self.documents) , desc="Filtering documents, leaving types/classes ...") + self.documents_of_types = [doc for doc in tqdm(documents, total=len(documents) , desc="Filtering documents, leaving types/classes ...") if self.language_parser.is_doc_type(doc)] save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_types, "documents_of_types") else: @@ -107,10 +107,14 @@ def __init__(self, documents: List[Document], # boolean attribute that indicates whether a path was found or not, initially set to False. self.found_path = False # Filter out all documents but full source docs. - self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in self.documents + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in documents if doc.metadata.get('content_type') == 'simplified_code'} - if not self.language_parser.is_script_language(): - self.documents = self.documents_of_functions + + # Create function list with the function existing in the dependencies + self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' + and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or + any(parent for parent in self.supported_packages if extract_jar_name(doc.metadata['source']) in parent))] + logger.debug("Chain of Calls Retriever - after documents_of_full_sources") self.last_visited_parent_package_indexes = dict() @@ -242,17 +246,26 @@ def __find_caller_function(self, document_function: Document, function_package: def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: """ Checks if a document is in the exclusions list based on its - function name, function body and source metadata. + function body and source metadata. """ + if not exclusions: + return False + + # Normalize once for the doc we are checking doc_function_content = doc.page_content.strip() doc_source = doc.metadata.get('source').strip() for exclusion_doc in exclusions: - exclusion_function_content = exclusion_doc.page_content.strip() + # First compare sources – cheap and often different exclusion_source = exclusion_doc.metadata.get('source').strip() + if exclusion_source != doc_source: + continue - if doc_function_content == exclusion_function_content and doc_source == exclusion_source: + # Only if source matches, compare (more expensive) function content + exclusion_function_content = exclusion_doc.page_content.strip() + if exclusion_function_content == doc_function_content: return True + return False def _is_method_excluded(self, function_name_to_search: str, target_class_names: frozenset[str], doc: Document, method_exclusions: dict) -> bool: @@ -276,7 +289,7 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] else: filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and - (self.language_parser.is_function(doc) or self.language_parser.is_script_language()) and + self.language_parser.is_function(doc) and not self._is_doc_excluded(doc, exclusions) and not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] @@ -349,6 +362,12 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: break self.tree_dict[package_name] = [parents, [], {}] + # Create function list with the function existing in the dependencies + self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' + and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or + any(parent for parent in self.tree_dict[package_name][PARENTS_INDEX] if extract_jar_name(doc.metadata['source']) in parent))] + + current_package_name = package_name # main loop. while not end_loop: @@ -471,9 +490,9 @@ def __find_initial_function(self, class_name: str, method_name: str, package_nam # TODO check if its correct to handle none 3rd party code here as well relevant_docs = [doc for doc in documents - if (doc.metadata.get('source').__contains__(self.language_parser.dir_name_for_3rd_party_packages()) - and doc.metadata.get('source').__contains__(jar_name) or - not doc.metadata.get('source').__contains__(self.language_parser.dir_name_for_3rd_party_packages())) and + if (self.language_parser.dir_name_for_3rd_party_packages() in doc.metadata.get('source') + and jar_name in doc.metadata.get('source') or + not self.language_parser.dir_name_for_3rd_party_packages() in doc.metadata.get('source')) and self.language_parser.get_class_name_from_class_function(doc) == class_name.rpartition('.')[2] and self.language_parser.get_function_name(doc) == method_name] package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] diff --git a/src/vuln_analysis/utils/java_utils.py b/src/vuln_analysis/utils/java_utils.py index 9b687ae6d..a79b1e8d9 100644 --- a/src/vuln_analysis/utils/java_utils.py +++ b/src/vuln_analysis/utils/java_utils.py @@ -1,13 +1,42 @@ import hashlib import re from dataclasses import dataclass +from functools import lru_cache from typing import Dict, List, Tuple, Optional, Any from langchain_core.documents import Document from tqdm import tqdm JAVA_ANNOTATION_SYMBOL = '@' -PRIMITIVE_TYPES = {"byte","short","int","long","float","double","boolean","char","void"} dummy_package_name = "dummy:dummy:1.0.0" +JAVA_METHOD_CLASS_KINDS = ("class", "interface", "enum", "record") +JAVA_METHOD_CLASS_MODS = { + "public", "protected", "private", "abstract", "final", "static", "sealed", "strictfp" +} +JAVA_METHOD_CTRL_WORDS = { + "if", "for", "while", "switch", "catch", "synchronized", "return", "new", "case" +} +JAVA_METHOD_METH_MODS = { + "public", "protected", "private", "abstract", "static", "final", "synchronized", + "native", "strictfp", "default", "transient" +} +JAVA_METHOD_PRIM_TYPES = { + "void", "int", "long", "double", "float", "short", "byte", "boolean", "char" +} +JAVA_METHOD_WS_CHARS = set(" \t\r\n\f\v") + +_ANNOTATION_RE = re.compile(r'@\w+(?:\s*\([^()]*\))?\s*') +_FINAL_RE = re.compile(r'(?.*?)(?P\s+)(?P[A-Za-z_$][\w$]*)(?P(?:\s*\[\s*\])*)\s*$', + re.DOTALL, +) +_ARRAY_BRACKETS_RE = re.compile(r'\[\s*\]') +_TRAILING_NAME_RE = re.compile(r'([A-Za-z_$][\w$]*)\s*$') + +# Precompiled regexes used by strip_java_generics +_GENERIC_SPACE_RE = re.compile(r'\s+') +_PUNCT_SPACE_RE = re.compile(r'\s*([\[\]\(\),.])\s*') +_VARARGS_SPACE_RE = re.compile(r'\s*(\.\.\.)') # Accepts: # - groupId: dot-separated segments; each segment alnum, may contain '-' or '_' inside @@ -70,6 +99,7 @@ def _convert_to_maven_artifact(package_name: str) -> str: return package_name +@lru_cache(maxsize=4096) def extract_jar_name(source: str) -> str: parts = source.split('/') @@ -277,7 +307,7 @@ def keep_type(typ: str) -> bool: base = base_type_of(typ) if not base: return False simple = base.split(".")[-1] - if simple.lower() in PRIMITIVE_TYPES: return False + if simple.lower() in JAVA_METHOD_PRIM_TYPES: return False if simple in WRAPS: return False if base.startswith("java.lang.") and simple in WRAPS: return False return True @@ -689,7 +719,8 @@ def collect_type(kind: str, name: str, lb: int, rb: int): return out -def get_type_name(the_type: Document) -> str: +@lru_cache(maxsize=8192) +def get_type_name(src: str) -> str: """ Return the simple name of the first class/interface/enum/record declared anywhere in the given Java source (top-level or nested). Makes no assumptions @@ -702,7 +733,6 @@ def get_type_name(the_type: Document) -> str: because the token `interface` is still present. - Returns the *simple* identifier (e.g., 'Point'). """ - src = the_type.page_content n = len(src) i = 0 @@ -717,7 +747,11 @@ def is_ws(ch: str) -> bool: def is_id_start(ch: str) -> bool: # Java allows '_' and '$'; also allow unicode letters - return ch == "_" or ch == "$" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) + return ( + ch == "_" or ch == "$" or + ("A" <= ch <= "Z") or ("a" <= ch <= "z") or + (ch >= "\u0080" and ch.isalpha()) + ) def is_id_part(ch: str) -> bool: return is_id_start(ch) or ("0" <= ch <= "9") @@ -728,7 +762,7 @@ def word_at(pos: int, word: str) -> bool: if end > n or src[pos:end] != word: return False # left boundary: start of file or not an identifier part - if pos > 0 and (src[pos-1].isalnum() or src[pos-1] in "_$"): + if pos > 0 and (src[pos - 1].isalnum() or src[pos - 1] in "_$"): return False # right boundary: end of file or not an identifier part if end < n and (src[end].isalnum() or src[end] in "_$"): @@ -772,7 +806,6 @@ def word_at(pos: int, word: str) -> bool: k += 1 return src[j:k] # If not a valid identifier here, keep scanning - # (defensive: malformed code or rare construct) i += 1 elif state == LINE: @@ -805,12 +838,8 @@ def word_at(pos: int, word: str) -> bool: return "" +@lru_cache(maxsize=16384) def is_java_method(source: str) -> bool: - """ - Return True iff the given Java source snippet is a *method* declaration, - or (fallback) contains a lambda ('->') outside of comments/strings. - Constructors are *not* considered methods here (no return type). - """ s = source n = len(s) i = 0 @@ -2213,10 +2242,10 @@ def extract_method_name_with_params(java_src: str) -> str | None: - The string begins at the actual method header OR a top-level lambda. - No comments/javadocs/annotations are present at the very beginning (before a lambda or header). """ - import re - s = java_src.lstrip() n = len(s) + if n == 0: + return None # ---------- helpers ---------- def is_id_start(ch: str) -> bool: @@ -2226,15 +2255,20 @@ def is_id_part(ch: str) -> bool: return ch.isalnum() or ch in "_$" def skip_ws(i: int) -> int: - while i < n and s[i].isspace(): + _s = s + _n = n + while i < _n and _s[i].isspace(): i += 1 return i def match_paren_close(start: int) -> int: """Given s[start] == '(', return index of its matching ')', else -1.""" - depth, i2 = 0, start - while i2 < n: - ch = s[i2] + depth = 0 + i2 = start + _s = s + _n = n + while i2 < _n: + ch = _s[i2] if ch == '(': depth += 1 elif ch == ')': @@ -2247,13 +2281,14 @@ def match_paren_close(start: int) -> int: # ---------- detect lambda ONLY if it is at the beginning ---------- i0 = skip_ws(0) if i0 < n: - if s[i0] == '(': # (params) -> ... + ch0 = s[i0] + if ch0 == '(': # (params) -> ... end = match_paren_close(i0) if end != -1: j = skip_ws(end + 1) if j + 1 < n and s[j] == '-' and s[j + 1] == '>': return "lambda" - elif is_id_start(s[i0]): # x -> ... + elif is_id_start(ch0): # x -> ... j = i0 + 1 while j < n and is_id_part(s[j]): j += 1 @@ -2268,67 +2303,84 @@ def match_paren_close(start: int) -> int: # Backtrack to method name just before '(' i = open_idx - 1 - while i >= 0 and s[i].isspace(): + _s = s # local alias + while i >= 0 and _s[i].isspace(): i -= 1 - if i < 0 or not is_id_part(s[i]): + if i < 0 or not is_id_part(_s[i]): return None start = i - while start - 1 >= 0 and is_id_part(s[start - 1]): + while start - 1 >= 0 and is_id_part(_s[start - 1]): start -= 1 - name = s[start:i + 1] + name = _s[start:i + 1] # Find matching ')' close_idx = match_paren_close(open_idx) if close_idx == -1: return None - raw_params = s[open_idx + 1:close_idx] + raw_params = _s[open_idx + 1:close_idx] # ---- split ONLY on top-level commas between parameters ---- - parts = [] - depth_p = depth_b = depth_a = 0 # () [] <> - in_str = in_chr = False - token_start = 0 - for idx, ch in enumerate(raw_params): - if in_str: - if ch == '\\': + if not raw_params: + return f"{name}()" + + # Fast path: no commas at all -> single parameter, so no need for deep scan + if ',' not in raw_params: + parts = [raw_params] + else: + parts: list[str] = [] + depth_p = depth_b = depth_a = 0 # () [] <> + in_str = in_chr = False + token_start = 0 + append_part = parts.append + + for idx, ch in enumerate(raw_params): + if in_str: + if ch == '\\': + continue + elif ch == '"': + in_str = False continue - elif ch == '"': - in_str = False - continue - if in_chr: - if ch == '\\': + if in_chr: + if ch == '\\': + continue + elif ch == "'": + in_chr = False continue - elif ch == "'": - in_chr = False - continue - if ch == '"': - in_str = True - continue - if ch == "'": - in_chr = True - continue + if ch == '"': + in_str = True + continue + if ch == "'": + in_chr = True + continue - if ch == '(': - depth_p += 1 - elif ch == ')': - depth_p -= 1 - elif ch == '[': - depth_b += 1 - elif ch == ']': - depth_b -= 1 - elif ch == '<': - depth_a += 1 - elif ch == '>' and depth_a > 0: - depth_a -= 1 - elif ch == ',' and depth_p == 0 and depth_b == 0 and depth_a == 0: - parts.append(raw_params[token_start:idx]) - token_start = idx + 1 - parts.append(raw_params[token_start:]) + if ch == '(': + depth_p += 1 + elif ch == ')': + depth_p -= 1 + elif ch == '[': + depth_b += 1 + elif ch == ']': + depth_b -= 1 + elif ch == '<': + depth_a += 1 + elif ch == '>' and depth_a > 0: + depth_a -= 1 + elif ch == ',' and depth_p == 0 and depth_b == 0 and depth_a == 0: + append_part(raw_params[token_start:idx]) + token_start = idx + 1 + parts.append(raw_params[token_start:]) # ---- per-parameter cleanup (preserve intra-type spacing; normalize name-side arrays) ---- - cleaned = [] + cleaned: list[str] = [] + cleaned_append = cleaned.append + ann_sub = _ANNOTATION_RE.sub + final_sub = _FINAL_RE.sub + param_main_match = _PARAM_MAIN_RE.match + trailing_name_search = _TRAILING_NAME_RE.search + array_brackets_findall = _ARRAY_BRACKETS_RE.findall + for p in parts: # Trim edges; preserve interior spacing p = p.strip() @@ -2337,24 +2389,20 @@ def match_paren_close(start: int) -> int: # Remove annotations like @Foo or @Foo(bar) anywhere in the parameter # (consume following whitespace so we don't leave double spaces). - p = re.sub(r'@\w+(?:\s*\([^()]*\))?\s*', '', p) + p = ann_sub('', p) # Remove standalone 'final' tokens (do not collapse other spacing) - p = re.sub(r'(?.*?)(?P\s+)(?P[A-Za-z_$][\w$]*)(?P(?:\s*\[\s*\])*)\s*$', - p, - flags=re.DOTALL, - ) + m = param_main_match(p) if not m: # Fallback: ensure there's at least one space before the trailing identifier - mname = re.search(r'([A-Za-z_$][\w$]*)\s*$', p) + mname = trailing_name_search(p) if mname: pname = mname.group(1) tpart = p[:mname.start()] @@ -2362,7 +2410,7 @@ def match_paren_close(start: int) -> int: p = f"{tpart} {pname}" else: p = f"{tpart}{pname}" - cleaned.append(p) + cleaned_append(p) continue tpart = m.group('t') @@ -2371,7 +2419,7 @@ def match_paren_close(start: int) -> int: if name_arr.strip(): # Move name-side brackets to the type side, canonicalized as '[]' per dim. - dims = len(re.findall(r'\[\s*\]', name_arr)) + dims = len(array_brackets_findall(name_arr)) t_clean = tpart.rstrip() # drop trailing space before appending t_with_arr = t_clean + '[]' * dims p = f"{t_with_arr} {pname}" @@ -2382,7 +2430,7 @@ def match_paren_close(start: int) -> int: else: p = f"{tpart}{pname}" - cleaned.append(p) + cleaned_append(p) params_norm = ', '.join(filter(None, cleaned)) return f"{name}({params_norm})" @@ -2399,6 +2447,7 @@ def find_function(caller_src:str, method_name:str, documents_of_functions:list[D def get_target_class_names(inheritance_list: List[Tuple[str, str]]) -> frozenset[str]: return frozenset([class_name for (class_name, source) in inheritance_list]) +@lru_cache(maxsize=4096) def strip_java_generics(type_str: str) -> str: """ Remove all balanced <...> generic sections from a Java type string. @@ -2413,6 +2462,15 @@ def strip_java_generics(type_str: str) -> str: "@Nonnull List<@A ? super T>" -> "@Nonnull List" """ s = type_str.strip() + + # Fast path: no generic markers at all, just normalize whitespace/punctuation + if '<' not in s and '>' not in s: + res = _GENERIC_SPACE_RE.sub(' ', s).strip() + res = _PUNCT_SPACE_RE.sub(r'\1', res) + res = _VARARGS_SPACE_RE.sub(r'\1', res) + return res + + # Slow path: remove balanced <...> sections out = [] depth = 0 for ch in s: @@ -2431,10 +2489,10 @@ def strip_java_generics(type_str: str) -> str: res = ''.join(out) # Collapse multiple spaces to one - res = re.sub(r'\s+', ' ', res).strip() + res = _GENERIC_SPACE_RE.sub(' ', res).strip() # Remove spaces around punctuation like dots, brackets, commas, and varargs - res = re.sub(r'\s*([\[\]\(\),.])\s*', r'\1', res) # e.g., "Map []" -> "Map[]", "Map . Entry" -> "Map.Entry" - res = re.sub(r'\s*(\.\.\.)', r'\1', res) # e.g., "List ..." -> "List..." + res = _PUNCT_SPACE_RE.sub(r'\1', res) # "Map []" -> "Map[]", "Map . Entry" -> "Map.Entry" + res = _VARARGS_SPACE_RE.sub(r'\1', res) # "List ..." -> "List..." return res \ No newline at end of file From b3e452489aa104ce139f5433193cebd3633a8d3a Mon Sep 17 00:00:00 2001 From: Theodor Mihalache Date: Wed, 3 Dec 2025 14:23:14 +0200 Subject: [PATCH 109/286] Fixed bug when filtering function documents according to dependencies Signed-off-by: Theodor Mihalache --- src/vuln_analysis/utils/java_chain_of_calls_retriever.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py index c4642f8a4..61857ee7a 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -113,7 +113,8 @@ def __init__(self, documents: List[Document], # Create function list with the function existing in the dependencies self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or - any(parent for parent in self.supported_packages if extract_jar_name(doc.metadata['source']) in parent))] + any(parent for parent in self.supported_packages + if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] logger.debug("Chain of Calls Retriever - after documents_of_full_sources") self.last_visited_parent_package_indexes = dict() @@ -365,7 +366,8 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # Create function list with the function existing in the dependencies self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or - any(parent for parent in self.tree_dict[package_name][PARENTS_INDEX] if extract_jar_name(doc.metadata['source']) in parent))] + any(parent for parent in self.tree_dict[package_name][PARENTS_INDEX] + if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] current_package_name = package_name From b1dd478d1da5b8a1e615b9e2bb7a761f0fc9d32c Mon Sep 17 00:00:00 2001 From: Theodor Mihalache Date: Wed, 3 Dec 2025 15:00:11 +0200 Subject: [PATCH 110/286] Fixed bug in standard lib parent construction handling Signed-off-by: Theodor Mihalache --- src/vuln_analysis/utils/java_chain_of_calls_retriever.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py index 61857ee7a..8d06f966f 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -349,12 +349,11 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() # find all parents ( all importing packages) of the input package so we'll have candidate pkgs to search in. parents = list({ - next( - (key for key in self.tree_dict.keys() if extract_jar_name(doc.metadata['source']) in key), - None # Default value if no key is found (optional) - ) + key for doc in importing_docs if doc.metadata['source'].startswith(prefix_of_3rd_parties_libs) + for key in self.tree_dict.keys() + if extract_jar_name(doc.metadata['source']) in key }) for doc in importing_docs: From 8345bdbd91f5beea5fae1dbdfc61ed6697aaea49 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 8 Dec 2025 11:30:52 +0200 Subject: [PATCH 111/286] Use Tekton for automation Testing on exploit-iq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enhancement enables the automated running of datasets within a pipeline environment to perform the following tasks: • Calculate confusion matrix metrics for vulnerability data --- .gitmodules | 3 + .tekton/on-cm-runner.yaml | 396 +++++++++++++++++++++++++++++++++++ .tests-automation | 1 + ci/scripts/config_fetcher.py | 79 +++++++ 4 files changed, 479 insertions(+) create mode 100644 .gitmodules create mode 100644 .tekton/on-cm-runner.yaml create mode 160000 .tests-automation create mode 100644 ci/scripts/config_fetcher.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..1717d5fe8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule ".tests-automation"] + path = .tests-automation + url = https://github.com/RHEcosystemAppEng/exploitiq-tests-automation.git diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml new file mode 100644 index 000000000..29f796eda --- /dev/null +++ b/.tekton/on-cm-runner.yaml @@ -0,0 +1,396 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + # [CHANGE 1] Updated Name + name: confusion-matrix-batch-runner + annotations: + # Trigger on comment "/test-heavy" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "issue_comment" && + body.action == "created" && + body.issue.pull_request && + body.comment.body.matches("^/test-heavy") + # We only need git-clone now, we removed buildah! + pipelinesascode.tekton.dev/task: "git-clone" + pipelinesascode.tekton.dev/max-keep-runs: "3" +spec: + timeouts: + pipeline: 10h30m0s # Timeout for the entire PipelineRun + params: + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + # [NEW] Capture PR Number for the tag name + - name: pr_number + value: "{{ pull_request_number }}" + # Capture the full comment text + - name: trigger_comment + value: "{{ trigger_comment }}" + # Point to the image ALREADY built by the PR pipeline + - name: target-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:on-pr-{{revision}} + + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: pr_number + - name: target-image + - name: trigger_comment + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + tasks: + # ------------------------------------------------ + # 1. CLONE + # ------------------------------------------------ + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + # Ensure submodules (like .tests-automation) are fetched + - name: submodules + value: "true" + + # ------------------------------------------------ + # 2. RUN TEST & TAGGING & RELEASING + # ------------------------------------------------ + - name: integration-test + timeout: 10h0m0s # Timeout for the task + runAfter: + - fetch-repository + workspaces: + - name: source + workspace: source + - name: basic-auth + workspace: basic-auth # Needed for pushing tags/releases + - name: exploit-iq-data + workspace: exploit-iq-data + params: + - name: CURRENT_REVISION + value: $(params.revision) + - name: PR_NUMBER + value: $(params.pr_number) + - name: TRIGGER_COMMENT + value: $(params.trigger_comment) + + taskSpec: + params: + - name: CURRENT_REVISION + type: string + - name: PR_NUMBER + type: string + - name: TRIGGER_COMMENT + type: string + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + volumes: + - name: google-creds-volume + secret: + secretName: google-sheets-secrets + + # >>> THE SERVER (Sidecar) <<< + sidecars: + - name: server-application + # [CHANGE 3] Use the image passed in params (the one built by the other pipeline) + image: $(params.target-image) + imagePullPolicy: Always + + volumeMounts: + - name: google-creds-volume + mountPath: /etc/secrets/google + readOnly: true + # Inject Env Vars (API Keys, Models, RedHat Creds) + - name: $(workspaces.exploit-iq-data.volume) + mountPath: /exploit-iq-data + envFrom: + - secretRef: + name: integration-tests + - configMapRef: + name: server-model-config + - secretRef: + name: redhat-app-creds + + env: + - name: PYTHONUNBUFFERED + value: "1" + - name: GOOGLE_SERVICE_ACCOUNT_FILE + value: "/etc/secrets/google/credentials.json" + - name: GSHEETS_INPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: input_sheet_id + # Pass the raw comment text into the container + - name: TRIGGER_COMMENT + value: "$(params.TRIGGER_COMMENT)" + - name: GOMODCACHE + value: "/exploit-iq-data/go/pkg/mod" + script: | + #!/bin/sh + set -e + # We use python to safely remove the 'telemetry' block + echo "--- Generating Config without Tracing ---" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + FETCHER_SCRIPT="ci/scripts/config_fetcher.py" + + # [CHANGE] Logic: Check if the comment contains "config=sheet" + # We use grep because the comment might be "/test-heavy config=sheet" + if echo "$TRIGGER_COMMENT" | grep -q "config=sheet"; then + echo ">> Trigger detected 'config=sheet'. using Dynamic Config." + pip install --user google-auth + if [ -f "$FETCHER_SCRIPT" ]; then + eval $(python "$FETCHER_SCRIPT") + else + echo "Error: Config fetcher script missing." + exit 1 + fi + else + echo ">> No custom config requested. Using Defaults." + fi + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="/exploit-iq-data" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + + # 2. [CRITICAL] Remove existing folder/file if the image created it + # This ensures 'ln' creates the link named 'am_cache', instead of putting it INSIDE 'am_cache' + if [ -e "${CACHE_DIR_TARGET}" ]; then + echo "Removing existing item at ${CACHE_DIR_TARGET} to replace with PVC link..." + rm -rf "${CACHE_DIR_TARGET}" + fi + + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" + + echo "--- Starting 'nat' Server ---" + exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 + readinessProbe: + tcpSocket: + port: 26466 + initialDelaySeconds: 10 + periodSeconds: 5 + + # >>> THE CLIENT (Test Runner) <<< + steps: + # ------------------------------------------------------- + # STEP A: Run the Python Automation Tests + # ------------------------------------------------------- + - name: run-test-suite + image: quay.io/ecosystem-appeng/auto-cm-testing:latest + imagePullPolicy: Always + workingDir: $(workspaces.source.path) + + volumeMounts: + - name: google-creds-volume + mountPath: /etc/secrets/google + readOnly: true + + env: + - name: GOOGLE_SERVICE_ACCOUNT_FILE + value: "/etc/secrets/google/credentials.json" + - name: GSHEETS_MODE + value: "both" + - name: GSHEETS_TAG + value: "Ver1" + - name: GSHEETS_INPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: input_sheet_id + - name: GSHEETS_OUTPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: output_sheet_id + + script: | + #!/bin/bash + set -e + + # 1. Construct the Tag Name + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- STARTING AUTOMATION TESTS ---" + echo "Generated Tag: $TAG_NAME" + echo "--- DEBUG: Current Directory ---" + pwd + + # 2. Run Python script with the tag + cd /app/ + python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" + + # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them + echo "--- Copying Reports to Shared Workspace ---" + # Ensure destination exists + mkdir -p $(workspaces.source.path)/src/reports + # Copy the archive folder from the container image (/app) to the PVC (/workspace) + cp -r /app/src/reports/archive $(workspaces.source.path)/src/reports + echo "--- TESTS FINISHED SUCCESSFULLY ---" + + # ------------------------------------------------------- + # STEP B: Tag the MAIN Repository + # ------------------------------------------------------- + - name: git-tag-main-repo + image: alpine/git + workingDir: $(workspaces.source.path) + script: | + #!/bin/sh + set -e + + # [FIX] Trust the workspace directory to avoid "dubious ownership" error + git config --global --add safe.directory $(workspaces.source.path) + + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- Configuring Git for Main Repo ---" + git config --global user.email "tekton-bot@redhat.com" + git config --global user.name "Tekton Pipeline" + + # ------------------------------------------------------- + # ROBUST AUTH LOGIC + # Handle both ".git-credentials" file AND "username/password" keys + # ------------------------------------------------------- + + if [ -f "/workspace/basic-auth/.git-credentials" ]; then + # SCENARIO A: Secret is a pre-made credential file (Local Fix) + echo "Found .git-credentials file. Using it." + git config --global credential.helper "store --file=/workspace/basic-auth/.git-credentials" + + elif [ -f "/workspace/basic-auth/password" ]; then + # SCENARIO B: Secret is broken into keys (Standard Production) + echo "Found username/password files. Constructing auth." + + # Default to 'git' if username is missing + GIT_USER="git" + if [ -f "/workspace/basic-auth/username" ]; then + GIT_USER=$(cat /workspace/basic-auth/username) + fi + GIT_PASS=$(cat /workspace/basic-auth/password) + + # Construct the URL manually with credentials + ORIGIN_URL=$(git remote get-url origin) + # Strip existing protocol (https://) + CLEAN_URL=${ORIGIN_URL#*//} + + # Set the remote to include the token + git remote set-url origin "https://${GIT_USER}:${GIT_PASS}@${CLEAN_URL}" + else + echo "WARNING: No auth found in /workspace/basic-auth. Push might fail." + fi + + echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" + git tag -f -a "$TAG_NAME" -m "Tests Passed for PR #$(params.PR_NUMBER)" + git push origin "$TAG_NAME" + + # ------------------------------------------------------- + # STEP C: Release & Upload Artifacts to AUTOMATION Repo + # ------------------------------------------------------- + - name: release-to-automation-repo + image: alpine/git + workingDir: $(workspaces.source.path) + script: | + #!/bin/sh + set -e + apk add --no-cache github-cli + + # Define the target repo explicitly + TARGET_REPO="RHEcosystemAppEng/exploitiq-tests-automation" + + + # --- Auth Setup --- + if [ -f "/workspace/basic-auth/.git-credentials" ]; then + # Extract token from URL format https://user:token@host + export GH_TOKEN=$(cat /workspace/basic-auth/.git-credentials | sed -n 's/.*:\(.*\)@.*/\1/p') + elif [ -f "/workspace/basic-auth/password" ]; then + export GH_TOKEN=$(cat /workspace/basic-auth/password) + fi + + # 2. Identify Tag and File + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + #[FIX] Point to the Shared Workspace path where Step A copied the files + # We look in src/reports relative to the workspace root + REPORT_PATH="src/reports/archive" + + echo "--- Searching for reports in: $(pwd)/$REPORT_PATH ---" + REPORT_FILE=$(ls -t "$REPORT_PATH"/*.tar 2>/dev/null | head -n 1) + + if [ -z "$REPORT_FILE" ]; then + echo "❌ Error: No report file found in $REPORT_PATH" + echo "Listing directory contents for debugging:" + ls -R src/ || echo "src directory not found" + exit 1 + fi + + echo "--- Found Report: $REPORT_FILE ---" + echo "--- Creating Release '$TAG_NAME' on Automation Repo ---" + + echo "--- Creating release in $TARGET_REPO ---" + gh release create "$TAG_NAME" \ + "$REPORT_FILE" \ + --repo "$TARGET_REPO" \ + --title "Report: $TAG_NAME" \ + --notes "Automated Test Report for Main Repo PR #$(params.PR_NUMBER)" \ + --target "main" + + # ------------------------------------------------------- + # STEP D: Cleanup Old Releases (Keep last 5) + # ------------------------------------------------------- + echo "--- Running Cleanup (Keeping last 5 releases) ---" + + KEEP_COUNT=5 + + gh release list --limit 50 | \ + tail -n +$((KEEP_COUNT + 1)) | \ + cut -f3 | \ + while read -r OLD_TAG; do + if [ -n "$OLD_TAG" ]; then + echo "Deleting old release: $OLD_TAG" + gh release delete "$OLD_TAG" --cleanup-tag --yes + fi + done + + echo "--- Release Process Complete ---" + + # ------------------------------------------------ + # WORKSPACE BINDINGS + # ------------------------------------------------ + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: exploit-iq-data + persistentVolumeClaim: + claimName: exploit-iq-cach-pvc diff --git a/.tests-automation b/.tests-automation new file mode 160000 index 000000000..ae5e9806c --- /dev/null +++ b/.tests-automation @@ -0,0 +1 @@ +Subproject commit ae5e9806cd15a20a170550fe4c1b69d5d425435f diff --git a/ci/scripts/config_fetcher.py b/ci/scripts/config_fetcher.py new file mode 100644 index 000000000..b4fe57f91 --- /dev/null +++ b/ci/scripts/config_fetcher.py @@ -0,0 +1,79 @@ +import os +import sys +import csv +import json +import requests +from google.oauth2 import service_account +from google.auth.transport.requests import Request + +def get_access_token(): + """Authenticates and returns a Bearer token.""" + + # 1. Get Credentials from the mounted volume + cred_path = os.getenv('GOOGLE_SERVICE_ACCOUNT_FILE') + if cred_path is None: + raise ValueError("GOOGLE_SERVICE_ACCOUNT_FILE environment variable is not set") + + scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'] + + try: + creds = service_account.Credentials.from_service_account_file( + cred_path, + scopes=scopes + ) + creds.refresh(Request()) + return creds.token + except Exception as e: + print(f"# Error loading credentials: {e}", file=sys.stderr) + return None + +def fetch_config(): + """ + Fetches configuration from a Google Sheet (CSV export) and prints + shell-compatible export statements. + """ + + # 2. Get Input Sheet ID + sheet_id = os.getenv('GSHEETS_INPUT_SHEET_ID') + if sheet_id is None: + raise ValueError("GSHEETS_INPUT_SHEET_ID environment variable is not set") + # We need the 'spreadsheets.readonly' scope to read the data + scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'] + tab_name = "LLM_Config" + # 1. Resolve Name -> GID + token = get_access_token() + # --- THE FIX: Use the 'values' API instead of 'export' URL --- + # We ask for columns A and B from the specific tab name + # URL encoded range: "TabName!A:B" + range_name = f"{tab_name}!A:B" + url = f"https://sheets.googleapis.com/v4/spreadsheets/{sheet_id}/values/{range_name}" + + headers = {'Authorization': f'Bearer {token}'} + + try: + response = requests.get(url, headers=headers, timeout=30) + response.raise_for_status() + data = response.json() + except Exception as e: + # If this fails, print the details (often helpful for debugging) + print(f"# API Request Failed: {e}", file=sys.stderr) + sys.exit(1) + + # The API returns JSON: { "values": [ ["Key", "Val"], ["Key2", "Val2"] ] } + rows = data.get('values', []) + for row in rows: + # We need at least 2 columns (Key and Value) + if len(row) >= 2: + key = row[0].strip() + value = row[1].strip() + + # 1. Skip empty keys + # 2. Skip the header row if it literally says "Key" + # 3. Skip comments (keys starting with #) + if key and key != "Key" and not key.startswith("#"): + # Escape single quotes to prevent shell syntax errors + safe_value = value.replace("'", "'\\''") + print(f"export {key}='{safe_value}'") + +if __name__ == "__main__": + fetch_config() \ No newline at end of file From 975711ae9d9c7bf1ee68c6b2262c16f5f60e7581 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf <98809100+TamarW0@users.noreply.github.com> Date: Mon, 8 Dec 2025 14:25:36 +0200 Subject: [PATCH 112/286] fix:Clean exclusions to enable repeated runs (#153) fix:Clean exclusions to enable repeated runs --------- Co-authored-by: Tamar Weisskopf --- .../tests/test_transitive_code_search.py | 22 +++++++++++++++++++ .../utils/chain_of_calls_retriever.py | 2 ++ 2 files changed, 24 insertions(+) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 2c9b2a82f..5e1a56115 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -237,7 +237,29 @@ async def test_python_transitive_search(): assert path_found == True assert len(list_path) == 2 +@pytest.mark.asyncio +async def test_repeated_runs_of_transitive_search(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/TamarW0/example-repo', + git_ref='17af124607ff06fdf95734419b7ecd3af769c2f8', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + input = "werkzeug,formparser.MultiPartParser" + result_1 = await transitive_code_search_runner_coroutine(input) + result_2 = await transitive_code_search_runner_coroutine(input) + + (path_found_1, list_path_1) = result_1 + (path_found_2, list_path_2) = result_2 + print(f"DEBUG: {path_found_1=}, {path_found_2=}") + print(f"DEBUG: {list_path_1=}, {list_path_2=}") + print(f"DEBUG: {len(list_path_1)=}, {len(list_path_2)=}") + assert path_found_1 == path_found_2 == True + assert len(list_path_1) == len(list_path_2) == 2 @pytest.mark.asyncio diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index 9bc2b1508..c30a4736d 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -474,6 +474,8 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: class_name, function = function.split(splitters[0]) found_package = False matching_documents = [] + for dependency in self.tree_dict.values(): + dependency[EXCLUSIONS_INDEX] = [] standard_libs_cache = StandardLibraryCache.get_instance() # If it's a standard library package, then skip checking the package in dependency tree. if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): From 88584d1e2118fd875a70381c07261539d73188b0 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf <98809100+TamarW0@users.noreply.github.com> Date: Mon, 8 Dec 2025 14:26:05 +0200 Subject: [PATCH 113/286] fix: Remove invalid character from query (#154) Co-authored-by: Tamar Weisskopf --- src/vuln_analysis/utils/chain_of_calls_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index c30a4736d..fe2abb6c6 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -466,7 +466,7 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" self.found_path = False - query = query.splitlines()[0].replace('"', '').replace("'", "").strip() + query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() (package_name, function) = tuple(query.split(",")) class_name = None splitters = [splitter for splitter in ['.'] if splitter in function] From 8b11d19c46e35e222e9e3b7b28b5b60d2f766f8a Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:06:46 +0200 Subject: [PATCH 114/286] fixed trigger for running cm testing (#159) --- .tekton/on-cm-runner.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 29f796eda..633a87e46 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -5,13 +5,9 @@ metadata: # [CHANGE 1] Updated Name name: confusion-matrix-batch-runner annotations: - # Trigger on comment "/test-heavy" - pipelinesascode.tekton.dev/on-cel-expression: | - event == "issue_comment" && - body.action == "created" && - body.issue.pull_request && - body.comment.body.matches("^/test-heavy") - # We only need git-clone now, we removed buildah! + pipelinesascode.tekton.dev/on-target-branch: "[rh-aiq-main, main]" + pipelinesascode.tekton.dev/on-comment: "^/test-heavy" + pipelinesascode.tekton.dev/task: "git-clone" pipelinesascode.tekton.dev/max-keep-runs: "3" spec: From c02a24dcaaf765750234959e010a508263e301d5 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:22:15 +0200 Subject: [PATCH 115/286] remove the regex on beginning on line --- .tekton/on-cm-runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 633a87e46..8cecaa368 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -6,8 +6,8 @@ metadata: name: confusion-matrix-batch-runner annotations: pipelinesascode.tekton.dev/on-target-branch: "[rh-aiq-main, main]" - pipelinesascode.tekton.dev/on-comment: "^/test-heavy" - + pipelinesascode.tekton.dev/on-comment: "/test-heavy" + pipelinesascode.tekton.dev/task: "git-clone" pipelinesascode.tekton.dev/max-keep-runs: "3" spec: From ca6b4f8e81956460578fe842d09f8de3b70c3d02 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Tue, 9 Dec 2025 14:37:27 +0200 Subject: [PATCH 116/286] update tekton gh token --- .tekton/on-cm-runner.yaml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 8cecaa368..a09f4ef74 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -308,6 +308,12 @@ spec: - name: release-to-automation-repo image: alpine/git workingDir: $(workspaces.source.path) + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: exploit-iq-automation-token + key: gh-token script: | #!/bin/sh set -e @@ -317,12 +323,17 @@ spec: TARGET_REPO="RHEcosystemAppEng/exploitiq-tests-automation" - # --- Auth Setup --- - if [ -f "/workspace/basic-auth/.git-credentials" ]; then - # Extract token from URL format https://user:token@host - export GH_TOKEN=$(cat /workspace/basic-auth/.git-credentials | sed -n 's/.*:\(.*\)@.*/\1/p') - elif [ -f "/workspace/basic-auth/password" ]; then - export GH_TOKEN=$(cat /workspace/basic-auth/password) + # [CHANGE 2] Priority Check: Use the Env Var if it exists + if [ -n "$GH_TOKEN" ]; then + echo "✅ Using provided GH_TOKEN from secret 'exploit-iq-automation-token'." + else + echo "⚠️ GH_TOKEN env var is missing. Falling back to basic-auth workspace (Might fail for cross-repo)..." + if [ -f "/workspace/basic-auth/.git-credentials" ]; then + # Extract token from URL format https://user:token@host + export GH_TOKEN=$(cat /workspace/basic-auth/.git-credentials | sed -n 's/.*:\(.*\)@.*/\1/p') + elif [ -f "/workspace/basic-auth/password" ]; then + export GH_TOKEN=$(cat /workspace/basic-auth/password) + fi fi # 2. Identify Tag and File From 31da49fd58a3033bc5a6ddc311a29f049db93d2d Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Wed, 10 Dec 2025 15:41:28 +0200 Subject: [PATCH 117/286] add ngnix support --- .tekton/on-cm-runner.yaml | 15 +++++++++ .../utils/function_name_locator.py | 33 ++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index a09f4ef74..0f867d7a9 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -136,6 +136,21 @@ spec: value: "$(params.TRIGGER_COMMENT)" - name: GOMODCACHE value: "/exploit-iq-data/go/pkg/mod" + - name: SERPAPI_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/search + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cwe-details + - name: FIRST_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/ghsa + - name: NVD_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd + - name: RHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + script: | #!/bin/sh set -e diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 9f28caa2d..534f2037a 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -37,12 +37,21 @@ def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.is_std_package = False self.is_package_valid = False self.stdlib_cache = StandardLibraryCache.get_instance() - + self.short_go_package_name = {} + if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + self.short_go_package_name = self.build_short_go_package_name() + + def build_short_go_package_name(self) -> dict: + short_go_package_name = {} + for package in self.coc_retriever.supported_packages: + short_go_package_name[package] = package.split("/")[-1] + return short_go_package_name + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: logger.info("package %s is not in supported packages", package) if self.coc_retriever.supported_packages is None: return [] - pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.3) + pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.5) std_close_matches = self.stdlib_cache.get_close_match_list(package, self.coc_retriever.ecosystem) if std_close_matches: logger.info("package %s is not in supported packages, but close matches are: %s", package, std_close_matches) @@ -120,6 +129,23 @@ def common_flow_control(self,input_function: str, package_docs) -> list[str]: return close_matches + def search_in_third_party_packages(self, package: str) -> bool: + """ + Search in third party packages + Args: + package: The package to search in + Returns: + True if package is found in third party packages, False otherwise + """ + + # Check in supported_packages list (handles None case) + long_path = False + if self.coc_retriever.supported_packages is not None: + long_path = package in self.coc_retriever.supported_packages + + # Check in short_go_package_name dict (keys) + short_path = package in (self.short_go_package_name or {}) + return long_path or short_path async def locate_functions(self, query: str) -> list[str]: """ @@ -155,8 +181,7 @@ async def locate_functions(self, query: str) -> list[str]: logger.info("Package and Function Locator: searching for (%s,%s)", package, function) # First validate package name exists in supported packages - if (self.coc_retriever.supported_packages is None or - package not in self.coc_retriever.supported_packages): + if (self.search_in_third_party_packages(package)): logger.info("Package '%s' not found in supported packages", package) is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) if is_standard_lib: From d86a780df5fd4c3a3ef6e86ccdb894ace5eaea60 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Wed, 10 Dec 2025 16:37:05 +0200 Subject: [PATCH 118/286] fixed endpoint of serp --- .tekton/on-cm-runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 0f867d7a9..a5bc49efa 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -137,7 +137,7 @@ spec: - name: GOMODCACHE value: "/exploit-iq-data/go/pkg/mod" - name: SERPAPI_BASE_URL - value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/search + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi - name: CVE_DETAILS_BASE_URL value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cve-details - name: CWE_DETAILS_BASE_URL From 31a00140a947c81d809f92aab447f9c0535e41da Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Wed, 10 Dec 2025 18:41:56 +0200 Subject: [PATCH 119/286] update git auth for tag --- .tekton/on-cm-runner.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index a5bc49efa..616d9c032 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -286,12 +286,9 @@ spec: # Handle both ".git-credentials" file AND "username/password" keys # ------------------------------------------------------- - if [ -f "/workspace/basic-auth/.git-credentials" ]; then - # SCENARIO A: Secret is a pre-made credential file (Local Fix) - echo "Found .git-credentials file. Using it." - git config --global credential.helper "store --file=/workspace/basic-auth/.git-credentials" - elif [ -f "/workspace/basic-auth/password" ]; then + + if [ -f "/workspace/basic-auth/password" ]; then # SCENARIO B: Secret is broken into keys (Standard Production) echo "Found username/password files. Constructing auth." From a61654608d6e83fe8cfa5c573daee9c8d3bbeed8 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Wed, 10 Dec 2025 20:23:22 +0200 Subject: [PATCH 120/286] creds --- .tekton/on-cm-runner.yaml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 616d9c032..67fcda7fd 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -282,32 +282,15 @@ spec: git config --global user.name "Tekton Pipeline" # ------------------------------------------------------- - # ROBUST AUTH LOGIC - # Handle both ".git-credentials" file AND "username/password" keys + # AUTH LOGIC: PAC ONLY (.git-credentials) # ------------------------------------------------------- - - - - if [ -f "/workspace/basic-auth/password" ]; then - # SCENARIO B: Secret is broken into keys (Standard Production) - echo "Found username/password files. Constructing auth." - - # Default to 'git' if username is missing - GIT_USER="git" - if [ -f "/workspace/basic-auth/username" ]; then - GIT_USER=$(cat /workspace/basic-auth/username) - fi - GIT_PASS=$(cat /workspace/basic-auth/password) - - # Construct the URL manually with credentials - ORIGIN_URL=$(git remote get-url origin) - # Strip existing protocol (https://) - CLEAN_URL=${ORIGIN_URL#*//} - - # Set the remote to include the token - git remote set-url origin "https://${GIT_USER}:${GIT_PASS}@${CLEAN_URL}" + if [ -f "/workspace/basic-auth/.git-credentials" ]; then + echo "✅ Found PAC credentials. Configuring git helper." + git config --global credential.helper "store --file=/workspace/basic-auth/.git-credentials" else - echo "WARNING: No auth found in /workspace/basic-auth. Push might fail." + echo "❌ CRITICAL ERROR: .git-credentials file missing in /workspace/basic-auth." + echo "This pipeline only supports Pipelines-as-Code authentication." + exit 1 fi echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" From 99a481db64645e2a3bde9c4ab12f23041a16511c Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Thu, 11 Dec 2025 10:10:43 +0200 Subject: [PATCH 121/286] more tests --- .tekton/on-cm-runner.yaml | 4 +++- src/vuln_analysis/utils/serp_api_wrapper.py | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 67fcda7fd..defeaabfb 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -286,7 +286,9 @@ spec: # ------------------------------------------------------- if [ -f "/workspace/basic-auth/.git-credentials" ]; then echo "✅ Found PAC credentials. Configuring git helper." - git config --global credential.helper "store --file=/workspace/basic-auth/.git-credentials" + cp /workspace/basic-auth/.git-credentials ~/.git-credentials + chmod 600 ~/.git-credentials + git config --global credential.helper "store --file=~/.git-credentials" else echo "❌ CRITICAL ERROR: .git-credentials file missing in /workspace/basic-auth." echo "This pipeline only supports Pipelines-as-Code authentication." diff --git a/src/vuln_analysis/utils/serp_api_wrapper.py b/src/vuln_analysis/utils/serp_api_wrapper.py index a412cc82d..032cfeb2c 100644 --- a/src/vuln_analysis/utils/serp_api_wrapper.py +++ b/src/vuln_analysis/utils/serp_api_wrapper.py @@ -16,11 +16,16 @@ import aiohttp from langchain.utils.env import get_from_env from langchain_community.utilities import SerpAPIWrapper +from numpy import e from pydantic import model_validator from vuln_analysis.utils.async_http_utils import retry_async from vuln_analysis.utils.url_utils import url_join +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +ERROR_OUT_OF_SEARCHES = "Your account has run out of searches" class MorpheusSerpAPIWrapper(SerpAPIWrapper): @@ -73,5 +78,8 @@ def _process_response(res: dict) -> str: """Catch the ValueError and return a message if no good search result found.""" try: return SerpAPIWrapper._process_response(res) - except ValueError: + except ValueError as e: + logger.error(f"Error processing response: {e}") + if ERROR_OUT_OF_SEARCHES in str(e): + return "Your account has run out of searches" return "No good search result found" From 26876b77d82f0a189cce2f16fae001ccd1801b4f Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Thu, 11 Dec 2025 11:29:18 +0200 Subject: [PATCH 122/286] more debug git token --- .tekton/on-cm-runner.yaml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index defeaabfb..da6bb34aa 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -286,9 +286,30 @@ spec: # ------------------------------------------------------- if [ -f "/workspace/basic-auth/.git-credentials" ]; then echo "✅ Found PAC credentials. Configuring git helper." - cp /workspace/basic-auth/.git-credentials ~/.git-credentials - chmod 600 ~/.git-credentials - git config --global credential.helper "store --file=~/.git-credentials" + #cp /workspace/basic-auth/.git-credentials ~/.git-credentials + #chmod 600 ~/.git-credentials + #git config --global credential.helper "store --file=~/.git-credentials" + # 1. Read the content (Format: https://user:token@github.com) + RAW_CRED=$(cat "$CRED_FILE") + + # 2. Extract the 'user:token' part using sed + # This regex grabs everything between 'https://' and '@' + AUTH_PART=$(echo "$RAW_CRED" | sed -n 's|https://\(.*\)@.*|\1|p') + + if [ -z "$AUTH_PART" ]; then + echo "❌ Error parsing credentials from file." + exit 1 + fi + + # 3. Get the current remote URL (e.g., https://github.com/org/repo.git) + CURRENT_URL=$(git remote get-url origin) + + # 4. Strip the 'https://' prefix + CLEAN_URL=${CURRENT_URL#https://} + + # 5. Set the new URL with auth injected + echo "Injecting credentials into git remote..." + git remote set-url origin "https://${AUTH_PART}@${CLEAN_URL}" else echo "❌ CRITICAL ERROR: .git-credentials file missing in /workspace/basic-auth." echo "This pipeline only supports Pipelines-as-Code authentication." From 8ef371c283f82c6343e28d5b83c72fec135c8820 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Thu, 11 Dec 2025 14:32:50 +0200 Subject: [PATCH 123/286] update --- .tekton/on-cm-runner.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index da6bb34aa..ee2f31c8e 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -284,7 +284,8 @@ spec: # ------------------------------------------------------- # AUTH LOGIC: PAC ONLY (.git-credentials) # ------------------------------------------------------- - if [ -f "/workspace/basic-auth/.git-credentials" ]; then + CRED_FILE="/workspace/basic-auth/.git-credentials" + if [ -f "$CRED_FILE" ]; then echo "✅ Found PAC credentials. Configuring git helper." #cp /workspace/basic-auth/.git-credentials ~/.git-credentials #chmod 600 ~/.git-credentials @@ -307,7 +308,7 @@ spec: # 4. Strip the 'https://' prefix CLEAN_URL=${CURRENT_URL#https://} - # 5. Set the new URL with auth injected + # 5. Set the new URL wi-th auth injected echo "Injecting credentials into git remote..." git remote set-url origin "https://${AUTH_PART}@${CLEAN_URL}" else From 921296f5aab826051c5f2da0e80607b061de9921 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Thu, 11 Dec 2025 15:49:45 +0200 Subject: [PATCH 124/286] remove locator changes --- .../utils/function_name_locator.py | 33 +++---------------- src/vuln_analysis/utils/serp_api_wrapper.py | 10 +----- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 534f2037a..9f28caa2d 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -37,21 +37,12 @@ def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.is_std_package = False self.is_package_valid = False self.stdlib_cache = StandardLibraryCache.get_instance() - self.short_go_package_name = {} - if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: - self.short_go_package_name = self.build_short_go_package_name() - - def build_short_go_package_name(self) -> dict: - short_go_package_name = {} - for package in self.coc_retriever.supported_packages: - short_go_package_name[package] = package.split("/")[-1] - return short_go_package_name - + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: logger.info("package %s is not in supported packages", package) if self.coc_retriever.supported_packages is None: return [] - pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.5) + pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.3) std_close_matches = self.stdlib_cache.get_close_match_list(package, self.coc_retriever.ecosystem) if std_close_matches: logger.info("package %s is not in supported packages, but close matches are: %s", package, std_close_matches) @@ -129,23 +120,6 @@ def common_flow_control(self,input_function: str, package_docs) -> list[str]: return close_matches - def search_in_third_party_packages(self, package: str) -> bool: - """ - Search in third party packages - Args: - package: The package to search in - Returns: - True if package is found in third party packages, False otherwise - """ - - # Check in supported_packages list (handles None case) - long_path = False - if self.coc_retriever.supported_packages is not None: - long_path = package in self.coc_retriever.supported_packages - - # Check in short_go_package_name dict (keys) - short_path = package in (self.short_go_package_name or {}) - return long_path or short_path async def locate_functions(self, query: str) -> list[str]: """ @@ -181,7 +155,8 @@ async def locate_functions(self, query: str) -> list[str]: logger.info("Package and Function Locator: searching for (%s,%s)", package, function) # First validate package name exists in supported packages - if (self.search_in_third_party_packages(package)): + if (self.coc_retriever.supported_packages is None or + package not in self.coc_retriever.supported_packages): logger.info("Package '%s' not found in supported packages", package) is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) if is_standard_lib: diff --git a/src/vuln_analysis/utils/serp_api_wrapper.py b/src/vuln_analysis/utils/serp_api_wrapper.py index 032cfeb2c..a412cc82d 100644 --- a/src/vuln_analysis/utils/serp_api_wrapper.py +++ b/src/vuln_analysis/utils/serp_api_wrapper.py @@ -16,16 +16,11 @@ import aiohttp from langchain.utils.env import get_from_env from langchain_community.utilities import SerpAPIWrapper -from numpy import e from pydantic import model_validator from vuln_analysis.utils.async_http_utils import retry_async from vuln_analysis.utils.url_utils import url_join -from vuln_analysis.logging.loggers_factory import LoggingFactory -logger = LoggingFactory.get_agent_logger(__name__) - -ERROR_OUT_OF_SEARCHES = "Your account has run out of searches" class MorpheusSerpAPIWrapper(SerpAPIWrapper): @@ -78,8 +73,5 @@ def _process_response(res: dict) -> str: """Catch the ValueError and return a message if no good search result found.""" try: return SerpAPIWrapper._process_response(res) - except ValueError as e: - logger.error(f"Error processing response: {e}") - if ERROR_OUT_OF_SEARCHES in str(e): - return "Your account has run out of searches" + except ValueError: return "No good search result found" From 4c9bb19cd421e503b448e12d257138a8330ebd58 Mon Sep 17 00:00:00 2001 From: Shimon Tanny Date: Thu, 11 Dec 2025 15:58:37 +0200 Subject: [PATCH 125/286] add something for create image --- src/vuln_analysis/utils/function_name_locator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 9f28caa2d..10b7c320c 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -24,7 +24,8 @@ logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") - +# FunctionNameLocator provides fuzzy matching capabilities to locate function names +# in source code across different programming languages and ecosystems. class FunctionNameLocator: """ A class dedicated to locating function names using fuzzy matching. From b68d8935d962ca1f4542691278fd8c546661ddbd Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Tue, 16 Dec 2025 09:44:08 +0200 Subject: [PATCH 126/286] add disable option disable web search (#163) * add disable option disable web search Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --------- Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --- .tekton/on-cm-runner.yaml | 2 +- kustomize/base/exploit-iq-config.yml | 1 + kustomize/config-http-openai-local.yml | 1 + src/vuln_analysis/configs/config-http-nim.yml | 1 + src/vuln_analysis/configs/config-http-openai.yml | 1 + src/vuln_analysis/configs/config-tracing.yml | 1 + src/vuln_analysis/configs/config.yml | 1 + src/vuln_analysis/functions/cve_agent.py | 3 +++ 8 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index ee2f31c8e..9a7660637 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -156,7 +156,7 @@ spec: set -e # We use python to safely remove the 'telemetry' block echo "--- Generating Config without Tracing ---" - python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" FETCHER_SCRIPT="ci/scripts/config_fetcher.py" # [CHANGE] Logic: Check if the comment contains "config=sheet" diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index f39fe866b..4038e49cf 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -105,6 +105,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false # transitive_search_tool_enabled: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 56c366d78..8cdb3e42b 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -101,6 +101,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false # transitive_search_tool_enabled: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 1ff242eb1..526196e8e 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -94,6 +94,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false # transitive_search_tool_enabled: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index fa3cdd7e4..a4470bf13 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -101,6 +101,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false # transitive_search_tool_enabled: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index ff11867da..c1eb0dd4e 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -104,6 +104,7 @@ functions: replace_exceptions: true replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index c12c5da4f..c3f230924 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -74,6 +74,7 @@ functions: replace_exceptions: true replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false + cve_web_search_enabled: true verbose: false cve_generate_cvss: _type: cve_generate_cvss diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index f1434aa00..93eed149f 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -63,6 +63,8 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") transitive_search_tool_enabled: bool = Field(default=True, description="Whether to enable transitive code search or not.") + cve_web_search_enabled: bool = Field(default=True, + description="Whether to enable CVE Web Search tool or not.") verbose: bool = Field(default=False, description="Set to true for verbose output") @@ -79,6 +81,7 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None) or + (tool.name == ToolNames.CVE_WEB_SEARCH and not config.cve_web_search_enabled) or (tool.name == ToolNames.CALL_CHAIN_ANALYZER and (not config.transitive_search_tool_enabled or state.code_index_path is None)) or (tool.name == ToolNames.FUNCTION_CALLER_FINDER and (not config.transitive_search_tool_enabled or From 2ffbb9508196bea7674c2b25cfdbad0171f34279 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Tue, 23 Dec 2025 14:09:47 +0200 Subject: [PATCH 127/286] update toolchain (#164) * update toolchain * fix go dict exception * fixed Escape identifier --- .tekton/on-cm-runner.yaml | 2 + .tekton/on-pull-request.yaml | 1 + Dockerfile | 2 + .../utils/function_name_locator.py | 43 ++++++++++++++++--- .../golang_functions_parsers.py | 4 +- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 9a7660637..9ad5894b9 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -122,6 +122,8 @@ spec: name: redhat-app-creds env: + - name: GOTOOLCHAIN + value: "auto" - name: PYTHONUNBUFFERED value: "1" - name: GOOGLE_SERVICE_ACCOUNT_FILE diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index e8ba883a1..6c2bc9176 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -204,6 +204,7 @@ spec: mkdir -p $HOME/go-sdk tar -C $HOME/go-sdk -xzf go.tar.gz export PATH=$HOME/go-sdk/go/bin:$PATH + export GOTOOLCHAIN=auto echo "Go version:" go version diff --git a/Dockerfile b/Dockerfile index a2b994916..11791b46e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,8 @@ RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24. && tar -C /usr/local -xzf /tmp/go1.24.1.linux-amd64.tar.gz \ && rm /tmp/go1.24.1.linux-amd64.tar.gz +ENV GOTOOLCHAIN=auto + # --- Temurin JDK 22 (amd64/x86_64) --- ARG JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz" ARG JDK_DIR="jdk-22.0.2+9" diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 10b7c320c..18c520874 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -24,8 +24,7 @@ logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") -# FunctionNameLocator provides fuzzy matching capabilities to locate function names -# in source code across different programming languages and ecosystems. + class FunctionNameLocator: """ A class dedicated to locating function names using fuzzy matching. @@ -38,12 +37,22 @@ def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.is_std_package = False self.is_package_valid = False self.stdlib_cache = StandardLibraryCache.get_instance() - + self.short_go_package_name = {} + if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + self.short_go_package_name = self.build_short_go_package_name() + + def build_short_go_package_name(self) -> dict: + short_go_package_name = {} + for package in self.coc_retriever.supported_packages: + short_name = package.split("/")[-1] + short_go_package_name[short_name] = package + return short_go_package_name + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: logger.info("package %s is not in supported packages", package) if self.coc_retriever.supported_packages is None: return [] - pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.3) + pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.5) std_close_matches = self.stdlib_cache.get_close_match_list(package, self.coc_retriever.ecosystem) if std_close_matches: logger.info("package %s is not in supported packages, but close matches are: %s", package, std_close_matches) @@ -121,6 +130,27 @@ def common_flow_control(self,input_function: str, package_docs) -> list[str]: return close_matches + def search_in_third_party_packages(self, package: str) -> bool: + """ + Search in third party packages + Args: + package: The package to search in + Returns: + True if package is found in third party packages, False otherwise + """ + + # Check in supported_packages list (handles None case) + long_path = False + if self.coc_retriever.supported_packages is not None: + long_path = package in self.coc_retriever.supported_packages + + # Check in short_go_package_name dict (keys) + short_path = package in (self.short_go_package_name or {}) + if long_path: + return True + if short_path: + return True + return False async def locate_functions(self, query: str) -> list[str]: """ @@ -156,8 +186,7 @@ async def locate_functions(self, query: str) -> list[str]: logger.info("Package and Function Locator: searching for (%s,%s)", package, function) # First validate package name exists in supported packages - if (self.coc_retriever.supported_packages is None or - package not in self.coc_retriever.supported_packages): + if (not self.search_in_third_party_packages(package)): logger.info("Package '%s' not found in supported packages", package) is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) if is_standard_lib: @@ -209,6 +238,8 @@ async def locate_functions(self, query: str) -> list[str]: package_docs = self.coc_retriever.sort_docs[package] else: # package_docs = self.coc_retriever.documents_of_functions + if package in self.short_go_package_name: + package = self.short_go_package_name[package] package_docs = [ doc for doc in (self.coc_retriever.documents_of_functions or []) diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py index 423bad8b2..5ffebf3e7 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py @@ -179,7 +179,7 @@ def create_map_of_local_vars(self, functions_methods_documents: list[Document]) right_side = func_method.page_content[index_of_start_if + 2: index_of_start_if + end_of_assignment - 1].strip() - all_vars[(left_side.strip())] = {" value": right_side.strip().replace + all_vars[(left_side.strip())] = {"value": right_side.strip().replace ("\n\t", "").replace("\t", ""), "type": LOCAL_IMPLICIT } @@ -453,7 +453,7 @@ def __check_identifier_resolved_to_callee_function_package(self, function: Docum for doc in code_documents: if function.metadata.get('source') == code_documents[doc].metadata.get('source'): # maybe identifier is the package itself in the file - regex = f"package {identifier}" + regex = f"package {re.escape(identifier)}" code_content = code_documents[doc].page_content matching = re.search(regex, code_content, re.MULTILINE) if matching and matching.group(0): From 40fa0ea6187a31620ad4a9fa2ceec43069f3469f Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Tue, 23 Dec 2025 14:47:34 +0200 Subject: [PATCH 128/286] update tekton script to override older tags incase of rerunning the cm tests (#165) --- .tekton/on-cm-runner.yaml | 16 +++++++++++++++- src/vuln_analysis/utils/dep_tree.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 9ad5894b9..05f467d16 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -320,8 +320,13 @@ spec: fi echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" + # 1. Create/Overwrite the tag locally + # The -f flag automatically overwrites the local tag if it exists git tag -f -a "$TAG_NAME" -m "Tests Passed for PR #$(params.PR_NUMBER)" - git push origin "$TAG_NAME" + + # 2. Push/Overwrite the tag remotely + # This one command handles both creation and updating + git push origin "refs/tags/$TAG_NAME" --force # ------------------------------------------------------- # STEP C: Release & Upload Artifacts to AUTOMATION Repo @@ -376,6 +381,15 @@ spec: fi echo "--- Found Report: $REPORT_FILE ---" + + # 2. Blind Delete (Simpler) + # We attempt to delete. If it fails (doesn't exist), we just continue. + # We redirect output to /dev/null so logs stay clean. + gh release delete "$TAG_NAME" \ + --repo "$TARGET_REPO" \ + --cleanup-tag \ + --yes >/dev/null 2>&1 || echo "No existing release to delete." + echo "--- Creating Release '$TAG_NAME' on Automation Repo ---" echo "--- Creating release in $TARGET_REPO ---" diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 1f9eec046..3aaf0adcf 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -746,7 +746,7 @@ def __init__(self): pass def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: - # Get go version from manifest + ## Get go version from manifest # TODO - download go binary of this version and run the go mod graph # using it for optimal performance. go_version = self.determine_go_version(manifest_path) From 89dc82177a0edc1d6bd7a695f0cf23c08e88e52d Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Tue, 23 Dec 2025 18:20:53 +0200 Subject: [PATCH 129/286] Fix 2 fqcn in different jars (#157) * Fix - docs using the vulnerable class were not included in the list of potential docs Signed-off-by: Theodor Mihalache --- .../java_functions_parsers.py | 2 +- src/vuln_analysis/utils/java_utils.py | 707 +++++++++++------- 2 files changed, 437 insertions(+), 272 deletions(-) diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py index c0cb77062..1596297de 100644 --- a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py @@ -1107,7 +1107,7 @@ def __trace_down_package(self, expression: str, type_documents: list[Document], def document_imports_package(self, documents: list[Document], package_name: str) -> list[Document]: importing_docs = [value for (file, value) in documents.items() - if self.is_package_imported(value.page_content, package_name)] + if self.is_package_imported(value.page_content, package_name) or package_name.rsplit('.', 1)[-1] in value.page_content] return importing_docs def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: diff --git a/src/vuln_analysis/utils/java_utils.py b/src/vuln_analysis/utils/java_utils.py index a79b1e8d9..378d9f99c 100644 --- a/src/vuln_analysis/utils/java_utils.py +++ b/src/vuln_analysis/utils/java_utils.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass from functools import lru_cache -from typing import Dict, List, Tuple, Optional, Any +from typing import Dict, List, Tuple, Optional, Any, Set from langchain_core.documents import Document from tqdm import tqdm @@ -1533,26 +1533,24 @@ def skip_type_list(p: int) -> int: def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: """ - Returns: - { - (SimpleTypeName, SourcePath): [ - ("Self", "SourcePath"), - ("Parent", "SourcePath-or-empty"), - ("GrandParent", "SourcePath-or-empty"), - ("Iface1", "SourcePath-or-empty"), - ... - ("ImplOrSubclass1", "SourcePath-or-empty"), - ("ImplOrSubclass2", "SourcePath-or-empty"), - ... - ] - } - - - Key is (type simple name, doc.metadata['source'] or 'path' or '') - - Handles 'Code for:' stubs and full files with comments/imports - - Order-preserving, no duplicates - - Explicit-imports-first resolution; star imports allowed - - Only DFS into interfaces present in the provided set (externals are leaves) - - Includes self, ancestors, implemented interfaces, and transitive implementers/subclasses + Build a jar-aware inheritance/implements map. + + Key: (simple_name, source_path) + Value: ordered, deduped list of (simple_name, source_path) including: + - self + - ancestors (class chain or interface parents) + - implemented interfaces (direct, via ancestors, and via super-interfaces) + - transitive subclasses / implementers (only for types present in the set) + + Jar-aware rule for resolvable related types: + - If an FQCN has a variant in the same jar, pick that. + - Else if it has exactly one variant globally, pick that. + - Else (multiple variants across jars and none in this jar), skip. + + External/unseen types (e.g., JDK’s Cloneable/Serializable/AbstractMap/Map) + are INCLUDED in the upward/interface lists as placeholders: + (simple_name, current_source_of_key) + Placeholders are NOT used to wire reverse edges (subclasses/implementers). """ # ---------------- helpers ---------------- @@ -1627,47 +1625,6 @@ def base_token(t: str) -> str: def simple_name_from_fqcn(fq: str) -> str: return fq.split('.')[-1] if fq else fq - def dedup_keep_order(items: List[str]) -> List[str]: - """ - Remove duplicates from a list of strings while preserving the order of their - first appearance. Falsy values (e.g., "", None) are ignored. - - Parameters - ---------- - items : List[str] - The input sequence of strings. Elements are considered duplicates if - their string values are equal. Whitespace differences matter—that is, - "Foo" and " Foo" are different. - - Returns - ------- - List[str] - A new list containing the first occurrence of each non-falsy, unique - string from `items`, in original order. - - Examples - -------- - >>> dedup_keep_order(["A", "B", "A", "C"]) - ['A', 'B', 'C'] - - >>> dedup_keep_order(["", "A", "", "A", "B"]) - ['A', 'B'] - - # Whitespace differences are preserved and treated as distinct - >>> dedup_keep_order(["Foo", " Foo", "Foo "]) - ['Foo', ' Foo', 'Foo '] - - >>> dedup_keep_order([]) - [] - """ - seen: set[str] = set() - ordered: List[str] = [] - for item in items: - if item and item not in seen: - seen.add(item) - ordered.append(item) - return ordered - def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: """ Remove duplicate (name, source) pairs while preserving the order @@ -1699,12 +1656,12 @@ def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: [] """ seen: set[Tuple[str, str]] = set() - deduped: List[Tuple[str, str]] = [] - for pair in pairs: - if pair and pair not in seen: - seen.add(pair) - deduped.append(pair) - return deduped + out: List[Tuple[str, str]] = [] + for p in pairs: + if p and p not in seen: + seen.add(p) + out.append(p) + return out def meta_source(doc) -> str: try: @@ -1815,9 +1772,23 @@ def segment_until_next(start: int) -> str | None: pkg_re = re.compile(r"\bpackage\s+([A-Za-z_][\w.]*)\s*;") import_re = re.compile(r"\bimport\s+(?!static\b)\s*([A-Za-z_][\w.]*?(?:\.\*|))\s*;") - # ---------------- first pass: parse & collect ---------------- + # ---- jar id extraction ---- + # Captures like: 'dependencies-sources/-sources' + jar_dir_re = re.compile(r"(?:^|/)(dependencies-sources/[^/]+)(?:/|$)") + + def source_jar_id(path: str) -> str: + """Extract a stable jar id directory from a source path.""" + if not path: + return "" + m = jar_dir_re.search(path) + if m: + return m.group(1) + parts = [p for p in path.split('/') if p] + return parts[0] if parts else "" + + # ---------------- first pass: collect per-FQCN variants ---------------- + variants_by_fq: Dict[str, List[Dict]] = {} known_fqcns: set[str] = set() - parsed: List[dict] = [] for doc in tqdm(java_types, total=len(java_types), desc="Building types inheritance map..."): src_all = doc.page_content or "" @@ -1839,64 +1810,52 @@ def segment_until_next(start: int) -> str | None: header_text = re.sub(r"\s+", " ", header_text).strip() m_name = name_kind_re.search(header_text) + + pkg = (pkg_re.search(src_all) or [None, ""])[1] + imps = import_re.findall(src_all) + src_path = meta_source(doc) + if not m_name: # fallback to filename file_simple = simple_name_from_filename(doc) if not file_simple: continue - pkg = (pkg_re.search(src_all) or [None, ""])[1] - imps = import_re.findall(src_all) fqcn = f"{pkg}.{file_simple}" if pkg else file_simple - parsed.append({ - "fqcn": fqcn, - "simple": file_simple, - "pkg": pkg, - "imports": imps, - "kind": "class", - "extends_list": [], - "implements_list": [], - "source": meta_source(doc), - }) - known_fqcns.add(fqcn) - continue - - kind, parsed_name = m_name.group(1), m_name.group(2) - - # Depth-aware, start-after-name extractor (robust for 'extends' after name) - ext_seg, impl_seg = extract_clauses_after_kindname(header_text) - - # ---------- LAST-RESORT: scan the de-commented full source for "class ... extends ..." ---------- - if (not ext_seg) and ("extends" in src_all): - s_nc = strip_comments_preserving_newlines(src_all) - # anchor on the same class name we already parsed - m_ext2 = re.search( - rf"\bclass\s+{re.escape(parsed_name)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", - s_nc - ) - if m_ext2: - ext_seg = m_ext2.group(1).strip() - # ---------- END LAST-RESORT ---------- - - extends_names = split_type_list_strict(ext_seg) if ext_seg else [] - implements_names = split_type_list_strict(impl_seg) if impl_seg else [] + kind = "class" + extends_names: List[str] = [] + implements_names: List[str] = [] + parsed_name = file_simple + else: + kind, parsed_name = m_name.group(1), m_name.group(2) + ext_seg, impl_seg = extract_clauses_after_kindname(header_text) + # LAST RESORT: scan de-commented full source for "class ... extends ..." + if (not ext_seg) and ("extends" in src_all): + s_nc = strip_comments_preserving_newlines(src_all) + m_ext2 = re.search( + rf"\bclass\s+{re.escape(parsed_name)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", + s_nc + ) + if m_ext2: + ext_seg = m_ext2.group(1).strip() + extends_names = split_type_list_strict(ext_seg) if ext_seg else [] + implements_names = split_type_list_strict(impl_seg) if impl_seg else [] file_simple = simple_name_from_filename(doc) simple_name = file_simple or parsed_name - - pkg = (pkg_re.search(src_all) or [None, ""])[1] - imps = import_re.findall(src_all) - fqcn = f"{pkg}.{simple_name}" if pkg else simple_name - parsed.append({ + + rec = { "fqcn": fqcn, "simple": simple_name, "pkg": pkg, "imports": imps, "kind": kind, - "extends_list": extends_names, - "implements_list": implements_names, - "source": meta_source(doc), - }) + "extends_raw": extends_names, + "implements_raw": implements_names, + "source": src_path, + "jar": source_jar_id(src_path), + } + variants_by_fq.setdefault(fqcn, []).append(rec) known_fqcns.add(fqcn) # simple -> known FQCNs @@ -1904,7 +1863,6 @@ def segment_until_next(start: int) -> str | None: for fq in known_fqcns: by_simple.setdefault(simple_name_from_fqcn(fq), []).append(fq) - # ---------------- resolver ---------------- def make_resolver(pkg: str, imports: List[str]): """ Build a simple type resolver for a compilation unit. @@ -2002,22 +1960,15 @@ def resolve(token: str) -> str: """ tok = base_token(token) if not tok: - return "" # empty/invalid token → empty string for caller logic - - # Already qualified + return "" # empty token if '.' in tok: return tok - - # Explicit import wins if tok in explicit: return explicit[tok] - - # Try same package if pkg: candidate = f"{pkg}.{tok}" if candidate in known_fqcns: return candidate - # Star imports – accept only a unique hit candidates: List[str] = [] for fq in by_simple.get(tok, []): @@ -2025,209 +1976,423 @@ def resolve(token: str) -> str: candidates.append(fq) if len(candidates) == 1: return candidates[0] - # Globally unique simple name across all known types all_candidates = by_simple.get(tok, []) if len(all_candidates) == 1: return all_candidates[0] - # Unknown or ambiguous → keep simple name return tok return resolve - # ---------------- build graph ---------------- - info: Dict[str, Dict] = {} - for h in parsed: - resolve = make_resolver(h["pkg"], h["imports"]) - - if h["kind"] == "record": - parents = ["java.lang.Record"] - elif h["kind"] == "class": - parents = [resolve(x) for x in (h.get("extends_list") or [])][:1] - elif h["kind"] == "enum": - parents = [] - else: # interface - parents = [resolve(x) for x in (h.get("extends_list") or [])] - - impls = [resolve(x) for x in (h.get("implements_list") or [])] - - info[h["fqcn"]] = { - "kind": h["kind"], - "parents": parents, - "impls": impls, # interfaces that this type implements - "simple": h["simple"], - "source": h.get("source", ""), - } + # resolve parents/impls per variant + for fq, var_list in variants_by_fq.items(): + for rec in var_list: + resolve = make_resolver(rec["pkg"], rec["imports"]) + if rec["kind"] == "record": + parents = ["java.lang.Record"] + elif rec["kind"] == "class": + parents = [resolve(x) for x in (rec.get("extends_raw") or [])][:1] + elif rec["kind"] == "enum": + parents = [] + else: # interface + parents = [resolve(x) for x in (rec.get("extends_raw") or [])] + impls = [resolve(x) for x in (rec.get("implements_raw") or [])] + rec["parents"] = parents + rec["impls"] = impls + + # index variants + def vid_of(rec: Dict) -> Tuple[str, str]: + return rec["fqcn"], rec["source"] + + variants: List[Dict] = [rec for lst in variants_by_fq.values() for rec in lst] + rec_by_vid: Dict[Tuple[str, str], Dict] = {vid_of(rec): rec for rec in variants} + rec_by_fq_and_jar: Dict[Tuple[str, str], Dict] = {} + for rec in variants: + rec_by_fq_and_jar[(rec["fqcn"], rec["jar"])] = rec + + def pick_variant(fq: str, jar: str) -> Dict | None: + """Pick the variant in the same jar if present; else unique globally; else None.""" + same = rec_by_fq_and_jar.get((fq, jar)) + if same: + return same + lst = variants_by_fq.get(fq) or [] + if len(lst) == 1: + return lst[0] + return None # ambiguous across jars, and not in this jar + + # reverse extends (jar-aware) + rev_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + rev_iface_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def is_iface_rec(rec: Dict) -> bool: + return rec["kind"] == "interface" + + for rec in variants: + jar = rec["jar"] + this_vid = vid_of(rec) + for p_fq in rec.get("parents", []): + if not p_fq: + continue + p_rec = pick_variant(p_fq, jar) + if not p_rec: + continue + p_vid = vid_of(p_rec) + if is_iface_rec(rec) and is_iface_rec(p_rec): + rev_iface_extends.setdefault(p_vid, []).append(this_vid) + else: + rev_extends.setdefault(p_vid, []).append(this_vid) + + # --------- upward chain & iface collectors with cycle guards --------- + extends_chain_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} - def is_iface(fq: str) -> bool: - return fq in info and info[fq]["kind"] == "interface" + def extends_chain_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + Upward chain for classes (linear) and interface parents (DFS), + jar-aware; placeholders where parents are unseen. + Returns list of (simple, source). + """ + if start_vid in extends_chain_pairs_memo: + return extends_chain_pairs_memo[start_vid] - # ---------------- ancestry (upwards) ---------------- - def extends_chain(fq: str) -> List[str]: - if fq not in info: + cur = rec_by_vid.get(start_vid) + if not cur: return [] - k = info[fq]["kind"] - chain: List[str] = [] - - if k == "interface": - seen = set() - def dfs_iface(x: str): - for p in info.get(x, {}).get("parents", []): - if p and p not in seen: - seen.add(p) - if is_iface(p): - dfs_iface(p) - chain.append(p) - dfs_iface(fq) - return dedup_keep_order(chain) - - # class/record/enum: linear chain - cur, visited = fq, set() + jar = cur["jar"] + chain: List[Tuple[str, str]] = [] + + if is_iface_rec(cur): + seen_vids = set() + + def dfs_iface(v: Tuple[str, str]): + r = rec_by_vid.get(v) + if not r: + return + for p_fq in r.get("parents", []): + if not p_fq: + continue + p_rec = pick_variant(p_fq, jar) + if p_rec: + p_vid = vid_of(p_rec) + if p_vid in seen_vids: + continue + seen_vids.add(p_vid) + if is_iface_rec(p_rec): + dfs_iface(p_vid) + chain.append((p_rec["simple"], p_rec["source"])) + else: + chain.append((simple_name_from_fqcn(p_fq), cur["source"])) + + dfs_iface(start_vid) + out = tup_dedup_keep_order(chain) + extends_chain_pairs_memo[start_vid] = out + return out + + # class/record/enum: linear with visited guard + cur_vid = start_vid + visited_vids = set() while True: - ps = info.get(cur, {}).get("parents", []) - if not ps: break - p = ps[0] - if not p or p in visited: break - visited.add(p) - chain.append(p) - if p in info and not is_iface(p): - cur = p + r = rec_by_vid.get(cur_vid) + if not r: + break + ps = r.get("parents", []) + if not ps: + break + p_fq = ps[0] + p_rec = pick_variant(p_fq, jar) if p_fq else None + if p_rec: + p_vid = vid_of(p_rec) + if p_vid in visited_vids or p_vid == cur_vid: + break + visited_vids.add(p_vid) + chain.append((p_rec["simple"], p_rec["source"])) + if not is_iface_rec(p_rec): + cur_vid = p_vid + else: + break else: + chain.append((simple_name_from_fqcn(p_fq), r["source"])) break - return dedup_keep_order(chain) - def all_interfaces(fq: str) -> List[str]: - """Interfaces implemented by a type (directly, via ancestors, and via super-interfaces).""" - ordered: List[str] = [] + out = tup_dedup_keep_order(chain) + extends_chain_pairs_memo[start_vid] = out + return out + + # vids only (memoized) for reverse implementers + iface_closure_vids_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def all_interfaces_vids(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For wiring reverse implementers: return interface vids reachable from a class + (direct implements + via ancestors + super-interfaces). Jar-aware. + """ + if start_vid in iface_closure_vids_memo: + return iface_closure_vids_memo[start_vid] + + rec = rec_by_vid.get(start_vid) + if not rec: + iface_closure_vids_memo[start_vid] = [] + return [] + jar = rec["jar"] + ordered_vids: List[Tuple[str, str]] = [] added, visiting = set(), set() - def add(i: str): - if i and i not in added: - added.add(i) - ordered.append(i) + def add_vid(v: Tuple[str, str]): + if v and v not in added: + added.add(v) + ordered_vids.append(v) - def dfs_ifaces(i: str): - if not i or i in visiting or i in added: + def dfs_iface_vid(v: Tuple[str, str]): + if not v or v in visiting or v in added: + return + r = rec_by_vid.get(v) + if not r: return - if i not in info or not is_iface(i): - add(i) + if not is_iface_rec(r): + add_vid(v) return - visiting.add(i) - for p in info.get(i, {}).get("parents", []): - dfs_ifaces(p) - visiting.remove(i) - add(i) - - for i in info.get(fq, {}).get("impls", []): - dfs_ifaces(i) - - for anc in extends_chain(fq): - if anc in info and not is_iface(anc): - for i in info.get(anc, {}).get("impls", []): - dfs_ifaces(i) - - if is_iface(fq): - for i in info.get(fq, {}).get("parents", []): - dfs_ifaces(i) - - return ordered - - # ---------------- reverse indices (downwards) ---------------- - rev_extends: Dict[str, List[str]] = {} # class -> direct subclasses - rev_iface_extends: Dict[str, List[str]] = {} # interface -> direct sub-interfaces - direct_impl: Dict[str, List[str]] = {} # interface -> classes that directly list it - - for fq, rec in info.items(): - for p in rec.get("parents", []): - if not p: continue - if is_iface(fq) and is_iface(p): - rev_iface_extends.setdefault(p, []).append(fq) + visiting.add(v) + for p_fq in r.get("parents", []): + p_rec = pick_variant(p_fq, jar) + if p_rec: + dfs_iface_vid(vid_of(p_rec)) + visiting.remove(v) + add_vid(v) + + # direct implements + for i_fq in rec.get("impls", []): + i_rec = pick_variant(i_fq, jar) + if i_rec: + dfs_iface_vid(vid_of(i_rec)) + + # via class ancestors (with visited guard on upward chain) + cur_r = rec + seen_up = set() + while True: + ps = cur_r.get("parents", []) + if not ps: + break + p_fq = ps[0] + p_rec = pick_variant(p_fq, jar) + if not p_rec or is_iface_rec(p_rec): + break + p_vid = vid_of(p_rec) + if p_vid in seen_up: + break + seen_up.add(p_vid) + for i_fq in p_rec.get("impls", []): + i2_rec = pick_variant(i_fq, jar) + if i2_rec: + dfs_iface_vid(vid_of(i2_rec)) + cur_r = p_rec + + # if start is an interface, include its super-interfaces + if is_iface_rec(rec): + for i_fq in rec.get("parents", []): + i_rec = pick_variant(i_fq, jar) + if i_rec: + dfs_iface_vid(vid_of(i_rec)) + + iface_closure_vids_memo[start_vid] = ordered_vids + return ordered_vids + + # --- helper: transitive super-interfaces as PAIRS, jar-aware, memoized --- + iface_super_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def iface_super_pairs(ik_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For an interface variant vid, return all transitive super-interfaces + as (simple, source) pairs, jar-aware. Memoized. + """ + if ik_vid in iface_super_pairs_memo: + return iface_super_pairs_memo[ik_vid] + + rec = rec_by_vid.get(ik_vid) + if not rec or rec["kind"] != "interface": + iface_super_pairs_memo[ik_vid] = [] + return [] + + jar = rec["jar"] + cur_src = rec["source"] + + ordered: List[Tuple[str, str]] = [] + added: Set[Tuple[str, str]] = set() + visiting: Set[Tuple[str, str]] = set() + + def add_pair(p: Tuple[str, str]): + if p and p not in added: + added.add(p) + ordered.append(p) + + def dfs_iface_vid(v: Tuple[str, str]): + if v in visiting: + return + r = rec_by_vid.get(v) + if not r or r["kind"] != "interface": + return + visiting.add(v) + for p_fq in r.get("parents", []): + p_rec = pick_variant(p_fq, jar) + if p_rec: + pk = (p_rec["fqcn"], p_rec["source"]) + dfs_iface_vid(pk) + add_pair((p_rec["simple"], p_rec["source"])) + else: + # unknown external interface -> placeholder bound to current interface's source + add_pair((simple_name_from_fqcn(p_fq), cur_src)) + visiting.remove(v) + + dfs_iface_vid(ik_vid) + out = tup_dedup_keep_order(ordered) + iface_super_pairs_memo[ik_vid] = out + return out + + # pairs version (placeholders for externals) with visited guard + all_ifaces_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def all_interfaces_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For a class/record/enum: direct interfaces + super-interfaces of those interfaces + + interfaces from ancestor classes (and their super-interfaces). + For an interface: its super-interfaces. + Jar-aware; placeholders only for interfaces not present in provided set. + """ + if start_vid in all_ifaces_pairs_memo: + return all_ifaces_pairs_memo[start_vid] + + rec = rec_by_vid.get(start_vid) + if not rec: + all_ifaces_pairs_memo[start_vid] = [] + return [] + + jar = rec["jar"] + cur_src = rec["source"] + + ordered: List[Tuple[str, str]] = [] + added: Set[Tuple[str, str]] = set() + + def add_pair(p: Tuple[str, str]): + if p and p not in added: + added.add(p) + ordered.append(p) + + # Helper: add one interface (jar-aware) and all its super-interfaces (pairs) + def add_iface_and_supers(i_fq: str): + i_rec = pick_variant(i_fq, jar) + if i_rec: + ik = (i_rec["fqcn"], i_rec["source"]) + add_pair((i_rec["simple"], i_rec["source"])) + for sup in iface_super_pairs(ik): + add_pair(sup) else: - rev_extends.setdefault(p, []).append(fq) - for i in rec.get("impls", []): - if i: - direct_impl.setdefault(i, []).append(fq) - - # Precompute all interfaces per-type (for fast reverse-implementers) - all_ifaces_map: Dict[str, List[str]] = { fq: all_interfaces(fq) for fq in info.keys() } - - # Interface -> all implementers (classes), including via superclass inheritance - rev_impl_all: Dict[str, List[str]] = {} - for c, ifaces in all_ifaces_map.items(): - if is_iface(c): # only classes + # external/unseen -> placeholder (cannot know its supers) + add_pair((simple_name_from_fqcn(i_fq), cur_src)) + + # Direct implements on this type + for i_fq in rec.get("impls", []): + add_iface_and_supers(i_fq) + + # Interfaces via ancestor classes (jar-aware). Guard against cycles. + cur_r = rec + seen_up: Set[Tuple[str, str]] = set() + while True: + ps = cur_r.get("parents", []) + if not ps: + break + p_fq = ps[0] + p_rec = pick_variant(p_fq, jar) + if not p_rec or is_iface_rec(p_rec): + break + p_vid = (p_rec["fqcn"], p_rec["source"]) + if p_vid in seen_up: + break + seen_up.add(p_vid) + for i_fq in p_rec.get("impls", []): + add_iface_and_supers(i_fq) + cur_r = p_rec + + # If starting node itself is an interface, include its super-interfaces. + if is_iface_rec(rec): + for sup in iface_super_pairs(start_vid): + add_pair(sup) + + out = ordered + all_ifaces_pairs_memo[start_vid] = out + return out + + # reverse: interface implementers (vids only, memoized closures) + rev_impl_all: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + for vid, r in rec_by_vid.items(): + if is_iface_rec(r): continue - for i in ifaces: - rev_impl_all.setdefault(i, []).append(c) + for i_vid in all_interfaces_vids(vid): + i_rec = rec_by_vid.get(i_vid) + if i_rec and is_iface_rec(i_rec): + rev_impl_all.setdefault(i_vid, []).append(vid) - def all_subclasses(fq: str) -> List[str]: - out: List[str] = [] + def all_subclasses_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] seen = set() - q = list(rev_extends.get(fq, [])) + q = list(rev_extends.get(parent_vid, [])) while q: x = q.pop(0) - if x in seen: continue + if x in seen: + continue seen.add(x) out.append(x) q.extend(rev_extends.get(x, []) or []) return out - def all_subinterfaces(fq: str) -> List[str]: - out: List[str] = [] + def all_subinterfaces_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] seen = set() - q = list(rev_iface_extends.get(fq, [])) + q = list(rev_iface_extends.get(parent_vid, [])) while q: x = q.pop(0) - if x in seen: continue + if x in seen: + continue seen.add(x) out.append(x) q.extend(rev_iface_extends.get(x, []) or []) return out - def all_implementers_of_interface(i_fq: str) -> List[str]: - impls: List[str] = [] - impls.extend(rev_impl_all.get(i_fq, [])) - for sub_i in all_subinterfaces(i_fq): + def all_implementers_of_interface_variant(i_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + impls: List[Tuple[str, str]] = [] + impls.extend(rev_impl_all.get(i_vid, [])) + for sub_i in all_subinterfaces_variant(i_vid): impls.extend(rev_impl_all.get(sub_i, [])) # stable dedup seen = set() - out: List[str] = [] - for x in impls: - if x and x not in seen: - seen.add(x) - out.append(x) + out: List[Tuple[str, str]] = [] + for v in impls: + if v and v not in seen: + seen.add(v) + out.append(v) return out # ---------------- final map ---------------- out: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} - for fq, rec in info.items(): + + for vid, rec in rec_by_vid.items(): self_name = rec["simple"] source = rec.get("source", "") - up_chain = [ - (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") - for x in extends_chain(fq) - ] - iface_list = [ - (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") - for x in all_ifaces_map[fq] - ] - - down_list: List[Tuple[str, str]] = [] - if is_iface(fq): - impls = all_implementers_of_interface(fq) - down_list.extend( - (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") - for x in impls + up_chain_pairs = extends_chain_pairs(vid) # may include placeholders + iface_pairs = all_interfaces_pairs(vid) # may include placeholders + + down_pairs: List[Tuple[str, str]] = [] + if is_iface_rec(rec): + impl_vids = all_implementers_of_interface_variant(vid) # vids only + down_pairs.extend( + (rec_by_vid[v]["simple"], rec_by_vid[v]["source"]) for v in impl_vids ) else: - subs = all_subclasses(fq) - down_list.extend( - (simple_name_from_fqcn(x), info[x]["source"] if x in info else "") - for x in subs + subs_vids = all_subclasses_variant(vid) + down_pairs.extend( + (rec_by_vid[v]["simple"], rec_by_vid[v]["source"]) for v in subs_vids ) - vals = tup_dedup_keep_order([(self_name, source)] + up_chain + iface_list + down_list) + vals = tup_dedup_keep_order([(self_name, source)] + up_chain_pairs + iface_pairs + down_pairs) out[(self_name, source)] = vals return out From 58385cfd11b352c10a94084c67dd45bec7a76a74 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Wed, 24 Dec 2025 18:10:24 +0200 Subject: [PATCH 130/286] Added members inheritance (#158) * Added members inheritance * Bug Fixes and speed improvements - Fixed a bug where multiple generics causes a wrong list of inheritance classes - Fixed a bug where functions were marked for exclusion even though they shouldn't be because different method called could be made from the function - Fixed a bug where transitive dependencies documents were not scanned because of the way the maven dependency tree is being built. - Added LRU caches to improve performance in frequently called methods Signed-off-by: Theodor Mihalache --- src/vuln_analysis/utils/dep_tree.py | 36 +- .../java_functions_parsers.py | 145 ++++- .../utils/java_chain_of_calls_retriever.py | 15 +- src/vuln_analysis/utils/java_utils.py | 494 ++++++++---------- 4 files changed, 382 insertions(+), 308 deletions(-) diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index 3aaf0adcf..fd983f839 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -25,6 +25,7 @@ from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager from vuln_analysis.logging.loggers_factory import LoggingFactory from vuln_analysis.utils.java_utils import add_missing_jar_string, is_maven_gav +from collections import deque logger = LoggingFactory.get_agent_logger(__name__) @@ -882,6 +883,7 @@ def __build_upside_down_dependency_graph( ) -> Tuple[str, Dict[str, List[str]]]: root: str = "" stack: List[str] = [] + # coord -> set of direct parents (possibly multiple) graph_sets: Dict[str, set] = {} for line in dependency_lines: @@ -910,10 +912,38 @@ def __build_upside_down_dependency_graph( stack.append(coord) - # no sorting, preserve insertion order of sets -> lists - graph: Dict[str, List[str]] = {k: list(v) for k, v in graph_sets.items()} - return root, graph + # ---------- second phase: all parents (direct + transitive) without duplicates ---------- + + def build_parent_chain(node: str) -> List[str]: + """ + For a given coord, return a flat list of *all* parents reachable + via any path up to the root, with no duplicates. + + Order: breadth-first from nearest parents outward. + """ + result: List[str] = [] + seen: set[str] = set() + + q = deque(graph_sets.get(node, ())) + while q: + parent = q.popleft() + if parent in seen: + continue + seen.add(parent) + result.append(parent) + # enqueue this parent's parents + for gp in graph_sets.get(parent, ()): + if gp not in seen: + q.append(gp) + + return result + + graph: Dict[str, List[str]] = { + coord: build_parent_chain(coord) for coord in graph_sets.keys() + } + + return root, graph def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[str]]: if not line.startswith("[INFO]"): diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py index 1596297de..28031273f 100644 --- a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py @@ -1,7 +1,7 @@ import hashlib import os import re -from typing import Dict, Tuple, Any, List, Optional +from typing import Dict, Tuple, List, Optional from tqdm import tqdm from langchain_core.documents import Document @@ -27,39 +27,142 @@ TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') class JavaLanguageFunctionsParser(LanguageFunctionsParser): - def parse_all_type_struct_class_to_fields(self, types: list[Document], inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]]) -> dict[tuple, list[tuple[str, str]]]: + def parse_all_type_struct_class_to_fields( + self, + types: list[Document], + inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]] + ) -> dict[tuple, list[tuple[str, str]]]: """ Returns (TypeSimpleName, doc.metadata['source']) -> [(memberName, memberType), ...] - Enums are ignored entirely. - - Interfaces: returns ONLY fields (constants). No default/static methods. - - Classes/records: returns fields. + - Interfaces: includes only fields (constants). + - Classes/records: includes fields. + - Also includes inherited fields (from superclasses and implemented interfaces) using BFS. + - Uses only the existing `inheritance_map`. + - Critically, it classifies *upstream* (parents/interfaces) via relative position indices to avoid pulling in descendants. """ + from collections import deque + from typing import Dict, Tuple, Any, List + results: Dict[Tuple[str, Any], List[Tuple[str, str]]] = {} + # 1) Collect own fields per document + per_key_fields: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} for doc in tqdm(types, total=len(types), desc="Parsing class/interface/enum/record documents for members..."): src = doc.page_content source = doc.metadata['source'] if not src: continue - - # One pass per document — returns {type_name: [(field, type), ...]} per_file = collect_fields_from_types(src, include_nested=True, include_anonymous=True) - for class_name, fields in per_file.items(): - results[(class_name, source)] = fields - - # # TODO add inherited fields support - # for (result_class_name, result_source), fields in results.items(): - # # Get inherited classes - # try: - # list_of_types = inheritance_map[(result_class_name, result_source)] - # except KeyError: - # i = 1 - # - # for (inherited_class_name, inherited_source) in list_of_types: - # if inherited_class_name != result_class_name and inherited_source != result_source: - # inherited_fields = results[(inherited_class_name, inherited_source)] - # fields.extend(inherited_fields) + per_key_fields[(class_name, source)] = fields + + # 2) Precompute: row lists and O(1) index lookups for each key + rows: Dict[Tuple[str, str], List[Tuple[str, str]]] = inheritance_map + row_index: Dict[Tuple[str, str], Dict[Tuple[str, str], int]] = {} + for k, lst in rows.items(): + # stable O(1) index lookup + row_index[k] = {v: i for i, v in enumerate(lst)} + + # 3) Upstream classifier (parents + interfaces) using relative index rule: + # candidate is upstream of type_key <=> candidate ∈ rows[type_key], + # type_key ∈ rows[candidate], and + # row_index[type_key][candidate] < row_index[candidate][type_key] + def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + Return the direct upstream types (superclasses and implemented interfaces) for `type_key` + using the relative-index rule over `rows` and `row_index`. + """ + type_row = rows.get(type_key) + if not type_row: + return [] + + # Fast lookup: neighbor -> index in type_key's row + type_index_map = row_index.get(type_key, {}) + + upstreams: List[Tuple[str, str]] = [] + seen_candidates: set[Tuple[str, str]] = set() + + for candidate in type_row: + if candidate == type_key or candidate in seen_candidates: + continue + + # Index of candidate within type_key's row + idx_candidate_in_type = type_index_map.get(candidate, 1 << 30) + + # Index of type_key within candidate's row + candidate_index_map = row_index.get(candidate) + if not candidate_index_map: + continue + idx_type_in_candidate = candidate_index_map.get(type_key) + if idx_type_in_candidate is None: + continue + + # Ancestor/interface test via relative positions + if idx_candidate_in_type < idx_type_in_candidate: + upstreams.append(candidate) + seen_candidates.add(candidate) + + return upstreams + + # 4) Merge own + inherited fields per type using BFS over upstream neighbors. + for type_key, own_fields in per_key_fields.items(): + merged: List[Tuple[str, str]] = [] + have_names: set[str] = set() # child-over-parent shadowing + + # Own first (preserve order) + for (fname, ftype) in own_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + + # BFS over upstreams + q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) + visited: set[Tuple[str, str]] = set() + + while q: + u = q.popleft() + if u in visited: + continue + visited.add(u) + + upstream_fields = per_key_fields.get(u) + if upstream_fields: + for (fname, ftype) in upstream_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + + for next_upstream in upstream_neighbors(u): + if next_upstream not in visited: + q.append(next_upstream) + + results[type_key] = merged + + # 5) Include types that had no own fields but may inherit some + for type_key in rows.keys(): + if type_key in results: + continue + merged: List[Tuple[str, str]] = [] + have_names: set[str] = set() + q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) + visited: set[Tuple[str, str]] = set() + while q: + u = q.popleft() + if u in visited: + continue + visited.add(u) + upstream_fields = per_key_fields.get(u) + if upstream_fields: + for (fname, ftype) in upstream_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + for next_upstream in upstream_neighbors(u): + if next_upstream not in visited: + q.append(next_upstream) + if merged: + results[type_key] = merged return results diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py index 8d06f966f..800126a42 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -110,8 +110,9 @@ def __init__(self, documents: List[Document], self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in documents if doc.metadata.get('content_type') == 'simplified_code'} - # Create function list with the function existing in the dependencies - self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' + # Filter function list with the function existing in the dependencies + self.documents = [doc for doc in tqdm(self.documents_of_functions, total=len(self.documents_of_functions) , desc="Filtering function list to functions existing in the dependencies") + if ('content_type' in doc.metadata and doc.metadata['content_type'] == 'functions_classes') and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or any(parent for parent in self.supported_packages if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] @@ -231,15 +232,15 @@ def __find_caller_function(self, document_function: Document, function_package: self.documents_of_functions, type_inheritance=self.type_inheritance) + method_name = extract_method_name_with_params(doc.page_content) + key = (doc.metadata['source'], function_name_to_search, target_class_names, method_name) + method_exclusions[key] = True + # If current document is found to be calling document_function in function_package, then return it as a # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. if function_is_being_called: package_exclusions.append(doc) return doc - else: - method_name = extract_method_name_with_params(doc.page_content) - key = (doc.metadata['source'], function_name_to_search, target_class_names, method_name) - method_exclusions[key] = True # If didn't find a matching caller function document, returns None. return None @@ -286,12 +287,10 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio if sources_location_packages: filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') and self.language_parser.is_function(doc) and - not self._is_doc_excluded(doc, exclusions) and not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] else: filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and self.language_parser.is_function(doc) and - not self._is_doc_excluded(doc, exclusions) and not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] diff --git a/src/vuln_analysis/utils/java_utils.py b/src/vuln_analysis/utils/java_utils.py index 378d9f99c..84253954f 100644 --- a/src/vuln_analysis/utils/java_utils.py +++ b/src/vuln_analysis/utils/java_utils.py @@ -99,7 +99,7 @@ def _convert_to_maven_artifact(package_name: str) -> str: return package_name -@lru_cache(maxsize=4096) +@lru_cache(maxsize=65536) def extract_jar_name(source: str) -> str: parts = source.split('/') @@ -719,7 +719,7 @@ def collect_type(kind: str, name: str, lb: int, rb: int): return out -@lru_cache(maxsize=8192) +@lru_cache(maxsize=150000) def get_type_name(src: str) -> str: """ Return the simple name of the first class/interface/enum/record declared @@ -838,7 +838,7 @@ def word_at(pos: int, word: str) -> bool: return "" -@lru_cache(maxsize=16384) +@lru_cache(maxsize=2500000) def is_java_method(source: str) -> bool: s = source n = len(s) @@ -1531,7 +1531,7 @@ def skip_type_list(p: int) -> int: return False -def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: +def create_inheritance_map(java_types: List["Document"]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: """ Build a jar-aware inheritance/implements map. @@ -1542,15 +1542,26 @@ def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], - implemented interfaces (direct, via ancestors, and via super-interfaces) - transitive subclasses / implementers (only for types present in the set) - Jar-aware rule for resolvable related types: - - If an FQCN has a variant in the same jar, pick that. + Jar-aware selection rule for related types: + - If an FQCN has a variant in the same jar as the declaring type, pick that. - Else if it has exactly one variant globally, pick that. - - Else (multiple variants across jars and none in this jar), skip. + - Else (multiple variants across jars and none in this jar), skip wiring to avoid ambiguity. - External/unseen types (e.g., JDK’s Cloneable/Serializable/AbstractMap/Map) - are INCLUDED in the upward/interface lists as placeholders: + External/unseen types (e.g., JDK classes/interfaces) are INCLUDED in the + upward/interface lists as placeholders with the current type's source path: (simple_name, current_source_of_key) Placeholders are NOT used to wire reverse edges (subclasses/implementers). + + Parsing guarantees: + - Robust to comments and annotations. + - Explicit-imports-first resolution; star imports allowed. + - Order-preserving, no duplicates. + - Critically, generic type-parameter bounds are ignored: + e.g., in `interface X extends A, B { ... }`, only `A, B` are + treated as super-interfaces. Any `extends` tokens inside the `<...>` generic + parameter list are NOT misinterpreted as inheritance clauses. + - As a final safety net, any residual tokens that still contain the word + `extends` (indicative of generic bounds accidentally leaking through) are dropped. """ # ---------------- helpers ---------------- @@ -1596,14 +1607,14 @@ def drop_leading_annotations(s: str) -> str: while i < n and (s[i].isalnum() or s[i] in ('_', '$', '.')): i += 1 if i < n and s[i] == '(': - depth = 1 + paren_depth = 1 i += 1 - while i < n and depth > 0: + while i < n and paren_depth > 0: ch = s[i] if ch == '(': - depth += 1 + paren_depth += 1 elif ch == ')': - depth -= 1 + paren_depth -= 1 i += 1 continue break @@ -1612,48 +1623,36 @@ def drop_leading_annotations(s: str) -> str: def base_token(t: str) -> str: """Strip generics and [] to get a core type token.""" t = (t or "").strip() - out, depth = [], 0 + out, generics_depth = [], 0 for ch in t: if ch == '<': - depth += 1 + generics_depth += 1 elif ch == '>': - depth = max(0, depth - 1) - elif depth == 0: + generics_depth = max(0, generics_depth - 1) + elif generics_depth == 0: out.append(ch) return ''.join(out).replace("[]", "").strip() - def simple_name_from_fqcn(fq: str) -> str: - return fq.split('.')[-1] if fq else fq + def simple_name_from_fqcn(fqcn: str) -> str: + return fqcn.split('.')[-1] if fqcn else fqcn + + def clean_type_token(tok: str) -> str: + """ + Defensive cleanup: drop tokens that look like generic bounds fragments, + e.g., 'T extends Foo' or 'B extends SdkBuilder'. + """ + t = tok.strip() + if not t: + return "" + # If a bare 'extends' remains in a token here, it can only be a generic bound. + if re.search(r"\bextends\b", t): + return "" + return t def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: """ Remove duplicate (name, source) pairs while preserving the order of their first appearance. - - Parameters - ---------- - pairs : List[Tuple[str, str]] - A list of (simple_type_name, source_path) tuples. Two entries are - considered duplicates only if BOTH the name and source are identical. - - Returns - ------- - List[Tuple[str, str]] - A new list containing the first occurrence of each unique pair, in the - same relative order as `pairs`. - - Examples - -------- - >>> tup_dedup_keep_order([("A", "/src/A.java"), ("B", "/src/B.java"), ("A", "/src/A.java")]) - [('A', '/src/A.java'), ('B', '/src/B.java')] - - # Different source_path → not a duplicate - >>> tup_dedup_keep_order([("A", "/src/A.java"), ("A", "/other/A.java")]) - [('A', '/src/A.java'), ('A', '/other/A.java')] - - # Empty input - >>> tup_dedup_keep_order([]) - [] """ seen: set[Tuple[str, str]] = set() out: List[Tuple[str, str]] = [] @@ -1680,73 +1679,99 @@ def split_type_list_strict(segment: str) -> List[str]: if not segment: return [] items, cur = [], [] - depth_ang = depth_par = 0 + generics_depth = 0 + paren_depth = 0 for ch in segment: - if ch == '<': depth_ang += 1 - elif ch == '>': depth_ang = max(0, depth_ang - 1) - elif ch == '(': depth_par += 1 - elif ch == ')': depth_par = max(0, depth_par - 1) - if ch == ',' and depth_ang == 0 and depth_par == 0: + if ch == '<': + generics_depth += 1 + elif ch == '>': + generics_depth = max(0, generics_depth - 1) + elif ch == '(': + paren_depth += 1 + elif ch == ')': + paren_depth = max(0, paren_depth - 1) + if ch == ',' and generics_depth == 0 and paren_depth == 0: tok = ''.join(cur).strip() - if tok: items.append(tok) + if tok: + items.append(tok) cur = [] else: cur.append(ch) last = ''.join(cur).strip() - if last: items.append(last) + if last: + items.append(last) return items # Clause extractor that starts scanning AFTER the 'kind name' match. name_kind_re = re.compile(r"""\b(class|interface|enum|record)\b\s+([A-Za-z_$][\w$]*)""", re.X) def extract_clauses_after_kindname(header: str) -> Tuple[str | None, str | None]: + """ + Structured scan for 'extends' and 'implements' clauses while ignoring + anything inside generic parameter lists after the type name. + """ m = name_kind_re.search(header) if not m: return None, None scan_start = m.end() # start right after 'class|interface|enum|record ' def is_word_at(pos: int, word: str) -> bool: - if pos < 0 or pos + len(word) > len(header): return False - if header[pos:pos+len(word)] != word: return False + if pos < 0 or pos + len(word) > len(header): + return False + if header[pos:pos+len(word)] != word: + return False left_ok = pos == 0 or not (header[pos-1].isalnum() or header[pos-1] == '_') right_ok = pos+len(word) == len(header) or not (header[pos+len(word)].isalnum() or header[pos+len(word)] == '_') return left_ok and right_ok - depth_ang = depth_par = 0 + generics_depth = 0 + paren_depth = 0 ext_start = impl_start = None i = scan_start while i < len(header): ch = header[i] - if ch == '<': depth_ang += 1 - elif ch == '>': depth_ang = max(0, depth_ang - 1) - elif ch == '(': depth_par += 1 - elif ch == ')': depth_par = max(0, depth_par - 1) - elif depth_ang == 0 and depth_par == 0: + if ch == '<': + generics_depth += 1 + elif ch == '>': + generics_depth = max(0, generics_depth - 1) + elif ch == '(': + paren_depth += 1 + elif ch == ')': + paren_depth = max(0, paren_depth - 1) + elif generics_depth == 0 and paren_depth == 0: if ext_start is None and is_word_at(i, "extends"): j = i + len("extends") - while j < len(header) and header[j].isspace(): j += 1 + while j < len(header) and header[j].isspace(): + j += 1 ext_start = j i = j continue if impl_start is None and is_word_at(i, "implements"): j = i + len("implements") - while j < len(header) and header[j].isspace(): j += 1 + while j < len(header) and header[j].isspace(): + j += 1 impl_start = j i = j continue i += 1 def segment_until_next(start: int) -> str | None: - if start is None: return None - depth_a = depth_p = 0 + if start is None: + return None + seg_generics_depth = 0 + seg_paren_depth = 0 k = start while k < len(header): - c = header[k] - if c == '<': depth_a += 1 - elif c == '>': depth_a = max(0, depth_a - 1) - elif c == '(': depth_p += 1 - elif c == ')': depth_p = max(0, depth_p - 1) - elif depth_a == 0 and depth_p == 0: + ch = header[k] + if ch == '<': + seg_generics_depth += 1 + elif ch == '>': + seg_generics_depth = max(0, seg_generics_depth - 1) + elif ch == '(': + seg_paren_depth += 1 + elif ch == ')': + seg_paren_depth = max(0, seg_paren_depth - 1) + elif seg_generics_depth == 0 and seg_paren_depth == 0: if is_word_at(k, "extends") or is_word_at(k, "implements"): break k += 1 @@ -1756,16 +1781,6 @@ def segment_until_next(start: int) -> str | None: ext_seg = segment_until_next(ext_start) impl_seg = segment_until_next(impl_start) - # Fallbacks: broad but safe, used only when structured scan yields nothing. - if ext_seg is None and "extends" in header: - m_ext = re.search(r"\bextends\b\s+([^{};]+?)(?:\bimplements\b|$)", header) - if m_ext: - ext_seg = m_ext.group(1).strip() - if impl_seg is None and "implements" in header: - m_impl = re.search(r"\bimplements\b\s+([^{};]+)$", header) - if m_impl: - impl_seg = m_impl.group(1).strip() - return ext_seg, impl_seg # Inline-friendly (work on full source) @@ -1797,7 +1812,8 @@ def source_jar_id(path: str) -> str: idx = src_all.lower().find("code for:") if idx != -1: line_end = src_all.find('\n', idx) - if line_end == -1: line_end = len(src_all) + if line_end == -1: + line_end = len(src_all) header_line = src_all[idx:line_end] header_text_raw = re.sub(r".*?Code\s*for\s*:\s*", "", header_line, flags=re.I).strip() else: @@ -1828,8 +1844,8 @@ def source_jar_id(path: str) -> str: else: kind, parsed_name = m_name.group(1), m_name.group(2) ext_seg, impl_seg = extract_clauses_after_kindname(header_text) - # LAST RESORT: scan de-commented full source for "class ... extends ..." - if (not ext_seg) and ("extends" in src_all): + # LAST RESORT (classes only): scan de-commented full source for "class ... extends ..." + if kind == "class" and (not ext_seg) and ("extends" in src_all): s_nc = strip_comments_preserving_newlines(src_all) m_ext2 = re.search( rf"\bclass\s+{re.escape(parsed_name)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", @@ -1840,6 +1856,10 @@ def source_jar_id(path: str) -> str: extends_names = split_type_list_strict(ext_seg) if ext_seg else [] implements_names = split_type_list_strict(impl_seg) if impl_seg else [] + # Defensive cleanup to avoid generic bounds leaking as parents/interfaces + extends_names = [clean_type_token(x) for x in extends_names if clean_type_token(x)] + implements_names = [clean_type_token(x) for x in implements_names if clean_type_token(x)] + file_simple = simple_name_from_filename(doc) simple_name = file_simple or parsed_name fqcn = f"{pkg}.{simple_name}" if pkg else simple_name @@ -1860,104 +1880,23 @@ def source_jar_id(path: str) -> str: # simple -> known FQCNs by_simple: Dict[str, List[str]] = {} - for fq in known_fqcns: - by_simple.setdefault(simple_name_from_fqcn(fq), []).append(fq) + for fqcn in known_fqcns: + by_simple.setdefault(simple_name_from_fqcn(fqcn), []).append(fqcn) def make_resolver(pkg: str, imports: List[str]): """ Build a simple type resolver for a compilation unit. - - Purpose - ------- - Given the package declaration (`pkg`) and the list of non-static import - statements (`imports`) from a single Java source file, this factory - returns a closure `resolve(token: str) -> str` that maps a *type token* - (possibly with generics/arrays and possibly unqualified) to a best-effort - fully qualified class name (FQCN) **string**, when it can be determined - unambiguously from what is already known. - - Process (high level) - -------------------- - 1) Precompute two import helpers: - - `explicit`: map of simple class name -> explicit import (e.g. "List" -> "java.util.List"). - - `stars`: list of star-imported packages (e.g. "java.util" for "java.util.*"). - - 2) The returned `resolve` closure: - - Normalizes the input token by stripping generics and `[]` to a base type. - - If the token is already qualified (contains a '.'), return it unchanged. - - If it matches an explicit import, return that FQCN. - - Otherwise try `.` if such a type is known in `known_fqcns`. - - Otherwise check star imports: if exactly one candidate in star packages exists, return it. - - Otherwise, if the simple name is globally unique among `known_fqcns`, return that FQCN. - - If none of the above rules yield an unambiguous answer, return the **simple name** unchanged. - - Return value of the closure - --------------------------- - The inner `resolve(token)` returns a **string**: - * either a fully qualified class name (e.g., "java.util.List") - * or, if resolution is ambiguous/unknown, the original *simple* type name - (e.g., "List"). It never returns `None`. - - Notes - ----- - - `known_fqcns` and `by_simple` are expected to be available in the enclosing scope: - * `known_fqcns: set[str]` — all FQCNs discovered across parsed units - * `by_simple: Dict[str, List[str]]` — simple name -> list of FQCNs having that simple name - - `base_token()` and `simple_name_from_fqcn()` are helpers also available in the enclosing scope. - - Parameters - ---------- - pkg : str - The package name declared in the current Java file (e.g., "org.example.util"). - imports : List[str] - A list of non-static import strings as they appear in the file - (e.g., ["java.util.List", "org.example.model.*"]). - - Returns - ------- - str - A resolver function that takes a type token and returns an FQCN string - or the original simple name if resolution is not unique. + See top-level docstring for resolution precedence. """ explicit: Dict[str, str] = {} - stars: List[str] = [] + star_import_packages: List[str] = [] for imp in imports: if imp.endswith(".*"): - stars.append(imp[:-2]) # package of a star import + star_import_packages.append(imp[:-2]) # package of a star import else: explicit[simple_name_from_fqcn(imp)] = imp # simple -> explicit FQCN def resolve(token: str) -> str: - """ - Resolve a type token to an FQCN when possible. - - Purpose - ------- - Convert a possibly decorated and/or unqualified Java type reference - (e.g., "List", "Foo[]", "Foo") into its canonical fully - qualified class name when it can be determined unambiguously using: - - explicit imports, - - the current package, - - star imports, - - or global uniqueness among known types. - - Process (concise) - ----------------- - 1) Normalize `token` to its base (strip generics and array suffixes). - 2) If already qualified, return as-is. - 3) Check explicit imports. - 4) Check current package (`pkg`). - 5) Check star-import packages, but only if it yields exactly one match. - 6) If the simple name is globally unique among `known_fqcns`, return that FQCN. - 7) Otherwise, return the **simple name** unchanged (ambiguous/unknown). - - Returns - ------- - str - Either a fully qualified class name (e.g., "java.util.Map") or, - if the type cannot be resolved uniquely, the original simple name - (e.g., "Map"). Never returns None. - """ tok = base_token(token) if not tok: return "" # empty token @@ -1971,9 +1910,9 @@ def resolve(token: str) -> str: return candidate # Star imports – accept only a unique hit candidates: List[str] = [] - for fq in by_simple.get(tok, []): - if any(fq.startswith(base + ".") for base in stars): - candidates.append(fq) + for fqcn in by_simple.get(tok, []): + if any(fqcn.startswith(base + ".") for base in star_import_packages): + candidates.append(fqcn) if len(candidates) == 1: return candidates[0] # Globally unique simple name across all known types @@ -1986,7 +1925,7 @@ def resolve(token: str) -> str: return resolve # resolve parents/impls per variant - for fq, var_list in variants_by_fq.items(): + for fqcn, var_list in variants_by_fq.items(): for rec in var_list: resolve = make_resolver(rec["pkg"], rec["imports"]) if rec["kind"] == "record": @@ -2011,12 +1950,14 @@ def vid_of(rec: Dict) -> Tuple[str, str]: for rec in variants: rec_by_fq_and_jar[(rec["fqcn"], rec["jar"])] = rec - def pick_variant(fq: str, jar: str) -> Dict | None: + def pick_variant_for_jar(type_fqcn: str, jar_id: str) -> Dict | None: """Pick the variant in the same jar if present; else unique globally; else None.""" - same = rec_by_fq_and_jar.get((fq, jar)) + if not type_fqcn: + return None + same = rec_by_fq_and_jar.get((type_fqcn, jar_id)) if same: return same - lst = variants_by_fq.get(fq) or [] + lst = variants_by_fq.get(type_fqcn) or [] if len(lst) == 1: return lst[0] return None # ambiguous across jars, and not in this jar @@ -2025,23 +1966,23 @@ def pick_variant(fq: str, jar: str) -> Dict | None: rev_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} rev_iface_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} - def is_iface_rec(rec: Dict) -> bool: - return rec["kind"] == "interface" + def is_interface_record(type_rec: Dict) -> bool: + return type_rec["kind"] == "interface" for rec in variants: - jar = rec["jar"] + jar_id = rec["jar"] this_vid = vid_of(rec) - for p_fq in rec.get("parents", []): - if not p_fq: + for parent_fqcn in rec.get("parents", []): + if not parent_fqcn: continue - p_rec = pick_variant(p_fq, jar) - if not p_rec: + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec: continue - p_vid = vid_of(p_rec) - if is_iface_rec(rec) and is_iface_rec(p_rec): - rev_iface_extends.setdefault(p_vid, []).append(this_vid) + parent_vid = vid_of(parent_rec) + if is_interface_record(rec) and is_interface_record(parent_rec): + rev_iface_extends.setdefault(parent_vid, []).append(this_vid) else: - rev_extends.setdefault(p_vid, []).append(this_vid) + rev_extends.setdefault(parent_vid, []).append(this_vid) # --------- upward chain & iface collectors with cycle guards --------- extends_chain_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} @@ -2058,30 +1999,30 @@ def extends_chain_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: cur = rec_by_vid.get(start_vid) if not cur: return [] - jar = cur["jar"] + jar_id = cur["jar"] chain: List[Tuple[str, str]] = [] - if is_iface_rec(cur): + if is_interface_record(cur): seen_vids = set() def dfs_iface(v: Tuple[str, str]): r = rec_by_vid.get(v) if not r: return - for p_fq in r.get("parents", []): - if not p_fq: + for parent_fqcn in r.get("parents", []): + if not parent_fqcn: continue - p_rec = pick_variant(p_fq, jar) - if p_rec: - p_vid = vid_of(p_rec) - if p_vid in seen_vids: + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + parent_vid = vid_of(parent_rec) + if parent_vid in seen_vids: continue - seen_vids.add(p_vid) - if is_iface_rec(p_rec): - dfs_iface(p_vid) - chain.append((p_rec["simple"], p_rec["source"])) + seen_vids.add(parent_vid) + if is_interface_record(parent_rec): + dfs_iface(parent_vid) + chain.append((parent_rec["simple"], parent_rec["source"])) else: - chain.append((simple_name_from_fqcn(p_fq), cur["source"])) + chain.append((simple_name_from_fqcn(parent_fqcn), cur["source"])) dfs_iface(start_vid) out = tup_dedup_keep_order(chain) @@ -2098,20 +2039,20 @@ def dfs_iface(v: Tuple[str, str]): ps = r.get("parents", []) if not ps: break - p_fq = ps[0] - p_rec = pick_variant(p_fq, jar) if p_fq else None - if p_rec: - p_vid = vid_of(p_rec) - if p_vid in visited_vids or p_vid == cur_vid: + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) if parent_fqcn else None + if parent_rec: + parent_vid = vid_of(parent_rec) + if parent_vid in visited_vids or parent_vid == cur_vid: break - visited_vids.add(p_vid) - chain.append((p_rec["simple"], p_rec["source"])) - if not is_iface_rec(p_rec): - cur_vid = p_vid + visited_vids.add(parent_vid) + chain.append((parent_rec["simple"], parent_rec["source"])) + if not is_interface_record(parent_rec): + cur_vid = parent_vid else: break else: - chain.append((simple_name_from_fqcn(p_fq), r["source"])) + chain.append((simple_name_from_fqcn(parent_fqcn), r["source"])) break out = tup_dedup_keep_order(chain) @@ -2133,7 +2074,7 @@ def all_interfaces_vids(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: if not rec: iface_closure_vids_memo[start_vid] = [] return [] - jar = rec["jar"] + jar_id = rec["jar"] ordered_vids: List[Tuple[str, str]] = [] added, visiting = set(), set() @@ -2148,22 +2089,22 @@ def dfs_iface_vid(v: Tuple[str, str]): r = rec_by_vid.get(v) if not r: return - if not is_iface_rec(r): + if not is_interface_record(r): add_vid(v) return visiting.add(v) - for p_fq in r.get("parents", []): - p_rec = pick_variant(p_fq, jar) - if p_rec: - dfs_iface_vid(vid_of(p_rec)) + for parent_fqcn in r.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + dfs_iface_vid(vid_of(parent_rec)) visiting.remove(v) add_vid(v) # direct implements - for i_fq in rec.get("impls", []): - i_rec = pick_variant(i_fq, jar) - if i_rec: - dfs_iface_vid(vid_of(i_rec)) + for iface_fqcn in rec.get("impls", []): + iface_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface_rec: + dfs_iface_vid(vid_of(iface_rec)) # via class ancestors (with visited guard on upward chain) cur_r = rec @@ -2172,26 +2113,26 @@ def dfs_iface_vid(v: Tuple[str, str]): ps = cur_r.get("parents", []) if not ps: break - p_fq = ps[0] - p_rec = pick_variant(p_fq, jar) - if not p_rec or is_iface_rec(p_rec): + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec or is_interface_record(parent_rec): break - p_vid = vid_of(p_rec) - if p_vid in seen_up: + parent_vid = vid_of(parent_rec) + if parent_vid in seen_up: break - seen_up.add(p_vid) - for i_fq in p_rec.get("impls", []): - i2_rec = pick_variant(i_fq, jar) - if i2_rec: - dfs_iface_vid(vid_of(i2_rec)) - cur_r = p_rec + seen_up.add(parent_vid) + for iface_fqcn in parent_rec.get("impls", []): + iface2_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface2_rec: + dfs_iface_vid(vid_of(iface2_rec)) + cur_r = parent_rec # if start is an interface, include its super-interfaces - if is_iface_rec(rec): - for i_fq in rec.get("parents", []): - i_rec = pick_variant(i_fq, jar) - if i_rec: - dfs_iface_vid(vid_of(i_rec)) + if is_interface_record(rec): + for parent_fqcn in rec.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + dfs_iface_vid(vid_of(parent_rec)) iface_closure_vids_memo[start_vid] = ordered_vids return ordered_vids @@ -2199,20 +2140,20 @@ def dfs_iface_vid(v: Tuple[str, str]): # --- helper: transitive super-interfaces as PAIRS, jar-aware, memoized --- iface_super_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} - def iface_super_pairs(ik_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + def iface_super_pairs(iface_vid: Tuple[str, str]) -> List[Tuple[str, str]]: """ For an interface variant vid, return all transitive super-interfaces as (simple, source) pairs, jar-aware. Memoized. """ - if ik_vid in iface_super_pairs_memo: - return iface_super_pairs_memo[ik_vid] + if iface_vid in iface_super_pairs_memo: + return iface_super_pairs_memo[iface_vid] - rec = rec_by_vid.get(ik_vid) + rec = rec_by_vid.get(iface_vid) if not rec or rec["kind"] != "interface": - iface_super_pairs_memo[ik_vid] = [] + iface_super_pairs_memo[iface_vid] = [] return [] - jar = rec["jar"] + jar_id = rec["jar"] cur_src = rec["source"] ordered: List[Tuple[str, str]] = [] @@ -2231,20 +2172,20 @@ def dfs_iface_vid(v: Tuple[str, str]): if not r or r["kind"] != "interface": return visiting.add(v) - for p_fq in r.get("parents", []): - p_rec = pick_variant(p_fq, jar) - if p_rec: - pk = (p_rec["fqcn"], p_rec["source"]) + for parent_fqcn in r.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + pk = (parent_rec["fqcn"], parent_rec["source"]) dfs_iface_vid(pk) - add_pair((p_rec["simple"], p_rec["source"])) + add_pair((parent_rec["simple"], parent_rec["source"])) else: # unknown external interface -> placeholder bound to current interface's source - add_pair((simple_name_from_fqcn(p_fq), cur_src)) + add_pair((simple_name_from_fqcn(parent_fqcn), cur_src)) visiting.remove(v) - dfs_iface_vid(ik_vid) + dfs_iface_vid(iface_vid) out = tup_dedup_keep_order(ordered) - iface_super_pairs_memo[ik_vid] = out + iface_super_pairs_memo[iface_vid] = out return out # pairs version (placeholders for externals) with visited guard @@ -2265,7 +2206,7 @@ def all_interfaces_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: all_ifaces_pairs_memo[start_vid] = [] return [] - jar = rec["jar"] + jar_id = rec["jar"] cur_src = rec["source"] ordered: List[Tuple[str, str]] = [] @@ -2277,20 +2218,20 @@ def add_pair(p: Tuple[str, str]): ordered.append(p) # Helper: add one interface (jar-aware) and all its super-interfaces (pairs) - def add_iface_and_supers(i_fq: str): - i_rec = pick_variant(i_fq, jar) - if i_rec: - ik = (i_rec["fqcn"], i_rec["source"]) - add_pair((i_rec["simple"], i_rec["source"])) + def add_iface_and_supers(iface_fqcn: str): + iface_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface_rec: + ik = (iface_rec["fqcn"], iface_rec["source"]) + add_pair((iface_rec["simple"], iface_rec["source"])) for sup in iface_super_pairs(ik): add_pair(sup) else: # external/unseen -> placeholder (cannot know its supers) - add_pair((simple_name_from_fqcn(i_fq), cur_src)) + add_pair((simple_name_from_fqcn(iface_fqcn), cur_src)) # Direct implements on this type - for i_fq in rec.get("impls", []): - add_iface_and_supers(i_fq) + for iface_fqcn in rec.get("impls", []): + add_iface_and_supers(iface_fqcn) # Interfaces via ancestor classes (jar-aware). Guard against cycles. cur_r = rec @@ -2299,20 +2240,20 @@ def add_iface_and_supers(i_fq: str): ps = cur_r.get("parents", []) if not ps: break - p_fq = ps[0] - p_rec = pick_variant(p_fq, jar) - if not p_rec or is_iface_rec(p_rec): + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec or is_interface_record(parent_rec): break - p_vid = (p_rec["fqcn"], p_rec["source"]) - if p_vid in seen_up: + parent_vid = (parent_rec["fqcn"], parent_rec["source"]) + if parent_vid in seen_up: break - seen_up.add(p_vid) - for i_fq in p_rec.get("impls", []): - add_iface_and_supers(i_fq) - cur_r = p_rec + seen_up.add(parent_vid) + for iface_fqcn in parent_rec.get("impls", []): + add_iface_and_supers(iface_fqcn) + cur_r = parent_rec # If starting node itself is an interface, include its super-interfaces. - if is_iface_rec(rec): + if is_interface_record(rec): for sup in iface_super_pairs(start_vid): add_pair(sup) @@ -2323,11 +2264,11 @@ def add_iface_and_supers(i_fq: str): # reverse: interface implementers (vids only, memoized closures) rev_impl_all: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} for vid, r in rec_by_vid.items(): - if is_iface_rec(r): + if is_interface_record(r): continue for i_vid in all_interfaces_vids(vid): i_rec = rec_by_vid.get(i_vid) - if i_rec and is_iface_rec(i_rec): + if i_rec and is_interface_record(i_rec): rev_impl_all.setdefault(i_vid, []).append(vid) def all_subclasses_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, str]]: @@ -2356,10 +2297,10 @@ def all_subinterfaces_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, st q.extend(rev_iface_extends.get(x, []) or []) return out - def all_implementers_of_interface_variant(i_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + def all_implementers_of_interface_variant(iface_vid: Tuple[str, str]) -> List[Tuple[str, str]]: impls: List[Tuple[str, str]] = [] - impls.extend(rev_impl_all.get(i_vid, [])) - for sub_i in all_subinterfaces_variant(i_vid): + impls.extend(rev_impl_all.get(iface_vid, [])) + for sub_i in all_subinterfaces_variant(iface_vid): impls.extend(rev_impl_all.get(sub_i, [])) # stable dedup seen = set() @@ -2381,7 +2322,7 @@ def all_implementers_of_interface_variant(i_vid: Tuple[str, str]) -> List[Tuple[ iface_pairs = all_interfaces_pairs(vid) # may include placeholders down_pairs: List[Tuple[str, str]] = [] - if is_iface_rec(rec): + if is_interface_record(rec): impl_vids = all_implementers_of_interface_variant(vid) # vids only down_pairs.extend( (rec_by_vid[v]["simple"], rec_by_vid[v]["source"]) for v in impl_vids @@ -2397,6 +2338,7 @@ def all_implementers_of_interface_variant(i_vid: Tuple[str, str]) -> List[Tuple[ return out +@lru_cache(maxsize=2500000) def extract_method_name_with_params(java_src: str) -> str | None: """ Return 'methodName()' from a Java method header string. @@ -2612,7 +2554,7 @@ def find_function(caller_src:str, method_name:str, documents_of_functions:list[D def get_target_class_names(inheritance_list: List[Tuple[str, str]]) -> frozenset[str]: return frozenset([class_name for (class_name, source) in inheritance_list]) -@lru_cache(maxsize=4096) +@lru_cache(maxsize=80000) def strip_java_generics(type_str: str) -> str: """ Remove all balanced <...> generic sections from a Java type string. From f5f57dc9721034958ac0deaba3e319a65681cf84 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Mon, 29 Dec 2025 11:07:29 +0200 Subject: [PATCH 131/286] Java transitive search constructor support (#169) * Added constructor support to java transitive search Signed-off-by: Theodor Mihalache --- .../tests/test_transitive_code_search.py | 73 ++- .../java_functions_parsers.py | 250 +++++++--- .../utils/java_chain_of_calls_retriever.py | 69 ++- .../utils/java_segmenters_with_methods.py | 82 +-- src/vuln_analysis/utils/java_utils.py | 470 ++++++++++++------ 5 files changed, 644 insertions(+), 300 deletions(-) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 5e1a56115..2ff08c8eb 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -372,27 +372,52 @@ async def test_transitive_search_java_2(): assert 'src/main/java/io/cryostat' in document.metadata['source'] assert 'StringUtils.isBlank(' in document.page_content -# @pytest.mark.asyncio -# async def test_transitive_search_java_3(): -# transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() -# set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", -# git_ref="8f753753379e9381429b476aacbf6890ef101438", -# included_extensions=["**/*.java"], -# excluded_extensions=["target/**/*", -# "build/**/*", -# "*.class", -# ".gradle/**/*", -# ".mvn/**/*", -# ".gitignore", -# "test/**/*", -# "tests/**/*", -# "src/test/**/*", -# "pom.xml", -# "build.gradle"]) -# result = await transitive_code_search_runner_coroutine(",java.net.URLEncoder.encode") -# (path_found, list_path) = result -# print(result) -# assert path_found is True -# assert len(list_path) > 1 -# document = list_path[-1] -# assert 'src/main/java/io/cryostat' in document.metadata['source'] \ No newline at end of file +@pytest.mark.asyncio +async def test_transitive_search_java_3(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine(",java.net.URLEncoder.encode") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) > 1 + document = list_path[-1] + assert 'src/main/java/io/cryostat' in document.metadata['source'] + +@pytest.mark.asyncio +async def test_transitive_search_java_4(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("org.openjdk.jmc:flightrecorder.configuration:9.0.0,org.openjdk.jmc.flightrecorder.configuration.events.SchemaVersion.fromBeanVersion") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) > 1 + document = list_path[-1] + assert 'src/main/java/io/cryostat' in document.metadata['source'] \ No newline at end of file diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py index 28031273f..ba8259e96 100644 --- a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py @@ -214,6 +214,13 @@ def get_function_name(self, function: Document) -> str: # Disallow these as the *first* token of a "type" to avoid false positives (e.g., record header) DISALLOW_TYPE_HEAD = {"class","interface","enum","record","new","return"} + # Method modifiers we may see before a return type / constructor name + MODIFIERS = { + "public","protected","private","static","final","abstract","synchronized", + "native","strictfp","default","transient","volatile","sealed" + # 'non-sealed' is two tokens; irrelevant for methods here + } + WS = " \t\r\n\f\v" DIG = "0123456789" IDF_EXTRA = "_$" @@ -267,9 +274,6 @@ def prev_non_ws(s: str, idx: int): s = ''.join(out) n = len(s) - # Quick lambda presence flag (after stripping strings/comments) - saw_lambda_arrow = '->' in s - # --- tiny scanners on cleaned source --- def skip_ws(idx: int) -> int: while idx < n and s[idx] in WS: @@ -324,6 +328,25 @@ def skip_annotation(i: int) -> int: j = end + 1 return j + def begins_with_lambda() -> bool: + """Return True iff the snippet itself starts with a lambda: '(...) ->' or 'x ->'.""" + i0 = skip_ws(0) + if i0 >= n: + return False + if s[i0] == '(': + end = match_paren_close(i0) + if end != -1: + j = skip_ws(end + 1) + return j + 1 < n and s[j] == '-' and s[j + 1] == '>' + return False + if is_id_start(s[i0]): + j = i0 + 1 + while j < n and is_id_part(s[j]): + j += 1 + k = skip_ws(j) + return k + 1 < n and s[k] == '-' and s[k + 1] == '>' + return False + def parse_return_type(hstart: int, name_start: int) -> int: """ Try to parse a return type in [hstart, name_start). @@ -429,6 +452,48 @@ def parse_return_type(hstart: int, name_start: int) -> int: return -1 return k + def is_constructor_header(hstart: int, name_start: int) -> bool: + """ + Return True iff after skipping annotations, modifiers, and leading method type-parameters, + the next non-ws token starts exactly at 'name_start' (i.e., no return type present). + """ + i = skip_ws(hstart) + # skip annotations + modifiers + while True: + i = skip_ws(i) + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i); continue + j = i + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + if s[i:j] in MODIFIERS: + i = j + continue + break + + i = skip_ws(i) + # leading method type-parameters: + if i < n and s[i] == '<': + ni = skip_angle(i) + if ni == i: + return False + i = ni + i = skip_ws(i) + + # type-use annotations (rare before constructor name, but tolerate) + while i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i) + i = skip_ws(i) + + # Now, we should be exactly at the constructor simple name if it's a ctor + return i == name_start + + # --- lambda-at-beginning guard (do NOT classify methods that merely contain lambdas) --- + if begins_with_lambda(): + return "lambda" # TODO handle lambda correctly + # --- main scan: seek '(' and validate the candidate as a declaration --- i = 0 while True: @@ -467,8 +532,8 @@ def parse_return_type(hstart: int, name_start: int) -> int: # after ')': optional ws + optional 'throws ...' + then '{' or ';' p = skip_ws(close_idx + 1) - # lambda guard: "() ->" - if p < n and s[p] == '-' and p + 1 < n and s[p + 1] == '>': + # lambda guard: "() ->" pattern directly after params + if p + 1 < n and s[p] == '-' and s[p + 1] == '>': # it's a lambda parameter list, not a method decl i = open_idx + 1 continue @@ -522,18 +587,14 @@ def parse_return_type(hstart: int, name_start: int) -> int: h -= 1 h_start = h + 1 - # Require a valid return type and no extra identifier between type and name - if parse_return_type(h_start, start) == -1: - i = open_idx + 1 - continue - - return name + # Either a regular method (has a return type) OR a constructor (no return type) + if parse_return_type(h_start, start) != -1 or is_constructor_header(h_start, start): + return name - # --- fallback: if we saw a lambda arrow anywhere, treat this as a lambda-only snippet --- - if saw_lambda_arrow: - logger.info("get_function_name is called on a lambda function") - return "lambda" # e.g., "o -> o.getParameterTypes().length" TODO handle lambda correctly + # otherwise, keep scanning + i = open_idx + 1 + # No declaration found, and snippet didn't begin with a lambda return "" def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, @@ -637,52 +698,55 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: except ValueError: caller_function_body = src - # Fast regex: name followed by '(' with some receiver context allowed before it - pattern = re.compile( + # --- patterns --- + # Regular method calls to specific callee + method_pat = re.compile( rf'(?:\breturn\b\s*\(?[^;]*?)?(?:[\w$()\[\].]*?\.?)?\b{re.escape(callee_function_name)}\s*\(', re.MULTILINE ) - # Dedup by the (start,end) slice of the exact call expression we extracted + # Constructor calls for declaring type (only if the callee is a constructor) + declaring_simple = self.get_class_name_from_class_function(callee_function) + is_ctor_target = callee_function_name in declaring_simple + # Note: we capture any qualified name ending in declaring_simple; anonymous-class body after ')' is OK. + ctor_pat = re.compile( + rf'\bnew\s+((?:[A-Za-z_$][\w$]*\.)*{re.escape(declaring_simple)})\s*\(' + ) if is_ctor_target else None + + # Dedup by (start,end) slice we attempt to resolve seen_slices = set() # Target class names for the resolver - target_class_names: frozenset[str] - class_name = self.get_class_name_from_class_function(callee_function) - key = (class_name, callee_function.metadata['source']) + key = (declaring_simple, callee_function.metadata['source']) if "dummy" not in callee_function_file_name: target_class_names = get_target_class_names(type_inheritance[key]) else: - target_class_names = frozenset([class_name]) - - for m in pattern.finditer(caller_function_body): - call = m.group(0) - if not m or not call: - continue + target_class_names = frozenset([declaring_simple]) - open_paren_pos = m.end(0) - 1 # the '(' - # Balance to closing ')' + # --- helper to process a match tuple uniformly --- + def _process_call(start_idx: int, open_paren_pos: int) -> bool: close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) if close_paren_pos == -1: - continue - - # This filters out method declarations (headers) - nxt = _next_token_after(caller_function_body, close_paren_pos) - if nxt == '{' or nxt == 'throws': - continue + return False - # Precisely slice from the start of the owning expression to the close paren - start_ctx = _expr_start_left(caller_function_body, m.start(), max_back=512) + # For methods we drop declarations by looking ahead; for constructors we keep + # anonymous class bodies, so we only apply this filter to method matches. + # The caller passes start_idx accordingly (method vs. ctor). + # Decide slice bounds and dedup + start_ctx = _expr_start_left(caller_function_body, start_idx, max_back=512) end_ctx = close_paren_pos + 1 slice_key = (start_ctx, end_ctx) if slice_key in seen_slices: - continue + return False seen_slices.add(slice_key) - # Robust resolver (handles casts, chains, static roots, helper returns, etc.) + # Build the identifier snippet we feed into the resolver: + # take the exact text from the expression start to '(' (inclusive) + ident_snippet = caller_function_body[start_ctx:open_paren_pos + 1] + if self.__check_identifier_resolved_to_callee_function_package( function=caller_function, - identifier_function=call, + identifier_function=ident_snippet, callee_package=callee_function_package, code_documents=code_documents, type_documents=type_documents, @@ -694,16 +758,45 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: callee_function_name=callee_function_name, type_inheritance=type_inheritance ): - logger.debug("__check_identifier_resolved_to_callee_function_package resolved successfully - callee_function_name = " - + callee_function_name + ", identifier_function = " - + call + ", target_class_names = " + str(target_class_names) + ", \ncaller_function_source = " + caller_function.metadata['source'] - + ", \ncaller_function = " + caller_function.page_content) + logger.debug("__check_identifier_resolved_to_callee_function_package resolved successfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}") return True - logger.debug("__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - callee_function_name = " - + callee_function_name + ", identifier_function = " - + call + ", target_class_names = " + str(target_class_names) + ", \ncaller_function_source = " + caller_function.metadata['source'] - + ", \ncaller_function = " + caller_function.page_content) + logger.debug("__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}") + return False + + if is_ctor_target and ctor_pat: + # --- Constructor matches (only when target is a ctor) --- + for m in ctor_pat.finditer(caller_function_body): + # m ends right after '(' of "new ... (" + open_paren_pos = m.end(0) - 1 + if _process_call(m.start(), open_paren_pos): + return True + else: + # --- Regular method matches --- + for m in method_pat.finditer(caller_function_body): + call = m.group(0) + if not call: + continue + + open_paren_pos = m.end(0) - 1 # '(' + close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) + if close_paren_pos == -1: + continue + + # Filter out method declarations (headers) + nxt = _next_token_after(caller_function_body, close_paren_pos) + if nxt == '{' or nxt == 'throws': + continue + + if _process_call(m.start(), open_paren_pos): + return True + return False def __check_identifier_resolved_to_callee_function_package( @@ -723,18 +816,17 @@ def __check_identifier_resolved_to_callee_function_package( ) -> bool: """ Decide if the found call expression (`identifier_function`) in `function` actually targets - the method in `callee_package`, supporting: + the method/constructor in `callee_package`, supporting: - Unqualified calls inside same package. - Static calls: ClassName.method(…) or pkg.ClassName.method(…) - - 'new' receivers: new Type(...).method(…) + - Constructor receivers: new Type(...).method(…) + - Direct constructors: new pkg.Type(...) - Arbitrary-depth chains: A.staticB(...).c().d(...).targetMethod(…) - Receiver that is a method call: foo().bar(…) → resolve foo()'s return type. - Unqualified helper call in same class: helper(…) → resolve helper() return type. - Receiver that is a cast: ((Type) x).target(…) → use Type as a strong hint. - SUPER handling uses `target_class_names` (parents/children set) — no file header parsing. """ - import re, hashlib - def _strip_return_and_ws(s: str) -> str: s = s.strip() if s.startswith("return"): @@ -876,13 +968,18 @@ def _peel_leading_cast(expr: str) -> tuple[str | None, str]: rest = m.group(2).strip() return last_type, rest - # reduce snippet to the actual callee + its immediate receiver + # SAFER: extract receiver + method; never builds an empty list or raises IndexError def _extract_recv_and_method(snippet: str) -> tuple[str | None, str | None]: s = snippet.strip() - m = re.search(r'([A-Za-z_$][\w$]*)\s*\($', s) or \ - (list(re.finditer(r'([A-Za-z_$][\w$]*)\s*\(', s))[-1] if re.search(r'\(', s) else None) + m = re.search(r'([A-Za-z_$][\w$]*)\s*\($', s) + if not m: + last = None + for mm in re.finditer(r'([A-Za-z_$][\w$]*)\s*\(', s): + last = mm + m = last if not m: return None, None + method = m.group(1) method_start = m.start(1) @@ -966,6 +1063,7 @@ def _type_token_matches_callee(type_token: str) -> bool: tt = re.sub(r'<[^<>]*>', '', type_token).strip() tt = re.sub(r'\s*(\[\])+$', '', tt) simple_tt = tt.rsplit('.', 1)[-1] + # IMPORTANT: If allow-list exists and the token's simple name is not in it, short-circuit False. if target_class_names and strip_java_generics(simple_tt) not in target_class_names: return False code_text = caller_text @@ -986,6 +1084,45 @@ def _caller_key() -> str: expr = _strip_return_and_ws(_strip_leading_this(identifier_function)) expr = re.sub(r'\s+', ' ', expr).strip() + def _extract_ctor_type(expr: str) -> str: + """ + Given an expression that starts with 'new ', return the constructor type token. + Handles qualified names, generics (including diamond), and array brackets. + Stops at the first '(' or '{' seen at top level (outside '<...>'). + Examples: + - 'new Foo(' -> 'Foo' + - 'new a.b.C(' -> 'a.b.C' + - 'new HashMap>(' -> 'HashMap>' + - 'new int[] {' -> 'int[]' + """ + i = 4 # skip 'new ' + n = len(expr) + # skip leading spaces + while i < n and expr[i].isspace(): + i += 1 + buf = [] + angle = 0 + while i < n: + ch = expr[i] + if ch == '<': + angle += 1 + buf.append(ch); i += 1; continue + if ch == '>': + if angle > 0: + angle -= 1 + buf.append(ch); i += 1; continue + if (ch == '(' or ch == '{') and angle == 0: + break + # include identifiers, dots, '$', arrays, spaces between [] if present + buf.append(ch); i += 1 + return ''.join(buf).strip() + + # ====== EARLY: direct constructor snippet 'new Type(...' ====== + # This covers cases where search_for_called_function fed us a constructor occurrence directly. + if expr.startswith('new '): + ctor_type_token = _extract_ctor_type(expr) + return _type_token_matches_callee(ctor_type_token) + recv_norm, method_norm = _extract_recv_and_method(expr) if method_norm: expr = (f"{recv_norm}.{method_norm}(" if recv_norm else f"{method_norm}(").strip() @@ -1006,7 +1143,6 @@ def _caller_key() -> str: # plain super.method(…) if recv_trim == 'super': caller_inheritance_list = get_target_class_names(type_inheritance[(caller_class_name, caller_src)]) - # Use target_class_names (parents/children set). Prefer excluding current class name if present. for cand in caller_inheritance_list: if cand == strip_java_generics(caller_class_name): continue @@ -1092,7 +1228,7 @@ def _caller_key() -> str: if caller_class_name and _type_token_matches_callee(caller_class_name): return True - # Case A: any static-style class root in the chain + # Case A: any static-style class root in the chain (includes 'new Type(...)' receivers) for seg in chain[:-1]: seg_core = _strip_call_parens(seg).strip() if seg_core.startswith("new "): diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py index 800126a42..5730d6d7f 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/java_chain_of_calls_retriever.py @@ -285,15 +285,17 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio target_class_names: frozenset[str], method_exclusions: dict) -> (list[Document], bool): if sources_location_packages: - filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') + result = [doc for doc in self.documents if package in doc.metadata.get('source') and self.language_parser.is_function(doc) and - not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] + not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + f"{function_name_to_search}(" in doc.page_content] else: - filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and + result = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and self.language_parser.is_function(doc) and - not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions)] + not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + f"{function_name_to_search}(" in doc.page_content] - return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] + return result # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). @@ -362,7 +364,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: self.tree_dict[package_name] = [parents, [], {}] # Create function list with the function existing in the dependencies - self.documents = [doc for doc in self.documents_of_functions if doc.metadata['content_type'] == 'functions_classes' + self.documents = [doc for doc in tqdm(self.documents_of_functions, total=len(self.documents_of_functions) , desc="Filtering function list to functions existing in the dependencies") if doc.metadata['content_type'] == 'functions_classes' and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or any(parent for parent in self.tree_dict[package_name][PARENTS_INDEX] if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] @@ -488,7 +490,6 @@ def __determine_doc_package_name(self, target_function_doc): def __find_initial_function(self, class_name: str, method_name: str, package_name: str ,documents: list[Document]) -> Document | None: jar_name = convert_from_maven_artifact(package_name) - # TODO check if its correct to handle none 3rd party code here as well relevant_docs = [doc for doc in documents if (self.language_parser.dir_name_for_3rd_party_packages() in doc.metadata.get('source') and jar_name in doc.metadata.get('source') or @@ -522,9 +523,14 @@ def function_called_from_caller_body(self, document: Document, function_to_searc Return True iff the Java method body (document.page_content) contains a call to `function_to_search`. Assumptions: the source is a single method (no comments/annotations/javadocs before it). - """ - import re + Supported matches: + - Bare method calls: getProperty(...) + - Dotted method calls: obj.getProperty(...), BeanUtils.getProperty(...) + - Chained method calls: PropertyUtilsBean.getInstance().getProperty(...) + - Generic method calls: obj.getProperty(...) + - Constructor calls: new Test(...), new pkg.Test<>(...), new Test(){...} + """ name_raw = (function_to_search or "").strip() if not name_raw: return True # preserve original behavior @@ -533,18 +539,28 @@ def function_called_from_caller_body(self, document: Document, function_to_searc base = re.sub(r"\s*\(.*\)\s*$", "", name_raw) m = re.match(r"^(?:(?P[\w$.]+)\.)?(?P[A-Za-z_$][\w$]*)$", base) if m: - target_qual = m.group("qual") - target_method = m.group("mname") + target_qual = m.group("qual") # e.g., "org.foo.Bar" + target_simple = m.group("mname") # e.g., "Bar" or a method name like "getProperty" else: - target_qual, target_method = None, re.sub(r"[^\w$]", "", base) + target_qual, target_simple = None, re.sub(r"[^\w$]", "", base) src = document.page_content or "" - # Search only in the method *body* to avoid matching the header. + # ---------- fast early exit ---------- + # If neither the simple token nor the qualifier shows up anywhere in the text, + # and there isn't a 'new ' token (constructor), bail out quickly. + if ( + target_simple not in src + and (not target_qual or target_qual not in src) + and f"new {target_simple}" not in src + ): + return False + + # ---------- restrict to method body (avoid matching header) ---------- body_start = src.find("{") body = src[body_start + 1:] if body_start != -1 else src - # Lightly mask string/char literals to avoid false positives inside quotes. + # ---------- mask string/char literals (cheap) ---------- def _strip_strings(s: str) -> str: out = [] i, n = 0, len(s) @@ -579,16 +595,31 @@ def _strip_strings(s: str) -> str: body = _strip_strings(body) - name_esc = re.escape(target_method) + # ---------- assemble patterns ---------- + simple_esc = re.escape(target_simple) + + # Method-call patterns (unchanged, plus optional before method name for generics) parts = [ - rf"\.\s*{name_esc}\s*\(", # chained call: ... .getProperty( - rf"(?]*>\s*)?{simple_esc}\s*\(", # chained/dotted: obj.name( + rf"(?]*>\s*)?{simple_esc}\s*\(") + parts.append(rf"(?]*>\s*)?{simple_esc}\s*\(") + + # Constructor-call patterns: new (...), allowing optional generics and anonymous class + # We only treat the target as a constructor if its "name" is a valid type identifier. + # This aligns with Java: constructor name == simple type name. + # Simple: new Test(...), new Test<...>(...), new Test(){...} + parts.append(rf"(?]*>)?\s*\(") + # Qualified: new pkg.Test(...), new pkg.Outer.Inner<...>(...) + if target_qual: + parts.append(rf"(?]*>)?\s*\(") call_re = re.compile("|".join(parts), flags=re.MULTILINE) return bool(call_re.search(body)) diff --git a/src/vuln_analysis/utils/java_segmenters_with_methods.py b/src/vuln_analysis/utils/java_segmenters_with_methods.py index 19e9bf4cc..6b54c00da 100644 --- a/src/vuln_analysis/utils/java_segmenters_with_methods.py +++ b/src/vuln_analysis/utils/java_segmenters_with_methods.py @@ -1286,9 +1286,14 @@ def _extract_methods_anywhere( ) -> List[str]: """ Extract Java method definitions (with bodies) anywhere within [start, end), - excluding constructors. Anonymous/inner/local class bodies are recursively scanned. + INCLUDING constructors. Anonymous/inner/local class bodies are recursively scanned. Method-level annotations preceding a signature are excluded from the returned slice. Lambdas are NOT returned here (they are gathered in a separate pass). + + Performance: + - Single forward scan with O(1) delimiter lookups when _set_delim_index() cache + is active. + - Avoids quadratic behavior by skipping ignored spans and using balanced matchers. """ if end is None: end = len(src) @@ -1315,7 +1320,7 @@ def _enclosing_region(pos: int) -> Optional[_TypeRegion]: idx -= 1 return None - # quick enum-constants section end cache + # Quick enum-constants section end cache (keyed by enum body '{' position) enum_term_cache: Dict[int, int] = {} def _region_kind(reg: _TypeRegion) -> str: @@ -1432,7 +1437,7 @@ def _scan_paren_internals_for_anonymous_classes(open_idx: int, close_idx: int, o break k2 = pos - # skip false positives inside ignored spans + # Skip false positives inside ignored spans sp = _ignored_span_at(k2) if _DELIM_SRC_ID == id(src) else None if sp and sp[0] <= k2 < sp[1]: k2 = sp[1]; continue @@ -1441,7 +1446,7 @@ def _scan_paren_internals_for_anonymous_classes(open_idx: int, close_idx: int, o post = src[k2 + 3] if k2 + 3 < close_idx else '' if (k2 == open_idx + 1 or not _is_ident_part(pre)) and (k2 + 3 >= close_idx or not _is_ident_part(post)): t = _skip_ws_comments(src, k2 + 3) - # qualified type with optional type params + # Qualified type with optional type params while t < close_idx: if _is_ident_start(src[t]): t += 1 @@ -1497,12 +1502,14 @@ def _body_might_have_local_types(beg: int, end_: int) -> bool: i = close_p + 1 continue + # Record headers are NOT methods/constructors; skip them explicitly. if _prev_word(src, name_start) == 'record': if close_p < n: _scan_paren_internals_for_anonymous_classes(i, close_p, out) i = close_p + 1 continue + # Control-flow invocations like `if (...)`, `for (...)` etc. are not methods. if name in _NON_METHOD_TOKENS: if close_p < n: _scan_paren_internals_for_anonymous_classes(i, close_p, out) @@ -1512,6 +1519,7 @@ def _body_might_have_local_types(beg: int, end_: int) -> bool: left_bound = _find_left_boundary(src, name_start) # "new" between boundary and name means likely a constructor call or anonymous class literal + # (i.e., not a declaration). If found, optionally recurse into the anonymous class body. def _has_new_token_between(s: str, left: int, right: int) -> bool: left = max(0, left); right = min(len(s), right) i2 = left @@ -1533,6 +1541,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: _scan_paren_internals_for_anonymous_classes(i, close_p, out) kk = _skip_ws_comments(src, close_p + 1) if kk < n and src[kk] == '{': + # Anonymous inner class: collect methods from its body. body_end = _match_balanced_braces(src, kk) if kk + 1 <= body_end: out.extend(_extract_methods_anywhere(src, start=kk + 1, end=body_end, types=types)) @@ -1541,6 +1550,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: i = close_p + 1 continue + # Enum constant bodies (class bodies inside the constants section) — recurse. k = _skip_ws_comments(src, close_p + 1) if k < n and src[k] == '{' and _in_enum_constants_section(name_start): if close_p < n: @@ -1551,7 +1561,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: i = body_end + 1 continue - # must have method body "{" + # We expect a method/constructor body here. k = _skip_throws_clause(src, k) if k >= n or src[k] != '{': if close_p < n: @@ -1562,16 +1572,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: sig_guess = _find_left_boundary(src, name_start) sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) - # constructor? compare with enclosing type simple name - encl_reg = _enclosing_region(name_start) - if encl_reg and name == encl_reg.name: - # do not emit constructor, but scan its body - body_end = _match_balanced_braces(src, k) - if k + 1 <= body_end and _body_might_have_local_types(k + 1, body_end): - out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) - i = body_end + 1 - continue - + # Constructors are now included: if name == enclosing type name, we still emit it. body_end = _match_balanced_braces(src, k) sig_start = _skip_ws_comments(src, sig_start) # final guard @@ -1587,6 +1588,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: seen_spans.add(span) out.append(method_text) + # Recurse into body for local/anonymous types if k + 1 <= body_end and _body_might_have_local_types(k + 1, body_end): out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) @@ -1602,7 +1604,7 @@ def _has_new_token_between(s: str, left: int, right: int) -> bool: name_start, name = _back_ident(src, open_p) if name and name not in _NON_METHOD_TOKENS: left_bound = _find_left_boundary(src, name_start) - # filter out anonymous class/ctor and constructors + # Filter out anonymous class/ctor callsites; constructors are INCLUDED. def _has_new_between(s, a, b): a = max(0, a); b = min(len(s), b) pos = s.find('new', a, b) @@ -1610,35 +1612,33 @@ def _has_new_between(s, a, b): sp = _ignored_span_at(pos) if _DELIM_SRC_ID == id(s) else None if not sp or not (sp[0] <= pos < sp[1]): pre = s[pos - 1] if pos > a else '' - post = s[pos + 3] if pos + 3 < b else '' + post = s[pos + 3] if s and (pos + 3) < b else '' if (pos == a or not _is_ident_part(pre)) and (pos + 3 >= b or not _is_ident_part(post)): return True pos = s.find('new', pos + 3, b) return False - if not _has_new_between(src, left_bound, name_start): - encl_reg = _enclosing_region(name_start) - if not (encl_reg and name == encl_reg.name) and _prev_word(src, name_start) != 'record': - body_end = _match_balanced_braces(src, i) - sig_guess = _find_left_boundary(src, name_start) - sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) - sig_start = _skip_ws_comments(src, sig_start) - - header_no_ann = _strip_annotations_in_range(src, sig_start, i) - clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) - - clean_body = _strip_body_comments_and_annotations(src, i, body_end) - - method_text = clean_header + clean_body - - span = (sig_start, body_end) - if span not in seen_spans: - seen_spans.add(span) - out.append(method_text) - - if i + 1 <= body_end and _body_might_have_local_types(i + 1, body_end): - out.extend(_extract_methods_anywhere(src, start=i + 1, end=body_end, types=types)) - i = body_end + 1 - continue + if not _has_new_between(src, left_bound, name_start) and _prev_word(src, name_start) != 'record': + body_end = _match_balanced_braces(src, i) + sig_guess = _find_left_boundary(src, name_start) + sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) + sig_start = _skip_ws_comments(src, sig_start) + + header_no_ann = _strip_annotations_in_range(src, sig_start, i) + clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) + + clean_body = _strip_body_comments_and_annotations(src, i, body_end) + + method_text = clean_header + clean_body + + span = (sig_start, body_end) + if span not in seen_spans: + seen_spans.add(span) + out.append(method_text) + + if i + 1 <= body_end and _body_might_have_local_types(i + 1, body_end): + out.extend(_extract_methods_anywhere(src, start=i + 1, end=body_end, types=types)) + i = body_end + 1 + continue i += 1 return out diff --git a/src/vuln_analysis/utils/java_utils.py b/src/vuln_analysis/utils/java_utils.py index 84253954f..47f6c33ab 100644 --- a/src/vuln_analysis/utils/java_utils.py +++ b/src/vuln_analysis/utils/java_utils.py @@ -8,6 +8,7 @@ JAVA_ANNOTATION_SYMBOL = '@' dummy_package_name = "dummy:dummy:1.0.0" +_BRACE_NORMAL_SPECIAL_RE = re.compile(r'[/"\'{}-]') JAVA_METHOD_CLASS_KINDS = ("class", "interface", "enum", "record") JAVA_METHOD_CLASS_MODS = { "public", "protected", "private", "abstract", "final", "static", "sealed", "strictfp" @@ -840,7 +841,21 @@ def word_at(pos: int, word: str) -> bool: @lru_cache(maxsize=2500000) def is_java_method(source: str) -> bool: + """ + Return True iff the given Java source snippet is a *method or constructor* + declaration, or (fallback) contains a lambda ('->') outside of comments/strings. + + Notes: + - Methods with explicit return type: detected. + - Constructors (no return type) are treated as methods too. + - Control-flow constructs are excluded. + - A lambda arrow outside comments/strings/text-blocks returns True. + """ s = source + # Safe early reject: no method/ctor is possible without '('; no lambda possible without '->'. + if "(" not in s and "->" not in s: + return False + n = len(s) i = 0 @@ -849,8 +864,9 @@ def is_java_method(source: str) -> bool: brace = 0 saw_lambda = False - def is_ws(ch: str) -> bool: - return ch <= " " and ch in " \t\r\n\f\v" + find_in_source = s.find + startswith = s.startswith + ws = JAVA_METHOD_WS_CHARS def is_id_start(ch: str) -> bool: return ch == "_" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) @@ -862,12 +878,24 @@ def is_id_part(ch: str) -> bool: (ch >= "\u0080" and ch.isalnum()) ) - CLASS_KINDS = ("class", "interface", "enum", "record") - CLASS_MODS = {"public","protected","private","abstract","final","static","sealed","strictfp"} - CTRL_WORDS = {"if","for","while","switch","catch","synchronized","return","new","case"} - METH_MODS = {"public","protected","private","abstract","static","final","synchronized", - "native","strictfp","default","transient"} - PRIM_TYPES = {"void","int","long","double","float","short","byte","boolean","char"} + def _skip_quoted(j: int, quote: str) -> int: + """ + Skip to the first unescaped closing quote starting at j (j is after opening quote). + Returns index right after the closing quote, or n if none. + """ + while j < n: + end = find_in_source(quote, j) + if end == -1: + return n + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1 + k -= 1 + if back % 2 == 0: + return end + 1 + j = end + 1 + return n def skip_paren(j: int) -> int: depth = 1 @@ -878,50 +906,62 @@ def skip_paren(j: int) -> int: if ch == "/": if j + 1 < n: c2 = s[j + 1] - if c2 == "/": st = LINE; j += 2; continue - if c2 == "*": st = BLOCK; j += 2; continue + if c2 == "/": + st = LINE + j += 2 + continue + if c2 == "*": + st = BLOCK + j += 2 + continue elif ch == '"': if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': - st = TEXT; j += 3; continue - st = STRING; j += 1; continue + st = TEXT + j += 3 + continue + st = STRING + j += 1 + continue elif ch == "'": - st = CHAR; j += 1; continue + st = CHAR + j += 1 + continue elif ch == "(": - depth += 1; j += 1; continue + depth += 1 + j += 1 + continue elif ch == ")": - depth -= 1; j += 1; continue + depth -= 1 + j += 1 + continue else: - j += 1; continue + j += 1 + continue elif st == LINE: - nl = s.find("\n", j) + nl = find_in_source("\n", j) j = n if nl == -1 else nl + 1 st = NORMAL elif st == BLOCK: - end = s.find("*/", j) + end = find_in_source("*/", j) j = n if end == -1 else end + 2 st = NORMAL elif st == STRING: - while j < n: - c2 = s[j]; j += 1 - if c2 == "\\": j += 1 - elif c2 == '"': break + j = _skip_quoted(j, '"') st = NORMAL elif st == CHAR: - while j < n: - c2 = s[j]; j += 1 - if c2 == "\\": j += 1 - elif c2 == "'": break + j = _skip_quoted(j, "'") st = NORMAL else: # TEXT while True: - end = s.find('"""', j) + end = find_in_source('"""', j) if end == -1: j = n break k = end - 1 back = 0 while k >= 0 and s[k] == "\\": - back += 1; k -= 1 + back += 1 + k -= 1 j = end + 3 if back % 2 == 0: break @@ -929,32 +969,40 @@ def skip_paren(j: int) -> int: return j def read_ident(j: int) -> tuple[Optional[str], int]: - if j >= n or not is_id_start(s[j]): return None, j + if j >= n or not is_id_start(s[j]): + return None, j k = j + 1 - while k < n and is_id_part(s[k]): k += 1 + while k < n and is_id_part(s[k]): + k += 1 return s[j:k], k def read_qual_ident(j: int) -> tuple[Optional[str], int]: name, k = read_ident(j) - if not name: return None, j + if not name: + return None, j while k < n and s[k] == ".": nm2, k2 = read_ident(k + 1) - if not nm2: break + if not nm2: + break k = k2 return name, k def skip_ws_comments(j: int) -> int: while j < n: ch = s[j] - if is_ws(ch): - j += 1; continue + if ch in ws: + j += 1 + continue if ch == "/" and j + 1 < n: c2 = s[j + 1] if c2 == "/": - j = (s.find("\n", j + 2) + 1) or n; continue + nl = find_in_source("\n", j + 2) + j = n if nl == -1 else nl + 1 + continue if c2 == "*": - end = s.find("*/", j + 2) - j = n if end == -1 else end + 2; continue + end = find_in_source("*/", j + 2) + j = n if end == -1 else end + 2 + continue break return j @@ -962,28 +1010,36 @@ def has_dot_between(j: int, k: int) -> bool: j = skip_ws_comments(j) while j < k: ch = s[j] - if ch == ".": return True - if is_ws(ch): - j += 1; continue + if ch == ".": + return True + if ch in ws: + j += 1 + continue if ch == "/" and j + 1 < n: - if s[j+1] == "/": - j = (s.find("\n", j + 2) + 1) or n; continue - if s[j+1] == "*": - end = s.find("*/", j + 2) - j = n if end == -1 else end + 2; continue + c2 = s[j + 1] + if c2 == "/": + nl = find_in_source("\n", j + 2) + j = n if nl == -1 else nl + 1 + continue + if c2 == "*": + end = find_in_source("*/", j + 2) + j = n if end == -1 else end + 2 + continue j += 1 return False def plausible_method_decl(name_start: int) -> bool: """ - Must see a real return type token before the name (excludes constructors). + Must see a real return type token before the name (method), + OR see only annotations/modifiers/generics/comments before the name (constructor). """ j = name_start j0 = max(s.rfind("\n", 0, j), s.rfind("{", 0, j), s.rfind(";", 0, j)) + 1 j = skip_ws_comments(j0) + broken = False + while j < name_start: - # annotations if s[j] == JAVA_ANNOTATION_SYMBOL: _, j = read_qual_ident(j + 1) j = skip_ws_comments(j) @@ -992,58 +1048,73 @@ def plausible_method_decl(name_start: int) -> bool: j = skip_ws_comments(j) continue - # type params if s[j] == "<": - depth = 1; j += 1 + depth = 1 + j += 1 while j < n and depth > 0: - if s[j] == "<": depth += 1 - elif s[j] == ">": depth -= 1 + if s[j] == "<": + depth += 1 + elif s[j] == ">": + depth -= 1 j += 1 j = skip_ws_comments(j) continue tok, k = read_ident(j) if tok: - if tok in PRIM_TYPES: - return True # primitive (incl. void) return + if tok in JAVA_METHOD_PRIM_TYPES: + return True - if tok in METH_MODS: + if tok in JAVA_METHOD_METH_MODS: j = skip_ws_comments(k) continue - # reference type start + if tok == "non": + k2 = skip_ws_comments(k) + if k2 < n and s[k2] == "-": + k3 = skip_ws_comments(k2 + 1) + id2, j2 = read_ident(k3) + if id2 == "sealed": + j = skip_ws_comments(j2) + continue + + # reference return type start j = k j = skip_ws_comments(j) - # optional generics if j < n and s[j] == "<": - depth = 1; j += 1 + depth = 1 + j += 1 while j < n and depth > 0: - if s[j] == "<": depth += 1 - elif s[j] == ">": depth -= 1 + if s[j] == "<": + depth += 1 + elif s[j] == ">": + depth -= 1 j += 1 j = skip_ws_comments(j) - # optional arrays while j + 1 < n and s[j] == "[" and s[j + 1] == "]": j += 2 j = skip_ws_comments(j) - # optional intersections: A & B & ... tcount = 0 while tcount < 4: j = skip_ws_comments(j) if j < n and s[j] == "&": j = skip_ws_comments(j + 1) nm2, k2 = read_ident(j) - if not nm2: break + if not nm2: + break j = k2 j = skip_ws_comments(j) if j < n and s[j] == "<": - depth = 1; j += 1 + depth = 1 + j += 1 while j < n and depth > 0: - if s[j] == "<": depth += 1 - elif s[j] == ">": depth -= 1 + if s[j] == "<": + depth += 1 + elif s[j] == ">": + depth -= 1 j += 1 j = skip_ws_comments(j) while j + 1 < n and s[j] == "[" and s[j + 1] == "]": @@ -1053,15 +1124,77 @@ def plausible_method_decl(name_start: int) -> bool: else: break - return True # saw a bona fide reference return type + return True + broken = True break - return False - while i < n: - ch = s[i] + # No return type found, but nothing invalid encountered -> constructor + return not broken + while i < n: if state == NORMAL: + # Big win: when inside braces, skip boring characters in C (regex search), + # only landing on characters that can change state or matter for lambda/braces. + if brace > 0: + m = _BRACE_NORMAL_SPECIAL_RE.search(s, i) + if not m: + break + i = m.start() + ch = s[i] + + # keep ordering consistent with original: lambda before comment/string checks + if ch == "-" and i + 1 < n and s[i + 1] == ">": + saw_lambda = True + i += 2 + continue + + if ch == "/": + if i + 1 < n: + c2 = s[i + 1] + if c2 == "/": + state = LINE + i += 2 + continue + if c2 == "*": + state = BLOCK + i += 2 + continue + i += 1 + continue + + if ch == '"': + if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + + if ch == "'": + state = CHAR + i += 1 + continue + + if ch == "{": + brace += 1 + i += 1 + continue + if ch == "}": + brace -= 1 + if brace < 0: + brace = 0 + i += 1 + continue + + # non-handled special (should not happen) + i += 1 + continue + + # brace == 0: full logic (header-level) + ch = s[i] + if ch == "-" and i + 1 < n and s[i + 1] == ">": saw_lambda = True i += 2 @@ -1070,142 +1203,161 @@ def plausible_method_decl(name_start: int) -> bool: if ch == "/": if i + 1 < n: c2 = s[i + 1] - if c2 == "/": state = LINE; i += 2; continue - if c2 == "*": state = BLOCK; i += 2; continue + if c2 == "/": + state = LINE + i += 2 + continue + if c2 == "*": + state = BLOCK + i += 2 + continue elif ch == '"': if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': - state = TEXT; i += 3; continue - state = STRING; i += 1; continue + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue elif ch == "'": - state = CHAR; i += 1; continue + state = CHAR + i += 1 + continue elif ch == "{": - brace += 1; i += 1; continue + brace += 1 + i += 1 + continue elif ch == "}": - brace = max(0, brace - 1); i += 1; continue + brace = max(0, brace - 1) + i += 1 + continue - if brace == 0: - if is_ws(ch): - i += 1; continue + # brace==0 path + if ch in ws: + i += 1 + continue - # skip annotations at top level - if ch == JAVA_ANNOTATION_SYMBOL: - _, i = read_qual_ident(i + 1) - i = skip_ws_comments(i) - if i < n and s[i] == "(": - i = skip_paren(i + 1) + # skip annotations at top level + if ch == JAVA_ANNOTATION_SYMBOL: + _, i = read_qual_ident(i + 1) + i = skip_ws_comments(i) + if i < n and s[i] == "(": + i = skip_paren(i + 1) + continue + + # skip modifiers & detect type headers early + if is_id_start(ch): + ident, j = read_ident(i) + if ident in JAVA_METHOD_CLASS_MODS: + i = skip_ws_comments(j) + continue + if ident == "non": + k = skip_ws_comments(j) + if k < n and s[k] == "-": + k2 = skip_ws_comments(k + 1) + id2, j2 = read_ident(k2) + if id2 == "sealed": + i = skip_ws_comments(j2) + continue + if ident in JAVA_METHOD_CLASS_KINDS: + i2 = skip_ws_comments(j) + name, j2 = read_ident(i2) + if name: + return False + i = j continue - # skip modifiers & detect type headers early (we only care to *not* misclassify) - if is_id_start(ch): - ident, j = read_ident(i) - if ident in CLASS_MODS: - i = skip_ws_comments(j); continue - if ident == "non": - k = skip_ws_comments(j) - if k < n and s[k] == "-": - k2 = skip_ws_comments(k + 1) - id2, j2 = read_ident(k2) - if id2 == "sealed": - i = skip_ws_comments(j2); continue - if ident in CLASS_KINDS: - i = skip_ws_comments(j) - name, j2 = read_ident(i) - if name: - return False - i = j - continue + # potential method/ctor header + if ch == "(": + j = i - 1 + while j >= 0 and s[j] in ws: + j -= 1 + end = j + 1 + while j >= 0 and is_id_part(s[j]): + j -= 1 + name = s[j+1:end] if end > (j + 1) else None + + if name and (name not in JAVA_METHOD_CTRL_WORDS) and not has_dot_between(end, i): + k = skip_paren(i + 1) + q = skip_ws_comments(k) + + # optional throws + if startswith("throws", q) and (q + 6 == n or not is_id_part(s[q + 6])): + q = skip_ws_comments(q + 6) + while q < n and s[q] not in "{;": + cq = s[q] + if cq in ws or cq in ".,": # whitespace, commas, dots + q += 1 + continue + if cq == "<": + depth = 1 + q += 1 + while q < n and depth > 0: + if s[q] == "<": + depth += 1 + elif s[q] == ">": + depth -= 1 + q += 1 + continue + if is_id_start(cq): + _, q = read_ident(q) + continue + if cq == "[" and q + 1 < n and s[q + 1] == "]": + q += 2 + continue + break + q = skip_ws_comments(q) - # potential method header: look for '(' with a plain name just before it - if ch == "(": - j = i - 1 - while j >= 0 and is_ws(s[j]): j -= 1 - end = j + 1 - while j >= 0 and is_id_part(s[j]): j -= 1 - name = s[j+1:end] if end > (j + 1) else None - - if name and (name not in CTRL_WORDS) and not has_dot_between(end, i): - k = skip_paren(i + 1) - q = skip_ws_comments(k) - - # optional 'throws ...' clause - if s.startswith("throws", q) and (q+6 == n or not is_id_part(s[q+6])): - q = skip_ws_comments(q + 6) - while q < n and s[q] not in "{;": - if is_ws(s[q]) or s[q] in ".,": q += 1; continue - if s[q] == "<": - depth = 1; q += 1 - while q < n and depth > 0: - if s[q] == "<": depth += 1 - elif s[q] == ">": depth -= 1 - q += 1 - continue - if is_id_start(s[q]): - _, q = read_ident(q); continue - if s[q] == "[" and q + 1 < n and s[q+1] == "]": - q += 2; continue - break - q = skip_ws_comments(q) - - # must end with '{' or ';' to be a decl, then validate return type - if q < n and s[q] in "{;": - if plausible_method_decl(j + 1): - return True # it's a method + if q < n and s[q] in "{;": + if plausible_method_decl(j + 1): + return True - i += 1 - continue + i += 1 + continue i += 1 continue elif state == LINE: - nl = s.find("\n", i) + nl = find_in_source("\n", i) i = n if nl == -1 else nl + 1 state = NORMAL continue elif state == BLOCK: - end = s.find("*/", i) + end = find_in_source("*/", i) i = n if end == -1 else end + 2 state = NORMAL continue elif state == STRING: - while i < n: - c2 = s[i]; i += 1 - if c2 == "\\": i += 1 - elif c2 == '"': break + i = _skip_quoted(i, '"') state = NORMAL continue elif state == CHAR: - while i < n: - c2 = s[i]; i += 1 - if c2 == "\\": i += 1 - elif c2 == "'": break + i = _skip_quoted(i, "'") state = NORMAL continue else: # TEXT while True: - end = s.find('"""', i) + end = find_in_source('"""', i) if end == -1: i = n break k = end - 1 back = 0 while k >= 0 and s[k] == "\\": - back += 1; k -= 1 + back += 1 + k -= 1 i = end + 3 if back % 2 == 0: break state = NORMAL continue - # lambda fallback → treat snippet as "method-like" - if saw_lambda: - return True - - return False + return saw_lambda def is_java_type(source: str) -> bool: """ @@ -1531,7 +1683,7 @@ def skip_type_list(p: int) -> int: return False -def create_inheritance_map(java_types: List["Document"]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: +def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: """ Build a jar-aware inheritance/implements map. From 5e6e9606a911cf97990a403ea10e932490e4db76 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 30 Dec 2025 10:37:23 +0200 Subject: [PATCH 132/286] chore: add /api/v1 prefix to API endpoints in configs Signed-off-by: Vladimir Belousov --- kustomize/README.md | 8 ++++---- kustomize/base/exploit-iq-config.yml | 4 ++-- kustomize/config-http-openai-local.yml | 4 ++-- src/vuln_analysis/configs/config-http-nim.yml | 4 ++-- src/vuln_analysis/configs/config-http-openai.yml | 4 ++-- src/vuln_analysis/configs/config-tracing.yml | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 3ca9e4b71..ae5dac46b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -180,14 +180,14 @@ grantMethod: prompt secret: $OAUTH_CLIENT_SECRET redirectURIs: - "http://exploit-iq-client:8080" - - "https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')/app/index.html" - - "http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')/app/index.html" + - "https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')" + - "http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')" EOF ``` Otherwise, just add your route to the existing `OAuthClient` CR object: ```shell -export HTTPS_ROUTE=https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')/app/index.html -export HTTP_ROUTE=http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')/app/index.html +export HTTPS_ROUTE=https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') +export HTTP_ROUTE=http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') oc patch oauthclient exploit-iq-client -p '{"redirectURIs":["'$HTTP_ROUTE'","'$HTTPS_ROUTE'"]}' ``` ```shell diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 4038e49cf..4cdcf0469 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -47,7 +47,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: CALLBACK_URL_PLACEHOLDER/vulnerabilities/{vuln_id}/comments + endpoint: CALLBACK_URL_PLACEHOLDER/api/v1/vulnerabilities/{vuln_id}/comments token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt @@ -132,7 +132,7 @@ functions: cve_http_output: _type: cve_http_output url: CALLBACK_URL_PLACEHOLDER - endpoint: /reports + endpoint: /api/v1/reports auth_type: bearer token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 8cdb3e42b..a70657af6 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -46,7 +46,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : @@ -133,7 +133,7 @@ functions: cve_http_output: _type: cve_http_output url: http://localhost:8080 - endpoint: /reports + endpoint: /api/v1/reports cve_file_output: _type: cve_file_output file_path: .tmp/output.json diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 526196e8e..8ef818111 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -39,7 +39,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : @@ -121,7 +121,7 @@ functions: cve_http_output: _type: cve_http_output url: http://localhost:8080 - endpoint: /reports + endpoint: /api/v1/reports cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index a4470bf13..1ff998ba1 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -46,7 +46,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : @@ -128,7 +128,7 @@ functions: cve_http_output: _type: cve_http_output url: http://localhost:8080 - endpoint: /reports + endpoint: /api/v1/reports cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index c1eb0dd4e..b18d815c9 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -50,7 +50,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: http://localhost:8080/vulnerabilities/{vuln_id}/comments + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom cve_check_vuln_deps : From e354c8cdcb12a2c38a696f528834bfca2fd06ee3 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 31 Dec 2025 16:05:43 +0200 Subject: [PATCH 133/286] security(mongo): add DB authentication and RBAC Signed-off-by: Vladimir Belousov --- kustomize/README.md | 27 ++++++++++++------ kustomize/base/exploit_iq_client.yaml | 15 +++++++++- kustomize/base/exploit_iq_client_db.yaml | 27 ++++++++++++++++++ kustomize/base/kustomization.yaml | 6 ++++ kustomize/base/mongodb_init_configmap.yaml | 27 ++++++++++++++++++ kustomize/base/mongodb_network_policy.yaml | 33 ++++++++++++++++++++++ 6 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 kustomize/base/mongodb_init_configmap.yaml create mode 100644 kustomize/base/mongodb_network_policy.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 3ca9e4b71..a991e8606 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -145,15 +145,26 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -7. Update `ExploitIQ` configuration file with the correct callback URL for the client service. +7. Create MongoDB credentials: + +```shell +cat > base/mongodb-credentials.env << EOF +admin-user=mongoadmin +admin-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) +exploit-iq-user=exploit-iq-user +exploit-iq-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) +EOF +``` + +8. Update `ExploitIQ` configuration file with the correct callback URL for the client service. ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` >[!IMPORTANT] -You should only run one of 7,8 steps, depends on if you want to run the service with a self hosted LLM or not. -8. To deploy `EXploitIQ` with a self-hosted LLM , run: +You should only run one of 9,10 steps, depends on if you want to run the service with a self hosted LLM or not. +9. To deploy `ExploitIQ` with a self-hosted LLM , run: ```shell # Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM @@ -161,14 +172,14 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ``` -9. Alternatively, to deploy `EXploitIQ` with a fully remote nim LLM, run: +10. Alternatively, to deploy `ExploitIQ` with a fully remote nim LLM, run: ```shell # Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` >[!WARNING] -Without completing the following step with the correct secret from step 5, authentication and logging into the UI App will fail! -10. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 5) and generated route +Without completing the following step with the correct secret from step 6, authentication and logging into the UI App will fail! +11. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 6) and generated route ```bash oc create -f - < Date: Thu, 1 Jan 2026 11:13:22 +0200 Subject: [PATCH 134/286] solve 1 hour limit tekton pac gh token (#176) * solve 1 hour limit tekton pac gh token --- .tekton/on-cm-runner.yaml | 125 ++++++++++++++-------------- src/vuln_analysis/utils/dep_tree.py | 2 +- 2 files changed, 62 insertions(+), 65 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 05f467d16..c99acc4a0 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -60,6 +60,8 @@ spec: - name: submodules value: "true" + + # ------------------------------------------------ # 2. RUN TEST & TAGGING & RELEASING # ------------------------------------------------ @@ -207,8 +209,66 @@ spec: # >>> THE CLIENT (Test Runner) <<< steps: + + # ------------------------------------------------------- + # STEP A: Tag the MAIN Repository (RUNS FIRST) + # ------------------------------------------------------- + - name: git-tag-main-repo + image: alpine/git + workingDir: $(workspaces.source.path) + script: | + #!/bin/sh + set -e + + # 1. Trust workspace + git config --global --add safe.directory $(workspaces.source.path) + + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- Configuring Git for Main Repo ---" + git config --global user.email "tekton-bot@redhat.com" + git config --global user.name "Tekton Pipeline" + + # ------------------------------------------------------- + # AUTH LOGIC: PAC ONLY + # ------------------------------------------------------- + CRED_FILE="/workspace/basic-auth/.git-credentials" + if [ -f "$CRED_FILE" ]; then + echo "✅ Found PAC credentials. Configuring..." + + # [FIX 1] Use head to safely get the first line + RAW_CRED=$(head -n 1 "$CRED_FILE") + + # Extract 'user:token' + AUTH_PART=$(echo "$RAW_CRED" | sed -n 's|https://\(.*\)@.*|\1|p') + + if [ -z "$AUTH_PART" ]; then + echo "❌ Error parsing credentials." + exit 1 + fi + + # [FIX 2] Robust URL Cleaning + # Remove 'https://' AND any existing 'user:pass@' from the cloned URL + CURRENT_URL=$(git remote get-url origin) + CLEAN_URL=$(echo "$CURRENT_URL" | sed 's|https://||' | sed 's|.*@||') + + echo "Injecting credentials into git remote..." + git remote set-url origin "https://${AUTH_PART}@${CLEAN_URL}" + else + echo "❌ CRITICAL ERROR: .git-credentials file missing." + exit 1 + fi + + echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" + + # [FIX 3] Updated Message (Since tests haven't run yet) + git tag -f -a "$TAG_NAME" -m "Automated Build Tag for PR #$(params.PR_NUMBER)" + + # Push + git push origin "refs/tags/$TAG_NAME" --force # ------------------------------------------------------- - # STEP A: Run the Python Automation Tests + # STEP B: Run the Python Automation Tests # ------------------------------------------------------- - name: run-test-suite image: quay.io/ecosystem-appeng/auto-cm-testing:latest @@ -263,70 +323,7 @@ spec: cp -r /app/src/reports/archive $(workspaces.source.path)/src/reports echo "--- TESTS FINISHED SUCCESSFULLY ---" - # ------------------------------------------------------- - # STEP B: Tag the MAIN Repository - # ------------------------------------------------------- - - name: git-tag-main-repo - image: alpine/git - workingDir: $(workspaces.source.path) - script: | - #!/bin/sh - set -e - - # [FIX] Trust the workspace directory to avoid "dubious ownership" error - git config --global --add safe.directory $(workspaces.source.path) - - SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) - TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" - - echo "--- Configuring Git for Main Repo ---" - git config --global user.email "tekton-bot@redhat.com" - git config --global user.name "Tekton Pipeline" - - # ------------------------------------------------------- - # AUTH LOGIC: PAC ONLY (.git-credentials) - # ------------------------------------------------------- - CRED_FILE="/workspace/basic-auth/.git-credentials" - if [ -f "$CRED_FILE" ]; then - echo "✅ Found PAC credentials. Configuring git helper." - #cp /workspace/basic-auth/.git-credentials ~/.git-credentials - #chmod 600 ~/.git-credentials - #git config --global credential.helper "store --file=~/.git-credentials" - # 1. Read the content (Format: https://user:token@github.com) - RAW_CRED=$(cat "$CRED_FILE") - - # 2. Extract the 'user:token' part using sed - # This regex grabs everything between 'https://' and '@' - AUTH_PART=$(echo "$RAW_CRED" | sed -n 's|https://\(.*\)@.*|\1|p') - - if [ -z "$AUTH_PART" ]; then - echo "❌ Error parsing credentials from file." - exit 1 - fi - - # 3. Get the current remote URL (e.g., https://github.com/org/repo.git) - CURRENT_URL=$(git remote get-url origin) - - # 4. Strip the 'https://' prefix - CLEAN_URL=${CURRENT_URL#https://} - - # 5. Set the new URL wi-th auth injected - echo "Injecting credentials into git remote..." - git remote set-url origin "https://${AUTH_PART}@${CLEAN_URL}" - else - echo "❌ CRITICAL ERROR: .git-credentials file missing in /workspace/basic-auth." - echo "This pipeline only supports Pipelines-as-Code authentication." - exit 1 - fi - - echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" - # 1. Create/Overwrite the tag locally - # The -f flag automatically overwrites the local tag if it exists - git tag -f -a "$TAG_NAME" -m "Tests Passed for PR #$(params.PR_NUMBER)" - # 2. Push/Overwrite the tag remotely - # This one command handles both creation and updating - git push origin "refs/tags/$TAG_NAME" --force # ------------------------------------------------------- # STEP C: Release & Upload Artifacts to AUTOMATION Repo diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/vuln_analysis/utils/dep_tree.py index fd983f839..2a136f0d3 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/vuln_analysis/utils/dep_tree.py @@ -108,7 +108,7 @@ class CCppDependencyTreeBuilder(DependencyTreeBuilder): 'stdio.h', 'stdlib.h', 'string.h', 'math.h', 'unistd.h', 'sys/types.h', 'sys/stat.h', 'errno.h', 'fcntl.h', 'time.h' } - + #kernel lib - need only for bpf projects KERNEL_LIBS = ["libbpf"] KERNEL_LIBS_PATH_MAP = { "libbpf": "tools/lib/bpf/" From ebb0de24fbbaf1be9058eb736ff7603662d4fe71 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Fri, 2 Jan 2026 00:01:05 +0200 Subject: [PATCH 135/286] docs: add missing setup and prerequisites for local deployment Signed-off-by: Zvi Grinberg --- kustomize/README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index ca1280ad9..0e837506f 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -16,14 +16,14 @@ One can run ExploitIQ on his local machine ( No GPU dependency is required!), fo cd $(git rev-parse --show-toplevel) # Create and activate the virtual environment rm -rf .venv || true -uv venv --python 3.12 +uv venv --python 3.12 --no-cache source .venv/bin/activate ``` 4. **Install Dependencies**: Once the environment is active, install the required packages. ```shell -uv sync +uv sync --no-cache ``` 5. **Set Environment Variables**: Define the following environment variables. @@ -36,6 +36,7 @@ export DOC_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-A export JUSTIFY_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 export NVIDIA_API_BASE=http://YOUR_SELF_HOSTED_OPENAI_LLM_ADDRESSS/v1 export PYTHONUNBUFFERED=1 +export GOPROXY=https://proxy.golang.org,direct export SERPAPI_API_KEY=YOUR_SERPAPI_KEY export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 export REGISTRY_REDHAT_USERNAME="your_username" @@ -46,6 +47,17 @@ export REGISTRY_REDHAT_PASSWORD="your_password_or_token" ```shell nat --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 ``` +7. In another terminal, in the same venv python env, start Arize phoenix service to enable tracing for local deployment: +```shell +phoenix serve +``` + +### Prerequisites +One need to install the following tools in order to run the agent locally without problems, all tools binaries expected +to be on the system path: +- [Java JDK >=21](https://www.oracle.com/il-en/java/technologies/downloads/#java21) +- [Maven Package manager](https://maven.apache.org/install.html) +- [Golang](https://go.dev/doc/install) ### Container Source Download Configuration From 3f5d6e49f7ce5ec15e53517caf7112dbe8178d4c Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 4 Jan 2026 09:55:41 +0200 Subject: [PATCH 136/286] fix: change example from concrete one to generic one Signed-off-by: Zvi Grinberg --- src/vuln_analysis/tools/transitive_code_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 64bae4005..398943788 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -157,7 +157,7 @@ async def _arun(query: str) -> list: description=(""" Finds functions in a package that call a specific library function. GO ecosystem only. Input format: 'package_name,library.function(args_with_literals)'. - Example: 'github.com/golang-jwt/jwt,errors.New(\"Invalid Key: Key must be PEM encoded\")'. + Example: 'github.com/namespace/package_name,errors.New(\"text_literal'\")'. Returns: ['package,caller1', 'package,caller2'] or []. """)) From d6f28f2cb347816a8874736afae35e4815e89dbf Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Sun, 4 Jan 2026 11:21:05 +0200 Subject: [PATCH 137/286] ignore python 2 code since our ast only works on python 3 (#174) * ignore python 2 code since our ast only works on python 3 --- README.md | 7 ++++ .../python_segmenters_with_classes_methods.py | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/README.md b/README.md index 2cb666b2b..659a6a456 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ limitations under the License. - [Customizing the embedding model](#customizing-the-embedding-model) - [Steps to configure an alternate embedding provider](#steps-to-configure-an-alternate-embedding-provider) - [Customizing the Output](#customizing-the-output) +- [Limitations](#limitations) - [Troubleshooting](#troubleshooting) - [Git LFS issues](#git-lfs-issues) - [Container build issues](#container-build-issues) @@ -770,6 +771,12 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. +## Limitations + +The following limitations apply to this workflow: + +- **Python 2 Support**: Python 2 scripts and source code are not supported for analysis. The workflow only processes Python 3 code. Python 2 code will be automatically skipped during code repository processing. This limitation exists because Python 2 reached end-of-life in January 2020 and modern AST parsing libraries used by the workflow only support Python 3. + ## Troubleshooting Several common issues can arise when running the workflow. Here are some common issues and their solutions. diff --git a/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py b/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py index 78804e396..de20d3645 100644 --- a/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py +++ b/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py @@ -1,5 +1,6 @@ import ast from typing import List +import re from langchain_community.document_loaders.parsers.language.python import PythonSegmenter @@ -33,13 +34,49 @@ def parse_all_classes_methods(code: str) -> list[str]: return methods +# Compiled once for speed +PY2_PRINT_REGEX = re.compile(r'^\s*print (?![\(])', re.MULTILINE) +PY2_EXCEPT_REGEX = re.compile(r'^\s*except .+,', re.MULTILINE) # matches: except Exception, e: +PY2_RAW_INPUT_REGEX = re.compile(r'\braw_input\s*\(', re.MULTILINE) # matches: raw_input( +PY2_RAISE_REGEX = re.compile(r'^\s*raise\s+\w+\s*,', re.MULTILINE) # matches: raise Exception, + + +def is_python2_code(code: str) -> bool: + """Check if the code is Python 2 code. + Args: + code: The code to check. + Returns: + True if the code is Python 2 code, False otherwise. + Checks: + 1. Check Shebang (Instant) + 2. Check for 'print "string"' + 3. Check for old exception syntax 'except Exception, e:' + 4. Check for 'raw_input()' function + 5. Check for old-style 'raise Exception, "Error"' syntax + """ + if code.startswith("#!") and "python2" in code[:50]: + return True + if PY2_PRINT_REGEX.search(code): + return True + if PY2_EXCEPT_REGEX.search(code): + return True + if PY2_RAW_INPUT_REGEX.search(code): + return True + if PY2_RAISE_REGEX.search(code): + return True + + return False class PythonSegmenterWithClassesMethods(PythonSegmenter): def __init__(self, code: str): super().__init__(code) + self._code = code def extract_functions_classes(self) -> List[str]: + if is_python2_code(self._code): + return [] + function_classes = super().extract_functions_classes() classes_methods = [] for func in function_classes: From 9c538cc81d5bedb2e98acdbdd4c31048907d56cf Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 4 Jan 2026 14:41:59 +0200 Subject: [PATCH 138/286] fix: resolve invalid project version selection Signed-off-by: Ilona Shishov --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 20b33fb4b..ebebf5527 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ build-backend = "setuptools.build_meta" requires = ["setuptools >= 64", "setuptools-scm>=8"] [tool.setuptools_scm] -# intentionally empty, the section is required for setuptools_scm to work but we don't need to set anything +# Only match version tags (v0.0.1, v1.2.3, etc.), ignore PR tags like pr-162-4c9bb19 +git_describe_command = ["git", "describe", "--tags", "--match", "v*"] [project] name = "vuln_analysis" From 35c0bf69336a185cbc872b043022f5df4611f1ea Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 5 Jan 2026 11:22:21 +0200 Subject: [PATCH 139/286] ci: change target branch and tag name Signed-off-by: Zvi Grinberg --- .tekton/on-push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml index ffcb58960..47fc26807 100644 --- a/.tekton/on-push.yaml +++ b/.tekton/on-push.yaml @@ -9,7 +9,7 @@ metadata: pipelinesascode.tekton.dev/on-event: "[push]" # The branch or tag we are targeting (ie: main, refs/tags/*) - pipelinesascode.tekton.dev/on-target-branch: "[rh-aiq-main]" + pipelinesascode.tekton.dev/on-target-branch: "[main]" # Fetch the git-clone task from hub, we are able to reference later on it # with taskRef and it will automatically be embedded into our pipeline. @@ -26,7 +26,7 @@ spec: - name: revision value: "{{ revision }}" - name: output-image - value: quay.io/ecosystem-appeng/agent-morpheus-rh:nat + value: quay.io/ecosystem-appeng/agent-morpheus-rh:latest - name: path-context value: . - name: dockerfile From 7aee02166dd4fe2c78bc88315ee3b9118a059953 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 24 Dec 2025 10:41:19 +0200 Subject: [PATCH 140/286] ci: add integration tests Signed-off-by: Zvi Grinberg --- .tekton/on-cm-runner.yaml | 2 +- .tekton/on-pull-request.yaml | 157 +++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index c99acc4a0..6497a6ea8 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -432,4 +432,4 @@ spec: secretName: "{{ git_auth_secret }}" - name: exploit-iq-data persistentVolumeClaim: - claimName: exploit-iq-cach-pvc + claimName: exploit-iq-cache-pvc diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 6c2bc9176..aeff02593 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -53,6 +53,7 @@ spec: workspaces: - name: source - name: basic-auth + - name: exploit-iq-data tasks: - name: fetch-repository taskRef: @@ -241,6 +242,159 @@ spec: make test-unit print_banner "LINT AND TEST COMPLETE" + - name: integration-test + timeout: 10h0m0s # Timeout for the task + runAfter: + - buildah-pvc + workspaces: + - name: source + workspace: source + - name: basic-auth + workspace: basic-auth # Needed for pushing tags/releases + - name: exploit-iq-data + workspace: exploit-iq-data + params: + - name: CURRENT_REVISION + value: $(params.revision) + - name: PR_NUMBER + value: $(params.pr_number) + - name: TRIGGER_COMMENT + value: $(params.trigger_comment) + + taskSpec: + params: + - name: CURRENT_REVISION + type: string + - name: PR_NUMBER + type: string + - name: TRIGGER_COMMENT + type: string + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + volumes: + - name: google-creds-volume + secret: + secretName: google-sheets-secrets + + # >>> THE SERVER (Sidecar) <<< + sidecars: + - name: server-application + # [CHANGE 3] Use the image passed in params (the one built by the other pipeline) + image: $(params.target-image) + imagePullPolicy: Always + + volumeMounts: + # Inject Env Vars (API Keys, Models, RedHat Creds) + - name: $(workspaces.exploit-iq-data.volume) + mountPath: /exploit-iq-data + envFrom: + - secretRef: + name: integration-tests + - configMapRef: + name: server-model-config + - secretRef: + name: redhat-app-creds + + env: + - name: PYTHONUNBUFFERED + value: "1" + # Pass the raw comment text into the container + - name: GOMODCACHE + value: "/exploit-iq-data/go/pkg/mod" + - name: SERPAPI_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cwe-details + - name: FIRST_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/ghsa + - name: NVD_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd + - name: RHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + + script: | + #!/bin/sh + set -e + # We use python to safely remove the 'telemetry' block + echo "--- Generating Config without Tracing ---" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="/exploit-iq-data" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + + # 2. [CRITICAL] Remove existing folder/file if the image created it + # This ensures 'ln' creates the link named 'am_cache', instead of putting it INSIDE 'am_cache' + if [ -e "${CACHE_DIR_TARGET}" ]; then + echo "Removing existing item at ${CACHE_DIR_TARGET} to replace with PVC link..." + rm -rf "${CACHE_DIR_TARGET}" + fi + + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" + + echo "--- Starting 'nat' Server ---" + exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 + readinessProbe: + tcpSocket: + port: 26466 + initialDelaySeconds: 10 + periodSeconds: 5 + + # >>> THE CLIENT (Test Runner) <<< + steps: + # ------------------------------------------------------- + # STEP A: Run the Integration Tests + # ------------------------------------------------------- + - name: run-test-suite + image: quay.io/ecosystem-appeng/auto-cm-testing:latest + imagePullPolicy: Always + workingDir: $(workspaces.source.path) + + volumeMounts: + - name: google-creds-volume + mountPath: /etc/secrets/google + readOnly: true + +# env: + + script: | + #!/bin/bash + set -e + + # 1. Construct the Tag Name + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- STARTING INTEGRATION TESTS ---" + echo "--- DEBUG: Current Directory ---" + pwd + + # 2. Run Python script with the tag + cd /app/ + python3 src/main_integration_tests.py + + # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them + echo "--- Copying Reports to Shared Workspace ---" + # Ensure destination exists + mkdir -p $(workspaces.source.path)/src/reports + # Copy the archive folder from the container image (/app) to the PVC (/workspace) + cp -r /app/src/reports/archive $(workspaces.source.path)/src/reports + echo "--- TESTS FINISHED SUCCESSFULLY ---" + workspaces: - name: source volumeClaimTemplate: @@ -280,3 +434,6 @@ spec: - name: dockerconfig-ws secret: secretName: ecosystem-appeng-morpheus-quay + - name: exploit-iq-data + persistentVolumeClaim: + claimName: exploit-iq-cache-pvc From 2ca344ccdc891dfff55de0b1756f17dd2f8ef2d7 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 6 Jan 2026 01:59:09 +0200 Subject: [PATCH 141/286] ci: few fixes Signed-off-by: Zvi Grinberg --- .dockerignore | 2 ++ .tekton/on-cm-runner.yaml | 2 +- .tekton/on-pull-request.yaml | 37 +++++++----------------------------- Dockerfile | 4 ++-- 4 files changed, 12 insertions(+), 33 deletions(-) diff --git a/.dockerignore b/.dockerignore index 5c2354a19..78ccfecd5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,5 @@ kafka-docker #Ignore the deployment configuration ./kustomize/ +**/.venv +**/.git diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 6497a6ea8..c99acc4a0 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -432,4 +432,4 @@ spec: secretName: "{{ git_auth_secret }}" - name: exploit-iq-data persistentVolumeClaim: - claimName: exploit-iq-cache-pvc + claimName: exploit-iq-cach-pvc diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index aeff02593..2048e0cd9 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -32,6 +32,8 @@ spec: value: . - name: dockerfile value: ./Dockerfile + - name: pr_number + value: "{{ pull_request_number }}" pipelineSpec: params: - name: repo_url @@ -239,7 +241,7 @@ spec: make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME print_banner "RUNNING UNIT TESTS" - make test-unit + make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" print_banner "LINT AND TEST COMPLETE" - name: integration-test @@ -258,8 +260,6 @@ spec: value: $(params.revision) - name: PR_NUMBER value: $(params.pr_number) - - name: TRIGGER_COMMENT - value: $(params.trigger_comment) taskSpec: params: @@ -267,23 +267,17 @@ spec: type: string - name: PR_NUMBER type: string - - name: TRIGGER_COMMENT - type: string workspaces: - name: source - name: basic-auth - name: exploit-iq-data - volumes: - - name: google-creds-volume - secret: - secretName: google-sheets-secrets # >>> THE SERVER (Sidecar) <<< sidecars: - name: server-application # [CHANGE 3] Use the image passed in params (the one built by the other pipeline) - image: $(params.target-image) - imagePullPolicy: Always + image: $(params.output-image) + imagePullPolicy: IfNotPresent volumeMounts: # Inject Env Vars (API Keys, Models, RedHat Creds) @@ -364,35 +358,18 @@ spec: imagePullPolicy: Always workingDir: $(workspaces.source.path) - volumeMounts: - - name: google-creds-volume - mountPath: /etc/secrets/google - readOnly: true - -# env: - script: | #!/bin/bash set -e - # 1. Construct the Tag Name - SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) - TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" - echo "--- STARTING INTEGRATION TESTS ---" echo "--- DEBUG: Current Directory ---" pwd - # 2. Run Python script with the tag + # 2. Run IT Script cd /app/ python3 src/main_integration_tests.py - # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them - echo "--- Copying Reports to Shared Workspace ---" - # Ensure destination exists - mkdir -p $(workspaces.source.path)/src/reports - # Copy the archive folder from the container image (/app) to the PVC (/workspace) - cp -r /app/src/reports/archive $(workspaces.source.path)/src/reports echo "--- TESTS FINISHED SUCCESSFULLY ---" workspaces: @@ -436,4 +413,4 @@ spec: secretName: ecosystem-appeng-morpheus-quay - name: exploit-iq-data persistentVolumeClaim: - claimName: exploit-iq-cache-pvc + claimName: unit-test-shared-cache diff --git a/Dockerfile b/Dockerfile index 11791b46e..72ea1eefe 100755 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN echo $'\ FROM base AS runtime RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ - source /workspace/.venv/bin/activate && \ - uv pip install "jupyterlab>=4.2.5,<5" + source /workspace/.venv/bin/activate + CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] From a2158e5de0645420bec588cb4f7fd6a00298c6e6 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 6 Jan 2026 09:20:38 +0200 Subject: [PATCH 142/286] test:fixed failing java test Signed-off-by: Zvi Grinberg --- src/vuln_analysis/tools/tests/test_transitive_code_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 2ff08c8eb..a3d3985a4 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -344,7 +344,7 @@ async def test_transitive_search_java_1(): (path_found, list_path) = result print(result) assert path_found is False - assert len(list_path) is 1 + assert len(list_path) in [0,1] @pytest.mark.asyncio async def test_transitive_search_java_2(): @@ -420,4 +420,4 @@ async def test_transitive_search_java_4(): assert path_found is True assert len(list_path) > 1 document = list_path[-1] - assert 'src/main/java/io/cryostat' in document.metadata['source'] \ No newline at end of file + assert 'src/main/java/io/cryostat' in document.metadata['source'] From 2264b8a4c8fbc5491627bd2dc2d83d7b35f6c7e2 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 6 Jan 2026 14:21:07 +0200 Subject: [PATCH 143/286] ci: few improvements and fixes Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 15 +++-- ci/it/integration-tests-input.json | 94 ++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 ci/it/integration-tests-input.json diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 2048e0cd9..9ae42742e 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -294,6 +294,8 @@ spec: env: - name: PYTHONUNBUFFERED value: "1" + - name: GOPROXY + value: https://proxy.golang.org,direct # Pass the raw comment text into the container - name: GOMODCACHE value: "/exploit-iq-data/go/pkg/mod" @@ -315,9 +317,12 @@ spec: script: | #!/bin/sh set -e - # We use python to safely remove the 'telemetry' block + # We use python to: + #1. safely remove the 'telemetry' block. + #2. removing the cve_output_config_name function from the agent flow ( no need for callback, result will be returned to IT process which calls the agent. + #3. Making sure that Internet search tool is enabled ( part of the tests are relying on that). echo "--- Generating Config without Tracing ---" - python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = True;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" CACHE_DIR_TARGET=".cache/am_cache" CACHE_PVC_SOURCE="/exploit-iq-data" @@ -366,11 +371,13 @@ spec: echo "--- DEBUG: Current Directory ---" pwd - # 2. Run IT Script + # 2. Prepares the IT input for test + cp -f ci/it/integration-tests-input.json /app/src/input/scan_it.json cd /app/ + # 3. Run IT Script python3 src/main_integration_tests.py - echo "--- TESTS FINISHED SUCCESSFULLY ---" + echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" workspaces: - name: source diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json new file mode 100644 index 000000000..ea8c4f30b --- /dev/null +++ b/ci/it/integration-tests-input.json @@ -0,0 +1,94 @@ +{ + "origin": "Agent Repository", + "iterations": 1, + "tests": [ + { + "language": "c", + "vuln_id": "CVE-2025-1094", + "image": { + "name": "registry.redhat.io/rhel8/postgresql-13", + "tag": "1-196.1724180180" + }, + "git": { + "repo": "https://github.com/postgres/postgres", + "ref": "REL_13_14" + }, + "use_sbom": true, + "sbom_file": "sboms/postgresql-13-1-196.1724180180.sbom", + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "go", + "vuln_id": "CVE-2025-22865", + "git": { + "repo": "https://github.com/kuadrant/authorino", + "ref": "f792cd138891dc1ead99fd089aa757fbca3aace9" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "python", + "vuln_id": "CVE-2024-49767", + "git": { + "repo": "https://github.com/TamarW0/example-repo", + "ref": "17af124607ff06fdf95734419b7ecd3af769c2f8" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "go", + "vuln_id": "CVE-2024-51744", + "git": { + "repo": "https://github.com/openshift/assisted-installer", + "ref": "ab9e2ade2f45890033a140b314ef87e367317b51" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + + { + "language": "go", + "vuln_id": "CVE-2024-28180", + "git": { + "repo": "https://github.com/openshift/oc-mirror", + "ref": "9621d8f72ecc7a0a13e40b9709b5e19cc621117b" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","protected_by_mitigating_control"], + "skip": true + }, + + + { + "language": "java", + "vuln_id": "CVE-2025-48734", + "git": { + "repo": "https://github.com/cryostatio/cryostat", + "ref": "v4.0.1" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_present","code_not_reachable"], + "skip": true + } + + + ] +} From 13253a207c1f9701a06fcd035b5cc8e2fe864b23 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Sun, 11 Jan 2026 09:28:42 +0200 Subject: [PATCH 144/286] Add Resource limits for tekton and Fix C transitive search issue (#181) --- .tekton/on-cm-runner.yaml | 10 +++++++++- src/vuln_analysis/utils/c_segmenter_custom.py | 2 +- src/vuln_analysis/utils/chain_of_calls_retriever.py | 4 ++++ .../utils/functions_parsers/c_lang_function_parsers.py | 7 +++++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index c99acc4a0..08e308c74 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -108,6 +108,14 @@ spec: image: $(params.target-image) imagePullPolicy: Always + resources: + requests: + cpu: "1000m" # CPU request (1 core) + memory: "8Gi" # Memory request (8 gigabytes) + limits: + cpu: "2000m" # CPU limit (2 cores) + memory: "16Gi" # Memory limit (16 gigabytes) + volumeMounts: - name: google-creds-volume mountPath: /etc/secrets/google @@ -206,7 +214,7 @@ spec: port: 26466 initialDelaySeconds: 10 periodSeconds: 5 - + # >>> THE CLIENT (Test Runner) <<< steps: diff --git a/src/vuln_analysis/utils/c_segmenter_custom.py b/src/vuln_analysis/utils/c_segmenter_custom.py index 87af1d152..b1155150a 100644 --- a/src/vuln_analysis/utils/c_segmenter_custom.py +++ b/src/vuln_analysis/utils/c_segmenter_custom.py @@ -2,7 +2,7 @@ from langchain_community.document_loaders.parsers.language.c import CSegmenter from typing import List - +#class extened CSegmenter class CSegmenterExtended(CSegmenter): def __init__(self, code: str): diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/vuln_analysis/utils/chain_of_calls_retriever.py index fe2abb6c6..3e243003e 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/vuln_analysis/utils/chain_of_calls_retriever.py @@ -340,6 +340,8 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack if found and self.language_parser.is_call_allowed( pkg_docs, doc, document_function): log_entries.append((file_name, func_name, function_name_to_search)) relevant_docs_to_search_in.append(doc) + if self.language_parser.is_root_package(doc): + return relevant_docs_to_search_in except ValueError as ex: logger.error("doc %s / %s", doc, ex) @@ -501,6 +503,8 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # Try to create dummy package for ecosystem standard library function, in such case, build a document for # vulnerable function in standard lib package of language. page_content = self.language_parser.get_dummy_function(function) + if page_content is None: + return matching_documents, self.found_path if class_name: page_content = page_content + f'\n{self.language_parser.get_comment_line_notation()}(class: {class_name})' target_function_doc = Document(page_content=page_content diff --git a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py index 992ed6cfe..c560988f3 100644 --- a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py @@ -592,7 +592,10 @@ def get_package_name(self, function: Document, package_name: str) -> str: return package_name def is_root_package(self, function: Document) -> bool: - + pkgs = self.get_package_names(function) + if len(pkgs) > 0: + if pkgs[0] == self.dep_builder_tree.prj_name: + return True return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) def is_comment_line(self, line: str) -> bool: @@ -705,7 +708,7 @@ def create_dummy_for_standard_lib(self,package_name: str)-> bool: return False def get_dummy_function(self, function_name): - return f"void {function_name}() {{}}" + return None def is_call_allowed(self, pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: caller_pkg = self.get_package_names(caller_function)[0] From f874205235a023fe045a6ff16ce9b6501aee776c Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 5 Jan 2026 10:45:10 +0200 Subject: [PATCH 145/286] chore: implement exploit_iq_commons shared package Signed-off-by: Vladimir Belousov --- pyproject.toml | 4 ++ .../__init__.py | 0 .../data}/__init__.py | 2 +- .../data/standard_libs/__init__.py | 14 +++++++ .../standard_libs/standard_libraries.json | 0 .../data_models/__init__.py | 0 .../data_models/common.py | 0 .../data_models/cve_intel.py | 0 .../data_models/dependencies.py | 0 .../data_models/info.py | 4 +- .../data_models/input.py | 16 +++---- src/exploit_iq_commons/logging/__init__.py | 0 .../logging/loggers_factory.py | 0 src/exploit_iq_commons/pyproject.toml | 25 +++++++++++ src/exploit_iq_commons/utils/__init__.py | 0 .../utils/c_segmenter_custom.py | 0 .../utils/chain_of_calls_retriever.py | 10 ++--- .../utils/chain_of_calls_retriever_base.py | 6 +-- .../utils/chain_of_calls_retriever_factory.py | 10 ++--- .../utils/data_utils.py | 6 +-- .../utils/dep_tree.py | 8 ++-- .../utils/document_embedding.py | 22 +++++----- .../utils/functions_parsers/__init__.py | 0 .../c_lang_function_parsers.py | 6 +-- .../golang_functions_parsers.py | 2 +- .../java_functions_parsers.py | 6 +-- .../lang_functions_parsers.py | 0 .../lang_functions_parsers_factory.py | 12 +++--- .../python_functions_parser.py | 2 +- .../utils/git_utils.py | 0 .../utils/go_segmenters_with_methods.py | 0 .../utils/java_chain_of_calls_retriever.py | 14 +++---- .../utils/java_segmenters_with_methods.py | 0 .../utils/java_utils.py | 0 .../utils/js_extended_parser.py | 2 +- .../python_segmenters_with_classes_methods.py | 0 .../utils/source_code_git_loader.py | 4 +- .../utils/source_rpm_downloader.py | 4 +- .../utils/standard_library_cache.py | 4 +- .../utils/string_utils.py | 0 .../utils/transitive_code_searcher_tool.py | 6 +-- src/vuln_analysis/data_models/output.py | 2 +- src/vuln_analysis/data_models/plugin.py | 4 +- .../data_models/plugins/intel_plugin.py | 6 +-- src/vuln_analysis/data_models/state.py | 4 +- src/vuln_analysis/functions/cve_agent.py | 2 +- .../functions/cve_calculate_intel_score.py | 2 +- .../functions/cve_check_vuln_deps.py | 12 +++--- src/vuln_analysis/functions/cve_checklist.py | 4 +- .../functions/cve_fetch_intel.py | 4 +- .../functions/cve_file_output.py | 2 +- .../functions/cve_generate_vdbs.py | 22 +++++----- .../functions/cve_http_output.py | 2 +- src/vuln_analysis/functions/cve_justify.py | 2 +- .../functions/cve_process_sbom.py | 16 +++---- src/vuln_analysis/functions/cve_summarize.py | 4 +- src/vuln_analysis/register.py | 6 +-- .../tools/lexical_full_search.py | 2 +- src/vuln_analysis/tools/local_vdb.py | 2 +- src/vuln_analysis/tools/serp.py | 2 +- .../tools/tests/test_segmenter.py | 4 +- .../tests/test_source_code_git_loader.py | 2 +- .../tests/test_transitive_code_search.py | 10 ++--- .../tools/transitive_code_search.py | 16 +++---- src/vuln_analysis/utils/async_http_utils.py | 2 +- .../utils/checklist_prompt_generator.py | 4 +- .../utils/clients/first_client.py | 4 +- .../utils/clients/ghsa_client.py | 4 +- src/vuln_analysis/utils/clients/nvd_client.py | 4 +- .../utils/clients/rhsa_client.py | 4 +- .../utils/clients/ubuntu_client.py | 2 +- .../utils/error_handling_decorator.py | 2 +- src/vuln_analysis/utils/full_text_search.py | 2 +- .../utils/function_name_extractor.py | 4 +- .../utils/function_name_locator.py | 8 ++-- src/vuln_analysis/utils/http_utils.py | 2 +- src/vuln_analysis/utils/intel_retriever.py | 12 +++--- src/vuln_analysis/utils/intel_source_score.py | 4 +- src/vuln_analysis/utils/intel_utils.py | 2 +- .../utils/justification_parser.py | 2 +- src/vuln_analysis/utils/llm_engine_utils.py | 10 ++--- src/vuln_analysis/utils/output_formatter.py | 2 +- .../utils/vulnerable_dependency_checker.py | 8 ++-- tests/test_java_script_extended.py | 2 +- uv.lock | 42 +++++++++++++++++-- 85 files changed, 263 insertions(+), 184 deletions(-) rename src/{vuln_analysis/logging => exploit_iq_commons}/__init__.py (100%) rename src/{vuln_analysis/utils/functions_parsers => exploit_iq_commons/data}/__init__.py (84%) create mode 100644 src/exploit_iq_commons/data/standard_libs/__init__.py rename src/{vuln_analysis => exploit_iq_commons}/data/standard_libs/standard_libraries.json (100%) create mode 100644 src/exploit_iq_commons/data_models/__init__.py rename src/{vuln_analysis => exploit_iq_commons}/data_models/common.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/data_models/cve_intel.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/data_models/dependencies.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/data_models/info.py (94%) rename src/{vuln_analysis => exploit_iq_commons}/data_models/input.py (92%) create mode 100644 src/exploit_iq_commons/logging/__init__.py rename src/{vuln_analysis => exploit_iq_commons}/logging/loggers_factory.py (100%) create mode 100644 src/exploit_iq_commons/pyproject.toml create mode 100644 src/exploit_iq_commons/utils/__init__.py rename src/{vuln_analysis => exploit_iq_commons}/utils/c_segmenter_custom.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/chain_of_calls_retriever.py (98%) rename src/{vuln_analysis => exploit_iq_commons}/utils/chain_of_calls_retriever_base.py (95%) rename src/{vuln_analysis => exploit_iq_commons}/utils/chain_of_calls_retriever_factory.py (74%) rename src/{vuln_analysis => exploit_iq_commons}/utils/data_utils.py (96%) rename src/{vuln_analysis => exploit_iq_commons}/utils/dep_tree.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/document_embedding.py (95%) create mode 100644 src/exploit_iq_commons/utils/functions_parsers/__init__.py rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/c_lang_function_parsers.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/golang_functions_parsers.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/java_functions_parsers.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/lang_functions_parsers.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/lang_functions_parsers_factory.py (63%) rename src/{vuln_analysis => exploit_iq_commons}/utils/functions_parsers/python_functions_parser.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/git_utils.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/go_segmenters_with_methods.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/java_chain_of_calls_retriever.py (98%) rename src/{vuln_analysis => exploit_iq_commons}/utils/java_segmenters_with_methods.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/java_utils.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/js_extended_parser.py (98%) rename src/{vuln_analysis => exploit_iq_commons}/utils/python_segmenters_with_classes_methods.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/source_code_git_loader.py (97%) rename src/{vuln_analysis => exploit_iq_commons}/utils/source_rpm_downloader.py (99%) rename src/{vuln_analysis => exploit_iq_commons}/utils/standard_library_cache.py (98%) rename src/{vuln_analysis => exploit_iq_commons}/utils/string_utils.py (100%) rename src/{vuln_analysis => exploit_iq_commons}/utils/transitive_code_searcher_tool.py (96%) diff --git a/pyproject.toml b/pyproject.toml index ebebf5527..e83eb0b1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ git_describe_command = ["git", "describe", "--tags", "--match", "v*"] name = "vuln_analysis" dynamic = ["version"] dependencies = [ + "exploit-iq-commons", "aiohttp-client-cache==0.11", "aioresponses==0.7.6", "nvidia-nat[langchain,profiling,telemetry]>=1.2.0rc8,<1.3.0", @@ -783,3 +784,6 @@ reportInvalidTypeForm = "warning" [tool.uv] managed = true + +[tool.uv.sources] +exploit-iq-commons = { path = "src/exploit_iq_commons", editable = true } diff --git a/src/vuln_analysis/logging/__init__.py b/src/exploit_iq_commons/__init__.py similarity index 100% rename from src/vuln_analysis/logging/__init__.py rename to src/exploit_iq_commons/__init__.py diff --git a/src/vuln_analysis/utils/functions_parsers/__init__.py b/src/exploit_iq_commons/data/__init__.py similarity index 84% rename from src/vuln_analysis/utils/functions_parsers/__init__.py rename to src/exploit_iq_commons/data/__init__.py index a1744724e..cf7c586a5 100644 --- a/src/vuln_analysis/utils/functions_parsers/__init__.py +++ b/src/exploit_iq_commons/data/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/exploit_iq_commons/data/standard_libs/__init__.py b/src/exploit_iq_commons/data/standard_libs/__init__.py new file mode 100644 index 000000000..cf7c586a5 --- /dev/null +++ b/src/exploit_iq_commons/data/standard_libs/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/data/standard_libs/standard_libraries.json b/src/exploit_iq_commons/data/standard_libs/standard_libraries.json similarity index 100% rename from src/vuln_analysis/data/standard_libs/standard_libraries.json rename to src/exploit_iq_commons/data/standard_libs/standard_libraries.json diff --git a/src/exploit_iq_commons/data_models/__init__.py b/src/exploit_iq_commons/data_models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/data_models/common.py b/src/exploit_iq_commons/data_models/common.py similarity index 100% rename from src/vuln_analysis/data_models/common.py rename to src/exploit_iq_commons/data_models/common.py diff --git a/src/vuln_analysis/data_models/cve_intel.py b/src/exploit_iq_commons/data_models/cve_intel.py similarity index 100% rename from src/vuln_analysis/data_models/cve_intel.py rename to src/exploit_iq_commons/data_models/cve_intel.py diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/exploit_iq_commons/data_models/dependencies.py similarity index 100% rename from src/vuln_analysis/data_models/dependencies.py rename to src/exploit_iq_commons/data_models/dependencies.py diff --git a/src/vuln_analysis/data_models/info.py b/src/exploit_iq_commons/data_models/info.py similarity index 94% rename from src/vuln_analysis/data_models/info.py rename to src/exploit_iq_commons/data_models/info.py index e94094350..a01f1dda7 100644 --- a/src/vuln_analysis/data_models/info.py +++ b/src/exploit_iq_commons/data_models/info.py @@ -15,8 +15,8 @@ from pydantic import BaseModel -from .cve_intel import CveIntel -from .dependencies import VulnerableDependencies +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.dependencies import VulnerableDependencies class SBOMPackage(BaseModel): diff --git a/src/vuln_analysis/data_models/input.py b/src/exploit_iq_commons/data_models/input.py similarity index 92% rename from src/vuln_analysis/data_models/input.py rename to src/exploit_iq_commons/data_models/input.py index 5ee3c1d11..870101dbd 100644 --- a/src/vuln_analysis/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -26,14 +26,14 @@ from pydantic import Tag from pydantic import field_validator -from ..utils.string_utils import is_valid_cve_id -from ..utils.string_utils import is_valid_ghsa_id -from ..utils.dep_tree import Ecosystem -from .common import AnalysisType -from .common import HashableModel -from .common import TypedBaseModel -from .info import AgentMorpheusInfo -from .info import SBOMPackage +from exploit_iq_commons.utils.string_utils import is_valid_cve_id +from exploit_iq_commons.utils.string_utils import is_valid_ghsa_id +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.common import HashableModel +from exploit_iq_commons.data_models.common import TypedBaseModel +from exploit_iq_commons.data_models.info import AgentMorpheusInfo +from exploit_iq_commons.data_models.info import SBOMPackage class SourceDocumentsInfo(HashableModel): diff --git a/src/exploit_iq_commons/logging/__init__.py b/src/exploit_iq_commons/logging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/logging/loggers_factory.py b/src/exploit_iq_commons/logging/loggers_factory.py similarity index 100% rename from src/vuln_analysis/logging/loggers_factory.py rename to src/exploit_iq_commons/logging/loggers_factory.py diff --git a/src/exploit_iq_commons/pyproject.toml b/src/exploit_iq_commons/pyproject.toml new file mode 100644 index 000000000..708bf96f4 --- /dev/null +++ b/src/exploit_iq_commons/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools >= 64", "setuptools-scm>=8"] + +[project] +name = "exploit-iq-commons" +version = "0.1.0" +description = "Common library for ExploitIQ - shared code for vulnerability analysis." +requires-python = ">=3.11,<3.13" + +dependencies = [ + "esprima==4.0.1", + "GitPython==3.1.44", + "langchain~=0.3.0", + "langchain-community~=0.3.0", + "pandas==2.3.1", + "tqdm==4.67.1", + "tree-sitter-languages==1.10.2", + "tree-sitter==0.21.3", + "pydantic>=2.0", +] + +[tool.setuptools.packages.find] +where = [".."] +include = ["exploit_iq_commons*"] diff --git a/src/exploit_iq_commons/utils/__init__.py b/src/exploit_iq_commons/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/utils/c_segmenter_custom.py b/src/exploit_iq_commons/utils/c_segmenter_custom.py similarity index 100% rename from src/vuln_analysis/utils/c_segmenter_custom.py rename to src/exploit_iq_commons/utils/c_segmenter_custom.py diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py similarity index 98% rename from src/vuln_analysis/utils/chain_of_calls_retriever.py rename to src/exploit_iq_commons/utils/chain_of_calls_retriever.py index 3e243003e..1b86a476a 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -6,14 +6,14 @@ from langchain_core.documents import Document -from .chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, PARENTS_INDEX, \ +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, PARENTS_INDEX, \ calculate_hashable_string_for_function, EXCLUSIONS_INDEX -from vuln_analysis.logging.loggers_factory import LoggingFactory -from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem -from .functions_parsers.lang_functions_parsers_factory import ( +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( get_language_function_parser, ) -from .standard_library_cache import StandardLibraryCache +from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever_base.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py similarity index 95% rename from src/vuln_analysis/utils/chain_of_calls_retriever_base.py rename to src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py index d9a68deec..53488164a 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever_base.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py @@ -4,8 +4,8 @@ from langchain_core.documents import Document -from vuln_analysis.utils.dep_tree import DependencyTree, Ecosystem -from vuln_analysis.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.dep_tree import DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser PARENTS_INDEX = 0 @@ -13,7 +13,7 @@ METHOD_EXCLUSIONS_INDEX = 2 -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/chain_of_calls_retriever_factory.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py similarity index 74% rename from src/vuln_analysis/utils/chain_of_calls_retriever_factory.py rename to src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py index 9c9e4d10a..ee5e0a698 100644 --- a/src/vuln_analysis/utils/chain_of_calls_retriever_factory.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py @@ -3,11 +3,11 @@ from langchain_core.documents import Document -from vuln_analysis.data_models.input import SourceDocumentsInfo -from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetriever -from vuln_analysis.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase -from vuln_analysis.utils.dep_tree import Ecosystem -from vuln_analysis.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever def get_chain_of_calls_retriever(ecosystem: Ecosystem, documents: List[Document], manifest_path: Path, query: str, code_source_info: SourceDocumentsInfo) -> ChainOfCallsRetrieverBase: diff --git a/src/vuln_analysis/utils/data_utils.py b/src/exploit_iq_commons/utils/data_utils.py similarity index 96% rename from src/vuln_analysis/utils/data_utils.py rename to src/exploit_iq_commons/utils/data_utils.py index dd74fcb25..ee2e0a2ee 100644 --- a/src/vuln_analysis/utils/data_utils.py +++ b/src/exploit_iq_commons/utils/data_utils.py @@ -24,8 +24,8 @@ from pydantic import BaseModel as BaseModel from pydantic.v1 import BaseModel as BaseModelV1 -from vuln_analysis.data_models.cve_intel import CveIntel -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -139,4 +139,4 @@ def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref, documents_name: str logger.debug(f"Cache Hit on Documents, Loaded existing docs at {cached_documents_path}") return documents, True - return [], False + return [], False \ No newline at end of file diff --git a/src/vuln_analysis/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py similarity index 99% rename from src/vuln_analysis/utils/dep_tree.py rename to src/exploit_iq_commons/utils/dep_tree.py index 2a136f0d3..f35bae99a 100644 --- a/src/vuln_analysis/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -17,14 +17,14 @@ import json import zipfile -from vuln_analysis.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from contextlib import contextmanager # C_DEP_LIBS_NAME moved here to avoid circular import C_DEP_LIBS_NAME = "rpm_libs" -from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager -from vuln_analysis.logging.loggers_factory import LoggingFactory -from vuln_analysis.utils.java_utils import add_missing_jar_string, is_maven_gav +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_utils import add_missing_jar_string, is_maven_gav from collections import deque logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py similarity index 95% rename from src/vuln_analysis/utils/document_embedding.py rename to src/exploit_iq_commons/utils/document_embedding.py index 9ccd55d90..9d92fc763 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -34,18 +34,18 @@ from langchain_community.vectorstores import FAISS from langchain_core.document_loaders.blob_loaders import Blob -from vuln_analysis.data_models.input import SourceDocumentsInfo -from vuln_analysis.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY, DEFAULT_GIT_DIRECTORY, \ +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY, DEFAULT_GIT_DIRECTORY, \ VDB_DIRECTORY, PathLike -from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods -from vuln_analysis.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods -from vuln_analysis.utils.java_segmenters_with_methods import JavaSegmenterWithMethods -from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter -from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader -from vuln_analysis.utils.git_utils import sanitize_git_url_for_path -from vuln_analysis.logging.loggers_factory import LoggingFactory - -from vuln_analysis.utils.c_segmenter_custom import CSegmenterExtended +from exploit_iq_commons.utils.go_segmenters_with_methods import GoSegmenterWithMethods +from exploit_iq_commons.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods +from exploit_iq_commons.utils.java_segmenters_with_methods import JavaSegmenterWithMethods +from exploit_iq_commons.utils.js_extended_parser import ExtendedJavaScriptSegmenter +from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover diff --git a/src/exploit_iq_commons/utils/functions_parsers/__init__.py b/src/exploit_iq_commons/utils/functions_parsers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py similarity index 99% rename from src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py rename to src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py index c560988f3..b8522eabb 100644 --- a/src/vuln_analysis/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py @@ -1,11 +1,11 @@ from typing import Tuple, List from langchain_core.documents import Document -from .lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser import re import regex import os -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -14,7 +14,7 @@ MIN_PARTS_FOR_PARSING = 2 # Minimum number of parts required for parsing STRING_NOT_FOUND = -1 # Return value when string.find() doesn't find the substring -from ..dep_tree import DependencyTree, CCppDependencyTreeBuilder, C_DEP_LIBS_NAME +from exploit_iq_commons.utils.dep_tree import DependencyTree, CCppDependencyTreeBuilder, C_DEP_LIBS_NAME def _remove_c_comments(code: str) -> str: # Remove all multi-line comments (/* ... */) diff --git a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py similarity index 99% rename from src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py rename to src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index 5ffebf3e7..b31644ba3 100644 --- a/src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -4,7 +4,7 @@ from langchain_core.documents import Document -from .lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser EMBEDDED_TYPE = "embedded_type" diff --git a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py similarity index 99% rename from src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py rename to src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index ba8259e96..64cb6f0dd 100644 --- a/src/vuln_analysis/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -6,11 +6,11 @@ from langchain_core.documents import Document -from .lang_functions_parsers import LanguageFunctionsParser -from ..java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, get_type_name, \ +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, get_type_name, \ is_java_type, is_java_method, extract_method_name_with_params, find_function, get_target_class_names, \ strip_java_generics, JAVA_ANNOTATION_SYMBOL -from ...logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py similarity index 100% rename from src/vuln_analysis/utils/functions_parsers/lang_functions_parsers.py rename to src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py diff --git a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py similarity index 63% rename from src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py rename to src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py index 4f1178c68..32445b170 100644 --- a/src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py @@ -1,9 +1,9 @@ -from ..dep_tree import Ecosystem, DependencyTree -from .golang_functions_parsers import GoLanguageFunctionsParser -from .python_functions_parser import PythonLanguageFunctionsParser -from .java_functions_parsers import JavaLanguageFunctionsParser -from .lang_functions_parsers import LanguageFunctionsParser -from .c_lang_function_parsers import CLanguageFunctionsParser +from exploit_iq_commons.utils.dep_tree import Ecosystem, DependencyTree +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.java_functions_parsers import JavaLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.c_lang_function_parsers import CLanguageFunctionsParser def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | None) -> LanguageFunctionsParser: """ diff --git a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py similarity index 99% rename from src/vuln_analysis/utils/functions_parsers/python_functions_parser.py rename to src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index 5007e8624..59b14f1a3 100644 --- a/src/vuln_analysis/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -5,7 +5,7 @@ from langchain_core.documents import Document -from vuln_analysis.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser PARAMETER = "parameter" RETURN_TYPES = "return_types" diff --git a/src/vuln_analysis/utils/git_utils.py b/src/exploit_iq_commons/utils/git_utils.py similarity index 100% rename from src/vuln_analysis/utils/git_utils.py rename to src/exploit_iq_commons/utils/git_utils.py diff --git a/src/vuln_analysis/utils/go_segmenters_with_methods.py b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py similarity index 100% rename from src/vuln_analysis/utils/go_segmenters_with_methods.py rename to src/exploit_iq_commons/utils/go_segmenters_with_methods.py diff --git a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py similarity index 98% rename from src/vuln_analysis/utils/java_chain_of_calls_retriever.py rename to src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index 5730d6d7f..e4c7ac955 100644 --- a/src/vuln_analysis/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -5,18 +5,18 @@ from langchain_core.documents import Document -from .chain_of_calls_retriever_base import PARENTS_INDEX, calculate_hashable_string_for_function, EXCLUSIONS_INDEX, \ +from exploit_iq_commons.utils.chain_of_calls_retriever_base import PARENTS_INDEX, calculate_hashable_string_for_function, EXCLUSIONS_INDEX, \ ChainOfCallsRetrieverBase, METHOD_EXCLUSIONS_INDEX -from .data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY -from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem -from .functions_parsers.lang_functions_parsers_factory import ( +from exploit_iq_commons.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY +from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( get_language_function_parser, ) -from vuln_analysis.logging.loggers_factory import LoggingFactory -from .java_utils import convert_from_maven_artifact, extract_jar_name, is_maven_gav, extract_method_name_with_params, \ +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_utils import convert_from_maven_artifact, extract_jar_name, is_maven_gav, extract_method_name_with_params, \ create_inheritance_map, get_target_class_names, dummy_package_name -from ..data_models.input import SourceDocumentsInfo +from exploit_iq_commons.data_models.input import SourceDocumentsInfo logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/java_segmenters_with_methods.py b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py similarity index 100% rename from src/vuln_analysis/utils/java_segmenters_with_methods.py rename to src/exploit_iq_commons/utils/java_segmenters_with_methods.py diff --git a/src/vuln_analysis/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py similarity index 100% rename from src/vuln_analysis/utils/java_utils.py rename to src/exploit_iq_commons/utils/java_utils.py diff --git a/src/vuln_analysis/utils/js_extended_parser.py b/src/exploit_iq_commons/utils/js_extended_parser.py similarity index 98% rename from src/vuln_analysis/utils/js_extended_parser.py rename to src/exploit_iq_commons/utils/js_extended_parser.py index cf7713125..56a03d7b4 100644 --- a/src/vuln_analysis/utils/js_extended_parser.py +++ b/src/exploit_iq_commons/utils/js_extended_parser.py @@ -21,7 +21,7 @@ import esprima from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/python_segmenters_with_classes_methods.py b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py similarity index 100% rename from src/vuln_analysis/utils/python_segmenters_with_classes_methods.py rename to src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py similarity index 97% rename from src/vuln_analysis/utils/source_code_git_loader.py rename to src/exploit_iq_commons/utils/source_code_git_loader.py index ae2b089dd..592caa2b8 100644 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -25,8 +25,8 @@ from langchain_core.document_loaders.blob_loaders import Blob from tqdm import tqdm -from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from exploit_iq_commons.logging.loggers_factory import LoggingFactory PathLike = typing.Union[str, os.PathLike] diff --git a/src/vuln_analysis/utils/source_rpm_downloader.py b/src/exploit_iq_commons/utils/source_rpm_downloader.py similarity index 99% rename from src/vuln_analysis/utils/source_rpm_downloader.py rename to src/exploit_iq_commons/utils/source_rpm_downloader.py index 051cbb2b6..09dc81078 100644 --- a/src/vuln_analysis/utils/source_rpm_downloader.py +++ b/src/exploit_iq_commons/utils/source_rpm_downloader.py @@ -12,8 +12,8 @@ from dataclasses import dataclass from typing import List, Dict, Any, Optional import threading -from vuln_analysis.data_models.info import SBOMPackage -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.data_models.info import SBOMPackage +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/standard_library_cache.py b/src/exploit_iq_commons/utils/standard_library_cache.py similarity index 98% rename from src/vuln_analysis/utils/standard_library_cache.py rename to src/exploit_iq_commons/utils/standard_library_cache.py index 7c67a4214..8d090c40d 100644 --- a/src/vuln_analysis/utils/standard_library_cache.py +++ b/src/exploit_iq_commons/utils/standard_library_cache.py @@ -14,8 +14,8 @@ import difflib # No typing imports needed; we use builtin generics -from vuln_analysis.logging.loggers_factory import LoggingFactory -from vuln_analysis.utils.dep_tree import Ecosystem +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.dep_tree import Ecosystem logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/string_utils.py b/src/exploit_iq_commons/utils/string_utils.py similarity index 100% rename from src/vuln_analysis/utils/string_utils.py rename to src/exploit_iq_commons/utils/string_utils.py diff --git a/src/vuln_analysis/utils/transitive_code_searcher_tool.py b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py similarity index 96% rename from src/vuln_analysis/utils/transitive_code_searcher_tool.py rename to src/exploit_iq_commons/utils/transitive_code_searcher_tool.py index 7c5053f08..8167f4405 100644 --- a/src/vuln_analysis/utils/transitive_code_searcher_tool.py +++ b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py @@ -17,8 +17,8 @@ from langchain.docstore.document import Document -from .chain_of_calls_retriever_base import ChainOfCallsRetrieverBase -from .dep_tree import ( +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import ( GOLANG_MANIFEST, JAVA_MANIFEST, JS_MANIFEST, @@ -32,7 +32,7 @@ C_CPLUSPLUS_MANIFEST_4, ) -from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index ba262f32b..232a98a20 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -18,7 +18,7 @@ from pydantic import BaseModel from pydantic import model_validator -from .input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput class AgentIntermediateStep(BaseModel): diff --git a/src/vuln_analysis/data_models/plugin.py b/src/vuln_analysis/data_models/plugin.py index ff09c0754..108e6c980 100644 --- a/src/vuln_analysis/data_models/plugin.py +++ b/src/vuln_analysis/data_models/plugin.py @@ -20,8 +20,8 @@ import aiohttp -from .common import TypedBaseModel -from .cve_intel import IntelPluginData +from exploit_iq_commons.data_models.common import TypedBaseModel +from exploit_iq_commons.data_models.cve_intel import IntelPluginData _T = typing.TypeVar('_T', bound='PluginSchema') diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py index b928c470d..cdad04751 100644 --- a/src/vuln_analysis/data_models/plugins/intel_plugin.py +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -12,10 +12,10 @@ import requests from pydantic import BaseModel -from ..cve_intel import IntelPluginData -from ..plugin import IntelPluginSchema +from exploit_iq_commons.data_models.cve_intel import IntelPluginData +from vuln_analysis.data_models.plugin import IntelPluginSchema -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index 3a52662a5..7b77e7f82 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -17,8 +17,8 @@ from pydantic import BaseModel -from vuln_analysis.data_models.cve_intel import CveIntel -from vuln_analysis.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput class AgentMorpheusEngineState(BaseModel): diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 93eed149f..60ca3a783 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -34,7 +34,7 @@ from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.utils.error_handling_decorator import ToolRaisedException from vuln_analysis.utils.prompting import get_agent_prompt -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/functions/cve_calculate_intel_score.py b/src/vuln_analysis/functions/cve_calculate_intel_score.py index 458af643b..ef1a25886 100644 --- a/src/vuln_analysis/functions/cve_calculate_intel_score.py +++ b/src/vuln_analysis/functions/cve_calculate_intel_score.py @@ -39,7 +39,7 @@ class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel @register_function(config_type=CVECalculateIntelScoreConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_calculate_intel_score(config: CVECalculateIntelScoreConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from vuln_analysis.utils.intel_source_score import IntelScorer async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 2da2185a7..4875e7926 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -24,8 +24,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.data_models.common import AnalysisType -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) @@ -42,10 +42,10 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep @register_function(config_type=CVEVulnerableDepsChecksConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_check_vuln_deps(config: CVEVulnerableDepsChecksConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.cve_intel import CveIntel - from vuln_analysis.data_models.dependencies import VulnerableDependencies - from vuln_analysis.data_models.dependencies import VulnerableSBOMPackage - from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.cve_intel import CveIntel + from exploit_iq_commons.data_models.dependencies import VulnerableDependencies + from exploit_iq_commons.data_models.dependencies import VulnerableSBOMPackage + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from vuln_analysis.utils.vulnerable_dependency_checker import VulnerableDependencyChecker async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 2c367d6ec..f0cd65662 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -24,8 +24,8 @@ from aiq.data_models.function import FunctionBaseConfig from aiq.data_models.component_ref import FunctionRef from pydantic import Field -from vuln_analysis.utils import data_utils -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils import data_utils +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index bf42f657d..1acc59e34 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -27,7 +27,7 @@ from vuln_analysis.data_models.plugin import PluginConfig -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) @@ -43,7 +43,7 @@ class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): @register_function(config_type=CVEFetchIntelConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_fetch_intel(config: CVEFetchIntelConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from vuln_analysis.utils.intel_retriever import IntelRetriever async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: diff --git a/src/vuln_analysis/functions/cve_file_output.py b/src/vuln_analysis/functions/cve_file_output.py index c3ea6788e..0d8562828 100644 --- a/src/vuln_analysis/functions/cve_file_output.py +++ b/src/vuln_analysis/functions/cve_file_output.py @@ -21,7 +21,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 53e4ca6fb..b37caa0f9 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -26,8 +26,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.data_models.common import AnalysisType -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id from vuln_analysis.tools.tool_names import ToolNames logger = LoggingFactory.get_agent_logger(__name__) @@ -60,17 +60,17 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): @register_function(config_type=CVEGenerateVDBsToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): - from vuln_analysis.data_models.info import AgentMorpheusInfo - from vuln_analysis.data_models.input import AgentMorpheusEngineInput - from vuln_analysis.data_models.input import AgentMorpheusInput - from vuln_analysis.data_models.input import SourceDocumentsInfo + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusInput + from exploit_iq_commons.data_models.input import SourceDocumentsInfo from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig - from vuln_analysis.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding from vuln_analysis.utils.full_text_search import FullTextSearch - from vuln_analysis.utils.git_utils import get_repo_from_path - from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager - from vuln_analysis.data_models.input import ManualSBOMInfoInput - from vuln_analysis.utils.standard_library_cache import StandardLibraryCache + from exploit_iq_commons.utils.git_utils import get_repo_from_path + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache agent_config = builder.get_function_config(config.agent_name) assert isinstance(agent_config, CVEAgentExecutorToolConfig) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index c9764a9bf..7a78c7c66 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -22,7 +22,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py index e7f50828a..fce509f0e 100644 --- a/src/vuln_analysis/functions/cve_justify.py +++ b/src/vuln_analysis/functions/cve_justify.py @@ -23,7 +23,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index ada78bb6c..65865167f 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -23,8 +23,8 @@ from pydantic import Field from pydantic import NonNegativeInt -from vuln_analysis.data_models.common import AnalysisType -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -38,12 +38,12 @@ class CVEProcessSBOMConfig(FunctionBaseConfig, name="cve_process_sbom"): @register_function(config_type=CVEProcessSBOMConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_process_sbom(config: CVEProcessSBOMConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.info import AgentMorpheusInfo - from vuln_analysis.data_models.input import AgentMorpheusEngineInput - from vuln_analysis.data_models.input import FileSBOMInfoInput - from vuln_analysis.data_models.input import HTTPSBOMInfoInput - from vuln_analysis.data_models.input import ManualSBOMInfoInput - from vuln_analysis.data_models.input import SBOMPackage + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import FileSBOMInfoInput + from exploit_iq_commons.data_models.input import HTTPSBOMInfoInput + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.data_models.input import SBOMPackage from vuln_analysis.utils import http_utils from vuln_analysis.utils.http_utils import HTTPMethod diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index af5161a3f..608fff839 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -23,9 +23,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.utils.string_utils import get_checklist_item_string +from exploit_iq_commons.utils.string_utils import get_checklist_item_string -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 520cdb9ab..48fe71d99 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -24,8 +24,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.data_models.input import AgentMorpheusInput +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusOutput from vuln_analysis.data_models.state import AgentMorpheusEngineState # pylint: disable=unused-import @@ -51,7 +51,7 @@ # pylint: enable=unused-import from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 33609f190..f2e92e664 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -22,7 +22,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory from vuln_analysis.utils.error_handling_decorator import catch_tool_errors LEXICAL_CODE_SEARCH = "lexical_code_search" diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 41a79ccd8..d6effc8dc 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -24,7 +24,7 @@ from vuln_analysis.data_models.vdb_type import VdbType -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory from vuln_analysis.utils.error_handling_decorator import catch_tool_errors LOCAL_VDB_RETRIEVER = "local_vdb_retriever" diff --git a/src/vuln_analysis/tools/serp.py b/src/vuln_analysis/tools/serp.py index 41981e0a7..fc117654f 100644 --- a/src/vuln_analysis/tools/serp.py +++ b/src/vuln_analysis/tools/serp.py @@ -20,7 +20,7 @@ from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory from vuln_analysis.utils.error_handling_decorator import catch_tool_errors SERP_WRAPPER = "serp_wrapper" diff --git a/src/vuln_analysis/tools/tests/test_segmenter.py b/src/vuln_analysis/tools/tests/test_segmenter.py index 45f32766b..18095b407 100644 --- a/src/vuln_analysis/tools/tests/test_segmenter.py +++ b/src/vuln_analysis/tools/tests/test_segmenter.py @@ -15,8 +15,8 @@ from pathlib import Path import pytest -from vuln_analysis.utils.c_segmenter_custom import CSegmenterExtended -from vuln_analysis.utils.functions_parsers.c_lang_function_parsers import _FUNCTION_PATTERN_REGEX +from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended +from exploit_iq_commons.utils.functions_parsers.c_lang_function_parsers import _FUNCTION_PATTERN_REGEX from langchain_community.document_loaders.parsers.language.c import CSegmenter def _get_function_name_from_segment(segment: str) -> str: diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py index 37daa023c..a0d8c9e8b 100644 --- a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -1,6 +1,6 @@ import pytest import shutil -from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader +from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader @pytest.mark.parametrize("refs", [ ("3.1.43", "3.1.42", "tag switch"), diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index a3d3985a4..c829f6e8b 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -3,10 +3,10 @@ import pytest from unittest.mock import patch, MagicMock -from vuln_analysis.data_models.common import AnalysisType +from exploit_iq_commons.data_models.common import AnalysisType from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig -from vuln_analysis.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, +from exploit_iq_commons.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput , SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) from vuln_analysis.runtime_context import ctx_state @@ -14,7 +14,7 @@ python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file) -from vuln_analysis.utils.source_rpm_downloader import RPMDependencyManager +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager transitive_code_search_runner_coroutine = None @@ -189,7 +189,7 @@ def mock_file_open(*args, **kwargs): "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file] } ]) -@patch('vuln_analysis.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) @patch('builtins.open', side_effect=mock_file_open) async def test_transitive_search_python_parameterized(mock_open, mock_run_command,test_case): """Parameterized test that runs all existing test cases with their respective configurations.""" @@ -203,7 +203,7 @@ async def test_transitive_search_python_parameterized(mock_open, mock_run_comman excluded_extensions=['**/*test*.py'] ) - with patch('vuln_analysis.utils.document_embedding.retrieve_from_cache', + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(test_case["mock_documents"], True)): result = await transitive_code_search_runner_coroutine(test_case["search_query"]) diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 398943788..6106b3178 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -15,7 +15,7 @@ import os from vuln_analysis.runtime_context import ctx_state -from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo @@ -25,17 +25,17 @@ from langchain.docstore.document import Document from vuln_analysis.data_models.state import AgentMorpheusEngineState -from vuln_analysis.utils.document_embedding import DocumentEmbedding -from ..data_models.input import SourceDocumentsInfo -from ..utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase -from ..utils.chain_of_calls_retriever_factory import get_chain_of_calls_retriever -from ..utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.document_embedding import DocumentEmbedding +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.chain_of_calls_retriever_factory import get_chain_of_calls_retriever +from exploit_iq_commons.utils.dep_tree import Ecosystem from ..utils.error_handling_decorator import catch_pipeline_errors_async, catch_tool_errors from ..utils.function_name_extractor import FunctionNameExtractor from ..utils.function_name_locator import FunctionNameLocator -from vuln_analysis.logging.loggers_factory import LoggingFactory -from ..utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME = "package_and_function_locator" diff --git a/src/vuln_analysis/utils/async_http_utils.py b/src/vuln_analysis/utils/async_http_utils.py index 288819b9c..030a2d802 100644 --- a/src/vuln_analysis/utils/async_http_utils.py +++ b/src/vuln_analysis/utils/async_http_utils.py @@ -21,7 +21,7 @@ import aiohttp -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 1822f3c56..8e515c550 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -23,9 +23,9 @@ from vuln_analysis.utils.prompting import MOD_FEW_SHOT from vuln_analysis.utils.prompting import additional_intel_prompting from vuln_analysis.utils.prompting import get_mod_examples -from vuln_analysis.utils.string_utils import attempt_fix_list_string +from exploit_iq_commons.utils.string_utils import attempt_fix_list_string -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/clients/first_client.py b/src/vuln_analysis/utils/clients/first_client.py index 1d7f1c446..1fd854332 100644 --- a/src/vuln_analysis/utils/clients/first_client.py +++ b/src/vuln_analysis/utils/clients/first_client.py @@ -18,11 +18,11 @@ import aiohttp -from vuln_analysis.data_models.cve_intel import CveIntelEpss +from exploit_iq_commons.data_models.cve_intel import CveIntelEpss from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/clients/ghsa_client.py b/src/vuln_analysis/utils/clients/ghsa_client.py index a17cf7f9e..466b4df7a 100644 --- a/src/vuln_analysis/utils/clients/ghsa_client.py +++ b/src/vuln_analysis/utils/clients/ghsa_client.py @@ -18,11 +18,11 @@ import aiohttp -from vuln_analysis.data_models.cve_intel import CveIntelGhsa +from exploit_iq_commons.data_models.cve_intel import CveIntelGhsa from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/clients/nvd_client.py b/src/vuln_analysis/utils/clients/nvd_client.py index 96ee6ed5a..c76bc5cac 100644 --- a/src/vuln_analysis/utils/clients/nvd_client.py +++ b/src/vuln_analysis/utils/clients/nvd_client.py @@ -20,7 +20,7 @@ import aiohttp from bs4 import BeautifulSoup -from vuln_analysis.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd from vuln_analysis.utils.async_http_utils import request_with_retry from vuln_analysis.utils.intel_utils import parse @@ -28,7 +28,7 @@ from vuln_analysis.utils.url_utils import url_join from vuln_analysis.utils.clients.intel_client import IntelClient -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/clients/rhsa_client.py b/src/vuln_analysis/utils/clients/rhsa_client.py index 956d92d04..404a1845e 100644 --- a/src/vuln_analysis/utils/clients/rhsa_client.py +++ b/src/vuln_analysis/utils/clients/rhsa_client.py @@ -18,11 +18,11 @@ import aiohttp -from vuln_analysis.data_models.cve_intel import CveIntelRhsa +from exploit_iq_commons.data_models.cve_intel import CveIntelRhsa from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.url_utils import url_join -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/clients/ubuntu_client.py b/src/vuln_analysis/utils/clients/ubuntu_client.py index d0964a63a..919799550 100644 --- a/src/vuln_analysis/utils/clients/ubuntu_client.py +++ b/src/vuln_analysis/utils/clients/ubuntu_client.py @@ -17,7 +17,7 @@ import aiohttp -from vuln_analysis.data_models.cve_intel import CveIntelUbuntu +from exploit_iq_commons.data_models.cve_intel import CveIntelUbuntu from vuln_analysis.utils.clients.intel_client import IntelClient from vuln_analysis.utils.clients.rhsa_client import logger from vuln_analysis.utils.url_utils import url_join diff --git a/src/vuln_analysis/utils/error_handling_decorator.py b/src/vuln_analysis/utils/error_handling_decorator.py index afabc02ed..3114b8764 100644 --- a/src/vuln_analysis/utils/error_handling_decorator.py +++ b/src/vuln_analysis/utils/error_handling_decorator.py @@ -1,7 +1,7 @@ import functools import traceback -from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index bb5c96582..eada7d0da 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -28,7 +28,7 @@ from tantivy import SchemaBuilder from tqdm import tqdm -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) variable_pattern = re.compile(r"([A-Z][a-z]+|[a-z]+|[A-Z]+(?=[A-Z]|$))") diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py index 0a8dfe109..64f734cda 100644 --- a/src/vuln_analysis/utils/function_name_extractor.py +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -1,8 +1,8 @@ import os import re -from vuln_analysis.utils.chain_of_calls_retriever import ChainOfCallsRetrieverBase -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetrieverBase +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 18c520874..3c16efa35 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -16,10 +16,10 @@ import difflib -from vuln_analysis.logging.loggers_factory import LoggingFactory -from vuln_analysis.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase -from vuln_analysis.utils.dep_tree import Ecosystem -from vuln_analysis.utils.standard_library_cache import StandardLibraryCache +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") diff --git a/src/vuln_analysis/utils/http_utils.py b/src/vuln_analysis/utils/http_utils.py index cae6ece36..3b2982538 100644 --- a/src/vuln_analysis/utils/http_utils.py +++ b/src/vuln_analysis/utils/http_utils.py @@ -22,7 +22,7 @@ import requests import urllib3 -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index ece0d3df2..8e3311f96 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -19,11 +19,11 @@ import aiohttp -from ..data_models.cve_intel import CveIntel, IntelPluginData -from ..data_models.cve_intel import CveIntelEpss -from ..data_models.cve_intel import CveIntelNvd -from ..data_models.cve_intel import CveIntelRhsa -from ..data_models.cve_intel import CveIntelUbuntu +from exploit_iq_commons.data_models.cve_intel import CveIntel, IntelPluginData +from exploit_iq_commons.data_models.cve_intel import CveIntelEpss +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelRhsa +from exploit_iq_commons.data_models.cve_intel import CveIntelUbuntu from .clients.first_client import FirstClient from .clients.ghsa_client import GHSAClient from .clients.nvd_client import NVDClient @@ -31,7 +31,7 @@ from .clients.ubuntu_client import UbuntuClient from ..data_models.plugin import PluginConfig, IntelPluginSchema -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 8a040a36c..596cb7987 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -15,9 +15,9 @@ from aiq.builder.builder import Builder from langchain_core.language_models.base import BaseLanguageModel -from ..data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.cve_intel import CveIntel from ..functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig -from ..utils import data_utils +from exploit_iq_commons.utils import data_utils from ..utils.prompting import additional_intel_prompting from aiq.builder.framework_enum import LLMFrameworkEnum diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 81cdfb9d2..ef7e7efb4 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -18,7 +18,7 @@ from pydpkg import Dpkg from pydpkg.exceptions import DpkgVersionError -from vuln_analysis.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd def update_version(incoming_version, current_version, compare): diff --git a/src/vuln_analysis/utils/justification_parser.py b/src/vuln_analysis/utils/justification_parser.py index 78fe26028..c0964ce5b 100644 --- a/src/vuln_analysis/utils/justification_parser.py +++ b/src/vuln_analysis/utils/justification_parser.py @@ -16,7 +16,7 @@ import logging from textwrap import dedent -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 78043e704..351b4253b 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -17,10 +17,10 @@ from ordered_set import OrderedSet -from vuln_analysis.data_models.common import AnalysisType -from vuln_analysis.data_models.dependencies import VulnerableDependencies -from vuln_analysis.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.data_models.input import AgentMorpheusInput +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.dependencies import VulnerableDependencies +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusEngineOutput from vuln_analysis.data_models.output import AgentMorpheusOutput from vuln_analysis.data_models.output import ChecklistItemOutput @@ -32,7 +32,7 @@ from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig -from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/output_formatter.py b/src/vuln_analysis/utils/output_formatter.py index 6604236ee..5bbbe5e60 100644 --- a/src/vuln_analysis/utils/output_formatter.py +++ b/src/vuln_analysis/utils/output_formatter.py @@ -20,7 +20,7 @@ from dateutil.parser import parse from vuln_analysis.data_models.output import AgentMorpheusOutput -from vuln_analysis.utils.data_utils import safe_getattr +from exploit_iq_commons.utils.data_utils import safe_getattr def generate_vulnerability_reports(model_dict: AgentMorpheusOutput, output_dir): diff --git a/src/vuln_analysis/utils/vulnerable_dependency_checker.py b/src/vuln_analysis/utils/vulnerable_dependency_checker.py index ac477f8e8..62eb9bd48 100644 --- a/src/vuln_analysis/utils/vulnerable_dependency_checker.py +++ b/src/vuln_analysis/utils/vulnerable_dependency_checker.py @@ -26,13 +26,13 @@ from tqdm import tqdm from univers import versions -from vuln_analysis.data_models.cve_intel import CveIntelNvd -from vuln_analysis.data_models.dependencies import DependencyPackage +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.dependencies import DependencyPackage from vuln_analysis.utils.clients.intel_client import IntelClient -from vuln_analysis.utils.string_utils import package_names_match +from exploit_iq_commons.utils.string_utils import package_names_match from vuln_analysis.utils.url_utils import url_join -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) REQUESTS_TIMEOUT = 2 diff --git a/tests/test_java_script_extended.py b/tests/test_java_script_extended.py index 7c7fd430f..51f535687 100644 --- a/tests/test_java_script_extended.py +++ b/tests/test_java_script_extended.py @@ -15,7 +15,7 @@ import pytest -from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter +from exploit_iq_commons.utils.js_extended_parser import ExtendedJavaScriptSegmenter TEST_CASES = [ { diff --git a/uv.lock b/uv.lock index 868d8a692..0b6084191 100644 --- a/uv.lock +++ b/uv.lock @@ -1007,6 +1007,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/ca/0753ba3a81255ac49748ec8b665ab01f8efcf711f74bbccb5457a6193acc/expandvars-1.1.1-py3-none-any.whl", hash = "sha256:09ca39e6bfcb0d899db8778a00dd3d89cfeb0080795c54f16f6279afd0ef8c5b", size = 7522, upload-time = "2025-07-12T07:46:18.984Z" }, ] +[[package]] +name = "exploit-iq-commons" +version = "0.1.0" +source = { editable = "src/exploit_iq_commons" } +dependencies = [ + { name = "esprima" }, + { name = "gitpython" }, + { name = "langchain" }, + { name = "langchain-community" }, + { name = "pandas" }, + { name = "pydantic" }, + { name = "tqdm" }, + { name = "tree-sitter" }, + { name = "tree-sitter-languages" }, +] + +[package.metadata] +requires-dist = [ + { name = "esprima", specifier = "==4.0.1" }, + { name = "gitpython", specifier = "==3.1.44" }, + { name = "langchain", specifier = "~=0.3.0" }, + { name = "langchain-community", specifier = "~=0.3.0" }, + { name = "pandas", specifier = "==2.3.1" }, + { name = "pydantic", specifier = ">=2.0" }, + { name = "tqdm", specifier = "==4.67.1" }, + { name = "tree-sitter", specifier = "==0.21.3" }, + { name = "tree-sitter-languages", specifier = "==1.10.2" }, +] + [[package]] name = "extratools" version = "0.8.2.1" @@ -1212,14 +1241,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.45" +version = "3.1.44" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196, upload-time = "2025-01-02T07:32:43.59Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599, upload-time = "2025-01-02T07:32:40.731Z" }, ] [[package]] @@ -1338,6 +1367,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, @@ -1347,6 +1378,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, ] @@ -5283,6 +5316,7 @@ dependencies = [ { name = "brotli" }, { name = "cvss" }, { name = "esprima" }, + { name = "exploit-iq-commons" }, { name = "faiss-cpu" }, { name = "gitpython" }, { name = "google-search-results" }, @@ -5327,6 +5361,7 @@ requires-dist = [ { name = "brotli" }, { name = "cvss", specifier = "==3.6" }, { name = "esprima" }, + { name = "exploit-iq-commons", editable = "src/exploit_iq_commons" }, { name = "faiss-cpu", specifier = "==1.9.0" }, { name = "gitpython" }, { name = "google-search-results", specifier = "==2.4" }, @@ -5354,6 +5389,7 @@ dev = [ { name = "pylint", specifier = "==3.3.*" }, { name = "pytest", specifier = "~=8.3" }, { name = "pytest-asyncio", specifier = "==0.24.*" }, + { name = "pytest-asyncio", specifier = "~=0.24.0" }, { name = "pytest-cov", specifier = "~=6.1" }, { name = "pytest-httpserver", specifier = "==1.1.*" }, { name = "pytest-pretty", specifier = "~=1.2.0" }, From 10ada84564fb01d08208e4e367b116fdee63efc0 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Tue, 13 Jan 2026 13:50:52 +0200 Subject: [PATCH 146/286] feat: added method reference support to java transitive search (#182) * feat: added method reference support to java transitive search Signed-off-by: Theodor Mihalache --- .../java_functions_parsers.py | 368 ++++++++++++++---- .../utils/java_chain_of_calls_retriever.py | 49 ++- .../tests/test_transitive_code_search.py | 24 ++ 3 files changed, 356 insertions(+), 85 deletions(-) diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index 64cb6f0dd..27ffa2c11 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -597,13 +597,34 @@ def is_constructor_header(hstart: int, name_start: int) -> bool: # No declaration found, and snippet didn't begin with a lambda return "" - def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, - callee_function_package: str, code_documents: list[Document], type_documents: list[Document], - callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict], - documents_of_functions: list[Document], - type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> bool: - + def search_for_called_function( + self, + caller_function: Document, + callee_function_name: str, + callee_function: Document, + callee_function_package: str, + code_documents: list[Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]], + ) -> bool: + """ + Scan `caller_function` for occurrences that may call `callee_function_name` (or a constructor) + and confirm via `__check_identifier_resolved_to_callee_function_package`. Supports: + + - Regular calls: `foo(...)`, `obj.foo(...)`, `Type.staticFoo(...)`, chains, casts, etc. + - Constructor calls: `new Type(...)` (when the callee is a constructor). + - Method references: `left-hand-side::method` → synthesized as `left-hand-side.method(` for resolution. + - Constructor references: `left-hand-side::new` → synthesized as `new left-hand-side(` for resolution. + + Fast-paths: + - Regex pre-filters for method names and constructor sites. + - Bounded left-scan to slice the owning expression. + - De-duplicate by (start, end) slice of the expression being resolved. + """ def _find_matching_paren(s: str, open_idx: int) -> int: """Find matching ')' for the '(' at open_idx. Ignores strings/char literals.""" depth = 0 @@ -614,17 +635,31 @@ def _find_matching_paren(s: str, open_idx: int) -> int: while i < n: ch = s[i] if in_str: - if esc: esc = False - elif ch == '\\': esc = True - elif ch == '"': in_str = False - i += 1; continue + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i += 1 + continue if in_chr: - if esc: esc = False - elif ch == '\\': esc = True - elif ch == "'": in_chr = False - i += 1; continue - if ch == '"': in_str = True; i += 1; continue - if ch == "'": in_chr = True; i += 1; continue + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i += 1 + continue + if ch == '"': + in_str = True + i += 1 + continue + if ch == "'": + in_chr = True + i += 1 + continue if ch == '(': depth += 1 elif ch == ')': @@ -640,7 +675,7 @@ def _next_token_after(s: str, idx: int) -> str: i = idx + 1 while i < n and s[i].isspace(): i += 1 - if s.startswith('throws', i) and (i + 6 == n or not s[i + 6].isalnum() and s[i + 6] != '_'): + if s.startswith('throws', i) and (i + 6 == n or (not s[i + 6].isalnum() and s[i + 6] != '_')): return 'throws' return s[i] if i < n else '' @@ -654,42 +689,190 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: i = pos - 1 in_str = in_chr = False esc = False - # depths for (), [], {} - dp = db = dbr = 0 + dp = db = dbr = 0 # depths for (), [], {} limit = max(0, pos - max_back) while i >= limit: ch = s[i] if in_str: - if esc: esc = False - elif ch == '\\': esc = True - elif ch == '"': in_str = False - i -= 1; continue + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i -= 1 + continue if in_chr: - if esc: esc = False - elif ch == '\\': esc = True - elif ch == "'": in_chr = False - i -= 1; continue - if ch == '"': in_str = True; i -= 1; continue - if ch == "'": in_chr = True; i -= 1; continue + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i -= 1 + continue + if ch == '"': + in_str = True + i -= 1 + continue + if ch == "'": + in_chr = True + i -= 1 + continue - # Track bracket depths (we treat scanning left; depths just indicate "inside something") - if ch == ')': dp += 1; i -= 1; continue - if ch == '(': dp = max(0, dp - 1); i -= 1; continue - if ch == ']': db += 1; i -= 1; continue - if ch == '[': db = max(0, db - 1); i -= 1; continue - if ch == '}': dbr += 1; i -= 1; continue - if ch == '{': dbr = max(0, dbr - 1); i -= 1; continue + if ch == ')': + dp += 1 + i -= 1 + continue + if ch == '(': + dp = max(0, dp - 1) + i -= 1 + continue + if ch == ']': + db += 1 + i -= 1 + continue + if ch == '[': + db = max(0, db - 1) + i -= 1 + continue + if ch == '}': + dbr += 1 + i -= 1 + continue + if ch == '{': + dbr = max(0, dbr - 1) + i -= 1 + continue - # Only consider boundaries at top level if dp == db == dbr == 0 and ch in BOUNDARY_CHARS: - return i + 1 # start right after the boundary + return i + 1 i -= 1 - return limit # we hit the scan cap; good enough + return limit - import re + # Method-ref left-hand-side extractor: find the minimal "left-hand-side" immediately before '::' + def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: + """ + Return the start index of the *left-hand-side* expression immediately preceding '::' at dc_idx. - # Extract body once + Unlike _expr_start_left (which finds the start of the whole owning expression), + this stops at top-level delimiters that commonly precede a method reference inside + an argument list (notably '(' and ','). + """ + i = dc_idx - 1 + n = len(s) + # skip whitespace + while i >= 0 and s[i].isspace(): + i -= 1 + + in_str = in_chr = False + esc = False + dp = db = dbr = da = 0 # (), [], {}, <> (generics) + limit = max(0, dc_idx - max_back) + + # boundaries that delimit an argument/expression at top-level for method references + # (include '(' and ',' to avoid capturing outer call prefixes like "foo(") + REF_BOUNDARY = set(';,=+-*/%!?&|^:\n,(') + + while i >= limit: + ch = s[i] + + if in_str: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i -= 1 + continue + if in_chr: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i -= 1 + continue + + if ch == '"': + in_str = True + i -= 1 + continue + if ch == "'": + in_chr = True + i -= 1 + continue + + # nesting tracking (reverse) + if ch == ')': + dp += 1 + i -= 1 + continue + if ch == '(': + if dp > 0: + dp -= 1 + i -= 1 + continue + # top-level '(' is a delimiter for the left-hand-side in method refs + if db == dbr == da == 0: + return i + 1 + i -= 1 + continue + + if ch == ']': + db += 1 + i -= 1 + continue + if ch == '[': + if db > 0: + db -= 1 + i -= 1 + continue + if dp == dbr == da == 0: + return i + 1 + i -= 1 + continue + + if ch == '}': + dbr += 1 + i -= 1 + continue + if ch == '{': + if dbr > 0: + dbr -= 1 + i -= 1 + continue + if dp == db == da == 0: + return i + 1 + i -= 1 + continue + + if ch == '>': + da += 1 + i -= 1 + continue + if ch == '<': + if da > 0: + da -= 1 + i -= 1 + continue + # top-level '<' (e.g., comparison) is a delimiter for our left-hand-side extraction + if dp == db == dbr == 0: + return i + 1 + i -= 1 + continue + + if dp == db == dbr == da == 0 and ch in REF_BOUNDARY: + return i + 1 + + i -= 1 + + return limit + + # Extract method body once src = caller_function.page_content try: lo = src.index("{") @@ -699,22 +882,27 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: caller_function_body = src # --- patterns --- - # Regular method calls to specific callee method_pat = re.compile( rf'(?:\breturn\b\s*\(?[^;]*?)?(?:[\w$()\[\].]*?\.?)?\b{re.escape(callee_function_name)}\s*\(', re.MULTILINE ) - # Constructor calls for declaring type (only if the callee is a constructor) declaring_simple = self.get_class_name_from_class_function(callee_function) - is_ctor_target = callee_function_name in declaring_simple - # Note: we capture any qualified name ending in declaring_simple; anonymous-class body after ')' is OK. + + # Treat as constructor-target if caller asked the declaring type name + is_ctor_target = callee_function_name == declaring_simple + ctor_pat = re.compile( rf'\bnew\s+((?:[A-Za-z_$][\w$]*\.)*{re.escape(declaring_simple)})\s*\(' ) if is_ctor_target else None - # Dedup by (start,end) slice we attempt to resolve - seen_slices = set() + # Method reference patterns (allow optional method type args: Type::m) + methodref_pat = re.compile( + rf'::\s*(?:<[^>]*>\s*)?{re.escape(callee_function_name)}\b' + ) + ctorref_pat = re.compile(r'::\s*new\b') if is_ctor_target else None + + seen_slices: set[tuple[int, int]] = set() # Target class names for the resolver key = (declaring_simple, callee_function.metadata['source']) @@ -723,16 +911,11 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: else: target_class_names = frozenset([declaring_simple]) - # --- helper to process a match tuple uniformly --- def _process_call(start_idx: int, open_paren_pos: int) -> bool: close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) if close_paren_pos == -1: return False - # For methods we drop declarations by looking ahead; for constructors we keep - # anonymous class bodies, so we only apply this filter to method matches. - # The caller passes start_idx accordingly (method vs. ctor). - # Decide slice bounds and dedup start_ctx = _expr_start_left(caller_function_body, start_idx, max_back=512) end_ctx = close_paren_pos + 1 slice_key = (start_ctx, end_ctx) @@ -740,8 +923,6 @@ def _process_call(start_idx: int, open_paren_pos: int) -> bool: return False seen_slices.add(slice_key) - # Build the identifier snippet we feed into the resolver: - # take the exact text from the expression start to '(' (inclusive) ident_snippet = caller_function_body[start_ctx:open_paren_pos + 1] if self.__check_identifier_resolved_to_callee_function_package( @@ -756,40 +937,70 @@ def _process_call(start_idx: int, open_paren_pos: int) -> bool: target_class_names=target_class_names, documents_of_functions=documents_of_functions, callee_function_name=callee_function_name, - type_inheritance=type_inheritance + type_inheritance=type_inheritance, ): - logger.debug("__check_identifier_resolved_to_callee_function_package resolved successfully - " - f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " - f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" - f", \ncaller_function={caller_function.page_content}") + logger.debug( + "__check_identifier_resolved_to_callee_function_package resolved successfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}" + ) return True - logger.debug("__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - " - f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " - f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" - f", \ncaller_function={caller_function.page_content}") + logger.debug( + "__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}" + ) return False + def _process_method_ref(dc_idx: int, ref_len: int, make_ctor: bool) -> bool: + """ + dc_idx: index of the first ':' in the '::' token + ref_len: length of the matched '::...' + """ + lhs_start = _method_ref_lhs_start(caller_function_body, dc_idx, max_back=512) + lhs = caller_function_body[lhs_start:dc_idx].strip() + if not lhs: + return False + + slice_key = (lhs_start, dc_idx + ref_len) + if slice_key in seen_slices: + return False + seen_slices.add(slice_key) + + ident_snippet = (f"new {lhs}(") if make_ctor else (f"{lhs}.{callee_function_name}(") + + return self.__check_identifier_resolved_to_callee_function_package( + function=caller_function, + identifier_function=ident_snippet, + callee_package=callee_function_package, + code_documents=code_documents, + type_documents=type_documents, + callee_function_file_name=callee_function_file_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + target_class_names=target_class_names, + documents_of_functions=documents_of_functions, + callee_function_name=callee_function_name, + type_inheritance=type_inheritance, + ) + + # 1) Constructor matches (only when target is a ctor) if is_ctor_target and ctor_pat: - # --- Constructor matches (only when target is a ctor) --- for m in ctor_pat.finditer(caller_function_body): - # m ends right after '(' of "new ... (" open_paren_pos = m.end(0) - 1 if _process_call(m.start(), open_paren_pos): return True else: - # --- Regular method matches --- + # 2) Regular method matches for m in method_pat.finditer(caller_function_body): - call = m.group(0) - if not call: - continue - - open_paren_pos = m.end(0) - 1 # '(' + open_paren_pos = m.end(0) - 1 close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) if close_paren_pos == -1: continue - # Filter out method declarations (headers) nxt = _next_token_after(caller_function_body, close_paren_pos) if nxt == '{' or nxt == 'throws': continue @@ -797,8 +1008,19 @@ def _process_call(start_idx: int, open_paren_pos: int) -> bool: if _process_call(m.start(), open_paren_pos): return True + # 3) Method reference matches + for m in methodref_pat.finditer(caller_function_body): + if _process_method_ref(m.start(), m.end() - m.start(), make_ctor=False): + return True + + if ctorref_pat: + for m in ctorref_pat.finditer(caller_function_body): + if _process_method_ref(m.start(), m.end() - m.start(), make_ctor=True): + return True + return False + def __check_identifier_resolved_to_callee_function_package( self, function: Document, diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index e4c7ac955..33c43afc2 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -288,12 +288,12 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio result = [doc for doc in self.documents if package in doc.metadata.get('source') and self.language_parser.is_function(doc) and not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and - f"{function_name_to_search}(" in doc.page_content] + (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] else: result = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and self.language_parser.is_function(doc) and not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and - f"{function_name_to_search}(" in doc.page_content] + (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] return result @@ -530,6 +530,8 @@ def function_called_from_caller_body(self, document: Document, function_to_searc - Chained method calls: PropertyUtilsBean.getInstance().getProperty(...) - Generic method calls: obj.getProperty(...) - Constructor calls: new Test(...), new pkg.Test<>(...), new Test(){...} + - Method references: obj::method, ClassName::method, pkg.Class::method + - Constructor references: ClassName::new, pkg.Class::new """ name_raw = (function_to_search or "").strip() if not name_raw: @@ -546,13 +548,24 @@ def function_called_from_caller_body(self, document: Document, function_to_searc src = document.page_content or "" + # Heuristic: treat as "type-like" if it looks like a class/simple type token (enables ::new) + def _looks_like_type_name(s: str) -> bool: + return bool(s) and (s[0].isupper() or s.endswith("[]")) + # ---------- fast early exit ---------- - # If neither the simple token nor the qualifier shows up anywhere in the text, - # and there isn't a 'new ' token (constructor), bail out quickly. + # If none of the key tokens appear (simple name, qualifier, new , ::, or ::new with class tokens), bail. if ( target_simple not in src and (not target_qual or target_qual not in src) and f"new {target_simple}" not in src + and f"::{target_simple}" not in src + and not ( + "::new" in src and ( + (_looks_like_type_name(target_simple)) or + (target_qual and target_qual in src) or + (target_qual and target_qual.split(".")[-1] in src) + ) + ) ): return False @@ -597,12 +610,13 @@ def _strip_strings(s: str) -> str: # ---------- assemble patterns ---------- simple_esc = re.escape(target_simple) + parts = [] # Method-call patterns (unchanged, plus optional before method name for generics) - parts = [ - rf"\.\s*(?:<[^>]*>\s*)?{simple_esc}\s*\(", # chained/dotted: obj.name( - rf"(?]*>\s*)?{simple_esc}\s*\(", # chained/dotted: obj.name( + rf"(? str: parts.append(rf"(?]*>\s*)?{simple_esc}\s*\(") # Constructor-call patterns: new (...), allowing optional generics and anonymous class - # We only treat the target as a constructor if its "name" is a valid type identifier. - # This aligns with Java: constructor name == simple type name. - # Simple: new Test(...), new Test<...>(...), new Test(){...} parts.append(rf"(?]*>)?\s*\(") - # Qualified: new pkg.Test(...), new pkg.Outer.Inner<...>(...) if target_qual: parts.append(rf"(?]*>)?\s*\(") + # ------- method reference patterns (::) ------- + # Unqualified/any-LHS method reference to the target method: + # this::method, super::method, obj::method, Type::method, pkg.Type::method + parts.append(rf"::\s*(?:<[^>]*>\s*)?{simple_esc}\b") + if target_qual: + parts.append(rf"(?]*>\s*)?{simple_esc}\b") + parts.append(rf"(?]*>\s*)?{simple_esc}\b") + + # Constructor reference patterns: + # If the searched name looks like a type, accept ClassName::new and pkg.Class::new + if _looks_like_type_name(target_simple): + parts.append(rf"(? 1 document = list_path[-1] assert 'src/main/java/io/cryostat' in document.metadata['source'] + +# Test method reference +@pytest.mark.asyncio +async def test_transitive_search_java_5(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("commons-io:commons-io:2.16.1,org.apache.commons.io.FileUtils.forceDeleteOnExit") + (path_found, list_path) = result + print(result) + assert path_found is False + assert len(list_path) is 1 \ No newline at end of file From 706d352647f4fc6924cb037f03d77819c8d316be Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Thu, 15 Jan 2026 16:45:17 +0200 Subject: [PATCH 147/286] Improve C performance test running (#183) --- src/exploit_iq_commons/utils/dep_tree.py | 3 +++ src/vuln_analysis/tools/tests/test_transitive_code_search.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index f35bae99a..74e89f883 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -507,6 +507,9 @@ def install_dependencies(self, manifest_path: Path): file_data = RPMDependencyManager.get_instance().container_image with open(os.path.join(download_path, "container_image.txt"), "w") as f: f.write(file_data) + #filter out kernel files from libbpf project by calling find_all_files method + #we want to use this before the creation of documents by the segmenter class + self.find_all_files(self.root_dir) def _install_dependencies_from_container(self, manifest_path: Path, download_path: str): diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index f1a43da00..80b881eaa 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -281,9 +281,13 @@ async def test_c_transitive_search(): included_extensions=['**/*.c','**/*.h'], excluded_extensions=['**/*test*.c',"**/docs/**/*","**/doc/**/*"] ) + # set rpm cache directory RPMDependencyManager.get_instance().set_rpm_cache_dir(".cache/am_cache/rpms") + # enable unit test mode RPMDependencyManager.get_instance().enableUnitTestMode() + # set sbom list RPMDependencyManager.get_instance().sbom = sbom_list + # run transitive code search result = await transitive_code_search_runner_coroutine("libxml2,xmlParseComment") (path_found, list_path) = result print(f"DEBUG: path_found = {path_found}") From 5804ce4cad50ea964ef92fb9077b676dbe4c163c Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 15 Jan 2026 18:06:18 +0200 Subject: [PATCH 148/286] fix: use Ecosystem enum value in SerpApi query Signed-off-by: Vladimir Belousov --- src/vuln_analysis/utils/function_name_locator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 3c16efa35..13f3adbc8 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -291,7 +291,7 @@ async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> b """ search = MorpheusSerpAPIWrapper(max_retries=2) - result = await search.arun(f"Is '{package_name}' part of the {ecosystem} standard library?") + result = await search.arun(f"Is '{package_name}' part of the {ecosystem.value} standard library?") logger.info("quick_standard_lib_check Standard library check result: %s", result) # Normalize result: if list, join into single string if isinstance(result, list): From 0af4763132666ed9d01a653e6d06c9956a34466d Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 11 Dec 2025 15:36:46 +0200 Subject: [PATCH 149/286] feat: add initial implementation for VEX generation using ABC, with CSAF format support Signed-off-by: Ilona Shishov --- pyproject.toml | 1 + src/vuln_analysis/data_models/output.py | 12 +- src/vuln_analysis/data_models/state.py | 1 + .../functions/cve_generate_vex.py | 53 +++++++ src/vuln_analysis/register.py | 12 +- src/vuln_analysis/utils/llm_engine_utils.py | 4 +- src/vuln_analysis/utils/vex/__init__.py | 14 ++ src/vuln_analysis/utils/vex/abc.py | 21 +++ .../utils/vex/implementations/__init__.py | 14 ++ .../vex/implementations/csaf_generator.py | 150 ++++++++++++++++++ src/vuln_analysis/utils/vex/loader.py | 16 ++ uv.lock | 25 ++- 12 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 src/vuln_analysis/functions/cve_generate_vex.py create mode 100644 src/vuln_analysis/utils/vex/__init__.py create mode 100644 src/vuln_analysis/utils/vex/abc.py create mode 100644 src/vuln_analysis/utils/vex/implementations/__init__.py create mode 100644 src/vuln_analysis/utils/vex/implementations/csaf_generator.py create mode 100644 src/vuln_analysis/utils/vex/loader.py diff --git a/pyproject.toml b/pyproject.toml index e83eb0b1d..b321ccedb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "tree-sitter-languages==1.10.2", "univers==30.12", "litellm<=1.75.8", + "csaf-tool==0.3.2", ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index 232a98a20..e77469a3f 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -81,12 +81,22 @@ class AgentMorpheusEngineOutput(BaseModel): cvss: CVSSOutput | None +class OutputPayload(BaseModel): + """ + Wrapper for final pipeline results. + - analysis: per-vulnerability analysis results + - vex: the vulnerability exploitability exchange document JSON + """ + analysis: list[AgentMorpheusEngineOutput] + vex: dict[str, typing.Any] | None + + class AgentMorpheusOutput(AgentMorpheusEngineInput): """" The final output of the Agent Morpheus pipeline. Contains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability. """ - output: list[AgentMorpheusEngineOutput] + output: OutputPayload @model_validator(mode="before") @classmethod diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index 7b77e7f82..38108bce3 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -34,5 +34,6 @@ class AgentMorpheusEngineState(BaseModel): justifications: dict[str, dict[str, str]] = {} poor_quality_intel_vul: dict[str, int] = {} cvss_results: dict[str, dict[str, str]] = {} + vex: dict[str, typing.Any] | None = None current_vuln_id: str | None = None diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py new file mode 100644 index 000000000..0179cdfd0 --- /dev/null +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import json + +from aiq.builder.builder import Builder +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.utils.vex.loader import load_vex_generator + +logger = logging.getLogger(__name__) + +class CVEGenerateVexConfig(FunctionBaseConfig, name="cve_generate_vex"): + """ + Defines a function that generates a custom VEX (Vendor Exploitability eXchange) document for vulnerable components following the OpenVEX specification. + """ + + skip: bool = Field(default=False, description="Whether or not the VEX generator should be skipped.") + vex_format: str = Field(default="csaf", description="VEX document format to generate.") + +@register_function(config_type=CVEGenerateVexConfig) +async def cve_generate_vex(config: CVEGenerateVexConfig, builder: Builder): + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + if config.skip: + logger.info("`config.skip` is set to True. Skipping VEX generation.") + return state + + generator = load_vex_generator(config.vex_format) + vex_doc = generator.generate(state) + state.vex = vex_doc + + return state + + yield FunctionInfo.from_fn(_arun, + input_schema=AgentMorpheusEngineState, + description="Generates a custom VEX document for vulnerable components.") diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 48fe71d99..0eaafac1f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -40,6 +40,7 @@ from vuln_analysis.functions import cve_process_sbom from vuln_analysis.functions import cve_summarize from vuln_analysis.functions import cve_generate_cvss +from vuln_analysis.functions import cve_generate_vex from vuln_analysis.functions import health_endpoint from vuln_analysis.tools import lexical_full_search # This is actually registers the tool in the type registry of NAT! @@ -69,6 +70,7 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): cve_agent_executor_name: str = Field(description="Function name to run CVE agent on checklist") cve_summarize_name: str = Field(description="Function name to generate summary") cve_justify_name: str = Field(description="Function to generate justifications for each CVE") + cve_generate_vex_name: str = Field(description="Function name to generate VEX for vulnerable components") cve_generate_cvss_name: str = Field(description="Function name to generate CVSS") cve_output_config_name: str | None = Field(default=None, description="Function to output workflow results " @@ -95,6 +97,7 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): cve_agent_executor_fn = builder.get_function(name=config.cve_agent_executor_name) cve_summary_fn = builder.get_function(name=config.cve_summarize_name) cve_justify_fn = builder.get_function(name=config.cve_justify_name) + cve_generate_vex_fn = builder.get_function(name=config.cve_generate_vex_name) cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None @@ -156,6 +159,11 @@ async def justify_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineSt return await cve_justify_fn.ainvoke(state.model_dump()) + async def generate_vex_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Generates VEX for vulnerable components""" + + return await cve_generate_vex_fn.ainvoke(state.model_dump()) + async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: """Generates CVSS for the results of the execution""" @@ -181,13 +189,15 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput subgraph_builder.add_node("agent_executor", agent_executor_node) subgraph_builder.add_node("summarize", summarize_node) subgraph_builder.add_node("justify", justify_node) + subgraph_builder.add_node("generate_vex", generate_vex_node) subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_edge(START, "checklist") subgraph_builder.add_edge("checklist", "agent_executor") subgraph_builder.add_edge("agent_executor", "summarize") subgraph_builder.add_edge("summarize", "justify") - subgraph_builder.add_edge("justify", "generate_cvss") + subgraph_builder.add_edge("justify", "generate_vex") + subgraph_builder.add_edge("generate_vex", "generate_cvss") subgraph = subgraph_builder.compile() @catch_pipeline_errors_async diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 351b4253b..727830212 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -23,6 +23,7 @@ from exploit_iq_commons.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusEngineOutput from vuln_analysis.data_models.output import AgentMorpheusOutput +from vuln_analysis.data_models.output import OutputPayload from vuln_analysis.data_models.output import ChecklistItemOutput from vuln_analysis.data_models.output import JustificationOutput from vuln_analysis.data_models.output import CVSSOutput @@ -265,7 +266,8 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, out.justification.label, out.cvss.score if out.cvss else "-") - return AgentMorpheusOutput(input=message.input, info=message.info, output=output) + payload = OutputPayload(analysis=output, vex=result.vex) + return AgentMorpheusOutput(input=message.input, info=message.info, output=payload) def finalize_preprocess_engine_input(message: AgentMorpheusEngineInput, engine_state: AgentMorpheusEngineState, builder: Builder) -> AgentMorpheusEngineState: config = builder.get_function_config("cve_calculate_intel_score") diff --git a/src/vuln_analysis/utils/vex/__init__.py b/src/vuln_analysis/utils/vex/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/abc.py b/src/vuln_analysis/utils/vex/abc.py new file mode 100644 index 000000000..98e8bfcf7 --- /dev/null +++ b/src/vuln_analysis/utils/vex/abc.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Dict + +from vuln_analysis.data_models.state import AgentMorpheusEngineState + + +class VexGenerator(ABC): + """ + Abstract base for VEX document generators. + """ + + @abstractmethod + def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: + """ + Generate a VEX document as a JSON-serializable dict from the engine state. + """ + pass + + diff --git a/src/vuln_analysis/utils/vex/implementations/__init__.py b/src/vuln_analysis/utils/vex/implementations/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/implementations/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py new file mode 100644 index 000000000..458de08f3 --- /dev/null +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from datetime import datetime +import json +import shutil +import subprocess +from typing import Any, Dict, List +import tempfile +import os + +from vuln_analysis.data_models.cve_intel import CveIntel +from vuln_analysis.data_models.state import AgentMorpheusEngineState + +from ..abc import VexGenerator +from csaf.generator import CSAFGenerator + + +def _choose_description(ci: CveIntel) -> str: + """ + Choose the description for the vulnerability. + """ + if ci and ci.ghsa and ci.ghsa.summary: + return ci.ghsa.summary + if ci and ci.rhsa and ci.rhsa.details: + return ci.rhsa.details + return "" + + +def _get_patched_package(v: dict) -> tuple[str | None, str | None]: + """ + Get the patched package from a GHSA vulnerability dict. + """ + pkg = v.get("package") or {} + return pkg.get("name"), pkg.get("first_patched_version") + + +def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: + """ + Build a patch recommendation from the GHSA data. + - If SBOM provided: return the first matching 'name:first_patched_version' where name is in SBOM, else return "". + - If no SBOM: return all unique 'name:first_patched_version' pairs as list, else return "". + """ + if not ci or not ci.ghsa or not ci.ghsa.vulnerabilities: + return "" + + vulns = ci.ghsa.vulnerabilities + + # SBOM present + if sbom_package_names is not None: + return next( + ( + f"{name}:{patch}" + for v in vulns + for (name, patch) in [(_get_patched_package(v))] + if name and patch and name in sbom_package_names + ), + "" + ) + + # No SBOM + name_to_version: dict[str, str] = {} + for v in vulns: + name, patch = _get_patched_package(v) + if not name or not patch: + continue + if name not in name_to_version: + name_to_version[name] = patch + + if not name_to_version: + return "" + return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) + + +class CsafVexGenerator(VexGenerator): + """ + CSAF VEX generator. Builds a CSAF JSON document and validates it with the csaf-tool. + """ + + def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: + + message: AgentMorpheusEngineInput = state.original_input + + csaf_gen = CSAFGenerator() + + product_name = message.input.image.name + product_tag = message.input.image.tag + + csaf_gen.set_title(f"ExploitIQ VEX Document - {product_name}:{product_tag}") + csaf_gen.set_header_title("ExploitIQ VEX Document") + + csaf_gen.set_value("notes",[ + { + "category": "disclaimer", + "text": "This CSAF document is generated as custom content for internal or experimental use. It is not an official Red Hat security document, has not undergone formal validation or review, and is not digitally signed. The information provided here should not be regarded as authoritative or replace official Red Hat advisories or VEX statements.", + "title": "Unofficial Content Notice" + } + ]) + + csaf_gen.set_value("author", "Red Hat Product Security") + csaf_gen.set_value("author_url", "https://access.redhat.com/security/") + + vendor = (product_name.split("/")[1] if "/" in product_name else "unknown") + csaf_gen.add_product(product_name=product_name, vendor=vendor, release=product_tag) + + intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in (message.info.intel or [])} + sbom_names: set[str] | None = None + if message.input.image.sbom_info and message.input.image.sbom_info.packages: + sbom_names = {pkg.name for pkg in message.input.image.sbom_info.packages} + + for vuln_id, summary in state.final_summaries.items(): + + ci = intel_map.get(vuln_id) + description = _choose_description(ci) + patch_recommendation = _build_patch_recommendation(ci, sbom_names) + comment = ( + f"Upgrade to the first patched version(s): {patch_recommendation}." + if patch_recommendation else "No remediation recommendation available for this vulnerability." + ) + csaf_gen.add_vulnerability( + product_name=product_name, + release=product_tag, + id=vuln_id, + description=description or summary, + status="known_affected", + comment=comment + ) + + csaf_gen.generate_csaf() + + with tempfile.TemporaryDirectory() as tmp_dir: + path = os.path.join(tmp_dir, "csaf.json") + csaf_gen.publish_csaf(path) + with open(path, "r") as f: + csaf_json = json.load(f) + + print("---------------------------------------------------------") + print(json.dumps(csaf_json, indent=2)) + print("---------------------------------------------------------") + # Validate/normalize with csaf-tool if available + # self._validate_with_csaf_tool(csaf_json) + return csaf_json + + # def _validate_with_csaf_tool(self, csaf_doc: Dict[str, Any]) -> None: + # """ + # Validate the CSAF document using csaf-tool CLI. This step ensures the produced JSON + # conforms to CSAF expectations. If csaf-tool is not available or validation fails, + # this method will not raise and the original document will be returned to the caller. + # """ + + # csaf_doc.analyse() diff --git a/src/vuln_analysis/utils/vex/loader.py b/src/vuln_analysis/utils/vex/loader.py new file mode 100644 index 000000000..316da7312 --- /dev/null +++ b/src/vuln_analysis/utils/vex/loader.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from .abc import VexGenerator +from .implementations.csaf_generator import CsafVexGenerator + +def load_vex_generator(fmt: str) -> VexGenerator: + """ + Return a VEX generator implementation for the requested format. + """ + + fmt_lower = fmt.lower() + + if fmt_lower == "csaf": + return CsafVexGenerator() + raise ValueError(f"Unsupported VEX format: {fmt_lower}") + diff --git a/uv.lock b/uv.lock index 0b6084191..152420d87 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", @@ -799,6 +799,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/76/cf8d69da8d0b5ecb0db406f24a63a3f69ba5e791a11b782aeeefef27ccbb/cryptography-45.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:629127cfdcdc6806dfe234734d7cb8ac54edaf572148274fa377a7d3405b0043", size = 3331874, upload-time = "2025-08-05T23:59:23.017Z" }, ] +[[package]] +name = "csaf-tool" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packageurl-python" }, + { name = "rich" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/24/b408082ef806581de3a2094caf0b9ed560a4fb8e8c14bb9c28462ecb023c/csaf_tool-0.3.2-py2.py3-none-any.whl", hash = "sha256:7e5559cb522eb76e3acad39a7bf9ba1b81e5a6224099d511a4c9c2dcf36caa16", size = 17629, upload-time = "2024-06-12T20:10:06.429Z" }, +] + [[package]] name = "cvss" version = "3.6" @@ -3377,6 +3389,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0d/73143ecb94ac4a5dcba223402139240a75dee0cc6ba8a543788a5646407a/ormsgpack-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:35fa9f81e5b9a0dab42e09a73f7339ecffdb978d6dbf9deb2ecf1e9fc7808722", size = 121401, upload-time = "2025-05-24T19:07:28.308Z" }, ] +[[package]] +name = "packageurl-python" +version = "0.17.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/d6/3b5a4e3cfaef7a53869a26ceb034d1ff5e5c27c814ce77260a96d50ab7bb/packageurl_python-0.17.6.tar.gz", hash = "sha256:1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25", size = 50618, upload-time = "2025-11-24T15:20:17.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/2f/c7277b7615a93f51b5fbc1eacfc1b75e8103370e786fd8ce2abf6e5c04ab/packageurl_python-0.17.6-py3-none-any.whl", hash = "sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9", size = 36776, upload-time = "2025-11-24T15:20:16.962Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -5314,6 +5335,7 @@ dependencies = [ { name = "aiohttp-client-cache" }, { name = "aioresponses" }, { name = "brotli" }, + { name = "csaf-tool" }, { name = "cvss" }, { name = "esprima" }, { name = "exploit-iq-commons" }, @@ -5359,6 +5381,7 @@ requires-dist = [ { name = "aiohttp-client-cache", specifier = "==0.11" }, { name = "aioresponses", specifier = "==0.7.6" }, { name = "brotli" }, + { name = "csaf-tool", specifier = "==0.3.2" }, { name = "cvss", specifier = "==3.6" }, { name = "esprima" }, { name = "exploit-iq-commons", editable = "src/exploit_iq_commons" }, From 65930e0ff9bf2d555b388a2546ad64207b30bad4 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 22 Dec 2025 13:01:17 +0200 Subject: [PATCH 150/286] chore: enrich CSAF with analysis data and set to generate for vulnerable CVEs only Signed-off-by: Ilona Shishov --- .../functions/cve_generate_vex.py | 4 + .../vex/implementations/csaf_generator.py | 140 +++++++++++++----- 2 files changed, 111 insertions(+), 33 deletions(-) diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index 0179cdfd0..7da96c1ed 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -42,6 +42,10 @@ async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: logger.info("`config.skip` is set to True. Skipping VEX generation.") return state + if not any(justification.get("justification_label") == "vulnerable" for justification in state.justifications.values()): + logger.info("No vulnerable CVE(s) found. Skipping VEX generation.") + return state + generator = load_vex_generator(config.vex_format) vex_doc = generator.generate(state) state.vex = vex_doc diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 458de08f3..4b07a8d3d 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -15,23 +15,12 @@ from csaf.generator import CSAFGenerator -def _choose_description(ci: CveIntel) -> str: - """ - Choose the description for the vulnerability. - """ - if ci and ci.ghsa and ci.ghsa.summary: - return ci.ghsa.summary - if ci and ci.rhsa and ci.rhsa.details: - return ci.rhsa.details - return "" - - def _get_patched_package(v: dict) -> tuple[str | None, str | None]: """ Get the patched package from a GHSA vulnerability dict. """ pkg = v.get("package") or {} - return pkg.get("name"), pkg.get("first_patched_version") + return pkg.get("name"), v.get("first_patched_version") def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: @@ -44,7 +33,6 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non return "" vulns = ci.ghsa.vulnerabilities - # SBOM present if sbom_package_names is not None: return next( @@ -56,7 +44,6 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non ), "" ) - # No SBOM name_to_version: dict[str, str] = {} for v in vulns: @@ -65,12 +52,82 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non continue if name not in name_to_version: name_to_version[name] = patch - if not name_to_version: return "" return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) +def _enrich_vulnerabilities_with_notes( + csaf_json: Dict[str, Any], + intel_map: Dict[str, CveIntel], + final_summaries: Dict[str, str], + justifications: Dict[str, Dict[str, str]] +) -> None: + """ + Enrich each vulnerability in the CSAF document with informational notes + from intel sources and analysis results. + + Returns the enriched csaf_json document. + """ + for v in csaf_json.get("vulnerabilities", []): + vuln_id = v.get("cve") + notes = v.get("notes", []) + + ci = intel_map.get(vuln_id) + + # Add GHSA description if available + ghsa_description = ci.ghsa.description if (ci and ci.ghsa and ci.ghsa.description) else None + if ghsa_description: + for note in notes: + if note.get("category") == "description": + note["title"] = "Vulnerability description" + note["text"] = ghsa_description + break + + # Add GHSA summary if available + ghsa_summary = ci.ghsa.summary if (ci and ci.ghsa and ci.ghsa.summary) else None + if ghsa_summary: + notes.append({ + "category": "summary", + "text": ghsa_summary, + "title": "Vulnerability summary" + }) + + # Add RHSA statement if available + rhsa_statement = ci.rhsa.statement if (ci and ci.rhsa and ci.rhsa.statement) else None + if rhsa_statement: + notes.append({ + "category": "general", + "text": rhsa_statement, + "title": "Red Hat Security Advisory Statement" + }) + + # Add ExploitIQ analysis summary + summary = final_summaries.get(vuln_id) + notes.append({ + "category": "analysis", + "title": "ExploitIQ Analysis Summary", + "text": summary + }) + + # Add ExploitIQ justification details + justification = justifications.get(vuln_id) + notes.append({ + "category": "analysis", + "title": "ExploitIQ Analysis Justification Reasoning", + "text": justification.get("justification") + }) + notes.append({ + "category": "analysis", + "title": "ExploitIQ Analysis Justification Label", + "text": justification.get("justification_label") + }) + + v["notes"] = notes + + return csaf_json + + class CsafVexGenerator(VexGenerator): """ CSAF VEX generator. Builds a CSAF JSON document and validates it with the csaf-tool. @@ -102,7 +159,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: vendor = (product_name.split("/")[1] if "/" in product_name else "unknown") csaf_gen.add_product(product_name=product_name, vendor=vendor, release=product_tag) - intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in (message.info.intel or [])} + intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in message.info.intel} sbom_names: set[str] | None = None if message.input.image.sbom_info and message.input.image.sbom_info.packages: sbom_names = {pkg.name for pkg in message.input.image.sbom_info.packages} @@ -110,20 +167,37 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: for vuln_id, summary in state.final_summaries.items(): ci = intel_map.get(vuln_id) - description = _choose_description(ci) - patch_recommendation = _build_patch_recommendation(ci, sbom_names) - comment = ( - f"Upgrade to the first patched version(s): {patch_recommendation}." - if patch_recommendation else "No remediation recommendation available for this vulnerability." - ) - csaf_gen.add_vulnerability( - product_name=product_name, - release=product_tag, - id=vuln_id, - description=description or summary, - status="known_affected", - comment=comment - ) + impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else "unknown" + + is_vulnerable = state.justifications.get(vuln_id).get("justification_label") == "vulnerable" + + if is_vulnerable: + patch_recommendation = _build_patch_recommendation(ci, sbom_names) + comment = ( + f"Upgrade to the first patched version(s): {patch_recommendation}." + if patch_recommendation else "No remediation recommendation available for this vulnerability." + ) + + csaf_gen.add_vulnerability( + product_name=product_name, + release=product_tag, + id=vuln_id, + status="known_affected", + description="", + comment=impact, + remediation="vendor_fix", + action=comment + ) + + else: + csaf_gen.add_vulnerability( + product_name=product_name, + release=product_tag, + id=vuln_id, + status="known_not_affected", + description="", + comment=impact, + ) csaf_gen.generate_csaf() @@ -133,11 +207,11 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: with open(path, "r") as f: csaf_json = json.load(f) - print("---------------------------------------------------------") - print(json.dumps(csaf_json, indent=2)) - print("---------------------------------------------------------") + _enrich_vulnerabilities_with_notes(csaf_json, intel_map, state.final_summaries, state.justifications) + # Validate/normalize with csaf-tool if available # self._validate_with_csaf_tool(csaf_json) + return csaf_json # def _validate_with_csaf_tool(self, csaf_doc: Dict[str, Any]) -> None: From 7cc8055392af5f0969a76e8ccef91158bf395892 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 22 Dec 2025 15:30:38 +0200 Subject: [PATCH 151/286] chore: add validation to generated CSAF document Signed-off-by: Ilona Shishov --- src/vuln_analysis/configs/config-http-nim.yml | 5 ++++ .../configs/config-http-openai.yml | 4 +++ src/vuln_analysis/configs/config-tracing.yml | 4 +++ src/vuln_analysis/configs/config.yml | 5 ++++ .../vex/implementations/csaf_generator.py | 30 +++++++++++-------- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 8ef818111..e14e0f60c 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -118,6 +118,10 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + vex_format: csaf cve_http_output: _type: cve_http_output url: http://localhost:8080 @@ -213,6 +217,7 @@ workflow: cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 1ff998ba1..0a8c4c51f 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -125,6 +125,9 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false cve_http_output: _type: cve_http_output url: http://localhost:8080 @@ -222,6 +225,7 @@ workflow: cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index b18d815c9..6c064d8bd 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -128,6 +128,9 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false cve_file_output: _type: cve_file_output file_path: .tmp/output.json @@ -218,6 +221,7 @@ workflow: cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index c3f230924..701dcd842 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -98,6 +98,10 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + vex_format: csaf cve_file_output: _type: cve_file_output file_path: .tmp/output.json @@ -186,6 +190,7 @@ workflow: cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 4b07a8d3d..90b380c04 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -7,13 +7,16 @@ from typing import Any, Dict, List import tempfile import os +import logging from vuln_analysis.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState from ..abc import VexGenerator from csaf.generator import CSAFGenerator +from csaf.analyser import CSAFAnalyser +logger = logging.getLogger(__name__) def _get_patched_package(v: dict) -> tuple[str | None, str | None]: """ @@ -204,21 +207,24 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: with tempfile.TemporaryDirectory() as tmp_dir: path = os.path.join(tmp_dir, "csaf.json") csaf_gen.publish_csaf(path) + + # Load CSAF JSON with open(path, "r") as f: csaf_json = json.load(f) - _enrich_vulnerabilities_with_notes(csaf_json, intel_map, state.final_summaries, state.justifications) + # Enrich the CSAF in memory + _enrich_vulnerabilities_with_notes( + csaf_json, intel_map, state.final_summaries, state.justifications + ) - # Validate/normalize with csaf-tool if available - # self._validate_with_csaf_tool(csaf_json) - - return csaf_json + # Overwrite the same file with enriched content + with open(path, "w") as f: + json.dump(csaf_json, f) - # def _validate_with_csaf_tool(self, csaf_doc: Dict[str, Any]) -> None: - # """ - # Validate the CSAF document using csaf-tool CLI. This step ensures the produced JSON - # conforms to CSAF expectations. If csaf-tool is not available or validation fails, - # this method will not raise and the original document will be returned to the caller. - # """ + # Validate the enriched CSAF + analyser = CSAFAnalyser(path) + if not analyser.validate(): + logger.error("CSAF document does not conform to CSAF 2.0 schema.") + analyser.analyse() # prints analysis to console - # csaf_doc.analyse() + return csaf_json \ No newline at end of file From ac13fb72ad6a44014253f360d1a2d278cf12e52e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 25 Dec 2025 12:30:58 +0200 Subject: [PATCH 152/286] chore: add error handling to vex generation Signed-off-by: Ilona Shishov --- src/vuln_analysis/functions/cve_generate_vex.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index 7da96c1ed..e5546147e 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -46,9 +46,14 @@ async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: logger.info("No vulnerable CVE(s) found. Skipping VEX generation.") return state - generator = load_vex_generator(config.vex_format) - vex_doc = generator.generate(state) - state.vex = vex_doc + try: + generator = load_vex_generator(config.vex_format) + vex_doc = generator.generate(state) + state.vex = vex_doc + except ValueError as e: + logger.error("VEX generator initialization failed: %s", e) + except Exception as e: + logger.error("VEX document generation failed: %s", e) return state From 81a9474406ee0716a13c0d24c18e8a8446a05b30 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 28 Dec 2025 10:17:55 +0200 Subject: [PATCH 153/286] test: add unit tests and error handling Signed-off-by: Ilona Shishov --- .../vex/implementations/csaf_generator.py | 11 +- tests/test_vex_csaf_helpers.py | 389 ++++++++++++++++++ tests/test_vex_loader.py | 55 +++ 3 files changed, 452 insertions(+), 3 deletions(-) create mode 100644 tests/test_vex_csaf_helpers.py create mode 100644 tests/test_vex_loader.py diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 90b380c04..8f2f18e86 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -36,7 +36,8 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non return "" vulns = ci.ghsa.vulnerabilities - # SBOM present + + # SBOM if sbom_package_names is not None: return next( ( @@ -47,6 +48,7 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non ), "" ) + # No SBOM name_to_version: dict[str, str] = {} for v in vulns: @@ -78,14 +80,17 @@ def _enrich_vulnerabilities_with_notes( ci = intel_map.get(vuln_id) - # Add GHSA description if available + # Update or remove GHSA description note ghsa_description = ci.ghsa.description if (ci and ci.ghsa and ci.ghsa.description) else None if ghsa_description: for note in notes: if note.get("category") == "description": note["title"] = "Vulnerability description" note["text"] = ghsa_description - break + break + else: + # Remove description note if no GHSA description available + notes[:] = [note for note in notes if note.get("category") != "description"] # Add GHSA summary if available ghsa_summary = ci.ghsa.summary if (ci and ci.ghsa and ci.ghsa.summary) else None diff --git a/tests/test_vex_csaf_helpers.py b/tests/test_vex_csaf_helpers.py new file mode 100644 index 000000000..5b71a0c42 --- /dev/null +++ b/tests/test_vex_csaf_helpers.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for VEX CSAF generator helper functions. +""" + +import pytest + +from vuln_analysis.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from vuln_analysis.utils.vex.implementations.csaf_generator import ( + _get_patched_package, + _build_patch_recommendation, + _enrich_vulnerabilities_with_notes, +) + + +# --- Fixtures for TestEnrichVulnerabilitiesWithNotes --- + +@pytest.fixture +def base_csaf_json(): + """Returns a fresh base CSAF JSON structure with one vulnerability.""" + return { + "vulnerabilities": [ + {"cve": "CVE-2024-1234", "notes": []} + ] + } + + +@pytest.fixture +def base_intel_map(): + """Returns a basic intel map with one CVE.""" + return {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234")} + + +@pytest.fixture +def base_final_summaries(): + """Returns basic final summaries dict.""" + return {"CVE-2024-1234": "Analysis summary"} + + +@pytest.fixture +def base_justifications(): + """Returns basic justifications dict.""" + return {"CVE-2024-1234": {"justification": "reasoning", "justification_label": "vulnerable"}} + + +class TestGetPatchedPackage: + """Unit tests for _get_patched_package() function.""" + + def test_valid_package_returns_name_and_version(self): + """Test extraction of name and version from valid vulnerability dict.""" + vuln = { + "package": {"name": "lodash"}, + "first_patched_version": "4.17.21" + } + result = _get_patched_package(vuln) + assert result == ("lodash", "4.17.21") + + def test_empty_dict_returns_none_tuple(self): + """Test that empty dict returns (None, None).""" + result = _get_patched_package({}) + assert result == (None, None) + + def test_missing_package_key_returns_none_name(self): + """Test that missing 'package' key returns None for name.""" + vuln = {"first_patched_version": "1.0.0"} + result = _get_patched_package(vuln) + assert result == (None, "1.0.0") + + def test_missing_version_returns_none_version(self): + """Test that missing version returns None for version.""" + vuln = {"package": {"name": "express"}} + result = _get_patched_package(vuln) + assert result == ("express", None) + + def test_null_package_returns_none_name(self): + """Test that null package value returns None for name.""" + vuln = {"package": None, "first_patched_version": "2.0.0"} + result = _get_patched_package(vuln) + assert result == (None, "2.0.0") + + def test_empty_package_dict_returns_none_name(self): + """Test that empty package dict returns None for name.""" + vuln = {"package": {}, "first_patched_version": "3.0.0"} + result = _get_patched_package(vuln) + assert result == (None, "3.0.0") + + +class TestBuildPatchRecommendation: + """Unit tests for _build_patch_recommendation() function.""" + + def test_returns_empty_when_intel_is_none(self): + """Test that None intel returns empty string.""" + result = _build_patch_recommendation(None, None) + assert result == "" + + def test_returns_empty_when_ghsa_is_none(self): + """Test that intel without GHSA returns empty string.""" + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=None) + result = _build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_none(self): + """Test that GHSA without vulnerabilities returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=None) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = _build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_empty(self): + """Test that empty vulnerabilities list returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=[]) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = _build_patch_recommendation(ci, None) + assert result == "" + + def test_with_sbom_returns_matching_package(self): + """Test that with SBOM, only matching package is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, {"lodash", "react"}) + assert result == "lodash:4.17.21" + + def test_with_sbom_returns_first_match_only(self): + """Test that with SBOM, only first matching package is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, {"lodash", "express"}) + # Should return first match + assert result == "lodash:4.17.21" + + def test_with_sbom_no_match_returns_empty(self): + """Test that with SBOM but no match, empty string is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, {"react", "vue"}) + assert result == "" + + def test_without_sbom_returns_all_packages(self): + """Test that without SBOM, all packages are returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, None) + assert "lodash:4.17.21" in result + assert "express:4.18.0" in result + + def test_without_sbom_deduplicates_packages(self): + """Test that duplicate package names are deduplicated.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "lodash"}, "first_patched_version": "4.17.22"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, None) + # First version should win + assert result == "lodash:4.17.21" + + def test_skips_vulnerabilities_without_name(self): + """Test that vulnerabilities without package name are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {}, "first_patched_version": "1.0.0"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + + def test_skips_vulnerabilities_without_version(self): + """Test that vulnerabilities without patched version are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = _build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + + +class TestEnrichVulnerabilitiesWithNotes: + """Unit tests for _enrich_vulnerabilities_with_notes() function.""" + + def test_adds_ghsa_summary_note(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that GHSA summary is added as a note.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + summary="Test vulnerability summary" + ) + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)} + + _enrich_vulnerabilities_with_notes(base_csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + summary_notes = [n for n in notes if n.get("category") == "summary"] + assert len(summary_notes) == 1 + assert summary_notes[0]["text"] == "Test vulnerability summary" + assert summary_notes[0]["title"] == "Vulnerability summary" + + def test_adds_rhsa_statement_note(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that RHSA statement is added as a note.""" + rhsa = CveIntelRhsa(statement="Red Hat security statement") + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", rhsa=rhsa)} + + _enrich_vulnerabilities_with_notes(base_csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + general_notes = [n for n in notes if n.get("category") == "general"] + assert len(general_notes) == 1 + assert general_notes[0]["text"] == "Red Hat security statement" + assert general_notes[0]["title"] == "Red Hat Security Advisory Statement" + + def test_adds_analysis_summary_note(self, base_csaf_json, base_intel_map, base_justifications): + """Test that analysis summary is added as a note.""" + final_summaries = {"CVE-2024-1234": "This is the analysis summary"} + + _enrich_vulnerabilities_with_notes(base_csaf_json, base_intel_map, final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + analysis_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Summary"] + assert len(analysis_notes) == 1 + assert analysis_notes[0]["text"] == "This is the analysis summary" + assert analysis_notes[0]["category"] == "analysis" + + def test_adds_justification_notes(self, base_csaf_json, base_intel_map, base_final_summaries): + """Test that justification reasoning and label are added as notes.""" + justifications = { + "CVE-2024-1234": { + "justification": "The vulnerable code path is reachable", + "justification_label": "vulnerable" + } + } + + _enrich_vulnerabilities_with_notes(base_csaf_json, base_intel_map, base_final_summaries, justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + + reasoning_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Justification Reasoning"] + assert len(reasoning_notes) == 1 + assert reasoning_notes[0]["text"] == "The vulnerable code path is reachable" + + label_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Justification Label"] + assert len(label_notes) == 1 + assert label_notes[0]["text"] == "vulnerable" + + def test_updates_existing_description_note(self, base_final_summaries, base_justifications): + """Test that existing description note is updated with GHSA description.""" + csaf_json = { + "vulnerabilities": [ + { + "cve": "CVE-2024-1234", + "notes": [ + {"category": "description", "text": "Original description", "title": ""} + ] + } + ] + } + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + description="GHSA detailed description" + ) + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)} + + _enrich_vulnerabilities_with_notes(csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = csaf_json["vulnerabilities"][0]["notes"] + desc_notes = [n for n in notes if n.get("category") == "description"] + assert len(desc_notes) == 1 + assert desc_notes[0]["text"] == "GHSA detailed description" + assert desc_notes[0]["title"] == "Vulnerability description" + + def test_removes_description_note_when_no_ghsa_description(self, base_intel_map, base_final_summaries, base_justifications): + """Test that description note is removed when no GHSA description is available.""" + csaf_json = { + "vulnerabilities": [ + { + "cve": "CVE-2024-1234", + "notes": [ + {"category": "description", "text": "Original description", "title": "Original"}, + {"category": "summary", "text": "Some summary", "title": "Summary"} + ] + } + ] + } + + _enrich_vulnerabilities_with_notes(csaf_json, base_intel_map, base_final_summaries, base_justifications) + + notes = csaf_json["vulnerabilities"][0]["notes"] + desc_notes = [n for n in notes if n.get("category") == "description"] + # Description note should be removed + assert len(desc_notes) == 0 + # Other notes should still be present + summary_notes = [n for n in notes if n.get("category") == "summary"] + assert len(summary_notes) >= 1 + + def test_handles_multiple_vulnerabilities(self): + """Test that multiple vulnerabilities are all enriched.""" + csaf_json = { + "vulnerabilities": [ + {"cve": "CVE-2024-1234", "notes": []}, + {"cve": "CVE-2024-5678", "notes": []}, + ] + } + intel_map = { + "CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234"), + "CVE-2024-5678": CveIntel(vuln_id="CVE-2024-5678"), + } + final_summaries = { + "CVE-2024-1234": "Summary 1", + "CVE-2024-5678": "Summary 2", + } + justifications = { + "CVE-2024-1234": {"justification": "reason1", "justification_label": "vulnerable"}, + "CVE-2024-5678": {"justification": "reason2", "justification_label": "not_vulnerable"}, + } + + _enrich_vulnerabilities_with_notes(csaf_json, intel_map, final_summaries, justifications) + + # Both vulnerabilities should have notes + for vuln in csaf_json["vulnerabilities"]: + assert len(vuln["notes"]) >= 3 # At least analysis summary + 2 justification notes + + def test_handles_missing_intel_for_vulnerability(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that missing intel for a vulnerability is handled gracefully.""" + # Should not raise an exception + _enrich_vulnerabilities_with_notes(base_csaf_json, {}, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + # Should still have analysis notes + assert len(notes) >= 3 + + def test_handles_empty_vulnerabilities_list(self): + """Test that empty vulnerabilities list is handled.""" + csaf_json = {"vulnerabilities": []} + + # Should not raise an exception + _enrich_vulnerabilities_with_notes(csaf_json, {}, {}, {}) + + assert csaf_json["vulnerabilities"] == [] \ No newline at end of file diff --git a/tests/test_vex_loader.py b/tests/test_vex_loader.py new file mode 100644 index 000000000..85ee4f380 --- /dev/null +++ b/tests/test_vex_loader.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for VEX generator loader/factory. + +Tests the load_vex_generator() function in loader.py. +""" + +import pytest + +from vuln_analysis.utils.vex.loader import load_vex_generator +from vuln_analysis.utils.vex.abc import VexGenerator +from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator + + +class TestLoadVexGenerator: + """Unit tests for load_vex_generator() factory function.""" + + def test_csaf_format_returns_csaf_generator(self): + """Test that 'csaf' format returns CsafVexGenerator instance.""" + generator = load_vex_generator("csaf") + assert isinstance(generator, CsafVexGenerator) + + def test_csaf_format_returns_vex_generator_subclass(self): + """Test that returned generator is a VexGenerator subclass.""" + generator = load_vex_generator("csaf") + assert isinstance(generator, VexGenerator) + + def test_csaf_uppercase_is_case_insensitive(self): + """Test that format matching is case insensitive (uppercase).""" + generator = load_vex_generator("CSAF") + assert isinstance(generator, CsafVexGenerator) + + def test_invalid_format_raises_value_error(self): + """Test that unsupported format raises ValueError.""" + with pytest.raises(ValueError, match="Unsupported VEX format"): + load_vex_generator("WrongFormat") + + def test_empty_format_raises_value_error(self): + """Test that empty format string raises ValueError.""" + with pytest.raises(ValueError, match="Unsupported VEX format"): + load_vex_generator("") \ No newline at end of file From f7111397044ccecdd8afe01299dfd5ae67ec69b8 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 28 Dec 2025 14:32:18 +0200 Subject: [PATCH 154/286] test: add integration tests to csaf vex generation Signed-off-by: Ilona Shishov --- .../vex/implementations/csaf_generator.py | 6 +- src/vuln_analysis/utils/vex/tests/__init__.py | 14 + .../tests/test_csaf_generator_integration.py | 361 ++++++++++++++++++ 3 files changed, 379 insertions(+), 2 deletions(-) create mode 100644 src/vuln_analysis/utils/vex/tests/__init__.py create mode 100644 src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 8f2f18e86..480309bb6 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -8,6 +8,7 @@ import tempfile import os import logging +import re from vuln_analysis.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState @@ -18,6 +19,8 @@ logger = logging.getLogger(__name__) +OCI_DIGEST_RE = re.compile(r"^[a-z0-9]+:[a-f0-9]{32,}$") + def _get_patched_package(v: dict) -> tuple[str | None, str | None]: """ Get the patched package from a GHSA vulnerability dict. @@ -150,8 +153,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: product_name = message.input.image.name product_tag = message.input.image.tag - csaf_gen.set_title(f"ExploitIQ VEX Document - {product_name}:{product_tag}") - csaf_gen.set_header_title("ExploitIQ VEX Document") + csaf_gen.set_header_title(f"ExploitIQ VEX Document - {product_name}{"@" if OCI_DIGEST_RE.fullmatch(product_tag) else ":"}{product_tag}") csaf_gen.set_value("notes",[ { diff --git a/src/vuln_analysis/utils/vex/tests/__init__.py b/src/vuln_analysis/utils/vex/tests/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/tests/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py new file mode 100644 index 000000000..6843db2f9 --- /dev/null +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -0,0 +1,361 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Integration tests for CSAF VEX generator. +""" + +import pytest + +from vuln_analysis.data_models.common import AnalysisType +from vuln_analysis.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from vuln_analysis.data_models.info import AgentMorpheusInfo, SBOMPackage +from vuln_analysis.data_models.input import ( + AgentMorpheusEngineInput, + AgentMorpheusInput, + ImageInfoInput, + ManualSBOMInfoInput, + ScanInfoInput, + SourceDocumentsInfo, + VulnInfo, +) +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator +from vuln_analysis.utils.vex.loader import load_vex_generator + + +_DEFAULT_SOURCE_INFO = [ + SourceDocumentsInfo( + type="code", + git_repo="https://github.com/example/repo", + ref="main", + include=["*.py"], + exclude=[], + ) +] +_DEFAULT_PRODUCT_NAME = "registry.example.com/myapp" +_DEFAULT_PRODUCT_TAG = "v1.0.0" +_DEFAULT_JUSTIFICATION = {"justification": "reason", "justification_label": "vulnerable"} +_DEFAULT_VULNS = ["CVE-2024-1234"] +_DEFAULT_SUMMARY = "Analysis summary for {v}" + +def create_mock_state( + vulns: list[str] = _DEFAULT_VULNS, + justification: dict[str, dict[str, str]] | None = None, + intel: list[CveIntel] | None = None, + product_name: str = _DEFAULT_PRODUCT_NAME, + product_tag: str = _DEFAULT_PRODUCT_TAG, + sbom_packages: list[SBOMPackage] = [SBOMPackage(name="test-package", version="1.0.0", system="npm")], +) -> AgentMorpheusEngineState: + """Create a mock AgentMorpheusEngineState for testing.""" + + intel = intel or [CveIntel(vuln_id=v) for v in vulns] + + image_info = ImageInfoInput( + name=product_name, + tag=product_tag, + analysis_type=AnalysisType.IMAGE, + source_info=_DEFAULT_SOURCE_INFO, + sbom_info=ManualSBOMInfoInput(packages=sbom_packages), + ) + + engine_input = AgentMorpheusEngineInput( + input=AgentMorpheusInput( + scan=ScanInfoInput(vulns=[VulnInfo(vuln_id=v) for v in vulns]), + image=image_info, + ), + info=AgentMorpheusInfo(intel=intel), + ) + + return AgentMorpheusEngineState( + cve_intel=intel, + original_input=engine_input, + final_summaries={v: _DEFAULT_SUMMARY.format(v=v) for v in vulns}, + justifications={v: justification or _DEFAULT_JUSTIFICATION for v in vulns}, + ) + + +class TestCsafVexGeneratorIntegration: + """Integration tests for CsafVexGenerator.generate() method.""" + + def test_loader_returns_working_generator(self): + """Test that load_vex_generator returns a functional generator.""" + state = create_mock_state() + + generator = load_vex_generator("csaf") + result = generator.generate(state) + + assert "document" in result + assert "product_tree" in result + assert "vulnerabilities" in result + + def test_document_has_correct_title(self): + """Test that document title contains product name and tag.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + title = result["document"].get("title") + assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + ":" + _DEFAULT_PRODUCT_TAG in title + + def test_document_has_disclaimer_note(self): + """Test that document includes the disclaimer note.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + notes = result["document"].get("notes", []) + disclaimer_notes = [n for n in notes if n.get("category") == "disclaimer"] + assert len(disclaimer_notes) == 1 + assert "Unofficial Content Notice" in disclaimer_notes[0]["title"] + + def test_document_has_author_note(self): + """Test that document includes the author note.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + publisher = result["document"].get("publisher") + assert "Red Hat Product Security" in publisher.get("name") + assert "https://access.redhat.com/security/" in publisher.get("namespace") + + def test_product_tree_contains_product(self): + """Test that product tree contains the analyzed product.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + product_tree = result["product_tree"] + assert _DEFAULT_PRODUCT_NAME in product_tree.get("branches")[0].get("branches")[0].get("name") + assert _DEFAULT_PRODUCT_TAG in product_tree.get("branches")[0].get("branches")[0].get("branches")[0].get("name") + + def test_vulnerable_cve_has_known_affected_status(self): + """Test that vulnerable CVEs get 'known_affected' status.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + product_status = vuln.get("product_status", {}) + assert "known_affected" in product_status + + def test_not_vulnerable_cve_has_known_not_affected_status(self): + """Test that non-vulnerable CVEs get 'known_not_affected' status.""" + state = create_mock_state( + justification={"justification": "Code path not reachable", "justification_label": "not_vulnerable"}, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + product_status = vuln.get("product_status", {}) + assert "known_not_affected" in product_status + + def test_vulnerable_cve_includes_remediation(self): + """Test that vulnerable CVEs include remediation information when patch is available.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + + state = create_mock_state( + intel=intel, + sbom_packages=[SBOMPackage(name="lodash", version="4.17.21", system="npm")], + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert len(remediations) > 0 + assert "lodash:4.17.21" in remediations[0].get("details") + + def test_no_patch_available_shows_fallback_message(self): + """Test 'No remediation recommendation available' message when no patch is available.""" + + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "No remediation recommendation available" in remediations[0].get("details") + + def test_multiple_vulnerabilities_all_included(self): + """Test that multiple vulnerabilities are all included in output.""" + state = create_mock_state( + vulns=["CVE-2024-1234", "CVE-2024-5678", "CVE-2024-9999"] + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + cve_ids = [v.get("cve") for v in result["vulnerabilities"]] + assert "CVE-2024-1234" in cve_ids + assert "CVE-2024-5678" in cve_ids + assert "CVE-2024-9999" in cve_ids + + def test_vulnerabilities_have_notes_enriched(self): + """Test that vulnerabilities have notes added from enrichment.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + summary="Test vulnerability summary", + description="Detailed description of the vulnerability", + ) + rhsa = CveIntelRhsa(threat_severity="Important", statement="RHSA statement.") + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa, rhsa=rhsa)] + + state = create_mock_state( + intel=intel, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + notes = vuln.get("notes", []) + + analysis_notes = [n for n in notes if n.get("category") == "analysis"] + assert len(analysis_notes) == 3 # analysis summary + justification reasoning + justification label + + description_note = [n for n in notes if n.get("category") == "description"] + assert len(description_note) == 1 # ghsa description + + summary_note = [n for n in notes if n.get("category") == "summary"] + assert len(summary_note) == 1 # ghsa summary + + general_notes = [n for n in notes if n.get("category") == "general"] + assert len(general_notes) == 1 # rhsa statement + + def test_description_note_removed_when_no_ghsa(self): + """Test that description note is removed when no GHSA data is available.""" + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + notes = vuln.get("notes", []) + description_notes = [n for n in notes if n.get("category") == "description"] + assert len(description_notes) == 0 # Description note should be removed + + def test_rhsa_threat_severity_used_as_impact(self): + """Test that RHSA threat severity is included as impact/comment.""" + rhsa = CveIntelRhsa(threat_severity="Important") + intel = [CveIntel(vuln_id="CVE-2024-1234", rhsa=rhsa)] + + state = create_mock_state( + intel=intel, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + impact = vuln.get("threats")[0].get("details") + assert "Important" in impact + + +class TestCsafVexGeneratorEdgeCases: + """Integration tests for edge cases and error handling.""" + + def test_handles_empty_sbom_packages(self): + """Test handling when SBOM has no packages.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + state = create_mock_state( + intel=intel, + sbom_packages=[], + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + assert "vulnerabilities" in result + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "lodash:4.17.21" in remediations[0].get("details") + assert "express:4.18.0" in remediations[0].get("details") + + def test_handles_sbom_packages_with_matching_and_non_matching_packages(self): + """Test handling when SBOM has a matching package and a non-matching package.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + state = create_mock_state( + intel=intel, + sbom_packages=[SBOMPackage(name="lodash", version="4.17.21", system="npm")], + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + assert "vulnerabilities" in result + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "lodash:4.17.21" in remediations[0].get("details") + assert "express:4.18.0" not in remediations[0].get("details") + + def test_handles_missing_rhsa_severity(self): + """Test handling when RHSA threat severity is missing.""" + + state = create_mock_state() + + generator = CsafVexGenerator() + result = generator.generate(state) + + assert "vulnerabilities" in result + + vuln = result["vulnerabilities"][0] + impact = vuln.get("threats")[0].get("details") + assert "unknown" in impact + + def test_handles_product_name_without_slash(self): + """Test vendor extraction when product name has no slash.""" + state = create_mock_state( + product_name="simpleapp", + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + product_tree = result["product_tree"] + assert "unknown" == product_tree.get("branches")[0].get("name") + assert "simpleapp" == product_tree.get("branches")[0].get("branches")[0].get("name") + + + From 51050e367af3a730349986fd5809461fd72f292a Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 28 Dec 2025 16:11:11 +0200 Subject: [PATCH 155/286] chore: Add module scoped fixtures and exract data to module level constants for reusability and readability Signed-off-by: Ilona Shishov --- .../vex/implementations/csaf_generator.py | 110 +++++++++++++----- src/vuln_analysis/utils/vex/loader.py | 11 +- .../tests/test_csaf_generator_integration.py | 65 +++++------ 3 files changed, 117 insertions(+), 69 deletions(-) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 480309bb6..e4ea1d044 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -19,7 +19,59 @@ logger = logging.getLogger(__name__) -OCI_DIGEST_RE = re.compile(r"^[a-z0-9]+:[a-f0-9]{32,}$") +# Regex pattern for OCI digest format +OCI_DIGEST_PATTERN = r"^[a-z0-9]+:[a-f0-9]{32,}$" +OCI_DIGEST_RE = re.compile(OCI_DIGEST_PATTERN) + +# Note categories +NOTE_CATEGORY_DESCRIPTION = "description" +NOTE_CATEGORY_SUMMARY = "summary" +NOTE_CATEGORY_GENERAL = "general" +NOTE_CATEGORY_ANALYSIS = "analysis" +NOTE_CATEGORY_DISCLAIMER = "disclaimer" + +# Note titles +NOTE_TITLE_VULNERABILITY_DESCRIPTION = "Vulnerability description" +NOTE_TITLE_VULNERABILITY_SUMMARY = "Vulnerability summary" +NOTE_TITLE_RHSA_STATEMENT = "Red Hat Security Advisory Statement" +NOTE_TITLE_EXPLOITIQ_SUMMARY = "ExploitIQ Analysis Summary" +NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_REASONING = "ExploitIQ Analysis Justification Reasoning" +NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_LABEL = "ExploitIQ Analysis Justification Label" +NOTE_TITLE_UNOFFICIAL_CONTENT = "Unofficial Content Notice" + +# Disclaimer text +DISCLAIMER_TEXT = ( + "This CSAF document is generated as custom content for internal or experimental use. " + "It is not an official Red Hat security document, has not undergone formal validation or review, " + "and is not digitally signed. The information provided here should not be regarded as authoritative " + "or replace official Red Hat advisories or VEX statements." +) + +# Author information +AUTHOR_NAME = "Red Hat Product Security" +AUTHOR_URL = "https://access.redhat.com/security/" + +# Default values +DEFAULT_VENDOR = "unknown" +DEFAULT_IMPACT = "unknown" + +# Justification labels +JUSTIFICATION_LABEL_VULNERABLE = "vulnerable" + +# Vulnerability statuses +STATUS_KNOWN_AFFECTED = "known_affected" +STATUS_KNOWN_NOT_AFFECTED = "known_not_affected" + +# Remediation +REMEDIATION_TYPE_VENDOR_FIX = "vendor_fix" +REMEDIATION_MESSAGE_TEMPLATE = "Upgrade to the first patched version(s): {patch_recommendation}." +REMEDIATION_MESSAGE_NO_RECOMMENDATION = "No remediation recommendation available for this vulnerability." + +# File names +CSAF_OUTPUT_FILENAME = "csaf.json" + +# Error messages +ERROR_CSAF_VALIDATION_FAILED = "CSAF document does not conform to CSAF 2.0 schema." def _get_patched_package(v: dict) -> tuple[str | None, str | None]: """ @@ -87,50 +139,50 @@ def _enrich_vulnerabilities_with_notes( ghsa_description = ci.ghsa.description if (ci and ci.ghsa and ci.ghsa.description) else None if ghsa_description: for note in notes: - if note.get("category") == "description": - note["title"] = "Vulnerability description" + if note.get("category") == NOTE_CATEGORY_DESCRIPTION: + note["title"] = NOTE_TITLE_VULNERABILITY_DESCRIPTION note["text"] = ghsa_description break else: # Remove description note if no GHSA description available - notes[:] = [note for note in notes if note.get("category") != "description"] + notes[:] = [note for note in notes if note.get("category") != NOTE_CATEGORY_DESCRIPTION] # Add GHSA summary if available ghsa_summary = ci.ghsa.summary if (ci and ci.ghsa and ci.ghsa.summary) else None if ghsa_summary: notes.append({ - "category": "summary", + "category": NOTE_CATEGORY_SUMMARY, "text": ghsa_summary, - "title": "Vulnerability summary" + "title": NOTE_TITLE_VULNERABILITY_SUMMARY }) # Add RHSA statement if available rhsa_statement = ci.rhsa.statement if (ci and ci.rhsa and ci.rhsa.statement) else None if rhsa_statement: notes.append({ - "category": "general", + "category": NOTE_CATEGORY_GENERAL, "text": rhsa_statement, - "title": "Red Hat Security Advisory Statement" + "title": NOTE_TITLE_RHSA_STATEMENT }) # Add ExploitIQ analysis summary summary = final_summaries.get(vuln_id) notes.append({ - "category": "analysis", - "title": "ExploitIQ Analysis Summary", + "category": NOTE_CATEGORY_ANALYSIS, + "title": NOTE_TITLE_EXPLOITIQ_SUMMARY, "text": summary }) # Add ExploitIQ justification details justification = justifications.get(vuln_id) notes.append({ - "category": "analysis", - "title": "ExploitIQ Analysis Justification Reasoning", + "category": NOTE_CATEGORY_ANALYSIS, + "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_REASONING, "text": justification.get("justification") }) notes.append({ - "category": "analysis", - "title": "ExploitIQ Analysis Justification Label", + "category": NOTE_CATEGORY_ANALYSIS, + "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_LABEL, "text": justification.get("justification_label") }) @@ -157,16 +209,16 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: csaf_gen.set_value("notes",[ { - "category": "disclaimer", - "text": "This CSAF document is generated as custom content for internal or experimental use. It is not an official Red Hat security document, has not undergone formal validation or review, and is not digitally signed. The information provided here should not be regarded as authoritative or replace official Red Hat advisories or VEX statements.", - "title": "Unofficial Content Notice" + "category": NOTE_CATEGORY_DISCLAIMER, + "text": DISCLAIMER_TEXT, + "title": NOTE_TITLE_UNOFFICIAL_CONTENT } ]) - csaf_gen.set_value("author", "Red Hat Product Security") - csaf_gen.set_value("author_url", "https://access.redhat.com/security/") + csaf_gen.set_value("author", AUTHOR_NAME) + csaf_gen.set_value("author_url", AUTHOR_URL) - vendor = (product_name.split("/")[1] if "/" in product_name else "unknown") + vendor = (product_name.split("/")[1] if "/" in product_name else DEFAULT_VENDOR) csaf_gen.add_product(product_name=product_name, vendor=vendor, release=product_tag) intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in message.info.intel} @@ -177,25 +229,25 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: for vuln_id, summary in state.final_summaries.items(): ci = intel_map.get(vuln_id) - impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else "unknown" + impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else DEFAULT_IMPACT - is_vulnerable = state.justifications.get(vuln_id).get("justification_label") == "vulnerable" + is_vulnerable = state.justifications.get(vuln_id).get("justification_label") == JUSTIFICATION_LABEL_VULNERABLE if is_vulnerable: patch_recommendation = _build_patch_recommendation(ci, sbom_names) comment = ( - f"Upgrade to the first patched version(s): {patch_recommendation}." - if patch_recommendation else "No remediation recommendation available for this vulnerability." + REMEDIATION_MESSAGE_TEMPLATE.format(patch_recommendation=patch_recommendation) + if patch_recommendation else REMEDIATION_MESSAGE_NO_RECOMMENDATION ) csaf_gen.add_vulnerability( product_name=product_name, release=product_tag, id=vuln_id, - status="known_affected", + status=STATUS_KNOWN_AFFECTED, description="", comment=impact, - remediation="vendor_fix", + remediation=REMEDIATION_TYPE_VENDOR_FIX, action=comment ) @@ -204,7 +256,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: product_name=product_name, release=product_tag, id=vuln_id, - status="known_not_affected", + status=STATUS_KNOWN_NOT_AFFECTED, description="", comment=impact, ) @@ -212,7 +264,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: csaf_gen.generate_csaf() with tempfile.TemporaryDirectory() as tmp_dir: - path = os.path.join(tmp_dir, "csaf.json") + path = os.path.join(tmp_dir, CSAF_OUTPUT_FILENAME) csaf_gen.publish_csaf(path) # Load CSAF JSON @@ -231,7 +283,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: # Validate the enriched CSAF analyser = CSAFAnalyser(path) if not analyser.validate(): - logger.error("CSAF document does not conform to CSAF 2.0 schema.") + logger.error(ERROR_CSAF_VALIDATION_FAILED) analyser.analyse() # prints analysis to console return csaf_json \ No newline at end of file diff --git a/src/vuln_analysis/utils/vex/loader.py b/src/vuln_analysis/utils/vex/loader.py index 316da7312..fd876e110 100644 --- a/src/vuln_analysis/utils/vex/loader.py +++ b/src/vuln_analysis/utils/vex/loader.py @@ -3,6 +3,13 @@ from .abc import VexGenerator from .implementations.csaf_generator import CsafVexGenerator +# Supported VEX formats +CSAF = "csaf" + +# Error messages +ERROR_UNSUPPORTED_VEX_FORMAT = "Unsupported VEX format: {fmt}" + + def load_vex_generator(fmt: str) -> VexGenerator: """ Return a VEX generator implementation for the requested format. @@ -10,7 +17,7 @@ def load_vex_generator(fmt: str) -> VexGenerator: fmt_lower = fmt.lower() - if fmt_lower == "csaf": + if fmt_lower == CSAF: return CsafVexGenerator() - raise ValueError(f"Unsupported VEX format: {fmt_lower}") + raise ValueError(ERROR_UNSUPPORTED_VEX_FORMAT.format(fmt=fmt_lower)) diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index 6843db2f9..6c25fd230 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -51,6 +51,15 @@ _DEFAULT_VULNS = ["CVE-2024-1234"] _DEFAULT_SUMMARY = "Analysis summary for {v}" + +@pytest.fixture(scope="module") +def mock_state() -> AgentMorpheusEngineState: + """Fixture providing a default mock AgentMorpheusEngineState for testing. + This state has one vulnerable CVE with a known affected status, with no GHSA or RHSA intel data. + """ + return create_mock_state() + + def create_mock_state( vulns: list[str] = _DEFAULT_VULNS, justification: dict[str, dict[str, str]] | None = None, @@ -90,67 +99,55 @@ def create_mock_state( class TestCsafVexGeneratorIntegration: """Integration tests for CsafVexGenerator.generate() method.""" - def test_loader_returns_working_generator(self): + def test_loader_returns_working_generator(self, mock_state): """Test that load_vex_generator returns a functional generator.""" - state = create_mock_state() - generator = load_vex_generator("csaf") - result = generator.generate(state) + result = generator.generate(mock_state) assert "document" in result assert "product_tree" in result assert "vulnerabilities" in result - def test_document_has_correct_title(self): + def test_document_has_correct_title(self, mock_state): """Test that document title contains product name and tag.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) title = result["document"].get("title") assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + ":" + _DEFAULT_PRODUCT_TAG in title - def test_document_has_disclaimer_note(self): + def test_document_has_disclaimer_note(self, mock_state): """Test that document includes the disclaimer note.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) notes = result["document"].get("notes", []) disclaimer_notes = [n for n in notes if n.get("category") == "disclaimer"] assert len(disclaimer_notes) == 1 assert "Unofficial Content Notice" in disclaimer_notes[0]["title"] - def test_document_has_author_note(self): + def test_document_has_author_note(self, mock_state): """Test that document includes the author note.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) publisher = result["document"].get("publisher") assert "Red Hat Product Security" in publisher.get("name") assert "https://access.redhat.com/security/" in publisher.get("namespace") - def test_product_tree_contains_product(self): + def test_product_tree_contains_product(self, mock_state): """Test that product tree contains the analyzed product.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) product_tree = result["product_tree"] assert _DEFAULT_PRODUCT_NAME in product_tree.get("branches")[0].get("branches")[0].get("name") assert _DEFAULT_PRODUCT_TAG in product_tree.get("branches")[0].get("branches")[0].get("branches")[0].get("name") - def test_vulnerable_cve_has_known_affected_status(self): + def test_vulnerable_cve_has_known_affected_status(self, mock_state): """Test that vulnerable CVEs get 'known_affected' status.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) vuln = result["vulnerabilities"][0] product_status = vuln.get("product_status", {}) @@ -192,13 +189,10 @@ def test_vulnerable_cve_includes_remediation(self): assert len(remediations) > 0 assert "lodash:4.17.21" in remediations[0].get("details") - def test_no_patch_available_shows_fallback_message(self): + def test_no_patch_available_shows_fallback_message(self, mock_state): """Test 'No remediation recommendation available' message when no patch is available.""" - - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) vuln = result["vulnerabilities"][0] remediations = vuln.get("remediations", []) @@ -250,12 +244,10 @@ def test_vulnerabilities_have_notes_enriched(self): general_notes = [n for n in notes if n.get("category") == "general"] assert len(general_notes) == 1 # rhsa statement - def test_description_note_removed_when_no_ghsa(self): + def test_description_note_removed_when_no_ghsa(self, mock_state): """Test that description note is removed when no GHSA data is available.""" - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) vuln = result["vulnerabilities"][0] notes = vuln.get("notes", []) @@ -330,13 +322,10 @@ def test_handles_sbom_packages_with_matching_and_non_matching_packages(self): assert "lodash:4.17.21" in remediations[0].get("details") assert "express:4.18.0" not in remediations[0].get("details") - def test_handles_missing_rhsa_severity(self): + def test_handles_missing_rhsa_severity(self, mock_state): """Test handling when RHSA threat severity is missing.""" - - state = create_mock_state() - generator = CsafVexGenerator() - result = generator.generate(state) + result = generator.generate(mock_state) assert "vulnerabilities" in result From 345c67942efb4bb98d00de7fee78cf06d403a33e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 29 Dec 2025 11:23:16 +0200 Subject: [PATCH 156/286] log: replaced logging import with LoggingFactory Signed-off-by: Ilona Shishov --- src/exploit_iq_commons/utils/dep_tree.py | 5 ++--- src/exploit_iq_commons/utils/js_extended_parser.py | 1 - src/exploit_iq_commons/utils/source_code_git_loader.py | 1 - src/vuln_analysis/data_models/plugins/intel_plugin.py | 1 - src/vuln_analysis/functions/cve_agent.py | 1 - src/vuln_analysis/functions/cve_calculate_intel_score.py | 5 ++--- src/vuln_analysis/functions/cve_check_vuln_deps.py | 1 - src/vuln_analysis/functions/cve_checklist.py | 1 - src/vuln_analysis/functions/cve_fetch_intel.py | 1 - src/vuln_analysis/functions/cve_file_output.py | 2 -- src/vuln_analysis/functions/cve_generate_cvss.py | 4 ++-- src/vuln_analysis/functions/cve_generate_vex.py | 4 ++-- src/vuln_analysis/functions/cve_http_output.py | 1 - src/vuln_analysis/functions/cve_justify.py | 1 - src/vuln_analysis/functions/cve_process_sbom.py | 2 -- src/vuln_analysis/functions/cve_summarize.py | 1 - src/vuln_analysis/register.py | 1 - src/vuln_analysis/tools/container_image_analysis_data.py | 4 ++-- src/vuln_analysis/tools/lexical_full_search.py | 2 -- src/vuln_analysis/tools/local_vdb.py | 2 -- src/vuln_analysis/tools/serp.py | 2 -- src/vuln_analysis/utils/async_http_utils.py | 1 - src/vuln_analysis/utils/checklist_prompt_generator.py | 3 +-- src/vuln_analysis/utils/clients/first_client.py | 1 - src/vuln_analysis/utils/clients/ghsa_client.py | 1 - src/vuln_analysis/utils/clients/nvd_client.py | 1 - src/vuln_analysis/utils/clients/rhsa_client.py | 1 - src/vuln_analysis/utils/full_text_search.py | 1 - src/vuln_analysis/utils/http_utils.py | 1 - src/vuln_analysis/utils/intel_source_score.py | 4 ++-- src/vuln_analysis/utils/justification_parser.py | 1 - .../utils/vex/implementations/csaf_generator.py | 4 ++-- src/vuln_analysis/utils/vulnerable_dependency_checker.py | 1 - 33 files changed, 15 insertions(+), 48 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 74e89f883..e4f84a41f 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -12,7 +12,6 @@ from packaging.specifiers import SpecifierSet from tqdm import tqdm -import logging import ast import json import zipfile @@ -180,7 +179,7 @@ def find_all_files(self, root_dir): try: full_path.rename(new_path) except Exception as e: - logging.warning( + logger.warning( "Rename failed: %s → %s: %s", full_path, new_path, e ) @@ -815,7 +814,7 @@ def get_go_mod_graph_tree(manifest_path) -> str: f"manifest wasn't found at {manifest_path}, error details => " f"{repr(e)}" ) - logging.error(error_message_exception) + logger.error(error_message_exception) raise e return process_object.stdout diff --git a/src/exploit_iq_commons/utils/js_extended_parser.py b/src/exploit_iq_commons/utils/js_extended_parser.py index 56a03d7b4..a8967ff7b 100644 --- a/src/exploit_iq_commons/utils/js_extended_parser.py +++ b/src/exploit_iq_commons/utils/js_extended_parser.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from typing import Any from typing import List from typing import Tuple diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 592caa2b8..a376d447d 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import typing from pathlib import Path diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py index cdad04751..9b85c6d11 100644 --- a/src/vuln_analysis/data_models/plugins/intel_plugin.py +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -7,7 +7,6 @@ # disclosure or distribution of this material and related documentation # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -import logging import requests from pydantic import BaseModel diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 60ca3a783..f4a3f62b9 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -15,7 +15,6 @@ import asyncio from vuln_analysis.runtime_context import ctx_state -import logging import typing from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum diff --git a/src/vuln_analysis/functions/cve_calculate_intel_score.py b/src/vuln_analysis/functions/cve_calculate_intel_score.py index ef1a25886..0ea840923 100644 --- a/src/vuln_analysis/functions/cve_calculate_intel_score.py +++ b/src/vuln_analysis/functions/cve_calculate_intel_score.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import aiohttp from aiq.builder.builder import Builder @@ -24,8 +23,8 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) - +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel_score"): """ diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index 4875e7926..4c3153e7e 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import aiohttp from aiq.builder.builder import Builder diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index f0cd65662..696c7fb9b 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import pandas as pd from aiq.builder.builder import Builder diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index 1acc59e34..26bb52c39 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import typing import aiohttp diff --git a/src/vuln_analysis/functions/cve_file_output.py b/src/vuln_analysis/functions/cve_file_output.py index 0d8562828..6e41d198c 100644 --- a/src/vuln_analysis/functions/cve_file_output.py +++ b/src/vuln_analysis/functions/cve_file_output.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index 424c24d22..7a21ee6b7 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -15,7 +15,6 @@ import asyncio from vuln_analysis.runtime_context import ctx_state -import logging import json import typing import re @@ -38,7 +37,8 @@ from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.utils.prompting import get_cvss_prompt -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" OUTPUT_CONTAIN_PARSING_ERROR = "Could not parse" diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index e5546147e..3a5e1a366 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import json from aiq.builder.builder import Builder @@ -24,7 +23,8 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.vex.loader import load_vex_generator -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class CVEGenerateVexConfig(FunctionBaseConfig, name="cve_generate_vex"): """ diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index 7a78c7c66..fcc3b172b 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import base64 -import logging from http import HTTPStatus from aiq.builder.builder import Builder diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py index fce509f0e..8124b4222 100644 --- a/src/vuln_analysis/functions/cve_justify.py +++ b/src/vuln_analysis/functions/cve_justify.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index 65865167f..c2b78d250 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index 608fff839..db7fba0b0 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 0eaafac1f..19541df1c 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from datetime import datetime from io import TextIOWrapper diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py index ea0a6d2d9..029afa1ad 100644 --- a/src/vuln_analysis/tools/container_image_analysis_data.py +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from typing import Any from aiq.builder.builder import Builder @@ -26,7 +25,8 @@ CONTAINER_IMAGE_ANALYSIS_DATA = "container_image_analysis_data" -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class ContainerImageAnalysisDataToolConfig(FunctionBaseConfig, name=("%s" % CONTAINER_IMAGE_ANALYSIS_DATA)): diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index f2e92e664..7166b5e5a 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index d6effc8dc..7549032fa 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo diff --git a/src/vuln_analysis/tools/serp.py b/src/vuln_analysis/tools/serp.py index fc117654f..7fe820b82 100644 --- a/src/vuln_analysis/tools/serp.py +++ b/src/vuln_analysis/tools/serp.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function diff --git a/src/vuln_analysis/utils/async_http_utils.py b/src/vuln_analysis/utils/async_http_utils.py index 030a2d802..5d2a3c1a3 100644 --- a/src/vuln_analysis/utils/async_http_utils.py +++ b/src/vuln_analysis/utils/async_http_utils.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import time import typing from contextlib import asynccontextmanager diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 8e515c550..3ed21b159 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -14,7 +14,6 @@ # limitations under the License. import ast -import logging import re from jinja2 import Template @@ -170,7 +169,7 @@ async def generate_checklist(prompt: str | None, return parsed_checklist.content except Exception as e: - logging.error(f" Error in generating checklist : {e}") + logger.error(f" Error in generating checklist : {e}") raise return gen_checklist.content diff --git a/src/vuln_analysis/utils/clients/first_client.py b/src/vuln_analysis/utils/clients/first_client.py index 1fd854332..f961b7fdc 100644 --- a/src/vuln_analysis/utils/clients/first_client.py +++ b/src/vuln_analysis/utils/clients/first_client.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import aiohttp diff --git a/src/vuln_analysis/utils/clients/ghsa_client.py b/src/vuln_analysis/utils/clients/ghsa_client.py index 466b4df7a..ce819833c 100644 --- a/src/vuln_analysis/utils/clients/ghsa_client.py +++ b/src/vuln_analysis/utils/clients/ghsa_client.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import aiohttp diff --git a/src/vuln_analysis/utils/clients/nvd_client.py b/src/vuln_analysis/utils/clients/nvd_client.py index c76bc5cac..1b13ff51c 100644 --- a/src/vuln_analysis/utils/clients/nvd_client.py +++ b/src/vuln_analysis/utils/clients/nvd_client.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import re diff --git a/src/vuln_analysis/utils/clients/rhsa_client.py b/src/vuln_analysis/utils/clients/rhsa_client.py index 404a1845e..30bea1f8e 100644 --- a/src/vuln_analysis/utils/clients/rhsa_client.py +++ b/src/vuln_analysis/utils/clients/rhsa_client.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import aiohttp diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index eada7d0da..3f09349bf 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import re from pathlib import Path diff --git a/src/vuln_analysis/utils/http_utils.py b/src/vuln_analysis/utils/http_utils.py index 3b2982538..f58f87ea6 100644 --- a/src/vuln_analysis/utils/http_utils.py +++ b/src/vuln_analysis/utils/http_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import time import typing from enum import Enum diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 596cb7987..53301acfb 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -10,7 +10,6 @@ # limitations under the License. import json import os -import logging import re from aiq.builder.builder import Builder from langchain_core.language_models.base import BaseLanguageModel @@ -21,7 +20,8 @@ from ..utils.prompting import additional_intel_prompting from aiq.builder.framework_enum import LLMFrameworkEnum -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class IntelScorer: def __init__(self, diff --git a/src/vuln_analysis/utils/justification_parser.py b/src/vuln_analysis/utils/justification_parser.py index c0964ce5b..ac94ed19f 100644 --- a/src/vuln_analysis/utils/justification_parser.py +++ b/src/vuln_analysis/utils/justification_parser.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from textwrap import dedent from exploit_iq_commons.logging.loggers_factory import LoggingFactory diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index e4ea1d044..286945018 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -7,7 +7,6 @@ from typing import Any, Dict, List import tempfile import os -import logging import re from vuln_analysis.data_models.cve_intel import CveIntel @@ -17,7 +16,8 @@ from csaf.generator import CSAFGenerator from csaf.analyser import CSAFAnalyser -logger = logging.getLogger(__name__) +from vuln_analysis.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) # Regex pattern for OCI digest format OCI_DIGEST_PATTERN = r"^[a-z0-9]+:[a-f0-9]{32,}$" diff --git a/src/vuln_analysis/utils/vulnerable_dependency_checker.py b/src/vuln_analysis/utils/vulnerable_dependency_checker.py index 62eb9bd48..3b459dd13 100644 --- a/src/vuln_analysis/utils/vulnerable_dependency_checker.py +++ b/src/vuln_analysis/utils/vulnerable_dependency_checker.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import os import re import urllib.parse From 8fb1103090f8f6b2767403d189b139830f71130e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 29 Dec 2025 11:34:33 +0200 Subject: [PATCH 157/286] refactor: rename vex modules for clarity Signed-off-by: Ilona Shishov --- src/vuln_analysis/functions/cve_generate_vex.py | 2 +- src/vuln_analysis/utils/vex/implementations/csaf_generator.py | 2 +- .../utils/vex/tests/test_csaf_generator_integration.py | 2 +- src/vuln_analysis/utils/vex/{abc.py => vex_generator_base.py} | 0 .../utils/vex/{loader.py => vex_generator_loader.py} | 2 +- tests/test_vex_loader.py | 4 ++-- 6 files changed, 6 insertions(+), 6 deletions(-) rename src/vuln_analysis/utils/vex/{abc.py => vex_generator_base.py} (100%) rename src/vuln_analysis/utils/vex/{loader.py => vex_generator_loader.py} (91%) diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index 3a5e1a366..b9475f4e5 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -21,7 +21,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field from vuln_analysis.data_models.state import AgentMorpheusEngineState -from vuln_analysis.utils.vex.loader import load_vex_generator +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator from vuln_analysis.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 286945018..5f590a865 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -12,7 +12,7 @@ from vuln_analysis.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState -from ..abc import VexGenerator +from ..vex_generator_base import VexGenerator from csaf.generator import CSAFGenerator from csaf.analyser import CSAFAnalyser diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index 6c25fd230..ad11c80e6 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -33,7 +33,7 @@ ) from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator -from vuln_analysis.utils.vex.loader import load_vex_generator +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator _DEFAULT_SOURCE_INFO = [ diff --git a/src/vuln_analysis/utils/vex/abc.py b/src/vuln_analysis/utils/vex/vex_generator_base.py similarity index 100% rename from src/vuln_analysis/utils/vex/abc.py rename to src/vuln_analysis/utils/vex/vex_generator_base.py diff --git a/src/vuln_analysis/utils/vex/loader.py b/src/vuln_analysis/utils/vex/vex_generator_loader.py similarity index 91% rename from src/vuln_analysis/utils/vex/loader.py rename to src/vuln_analysis/utils/vex/vex_generator_loader.py index fd876e110..aeb47e3b9 100644 --- a/src/vuln_analysis/utils/vex/loader.py +++ b/src/vuln_analysis/utils/vex/vex_generator_loader.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .abc import VexGenerator +from .vex_generator_base import VexGenerator from .implementations.csaf_generator import CsafVexGenerator # Supported VEX formats diff --git a/tests/test_vex_loader.py b/tests/test_vex_loader.py index 85ee4f380..76987d787 100644 --- a/tests/test_vex_loader.py +++ b/tests/test_vex_loader.py @@ -21,8 +21,8 @@ import pytest -from vuln_analysis.utils.vex.loader import load_vex_generator -from vuln_analysis.utils.vex.abc import VexGenerator +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator +from vuln_analysis.utils.vex.vex_generator_base import VexGenerator from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator From f5c64b1b9ea02012d58ed044fe4b29dad0797f4e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 29 Dec 2025 14:15:28 +0200 Subject: [PATCH 158/286] chore: update openapi spec with vex generation, cvss generation and cve intel score Signed-off-by: Ilona Shishov --- .../configs/openapi/openapi.json | 74 +++++++++++++++++-- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/src/vuln_analysis/configs/openapi/openapi.json b/src/vuln_analysis/configs/openapi/openapi.json index 2f2a49c54..79feca4cc 100644 --- a/src/vuln_analysis/configs/openapi/openapi.json +++ b/src/vuln_analysis/configs/openapi/openapi.json @@ -944,6 +944,20 @@ }, "justification": { "$ref": "#/components/schemas/JustificationOutput" + }, + "intel_score": { + "type": "integer", + "title": "Intel Score" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSSOutput" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -951,10 +965,12 @@ "vuln_id", "checklist", "summary", - "justification" + "justification", + "intel_score", + "cvss" ], "title": "AgentMorpheusEngineOutput", - "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision." + "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision.\n- intel_score: the intelligence score for the vulnerability.\n- cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability." }, "AgentMorpheusInfo": { "properties": { @@ -1054,11 +1070,7 @@ "$ref": "#/components/schemas/AgentMorpheusInfo" }, "output": { - "items": { - "$ref": "#/components/schemas/AgentMorpheusEngineOutput" - }, - "type": "array", - "title": "Output" + "$ref": "#/components/schemas/OutputPayload" } }, "type": "object", @@ -1218,6 +1230,25 @@ "type": "object", "title": "CVSS3" }, + "CVSSOutput": { + "properties": { + "vector_string": { + "type": "string", + "title": "Vector String" + }, + "score": { + "type": "string", + "title": "Score" + } + }, + "type": "object", + "required": [ + "vector_string", + "score" + ], + "title": "CVSSOutput", + "description": "CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image.\n- vector_string: The CVSS vector string that encodes the metric values used to calculate the score.\n- score: The calculated CVSS base score representing the severity of the vulnerability in the given image." + }, "CVSSV3": { "properties": { "attackComplexity": { @@ -2492,6 +2523,35 @@ "type": "object", "title": "Note" }, + "OutputPayload": { + "properties": { + "analysis": { + "items": { + "$ref": "#/components/schemas/AgentMorpheusEngineOutput" + }, + "type": "array", + "title": "Analysis" + }, + "vex": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vex" + } + }, + "type": "object", + "required": [ + "analysis", + "vex" + ], + "title": "OutputPayload", + "description": "Wrapper for final pipeline results.\n- analysis: per-vulnerability analysis results\n- vex: the vulnerability exploitability exchange document JSON" + }, "PackageState": { "properties": { "product_name": { From cf5eb4a2d5b7838a24478cfcab6a1c68a604ffc2 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 29 Dec 2025 15:17:25 +0200 Subject: [PATCH 159/286] chore: iterate over justifications in vex generation for efficiency Signed-off-by: Ilona Shishov --- src/vuln_analysis/utils/vex/implementations/csaf_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 5f590a865..0831f93e5 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -226,12 +226,12 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: if message.input.image.sbom_info and message.input.image.sbom_info.packages: sbom_names = {pkg.name for pkg in message.input.image.sbom_info.packages} - for vuln_id, summary in state.final_summaries.items(): + for vuln_id, justification in state.justifications.items(): ci = intel_map.get(vuln_id) impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else DEFAULT_IMPACT - is_vulnerable = state.justifications.get(vuln_id).get("justification_label") == JUSTIFICATION_LABEL_VULNERABLE + is_vulnerable = justification.get("justification_label") == JUSTIFICATION_LABEL_VULNERABLE if is_vulnerable: patch_recommendation = _build_patch_recommendation(ci, sbom_names) From 7c64d2e10829267d4263d3a9625a3637c9ad1b5c Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 29 Dec 2025 15:25:00 +0200 Subject: [PATCH 160/286] fix: return empty json if vex document fails validation Signed-off-by: Ilona Shishov --- src/vuln_analysis/utils/vex/implementations/csaf_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 0831f93e5..af8e4ea3a 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -285,5 +285,6 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: if not analyser.validate(): logger.error(ERROR_CSAF_VALIDATION_FAILED) analyser.analyse() # prints analysis to console + return {} return csaf_json \ No newline at end of file From ad2d46f37241fbe8ff25e03a4616c0761888c5b3 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 1 Jan 2026 16:51:54 +0200 Subject: [PATCH 161/286] fix: provide all relevant patch recommendations Signed-off-by: Ilona Shishov --- .../vex/implementations/csaf_generator.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index af8e4ea3a..8f085df3f 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -84,7 +84,7 @@ def _get_patched_package(v: dict) -> tuple[str | None, str | None]: def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: """ Build a patch recommendation from the GHSA data. - - If SBOM provided: return the first matching 'name:first_patched_version' where name is in SBOM, else return "". + - If SBOM provided: return all unique 'name:first_patched_version' pairs where name is in SBOM as list, else return "". - If no SBOM: return all unique 'name:first_patched_version' pairs as list, else return "". """ if not ci or not ci.ghsa or not ci.ghsa.vulnerabilities: @@ -92,26 +92,18 @@ def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | Non vulns = ci.ghsa.vulnerabilities - # SBOM - if sbom_package_names is not None: - return next( - ( - f"{name}:{patch}" - for v in vulns - for (name, patch) in [(_get_patched_package(v))] - if name and patch and name in sbom_package_names - ), - "" - ) - - # No SBOM name_to_version: dict[str, str] = {} for v in vulns: name, patch = _get_patched_package(v) if not name or not patch: continue - if name not in name_to_version: - name_to_version[name] = patch + if name in name_to_version: + continue + # If SBOM provided, only include packages that are in the SBOM + if sbom_package_names is not None and name not in sbom_package_names: + continue + name_to_version[name] = patch + if not name_to_version: return "" return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) From d17fd94f23c4e07cff4fe260c6b99342c9e622b6 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 13 Jan 2026 14:10:27 +0200 Subject: [PATCH 162/286] chore: update imports to exploit-iq-commons for transfered components Signed-off-by: Ilona Shishov --- src/vuln_analysis/functions/cve_calculate_intel_score.py | 2 +- src/vuln_analysis/functions/cve_generate_cvss.py | 2 +- src/vuln_analysis/functions/cve_generate_vex.py | 2 +- src/vuln_analysis/tools/container_image_analysis_data.py | 2 +- src/vuln_analysis/utils/intel_source_score.py | 2 +- .../utils/vex/implementations/csaf_generator.py | 4 ++-- .../utils/vex/tests/test_csaf_generator_integration.py | 8 ++++---- tests/test_vex_csaf_helpers.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/vuln_analysis/functions/cve_calculate_intel_score.py b/src/vuln_analysis/functions/cve_calculate_intel_score.py index 0ea840923..5f562e4d2 100644 --- a/src/vuln_analysis/functions/cve_calculate_intel_score.py +++ b/src/vuln_analysis/functions/cve_calculate_intel_score.py @@ -23,7 +23,7 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel_score"): diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py index 7a21ee6b7..8343d1c1b 100644 --- a/src/vuln_analysis/functions/cve_generate_cvss.py +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -37,7 +37,7 @@ from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.utils.prompting import get_cvss_prompt -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index b9475f4e5..62a9d0598 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -23,7 +23,7 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) class CVEGenerateVexConfig(FunctionBaseConfig, name="cve_generate_vex"): diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py index 029afa1ad..53495dac3 100644 --- a/src/vuln_analysis/tools/container_image_analysis_data.py +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -25,7 +25,7 @@ CONTAINER_IMAGE_ANALYSIS_DATA = "container_image_analysis_data" -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 53301acfb..479b3cc55 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -20,7 +20,7 @@ from ..utils.prompting import additional_intel_prompting from aiq.builder.framework_enum import LLMFrameworkEnum -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) class IntelScorer: diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 8f085df3f..d5f426694 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -9,14 +9,14 @@ import os import re -from vuln_analysis.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState from ..vex_generator_base import VexGenerator from csaf.generator import CSAFGenerator from csaf.analyser import CSAFAnalyser -from vuln_analysis.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) # Regex pattern for OCI digest format diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index ad11c80e6..8bd0e27ba 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -19,10 +19,10 @@ import pytest -from vuln_analysis.data_models.common import AnalysisType -from vuln_analysis.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa -from vuln_analysis.data_models.info import AgentMorpheusInfo, SBOMPackage -from vuln_analysis.data_models.input import ( +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from exploit_iq_commons.data_models.info import AgentMorpheusInfo, SBOMPackage +from exploit_iq_commons.data_models.input import ( AgentMorpheusEngineInput, AgentMorpheusInput, ImageInfoInput, diff --git a/tests/test_vex_csaf_helpers.py b/tests/test_vex_csaf_helpers.py index 5b71a0c42..a5a6d7a47 100644 --- a/tests/test_vex_csaf_helpers.py +++ b/tests/test_vex_csaf_helpers.py @@ -19,7 +19,7 @@ import pytest -from vuln_analysis.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa from vuln_analysis.utils.vex.implementations.csaf_generator import ( _get_patched_package, _build_patch_recommendation, From bf9a5496ad737ab03e02dc822bed9c3466bcc290 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Wed, 14 Jan 2026 14:14:08 +0200 Subject: [PATCH 163/286] fix: update csaf vex document catagories and add full schema validation Signed-off-by: Ilona Shishov --- pyproject.toml | 1 + .../vex/csaf/v2.0/csaf_json_schema.json | 1414 +++++++++++++++++ .../vex/implementations/csaf_generator.py | 63 +- 3 files changed, 1452 insertions(+), 26 deletions(-) create mode 100644 src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json diff --git a/pyproject.toml b/pyproject.toml index b321ccedb..0d4d36bd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "univers==30.12", "litellm<=1.75.8", "csaf-tool==0.3.2", + "jsonschema>=4.0.0,<5.0.0", ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json b/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json new file mode 100644 index 000000000..93ff152a2 --- /dev/null +++ b/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json @@ -0,0 +1,1414 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", + "title": "Common Security Advisory Framework", + "description": "Representation of security advisory information as a JSON document.", + "type": "object", + "$defs": { + "acknowledgments_t": { + "title": "List of acknowledgments", + "description": "Contains a list of acknowledgment elements.", + "type": "array", + "minItems": 1, + "items": { + "title": "Acknowledgment", + "description": "Acknowledges contributions by describing those that contributed.", + "type": "object", + "minProperties": 1, + "properties": { + "names": { + "title": "List of acknowledged names", + "description": "Contains the names of contributors being recognized.", + "type": "array", + "minItems": 1, + "items": { + "title": "Name of the contributor", + "description": "Contains the name of a single contributor being recognized.", + "type": "string", + "minLength": 1, + "examples": [ + "Albert Einstein", + "Johann Sebastian Bach" + ] + } + }, + "organization": { + "title": "Contributing organization", + "description": "Contains the name of a contributing organization being recognized.", + "type": "string", + "minLength": 1, + "examples": [ + "CISA", + "Google Project Zero", + "Talos" + ] + }, + "summary": { + "title": "Summary of the acknowledgment", + "description": "SHOULD represent any contextual details the document producers wish to make known about the acknowledgment or acknowledged parties.", + "type": "string", + "minLength": 1, + "examples": [ + "First analysis of Coordinated Multi-Stream Attack (CMSA)" + ] + }, + "urls": { + "title": "List of URLs", + "description": "Specifies a list of URLs or location of the reference to be acknowledged.", + "type": "array", + "minItems": 1, + "items": { + "title": "URL of acknowledgment", + "description": "Contains the URL or location of the reference to be acknowledged.", + "type": "string", + "format": "uri" + } + } + } + } + }, + "branches_t": { + "title": "List of branches", + "description": "Contains branch elements as children of the current element.", + "type": "array", + "minItems": 1, + "items": { + "title": "Branch", + "description": "Is a part of the hierarchical structure of the product tree.", + "type": "object", + "maxProperties": 3, + "minProperties": 3, + "required": [ + "category", + "name" + ], + "properties": { + "branches": { + "$ref": "#/$defs/branches_t" + }, + "category": { + "title": "Category of the branch", + "description": "Describes the characteristics of the labeled branch.", + "type": "string", + "enum": [ + "architecture", + "host_name", + "language", + "legacy", + "patch_level", + "product_family", + "product_name", + "product_version", + "product_version_range", + "service_pack", + "specification", + "vendor" + ] + }, + "name": { + "title": "Name of the branch", + "description": "Contains the canonical descriptor or 'friendly name' of the branch.", + "type": "string", + "minLength": 1, + "examples": [ + "10", + "365", + "Microsoft", + "Office", + "PCS 7", + "SIMATIC", + "Siemens", + "Windows" + ] + }, + "product": { + "$ref": "#/$defs/full_product_name_t" + } + } + } + }, + "full_product_name_t": { + "title": "Full product name", + "description": "Specifies information about the product and assigns the product_id.", + "type": "object", + "required": [ + "name", + "product_id" + ], + "properties": { + "name": { + "title": "Textual description of the product", + "description": "The value should be the product’s full canonical name, including version number and other attributes, as it would be used in a human-friendly document.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco AnyConnect Secure Mobility Client 2.3.185", + "Microsoft Host Integration Server 2006 Service Pack 1" + ] + }, + "product_id": { + "$ref": "#/$defs/product_id_t" + }, + "product_identification_helper": { + "title": "Helper to identify the product", + "description": "Provides at least one method which aids in identifying the product in an asset database.", + "type": "object", + "minProperties": 1, + "properties": { + "cpe": { + "title": "Common Platform Enumeration representation", + "description": "The Common Platform Enumeration (CPE) attribute refers to a method for naming platforms external to this specification.", + "type": "string", + "pattern": "^(cpe:2\\.3:[aho\\*\\-](:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#\\$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|\\}~]))+(\\?*|\\*?))|[\\*\\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\\*\\-]))(:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#\\$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|\\}~]))+(\\?*|\\*?))|[\\*\\-])){4})|([c][pP][eE]:/[AHOaho]?(:[A-Za-z0-9\\._\\-~%]*){0,6})$", + "minLength": 5 + }, + "hashes": { + "title": "List of hashes", + "description": "Contains a list of cryptographic hashes usable to identify files.", + "type": "array", + "minItems": 1, + "items": { + "title": "Cryptographic hashes", + "description": "Contains all information to identify a file based on its cryptographic hash values.", + "type": "object", + "required": [ + "file_hashes", + "filename" + ], + "properties": { + "file_hashes": { + "title": "List of file hashes", + "description": "Contains a list of cryptographic hashes for this file.", + "type": "array", + "minItems": 1, + "items": { + "title": "File hash", + "description": "Contains one hash value and algorithm of the file to be identified.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "title": "Algorithm of the cryptographic hash", + "description": "Contains the name of the cryptographic hash algorithm used to calculate the value.", + "type": "string", + "default": "sha256", + "minLength": 1, + "examples": [ + "blake2b512", + "sha256", + "sha3-512", + "sha384", + "sha512" + ] + }, + "value": { + "title": "Value of the cryptographic hash", + "description": "Contains the cryptographic hash value in hexadecimal representation.", + "type": "string", + "pattern": "^[0-9a-fA-F]{32,}$", + "minLength": 32, + "examples": [ + "37df33cb7464da5c7f077f4d56a32bc84987ec1d85b234537c1c1a4d4fc8d09dc29e2e762cb5203677bf849a2855a0283710f1f5fe1d6ce8d5ac85c645d0fcb3", + "4775203615d9534a8bfca96a93dc8b461a489f69124a130d786b42204f3341cc", + "9ea4c8200113d49d26505da0e02e2f49055dc078d1ad7a419b32e291c7afebbb84badfbd46dec42883bea0b2a1fa697c" + ] + } + } + } + }, + "filename": { + "title": "Filename", + "description": "Contains the name of the file which is identified by the hash values.", + "type": "string", + "minLength": 1, + "examples": [ + "WINWORD.EXE", + "msotadddin.dll", + "sudoers.so" + ] + } + } + } + }, + "model_numbers": { + "title": "List of models", + "description": "Contains a list of full or abbreviated (partial) model numbers.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Model number", + "description": "Contains a full or abbreviated (partial) model number of the component to identify.", + "type": "string", + "minLength": 1 + } + }, + "purl": { + "title": "package URL representation", + "description": "The package URL (purl) attribute refers to a method for reliably identifying and locating software packages external to this specification.", + "type": "string", + "format": "uri", + "pattern": "^pkg:[A-Za-z\\.\\-\\+][A-Za-z0-9\\.\\-\\+]*/.+", + "minLength": 7 + }, + "sbom_urls": { + "title": "List of SBOM URLs", + "description": "Contains a list of URLs where SBOMs for this product can be retrieved.", + "type": "array", + "minItems": 1, + "items": { + "title": "SBOM URL", + "description": "Contains a URL of one SBOM for this product.", + "type": "string", + "format": "uri" + } + }, + "serial_numbers": { + "title": "List of serial numbers", + "description": "Contains a list of full or abbreviated (partial) serial numbers.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Serial number", + "description": "Contains a full or abbreviated (partial) serial number of the component to identify.", + "type": "string", + "minLength": 1 + } + }, + "skus": { + "title": "List of stock keeping units", + "description": "Contains a list of full or abbreviated (partial) stock keeping units.", + "type": "array", + "minItems": 1, + "items": { + "title": "Stock keeping unit", + "description": "Contains a full or abbreviated (partial) stock keeping unit (SKU) which is used in the ordering process to identify the component.", + "type": "string", + "minLength": 1 + } + }, + "x_generic_uris": { + "title": "List of generic URIs", + "description": "Contains a list of identifiers which are either vendor-specific or derived from a standard not yet supported.", + "type": "array", + "minItems": 1, + "items": { + "title": "Generic URI", + "description": "Provides a generic extension point for any identifier which is either vendor-specific or derived from a standard not yet supported.", + "type": "object", + "required": [ + "namespace", + "uri" + ], + "properties": { + "namespace": { + "title": "Namespace of the generic URI", + "description": "Refers to a URL which provides the name and knowledge about the specification used or is the namespace in which these values are valid.", + "type": "string", + "format": "uri" + }, + "uri": { + "title": "URI", + "description": "Contains the identifier itself.", + "type": "string", + "format": "uri" + } + } + } + } + } + } + } + }, + "lang_t": { + "title": "Language type", + "description": "Identifies a language, corresponding to IETF BCP 47 / RFC 5646. See IETF language registry: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry", + "type": "string", + "pattern": "^(([A-Za-z]{2,3}(-[A-Za-z]{3}(-[A-Za-z]{3}){0,2})?|[A-Za-z]{4,8})(-[A-Za-z]{4})?(-([A-Za-z]{2}|[0-9]{3}))?(-([A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-[A-WY-Za-wy-z0-9](-[A-Za-z0-9]{2,8})+)*(-[Xx](-[A-Za-z0-9]{1,8})+)?|[Xx](-[A-Za-z0-9]{1,8})+|[Ii]-[Dd][Ee][Ff][Aa][Uu][Ll][Tt]|[Ii]-[Mm][Ii][Nn][Gg][Oo])$", + "examples": [ + "de", + "en", + "fr", + "frc", + "jp" + ] + }, + "notes_t": { + "title": "List of notes", + "description": "Contains notes which are specific to the current context.", + "type": "array", + "minItems": 1, + "items": { + "title": "Note", + "description": "Is a place to put all manner of text blobs related to the current context.", + "type": "object", + "required": [ + "category", + "text" + ], + "properties": { + "audience": { + "title": "Audience of note", + "description": "Indicates who is intended to read it.", + "type": "string", + "minLength": 1, + "examples": [ + "all", + "executives", + "operational management and system administrators", + "safety engineers" + ] + }, + "category": { + "title": "Note category", + "description": "Contains the information of what kind of note this is.", + "type": "string", + "enum": [ + "description", + "details", + "faq", + "general", + "legal_disclaimer", + "other", + "summary" + ] + }, + "text": { + "title": "Note content", + "description": "Holds the content of the note. Content varies depending on type.", + "type": "string", + "minLength": 1 + }, + "title": { + "title": "Title of note", + "description": "Provides a concise description of what is contained in the text of the note.", + "type": "string", + "minLength": 1, + "examples": [ + "Details", + "Executive summary", + "Technical summary", + "Impact on safety systems" + ] + } + } + } + }, + "product_group_id_t": { + "title": "Reference token for product group instance", + "description": "Token required to identify a group of products so that it can be referred to from other parts in the document. There is no predefined or required format for the product_group_id as long as it uniquely identifies a group in the context of the current document.", + "type": "string", + "minLength": 1, + "examples": [ + "CSAFGID-0001", + "CSAFGID-0002", + "CSAFGID-0020" + ] + }, + "product_groups_t": { + "title": "List of product_group_ids", + "description": "Specifies a list of product_group_ids to give context to the parent item.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_group_id_t" + } + }, + "product_id_t": { + "title": "Reference token for product instance", + "description": "Token required to identify a full_product_name so that it can be referred to from other parts in the document. There is no predefined or required format for the product_id as long as it uniquely identifies a product in the context of the current document.", + "type": "string", + "minLength": 1, + "examples": [ + "CSAFPID-0004", + "CSAFPID-0008" + ] + }, + "products_t": { + "title": "List of product_ids", + "description": "Specifies a list of product_ids to give context to the parent item.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_id_t" + } + }, + "references_t": { + "title": "List of references", + "description": "Holds a list of references.", + "type": "array", + "minItems": 1, + "items": { + "title": "Reference", + "description": "Holds any reference to conferences, papers, advisories, and other resources that are related and considered related to either a surrounding part of or the entire document and to be of value to the document consumer.", + "type": "object", + "required": [ + "summary", + "url" + ], + "properties": { + "category": { + "title": "Category of reference", + "description": "Indicates whether the reference points to the same document or vulnerability in focus (depending on scope) or to an external resource.", + "type": "string", + "default": "external", + "enum": [ + "external", + "self" + ] + }, + "summary": { + "title": "Summary of the reference", + "description": "Indicates what this reference refers to.", + "type": "string", + "minLength": 1 + }, + "url": { + "title": "URL of reference", + "description": "Provides the URL for the reference.", + "type": "string", + "format": "uri" + } + } + } + }, + "version_t": { + "title": "Version", + "description": "Specifies a version string to denote clearly the evolution of the content of the document. Format must be either integer or semantic versioning.", + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$|^((0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?)$", + "examples": [ + "1", + "4", + "0.9.0", + "1.4.3", + "2.40.0+21AF26D3" + ] + } + }, + "required": [ + "document" + ], + "properties": { + "document": { + "title": "Document level meta-data", + "description": "Captures the meta-data about this document describing a particular set of security advisories.", + "type": "object", + "required": [ + "category", + "csaf_version", + "publisher", + "title", + "tracking" + ], + "properties": { + "acknowledgments": { + "title": "Document acknowledgments", + "description": "Contains a list of acknowledgment elements associated with the whole document.", + "$ref": "#/$defs/acknowledgments_t" + }, + "aggregate_severity": { + "title": "Aggregate severity", + "description": "Is a vehicle that is provided by the document producer to convey the urgency and criticality with which the one or more vulnerabilities reported should be addressed. It is a document-level metric and applied to the document as a whole — not any specific vulnerability. The range of values in this field is defined according to the document producer's policies and procedures.", + "type": "object", + "required": [ + "text" + ], + "properties": { + "namespace": { + "title": "Namespace of aggregate severity", + "description": "Points to the namespace so referenced.", + "type": "string", + "format": "uri" + }, + "text": { + "title": "Text of aggregate severity", + "description": "Provides a severity which is independent of - and in addition to - any other standard metric for determining the impact or severity of a given vulnerability (such as CVSS).", + "type": "string", + "minLength": 1, + "examples": [ + "Critical", + "Important", + "Moderate" + ] + } + } + }, + "category": { + "title": "Document category", + "description": "Defines a short canonical name, chosen by the document producer, which will inform the end user as to the category of document.", + "type": "string", + "pattern": "^[^\\s\\-_\\.](.*[^\\s\\-_\\.])?$", + "minLength": 1, + "examples": [ + "csaf_base", + "csaf_security_advisory", + "csaf_vex", + "Example Company Security Notice" + ] + }, + "csaf_version": { + "title": "CSAF version", + "description": "Gives the version of the CSAF specification which the document was generated for.", + "type": "string", + "enum": [ + "2.0" + ] + }, + "distribution": { + "title": "Rules for sharing document", + "description": "Describe any constraints on how this document might be shared.", + "type": "object", + "minProperties": 1, + "properties": { + "text": { + "title": "Textual description", + "description": "Provides a textual description of additional constraints.", + "type": "string", + "minLength": 1, + "examples": [ + "Copyright 2021, Example Company, All Rights Reserved.", + "Distribute freely.", + "Share only on a need-to-know-basis only." + ] + }, + "tlp": { + "title": "Traffic Light Protocol (TLP)", + "description": "Provides details about the TLP classification of the document.", + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "title": "Label of TLP", + "description": "Provides the TLP label of the document.", + "type": "string", + "enum": [ + "AMBER", + "GREEN", + "RED", + "WHITE" + ] + }, + "url": { + "title": "URL of TLP version", + "description": "Provides a URL where to find the textual description of the TLP version which is used in this document. Default is the URL to the definition by FIRST.", + "type": "string", + "default": "https://www.first.org/tlp/", + "format": "uri", + "examples": [ + "https://www.us-cert.gov/tlp", + "https://www.bsi.bund.de/SharedDocs/Downloads/DE/BSI/Kritis/Merkblatt_TLP.pdf" + ] + } + } + } + } + }, + "lang": { + "title": "Document language", + "description": "Identifies the language used by this document, corresponding to IETF BCP 47 / RFC 5646.", + "$ref": "#/$defs/lang_t" + }, + "notes": { + "title": "Document notes", + "description": "Holds notes associated with the whole document.", + "$ref": "#/$defs/notes_t" + }, + "publisher": { + "title": "Publisher", + "description": "Provides information about the publisher of the document.", + "type": "object", + "required": [ + "category", + "name", + "namespace" + ], + "properties": { + "category": { + "title": "Category of publisher", + "description": "Provides information about the category of publisher releasing the document.", + "type": "string", + "enum": [ + "coordinator", + "discoverer", + "other", + "translator", + "user", + "vendor" + ] + }, + "contact_details": { + "title": "Contact details", + "description": "Information on how to contact the publisher, possibly including details such as web sites, email addresses, phone numbers, and postal mail addresses.", + "type": "string", + "minLength": 1, + "examples": [ + "Example Company can be reached at contact_us@example.com, or via our website at https://www.example.com/contact." + ] + }, + "issuing_authority": { + "title": "Issuing authority", + "description": "Provides information about the authority of the issuing party to release the document, in particular, the party's constituency and responsibilities or other obligations.", + "type": "string", + "minLength": 1 + }, + "name": { + "title": "Name of publisher", + "description": "Contains the name of the issuing party.", + "type": "string", + "minLength": 1, + "examples": [ + "BSI", + "Cisco PSIRT", + "Siemens ProductCERT" + ] + }, + "namespace": { + "title": "Namespace of publisher", + "description": "Contains a URL which is under control of the issuing party and can be used as a globally unique identifier for that issuing party.", + "type": "string", + "format": "uri", + "examples": [ + "https://csaf.io", + "https://www.example.com" + ] + } + } + }, + "references": { + "title": "Document references", + "description": "Holds a list of references associated with the whole document.", + "$ref": "#/$defs/references_t" + }, + "source_lang": { + "title": "Source language", + "description": "If this copy of the document is a translation then the value of this property describes from which language this document was translated.", + "$ref": "#/$defs/lang_t" + }, + "title": { + "title": "Title of this document", + "description": "This SHOULD be a canonical name for the document, and sufficiently unique to distinguish it from similar documents.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco IPv6 Crafted Packet Denial of Service Vulnerability", + "Example Company Cross-Site-Scripting Vulnerability in Example Generator" + ] + }, + "tracking": { + "title": "Tracking", + "description": "Is a container designated to hold all management attributes necessary to track a CSAF document as a whole.", + "type": "object", + "required": [ + "current_release_date", + "id", + "initial_release_date", + "revision_history", + "status", + "version" + ], + "properties": { + "aliases": { + "title": "Aliases", + "description": "Contains a list of alternate names for the same document.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Alternate name", + "description": "Specifies a non-empty string that represents a distinct optional alternative ID used to refer to the document.", + "type": "string", + "minLength": 1, + "examples": [ + "CVE-2019-12345" + ] + } + }, + "current_release_date": { + "title": "Current release date", + "description": "The date when the current revision of this document was released", + "type": "string", + "format": "date-time" + }, + "generator": { + "title": "Document generator", + "description": "Is a container to hold all elements related to the generation of the document. These items will reference when the document was actually created, including the date it was generated and the entity that generated it.", + "type": "object", + "required": [ + "engine" + ], + "properties": { + "date": { + "title": "Date of document generation", + "description": "This SHOULD be the current date that the document was generated. Because documents are often generated internally by a document producer and exist for a nonzero amount of time before being released, this field MAY be different from the Initial Release Date and Current Release Date.", + "type": "string", + "format": "date-time" + }, + "engine": { + "title": "Engine of document generation", + "description": "Contains information about the engine that generated the CSAF document.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "title": "Engine name", + "description": "Represents the name of the engine that generated the CSAF document.", + "type": "string", + "minLength": 1, + "examples": [ + "Red Hat rhsa-to-cvrf", + "Secvisogram", + "TVCE" + ] + }, + "version": { + "title": "Engine version", + "description": "Contains the version of the engine that generated the CSAF document.", + "type": "string", + "minLength": 1, + "examples": [ + "0.6.0", + "1.0.0-beta+exp.sha.a1c44f85", + "2" + ] + } + } + } + } + }, + "id": { + "title": "Unique identifier for the document", + "description": "The ID is a simple label that provides for a wide range of numbering values, types, and schemes. Its value SHOULD be assigned and maintained by the original document issuing authority.", + "type": "string", + "pattern": "^[\\S](.*[\\S])?$", + "minLength": 1, + "examples": [ + "Example Company - 2019-YH3234", + "RHBA-2019:0024", + "cisco-sa-20190513-secureboot" + ] + }, + "initial_release_date": { + "title": "Initial release date", + "description": "The date when this document was first published.", + "type": "string", + "format": "date-time" + }, + "revision_history": { + "title": "Revision history", + "description": "Holds one revision item for each version of the CSAF document, including the initial one.", + "type": "array", + "minItems": 1, + "items": { + "title": "Revision", + "description": "Contains all the information elements required to track the evolution of a CSAF document.", + "type": "object", + "required": [ + "date", + "number", + "summary" + ], + "properties": { + "date": { + "title": "Date of the revision", + "description": "The date of the revision entry", + "type": "string", + "format": "date-time" + }, + "legacy_version": { + "title": "Legacy version of the revision", + "description": "Contains the version string used in an existing document with the same content.", + "type": "string", + "minLength": 1 + }, + "number": { + "$ref": "#/$defs/version_t" + }, + "summary": { + "title": "Summary of the revision", + "description": "Holds a single non-empty string representing a short description of the changes.", + "type": "string", + "minLength": 1, + "examples": [ + "Initial version." + ] + } + } + } + }, + "status": { + "title": "Document status", + "description": "Defines the draft status of the document.", + "type": "string", + "enum": [ + "draft", + "final", + "interim" + ] + }, + "version": { + "$ref": "#/$defs/version_t" + } + } + } + } + }, + "product_tree": { + "title": "Product tree", + "description": "Is a container for all fully qualified product names that can be referenced elsewhere in the document.", + "type": "object", + "minProperties": 1, + "properties": { + "branches": { + "$ref": "#/$defs/branches_t" + }, + "full_product_names": { + "title": "List of full product names", + "description": "Contains a list of full product names.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/full_product_name_t" + } + }, + "product_groups": { + "title": "List of product groups", + "description": "Contains a list of product groups.", + "type": "array", + "minItems": 1, + "items": { + "title": "Product group", + "description": "Defines a new logical group of products that can then be referred to in other parts of the document to address a group of products with a single identifier.", + "type": "object", + "required": [ + "group_id", + "product_ids" + ], + "properties": { + "group_id": { + "$ref": "#/$defs/product_group_id_t" + }, + "product_ids": { + "title": "List of Product IDs", + "description": "Lists the product_ids of those products which known as one group in the document.", + "type": "array", + "minItems": 2, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_id_t" + } + }, + "summary": { + "title": "Summary of the product group", + "description": "Gives a short, optional description of the group.", + "type": "string", + "minLength": 1, + "examples": [ + "Products supporting Modbus.", + "The x64 versions of the operating system." + ] + } + } + } + }, + "relationships": { + "title": "List of relationships", + "description": "Contains a list of relationships.", + "type": "array", + "minItems": 1, + "items": { + "title": "Relationship", + "description": "Establishes a link between two existing full_product_name_t elements, allowing the document producer to define a combination of two products that form a new full_product_name entry.", + "type": "object", + "required": [ + "category", + "full_product_name", + "product_reference", + "relates_to_product_reference" + ], + "properties": { + "category": { + "title": "Relationship category", + "description": "Defines the category of relationship for the referenced component.", + "type": "string", + "enum": [ + "default_component_of", + "external_component_of", + "installed_on", + "installed_with", + "optional_component_of" + ] + }, + "full_product_name": { + "$ref": "#/$defs/full_product_name_t" + }, + "product_reference": { + "title": "Product reference", + "description": "Holds a Product ID that refers to the Full Product Name element, which is referenced as the first element of the relationship.", + "$ref": "#/$defs/product_id_t" + }, + "relates_to_product_reference": { + "title": "Relates to product reference", + "description": "Holds a Product ID that refers to the Full Product Name element, which is referenced as the second element of the relationship.", + "$ref": "#/$defs/product_id_t" + } + } + } + } + } + }, + "vulnerabilities": { + "title": "Vulnerabilities", + "description": "Represents a list of all relevant vulnerability information items.", + "type": "array", + "minItems": 1, + "items": { + "title": "Vulnerability", + "description": "Is a container for the aggregation of all fields that are related to a single vulnerability in the document.", + "type": "object", + "minProperties": 1, + "properties": { + "acknowledgments": { + "title": "Vulnerability acknowledgments", + "description": "Contains a list of acknowledgment elements associated with this vulnerability item.", + "$ref": "#/$defs/acknowledgments_t" + }, + "cve": { + "title": "CVE", + "description": "Holds the MITRE standard Common Vulnerabilities and Exposures (CVE) tracking number for the vulnerability.", + "type": "string", + "pattern": "^CVE-[0-9]{4}-[0-9]{4,}$" + }, + "cwe": { + "title": "CWE", + "description": "Holds the MITRE standard Common Weakness Enumeration (CWE) for the weakness associated.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "title": "Weakness ID", + "description": "Holds the ID for the weakness associated.", + "type": "string", + "pattern": "^CWE-[1-9]\\d{0,5}$", + "examples": [ + "CWE-22", + "CWE-352", + "CWE-79" + ] + }, + "name": { + "title": "Weakness name", + "description": "Holds the full name of the weakness as given in the CWE specification.", + "type": "string", + "minLength": 1, + "examples": [ + "Cross-Site Request Forgery (CSRF)", + "Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')", + "Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting')" + ] + } + } + }, + "discovery_date": { + "title": "Discovery date", + "description": "Holds the date and time the vulnerability was originally discovered.", + "type": "string", + "format": "date-time" + }, + "flags": { + "title": "List of flags", + "description": "Contains a list of machine readable flags.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Flag", + "description": "Contains product specific information in regard to this vulnerability as a single machine readable flag.", + "type": "object", + "required": [ + "label" + ], + "properties": { + "date": { + "title": "Date of the flag", + "description": "Contains the date when assessment was done or the flag was assigned.", + "type": "string", + "format": "date-time" + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "label": { + "title": "Label of the flag", + "description": "Specifies the machine readable label.", + "type": "string", + "enum": [ + "component_not_present", + "inline_mitigations_already_exist", + "vulnerable_code_cannot_be_controlled_by_adversary", + "vulnerable_code_not_in_execute_path", + "vulnerable_code_not_present" + ] + }, + "product_ids": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "ids": { + "title": "List of IDs", + "description": "Represents a list of unique labels or tracking IDs for the vulnerability (if such information exists).", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "ID", + "description": "Contains a single unique label or tracking ID for the vulnerability.", + "type": "object", + "required": [ + "system_name", + "text" + ], + "properties": { + "system_name": { + "title": "System name", + "description": "Indicates the name of the vulnerability tracking or numbering system.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco Bug ID", + "GitHub Issue" + ] + }, + "text": { + "title": "Text", + "description": "Is unique label or tracking ID for the vulnerability (if such information exists).", + "type": "string", + "minLength": 1, + "examples": [ + "CSCso66472", + "oasis-tcs/csaf#210" + ] + } + } + } + }, + "involvements": { + "title": "List of involvements", + "description": "Contains a list of involvements.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Involvement", + "description": "Is a container, that allows the document producers to comment on the level of involvement (or engagement) of themselves or third parties in the vulnerability identification, scoping, and remediation process.", + "type": "object", + "required": [ + "party", + "status" + ], + "properties": { + "date": { + "title": "Date of involvement", + "description": "Holds the date and time of the involvement entry.", + "type": "string", + "format": "date-time" + }, + "party": { + "title": "Party category", + "description": "Defines the category of the involved party.", + "type": "string", + "enum": [ + "coordinator", + "discoverer", + "other", + "user", + "vendor" + ] + }, + "status": { + "title": "Party status", + "description": "Defines contact status of the involved party.", + "type": "string", + "enum": [ + "completed", + "contact_attempted", + "disputed", + "in_progress", + "not_contacted", + "open" + ] + }, + "summary": { + "title": "Summary of the involvement", + "description": "Contains additional context regarding what is going on.", + "type": "string", + "minLength": 1 + } + } + } + }, + "notes": { + "title": "Vulnerability notes", + "description": "Holds notes associated with this vulnerability item.", + "$ref": "#/$defs/notes_t" + }, + "product_status": { + "title": "Product status", + "description": "Contains different lists of product_ids which provide details on the status of the referenced product related to the current vulnerability. ", + "type": "object", + "minProperties": 1, + "properties": { + "first_affected": { + "title": "First affected", + "description": "These are the first versions of the releases known to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "first_fixed": { + "title": "First fixed", + "description": "These versions contain the first fix for the vulnerability but may not be the recommended fixed versions.", + "$ref": "#/$defs/products_t" + }, + "fixed": { + "title": "Fixed", + "description": "These versions contain a fix for the vulnerability but may not be the recommended fixed versions.", + "$ref": "#/$defs/products_t" + }, + "known_affected": { + "title": "Known affected", + "description": "These versions are known to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "known_not_affected": { + "title": "Known not affected", + "description": "These versions are known not to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "last_affected": { + "title": "Last affected", + "description": "These are the last versions in a release train known to be affected by the vulnerability. Subsequently released versions would contain a fix for the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "recommended": { + "title": "Recommended", + "description": "These versions have a fix for the vulnerability and are the vendor-recommended versions for fixing the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "under_investigation": { + "title": "Under investigation", + "description": "It is not known yet whether these versions are or are not affected by the vulnerability. However, it is still under investigation - the result will be provided in a later release of the document.", + "$ref": "#/$defs/products_t" + } + } + }, + "references": { + "title": "Vulnerability references", + "description": "Holds a list of references associated with this vulnerability item.", + "$ref": "#/$defs/references_t" + }, + "release_date": { + "title": "Release date", + "description": "Holds the date and time the vulnerability was originally released into the wild.", + "type": "string", + "format": "date-time" + }, + "remediations": { + "title": "List of remediations", + "description": "Contains a list of remediations.", + "type": "array", + "minItems": 1, + "items": { + "title": "Remediation", + "description": "Specifies details on how to handle (and presumably, fix) a vulnerability.", + "type": "object", + "required": [ + "category", + "details" + ], + "properties": { + "category": { + "title": "Category of the remediation", + "description": "Specifies the category which this remediation belongs to.", + "type": "string", + "enum": [ + "mitigation", + "no_fix_planned", + "none_available", + "vendor_fix", + "workaround" + ] + }, + "date": { + "title": "Date of the remediation", + "description": "Contains the date from which the remediation is available.", + "type": "string", + "format": "date-time" + }, + "details": { + "title": "Details of the remediation", + "description": "Contains a thorough human-readable discussion of the remediation.", + "type": "string", + "minLength": 1 + }, + "entitlements": { + "title": "List of entitlements", + "description": "Contains a list of entitlements.", + "type": "array", + "minItems": 1, + "items": { + "title": "Entitlement of the remediation", + "description": "Contains any possible vendor-defined constraints for obtaining fixed software or hardware that fully resolves the vulnerability.", + "type": "string", + "minLength": 1 + } + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "product_ids": { + "$ref": "#/$defs/products_t" + }, + "restart_required": { + "title": "Restart required by remediation", + "description": "Provides information on category of restart is required by this remediation to become effective.", + "type": "object", + "required": [ + "category" + ], + "properties": { + "category": { + "title": "Category of restart", + "description": "Specifies what category of restart is required by this remediation to become effective.", + "type": "string", + "enum": [ + "connected", + "dependencies", + "machine", + "none", + "parent", + "service", + "system", + "vulnerable_component", + "zone" + ] + }, + "details": { + "title": "Additional restart information", + "description": "Provides additional information for the restart. This can include details on procedures, scope or impact.", + "type": "string", + "minLength": 1 + } + } + }, + "url": { + "title": "URL to the remediation", + "description": "Contains the URL where to obtain the remediation.", + "type": "string", + "format": "uri" + } + } + } + }, + "scores": { + "title": "List of scores", + "description": "Contains score objects for the current vulnerability.", + "type": "array", + "minItems": 1, + "items": { + "title": "Score", + "description": "Specifies information about (at least one) score of the vulnerability and for which products the given value applies.", + "type": "object", + "minProperties": 2, + "required": [ + "products" + ], + "properties": { + "cvss_v2": { + "$ref": "https://www.first.org/cvss/cvss-v2.0.json" + }, + "cvss_v3": { + "oneOf": [ + { + "$ref": "https://www.first.org/cvss/cvss-v3.0.json" + }, + { + "$ref": "https://www.first.org/cvss/cvss-v3.1.json" + } + ] + }, + "products": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "threats": { + "title": "List of threats", + "description": "Contains information about a vulnerability that can change with time.", + "type": "array", + "minItems": 1, + "items": { + "title": "Threat", + "description": "Contains the vulnerability kinetic information. This information can change as the vulnerability ages and new information becomes available.", + "type": "object", + "required": [ + "category", + "details" + ], + "properties": { + "category": { + "title": "Category of the threat", + "description": "Categorizes the threat according to the rules of the specification.", + "type": "string", + "enum": [ + "exploit_status", + "impact", + "target_set" + ] + }, + "date": { + "title": "Date of the threat", + "description": "Contains the date when the assessment was done or the threat appeared.", + "type": "string", + "format": "date-time" + }, + "details": { + "title": "Details of the threat", + "description": "Represents a thorough human-readable discussion of the threat.", + "type": "string", + "minLength": 1 + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "product_ids": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "title": { + "title": "Title", + "description": "Gives the document producer the ability to apply a canonical name or title to the vulnerability.", + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index d5f426694..830efe1f9 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -1,20 +1,21 @@ from __future__ import annotations from datetime import datetime +from functools import lru_cache import json -import shutil -import subprocess -from typing import Any, Dict, List -import tempfile import os +from pathlib import Path import re +import tempfile +from typing import Any, Dict, List + +from jsonschema import Draft202012Validator from exploit_iq_commons.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState from ..vex_generator_base import VexGenerator from csaf.generator import CSAFGenerator -from csaf.analyser import CSAFAnalyser from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -27,8 +28,8 @@ NOTE_CATEGORY_DESCRIPTION = "description" NOTE_CATEGORY_SUMMARY = "summary" NOTE_CATEGORY_GENERAL = "general" -NOTE_CATEGORY_ANALYSIS = "analysis" -NOTE_CATEGORY_DISCLAIMER = "disclaimer" +NOTE_CATEGORY_OTHER = "other" +NOTE_CATEGORY_LEGAL_DISCLAIMER = "legal_disclaimer" # Note titles NOTE_TITLE_VULNERABILITY_DESCRIPTION = "Vulnerability description" @@ -73,6 +74,19 @@ # Error messages ERROR_CSAF_VALIDATION_FAILED = "CSAF document does not conform to CSAF 2.0 schema." +# VEX schema path +SCHEMA_PATH = Path(__file__).resolve().parents[3] / "configs" / "vex" / "csaf" / "v2.0" / "csaf_json_schema.json" + +@lru_cache(maxsize=1) +def get_vex_validator() -> Draft202012Validator: + """ + Load schema and create a cached VEX document validator instance. + """ + with open(SCHEMA_PATH) as f: + schema = json.load(f) + logger.debug("VEX document schema loaded from %s", SCHEMA_PATH) + return Draft202012Validator(schema) + def _get_patched_package(v: dict) -> tuple[str | None, str | None]: """ Get the patched package from a GHSA vulnerability dict. @@ -160,7 +174,7 @@ def _enrich_vulnerabilities_with_notes( # Add ExploitIQ analysis summary summary = final_summaries.get(vuln_id) notes.append({ - "category": NOTE_CATEGORY_ANALYSIS, + "category": NOTE_CATEGORY_OTHER, "title": NOTE_TITLE_EXPLOITIQ_SUMMARY, "text": summary }) @@ -168,12 +182,12 @@ def _enrich_vulnerabilities_with_notes( # Add ExploitIQ justification details justification = justifications.get(vuln_id) notes.append({ - "category": NOTE_CATEGORY_ANALYSIS, + "category": NOTE_CATEGORY_OTHER, "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_REASONING, "text": justification.get("justification") }) notes.append({ - "category": NOTE_CATEGORY_ANALYSIS, + "category": NOTE_CATEGORY_OTHER, "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_LABEL, "text": justification.get("justification_label") }) @@ -201,7 +215,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: csaf_gen.set_value("notes",[ { - "category": NOTE_CATEGORY_DISCLAIMER, + "category": NOTE_CATEGORY_LEGAL_DISCLAIMER, "text": DISCLAIMER_TEXT, "title": NOTE_TITLE_UNOFFICIAL_CONTENT } @@ -263,20 +277,17 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: with open(path, "r") as f: csaf_json = json.load(f) - # Enrich the CSAF in memory - _enrich_vulnerabilities_with_notes( - csaf_json, intel_map, state.final_summaries, state.justifications - ) - - # Overwrite the same file with enriched content - with open(path, "w") as f: - json.dump(csaf_json, f) - - # Validate the enriched CSAF - analyser = CSAFAnalyser(path) - if not analyser.validate(): - logger.error(ERROR_CSAF_VALIDATION_FAILED) - analyser.analyse() # prints analysis to console - return {} + # Enrich the CSAF in memory + _enrich_vulnerabilities_with_notes( + csaf_json, intel_map, state.final_summaries, state.justifications + ) + + # Validate the CSAF document against the JSON schema + errors = list(get_vex_validator().iter_errors(csaf_json)) + if errors: + logger.error("%s Found %d error(s):", ERROR_CSAF_VALIDATION_FAILED, len(errors)) + for e in errors: + logger.error(" %s: %s", e.json_path, e.message) + return {} return csaf_json \ No newline at end of file From 82ea82531976baa88b281d4edc4226de74b4a99d Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Wed, 14 Jan 2026 15:24:47 +0200 Subject: [PATCH 164/286] chore: extract vex utils to dedicated file Signed-off-by: Ilona Shishov --- .../vex/implementations/csaf_generator.py | 60 ++------------ src/vuln_analysis/utils/vex/vex_utils.py | 81 +++++++++++++++++++ 2 files changed, 87 insertions(+), 54 deletions(-) create mode 100644 src/vuln_analysis/utils/vex/vex_utils.py diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 830efe1f9..21e4b3e72 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -1,20 +1,17 @@ from __future__ import annotations -from datetime import datetime -from functools import lru_cache import json import os from pathlib import Path import re import tempfile -from typing import Any, Dict, List - -from jsonschema import Draft202012Validator +from typing import Any, Dict from exploit_iq_commons.data_models.cve_intel import CveIntel from vuln_analysis.data_models.state import AgentMorpheusEngineState from ..vex_generator_base import VexGenerator +from ..vex_utils import get_vex_validator, build_patch_recommendation from csaf.generator import CSAFGenerator from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -74,53 +71,8 @@ # Error messages ERROR_CSAF_VALIDATION_FAILED = "CSAF document does not conform to CSAF 2.0 schema." -# VEX schema path -SCHEMA_PATH = Path(__file__).resolve().parents[3] / "configs" / "vex" / "csaf" / "v2.0" / "csaf_json_schema.json" - -@lru_cache(maxsize=1) -def get_vex_validator() -> Draft202012Validator: - """ - Load schema and create a cached VEX document validator instance. - """ - with open(SCHEMA_PATH) as f: - schema = json.load(f) - logger.debug("VEX document schema loaded from %s", SCHEMA_PATH) - return Draft202012Validator(schema) - -def _get_patched_package(v: dict) -> tuple[str | None, str | None]: - """ - Get the patched package from a GHSA vulnerability dict. - """ - pkg = v.get("package") or {} - return pkg.get("name"), v.get("first_patched_version") - - -def _build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: - """ - Build a patch recommendation from the GHSA data. - - If SBOM provided: return all unique 'name:first_patched_version' pairs where name is in SBOM as list, else return "". - - If no SBOM: return all unique 'name:first_patched_version' pairs as list, else return "". - """ - if not ci or not ci.ghsa or not ci.ghsa.vulnerabilities: - return "" - - vulns = ci.ghsa.vulnerabilities - - name_to_version: dict[str, str] = {} - for v in vulns: - name, patch = _get_patched_package(v) - if not name or not patch: - continue - if name in name_to_version: - continue - # If SBOM provided, only include packages that are in the SBOM - if sbom_package_names is not None and name not in sbom_package_names: - continue - name_to_version[name] = patch - - if not name_to_version: - return "" - return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) +# CSAF VEX schema path +CSAF_SCHEMA_PATH = Path(__file__).resolve().parents[3] / "configs" / "vex" / "csaf" / "v2.0" / "csaf_json_schema.json" def _enrich_vulnerabilities_with_notes( @@ -240,7 +192,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: is_vulnerable = justification.get("justification_label") == JUSTIFICATION_LABEL_VULNERABLE if is_vulnerable: - patch_recommendation = _build_patch_recommendation(ci, sbom_names) + patch_recommendation = build_patch_recommendation(ci, sbom_names) comment = ( REMEDIATION_MESSAGE_TEMPLATE.format(patch_recommendation=patch_recommendation) if patch_recommendation else REMEDIATION_MESSAGE_NO_RECOMMENDATION @@ -283,7 +235,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: ) # Validate the CSAF document against the JSON schema - errors = list(get_vex_validator().iter_errors(csaf_json)) + errors = list(get_vex_validator(CSAF_SCHEMA_PATH).iter_errors(csaf_json)) if errors: logger.error("%s Found %d error(s):", ERROR_CSAF_VALIDATION_FAILED, len(errors)) for e in errors: diff --git a/src/vuln_analysis/utils/vex/vex_utils.py b/src/vuln_analysis/utils/vex/vex_utils.py new file mode 100644 index 000000000..4a6668d08 --- /dev/null +++ b/src/vuln_analysis/utils/vex/vex_utils.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from functools import lru_cache +import json +from pathlib import Path + +from jsonschema import Draft202012Validator + +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + + +@lru_cache(maxsize=None) +def get_vex_validator(schema_path: Path) -> Draft202012Validator: + """ + Load schema and create a cached VEX document validator instance. + + Args: + schema_path: Path to the JSON schema file. + + Returns: + A Draft202012Validator instance for the schema. + """ + with open(schema_path) as f: + schema = json.load(f) + logger.debug("VEX document schema loaded from %s", schema_path) + return Draft202012Validator(schema) + + +def get_patched_package(vuln: dict) -> tuple[str | None, str | None]: + """ + Extract package name and first patched version from a GHSA vulnerability dict. + + Args: + vuln: A GHSA vulnerability dictionary. + + Returns: + Tuple of (package_name, first_patched_version), either may be None. + """ + pkg = vuln.get("package") or {} + return pkg.get("name"), vuln.get("first_patched_version") + + +def build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: + """ + Build a patch recommendation string from GHSA data. + + Args: + ci: CveIntel object containing GHSA vulnerability data. + sbom_package_names: Optional set of package names from SBOM to filter by. + + Returns: + Comma-separated string of "name:version" pairs, or empty string if none found. + + Notes: + - If SBOM provided: returns only patches for packages in the SBOM. + - If no SBOM: returns all unique patches. + """ + if not ci or not ci.ghsa or not ci.ghsa.vulnerabilities: + return "" + + vulns = ci.ghsa.vulnerabilities + + name_to_version: dict[str, str] = {} + for v in vulns: + name, patch = get_patched_package(v) + if not name or not patch: + continue + if name in name_to_version: + continue + # If SBOM provided, only include packages that are in the SBOM + if sbom_package_names is not None and name not in sbom_package_names: + continue + name_to_version[name] = patch + + if not name_to_version: + return "" + return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) + From 9fff785fcb32635a312c87d6f0e35499b3d02375 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Wed, 14 Jan 2026 16:34:39 +0200 Subject: [PATCH 165/286] test: update vex generation unit and IT tests Signed-off-by: Ilona Shishov --- .../tests/test_csaf_generator_integration.py | 6 +- tests/test_vex_csaf_helpers.py | 179 +------------- tests/test_vex_utils.py | 232 ++++++++++++++++++ 3 files changed, 238 insertions(+), 179 deletions(-) create mode 100644 tests/test_vex_utils.py diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index 8bd0e27ba..dcfa40fa5 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -122,7 +122,7 @@ def test_document_has_disclaimer_note(self, mock_state): result = generator.generate(mock_state) notes = result["document"].get("notes", []) - disclaimer_notes = [n for n in notes if n.get("category") == "disclaimer"] + disclaimer_notes = [n for n in notes if n.get("category") == "legal_disclaimer"] assert len(disclaimer_notes) == 1 assert "Unofficial Content Notice" in disclaimer_notes[0]["title"] @@ -232,8 +232,8 @@ def test_vulnerabilities_have_notes_enriched(self): vuln = result["vulnerabilities"][0] notes = vuln.get("notes", []) - analysis_notes = [n for n in notes if n.get("category") == "analysis"] - assert len(analysis_notes) == 3 # analysis summary + justification reasoning + justification label + other_notes = [n for n in notes if n.get("category") == "other"] + assert len(other_notes) == 3 # analysis summary + justification reasoning + justification label description_note = [n for n in notes if n.get("category") == "description"] assert len(description_note) == 1 # ghsa description diff --git a/tests/test_vex_csaf_helpers.py b/tests/test_vex_csaf_helpers.py index a5a6d7a47..687e52473 100644 --- a/tests/test_vex_csaf_helpers.py +++ b/tests/test_vex_csaf_helpers.py @@ -14,15 +14,13 @@ # limitations under the License. """ -Unit tests for VEX CSAF generator helper functions. +Unit tests for CSAF VEX generator helper functions. """ import pytest from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa from vuln_analysis.utils.vex.implementations.csaf_generator import ( - _get_patched_package, - _build_patch_recommendation, _enrich_vulnerabilities_with_notes, ) @@ -57,177 +55,6 @@ def base_justifications(): return {"CVE-2024-1234": {"justification": "reasoning", "justification_label": "vulnerable"}} -class TestGetPatchedPackage: - """Unit tests for _get_patched_package() function.""" - - def test_valid_package_returns_name_and_version(self): - """Test extraction of name and version from valid vulnerability dict.""" - vuln = { - "package": {"name": "lodash"}, - "first_patched_version": "4.17.21" - } - result = _get_patched_package(vuln) - assert result == ("lodash", "4.17.21") - - def test_empty_dict_returns_none_tuple(self): - """Test that empty dict returns (None, None).""" - result = _get_patched_package({}) - assert result == (None, None) - - def test_missing_package_key_returns_none_name(self): - """Test that missing 'package' key returns None for name.""" - vuln = {"first_patched_version": "1.0.0"} - result = _get_patched_package(vuln) - assert result == (None, "1.0.0") - - def test_missing_version_returns_none_version(self): - """Test that missing version returns None for version.""" - vuln = {"package": {"name": "express"}} - result = _get_patched_package(vuln) - assert result == ("express", None) - - def test_null_package_returns_none_name(self): - """Test that null package value returns None for name.""" - vuln = {"package": None, "first_patched_version": "2.0.0"} - result = _get_patched_package(vuln) - assert result == (None, "2.0.0") - - def test_empty_package_dict_returns_none_name(self): - """Test that empty package dict returns None for name.""" - vuln = {"package": {}, "first_patched_version": "3.0.0"} - result = _get_patched_package(vuln) - assert result == (None, "3.0.0") - - -class TestBuildPatchRecommendation: - """Unit tests for _build_patch_recommendation() function.""" - - def test_returns_empty_when_intel_is_none(self): - """Test that None intel returns empty string.""" - result = _build_patch_recommendation(None, None) - assert result == "" - - def test_returns_empty_when_ghsa_is_none(self): - """Test that intel without GHSA returns empty string.""" - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=None) - result = _build_patch_recommendation(ci, None) - assert result == "" - - def test_returns_empty_when_vulnerabilities_is_none(self): - """Test that GHSA without vulnerabilities returns empty string.""" - ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=None) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - result = _build_patch_recommendation(ci, None) - assert result == "" - - def test_returns_empty_when_vulnerabilities_is_empty(self): - """Test that empty vulnerabilities list returns empty string.""" - ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=[]) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - result = _build_patch_recommendation(ci, None) - assert result == "" - - def test_with_sbom_returns_matching_package(self): - """Test that with SBOM, only matching package is returned.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, - {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, {"lodash", "react"}) - assert result == "lodash:4.17.21" - - def test_with_sbom_returns_first_match_only(self): - """Test that with SBOM, only first matching package is returned.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, - {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, {"lodash", "express"}) - # Should return first match - assert result == "lodash:4.17.21" - - def test_with_sbom_no_match_returns_empty(self): - """Test that with SBOM but no match, empty string is returned.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, {"react", "vue"}) - assert result == "" - - def test_without_sbom_returns_all_packages(self): - """Test that without SBOM, all packages are returned.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, - {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, None) - assert "lodash:4.17.21" in result - assert "express:4.18.0" in result - - def test_without_sbom_deduplicates_packages(self): - """Test that duplicate package names are deduplicated.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, - {"package": {"name": "lodash"}, "first_patched_version": "4.17.22"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, None) - # First version should win - assert result == "lodash:4.17.21" - - def test_skips_vulnerabilities_without_name(self): - """Test that vulnerabilities without package name are skipped.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {}, "first_patched_version": "1.0.0"}, - {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, None) - assert result == "express:4.18.0" - - def test_skips_vulnerabilities_without_version(self): - """Test that vulnerabilities without patched version are skipped.""" - ghsa = CveIntelGhsa( - ghsa_id="GHSA-1234-5678-9012", - vulnerabilities=[ - {"package": {"name": "lodash"}}, - {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, - ] - ) - ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) - - result = _build_patch_recommendation(ci, None) - assert result == "express:4.18.0" - - class TestEnrichVulnerabilitiesWithNotes: """Unit tests for _enrich_vulnerabilities_with_notes() function.""" @@ -270,7 +97,7 @@ def test_adds_analysis_summary_note(self, base_csaf_json, base_intel_map, base_j analysis_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Summary"] assert len(analysis_notes) == 1 assert analysis_notes[0]["text"] == "This is the analysis summary" - assert analysis_notes[0]["category"] == "analysis" + assert analysis_notes[0]["category"] == "other" def test_adds_justification_notes(self, base_csaf_json, base_intel_map, base_final_summaries): """Test that justification reasoning and label are added as notes.""" @@ -386,4 +213,4 @@ def test_handles_empty_vulnerabilities_list(self): # Should not raise an exception _enrich_vulnerabilities_with_notes(csaf_json, {}, {}, {}) - assert csaf_json["vulnerabilities"] == [] \ No newline at end of file + assert csaf_json["vulnerabilities"] == [] diff --git a/tests/test_vex_utils.py b/tests/test_vex_utils.py new file mode 100644 index 000000000..1e7a28ba7 --- /dev/null +++ b/tests/test_vex_utils.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for VEX utility functions in vex_utils.py. +""" + +from pathlib import Path +import tempfile + +import pytest +from jsonschema import Draft202012Validator + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa +from vuln_analysis.utils.vex.implementations.csaf_generator import ( + CSAF_SCHEMA_PATH as vex_schema_path_example, +) +from vuln_analysis.utils.vex.vex_utils import ( + get_vex_validator, + get_patched_package, + build_patch_recommendation, +) + + +class TestGetVexValidator: + """Unit tests for get_vex_validator() function.""" + + def test_returns_draft202012_validator(self): + """Test that get_vex_validator returns a Draft202012Validator instance.""" + validator = get_vex_validator(vex_schema_path_example) + assert isinstance(validator, Draft202012Validator) + + def test_caching_returns_same_instance(self): + """Test that calling with same path returns the same cached validator (same object in memory and not just equal in vlaue).""" + validator1 = get_vex_validator(vex_schema_path_example) + validator2 = get_vex_validator(vex_schema_path_example) + assert validator1 is validator2 + + def test_different_paths_return_different_validators(self): + """Test that different schema paths return different validator instances.""" + minimal_schema = '{"type": "object"}' + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=True) as f: + f.write(minimal_schema) + f.flush() # Ensure all contents are written before reading + temp_path = Path(f.name) + validator1 = get_vex_validator(vex_schema_path_example) + validator2 = get_vex_validator(temp_path) + assert validator1 is not validator2 + + def test_validator_can_validate_documents(self): + """Test that returned validator can actually validate documents.""" + validator = get_vex_validator(vex_schema_path_example) + + errors = list(validator.iter_errors({})) + assert len(errors) > 0 + + def test_invalid_path_raises_file_not_found(self): + """Test that invalid schema path raises FileNotFoundError.""" + invalid_path = Path("/nonexistent/path/schema.json") + + with pytest.raises(FileNotFoundError): + get_vex_validator(invalid_path) + + +class TestGetPatchedPackage: + """Unit tests for get_patched_package() function.""" + + def test_valid_package_returns_name_and_version(self): + """Test extraction of name and version from valid vulnerability dict.""" + vuln = { + "package": {"name": "lodash"}, + "first_patched_version": "4.17.21" + } + result = get_patched_package(vuln) + assert result == ("lodash", "4.17.21") + + def test_empty_dict_returns_none_tuple(self): + """Test that empty dict returns (None, None).""" + result = get_patched_package({}) + assert result == (None, None) + + def test_missing_package_key_returns_none_name(self): + """Test that missing 'package' key returns None for name.""" + vuln = {"first_patched_version": "1.0.0"} + result = get_patched_package(vuln) + assert result == (None, "1.0.0") + + def test_missing_version_returns_none_version(self): + """Test that missing version returns None for version.""" + vuln = {"package": {"name": "express"}} + result = get_patched_package(vuln) + assert result == ("express", None) + + def test_null_package_returns_none_name(self): + """Test that null package value returns None for name.""" + vuln = {"package": None, "first_patched_version": "2.0.0"} + result = get_patched_package(vuln) + assert result == (None, "2.0.0") + + def test_empty_package_dict_returns_none_name(self): + """Test that empty package dict returns None for name.""" + vuln = {"package": {}, "first_patched_version": "3.0.0"} + result = get_patched_package(vuln) + assert result == (None, "3.0.0") + + +class TestBuildPatchRecommendation: + """Unit tests for build_patch_recommendation() function.""" + + def test_returns_empty_when_intel_is_none(self): + """Test that None intel returns empty string.""" + result = build_patch_recommendation(None, None) + assert result == "" + + def test_returns_empty_when_ghsa_is_none(self): + """Test that intel without GHSA returns empty string.""" + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=None) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_none(self): + """Test that GHSA without vulnerabilities returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=None) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_empty(self): + """Test that empty vulnerabilities list returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=[]) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_with_sbom_returns_matching_package(self): + """Test that with SBOM, only matching package is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, {"lodash", "express", "react"}) + assert result == "lodash:4.17.21, express:4.18.0" + + def test_with_sbom_no_match_returns_empty(self): + """Test that with SBOM but no match, empty string is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, {"react", "vue"}) + assert result == "" + + def test_without_sbom_returns_all_packages(self): + """Test that without SBOM, all packages are returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert "lodash:4.17.21" in result + assert "express:4.18.0" in result + + def test_without_sbom_deduplicates_packages(self): + """Test that duplicate package names are deduplicated.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "lodash"}, "first_patched_version": "4.17.22"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + # First version should win + assert result == "lodash:4.17.21" + + def test_skips_vulnerabilities_without_name(self): + """Test that vulnerabilities without package name are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {}, "first_patched_version": "1.0.0"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + + def test_skips_vulnerabilities_without_version(self): + """Test that vulnerabilities without patched version are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + From 462fd488c8c983c6e2a946c283d4db8c493ebbda Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Wed, 14 Jan 2026 17:26:45 +0200 Subject: [PATCH 166/286] fix: vendor selection Signed-off-by: Ilona Shishov --- src/vuln_analysis/utils/vex/implementations/csaf_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 21e4b3e72..dfd270d4b 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -176,7 +176,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: csaf_gen.set_value("author", AUTHOR_NAME) csaf_gen.set_value("author_url", AUTHOR_URL) - vendor = (product_name.split("/")[1] if "/" in product_name else DEFAULT_VENDOR) + vendor = (product_name.split("/")[-2] if "/" in product_name else "") or DEFAULT_VENDOR csaf_gen.add_product(product_name=product_name, vendor=vendor, release=product_tag) intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in message.info.intel} From 42216c78e87d8ddb9734967dca2b1d13ff2af863 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 15 Jan 2026 10:51:50 +0200 Subject: [PATCH 167/286] test: update and add additional integration tests for VEX doc validation Signed-off-by: Ilona Shishov --- .../tests/test_csaf_generator_integration.py | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index dcfa40fa5..64ec70c01 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -17,6 +17,8 @@ Integration tests for CSAF VEX generator. """ +from unittest.mock import patch + import pytest from exploit_iq_commons.data_models.common import AnalysisType @@ -50,6 +52,7 @@ _DEFAULT_JUSTIFICATION = {"justification": "reason", "justification_label": "vulnerable"} _DEFAULT_VULNS = ["CVE-2024-1234"] _DEFAULT_SUMMARY = "Analysis summary for {v}" +_DEFAULT_SBOM_PACKAGES = [SBOMPackage(name="test-package", version="1.0.0", system="npm")] @pytest.fixture(scope="module") @@ -66,18 +69,20 @@ def create_mock_state( intel: list[CveIntel] | None = None, product_name: str = _DEFAULT_PRODUCT_NAME, product_tag: str = _DEFAULT_PRODUCT_TAG, - sbom_packages: list[SBOMPackage] = [SBOMPackage(name="test-package", version="1.0.0", system="npm")], + sbom_packages: list[SBOMPackage] | None = _DEFAULT_SBOM_PACKAGES, ) -> AgentMorpheusEngineState: """Create a mock AgentMorpheusEngineState for testing.""" intel = intel or [CveIntel(vuln_id=v) for v in vulns] + sbom_info = None if sbom_packages is None else ManualSBOMInfoInput(packages=sbom_packages) + image_info = ImageInfoInput( name=product_name, tag=product_tag, analysis_type=AnalysisType.IMAGE, source_info=_DEFAULT_SOURCE_INFO, - sbom_info=ManualSBOMInfoInput(packages=sbom_packages), + sbom_info=sbom_info, ) engine_input = AgentMorpheusEngineInput( @@ -116,6 +121,19 @@ def test_document_has_correct_title(self, mock_state): title = result["document"].get("title") assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + ":" + _DEFAULT_PRODUCT_TAG in title + def test_oci_digest_tag_uses_at_separator(self): + """Test that OCI digest tags use @ separator instead of : in title.""" + oci_digest = "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + state = create_mock_state( + product_tag=oci_digest, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + title = result["document"].get("title") + assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + "@" + oci_digest in title + def test_document_has_disclaimer_note(self, mock_state): """Test that document includes the disclaimer note.""" generator = CsafVexGenerator() @@ -274,8 +292,9 @@ def test_rhsa_threat_severity_used_as_impact(self): class TestCsafVexGeneratorEdgeCases: """Integration tests for edge cases and error handling.""" - def test_handles_empty_sbom_packages(self): - """Test handling when SBOM has no packages.""" + @pytest.mark.parametrize("sbom_packages", [None, []], ids=["sbom_info_none", "empty_packages"]) + def test_includes_all_packages_when_no_sbom_filtering(self, sbom_packages): + """Test that all packages are included when SBOM is None or has no packages.""" ghsa = CveIntelGhsa( ghsa_id="GHSA-1234-5678-9012", vulnerabilities=[ @@ -285,8 +304,8 @@ def test_handles_empty_sbom_packages(self): ) intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] state = create_mock_state( - intel=intel, - sbom_packages=[], + intel=intel, + sbom_packages=sbom_packages ) generator = CsafVexGenerator() @@ -297,7 +316,7 @@ def test_handles_empty_sbom_packages(self): remediations = vuln.get("remediations", []) assert "lodash:4.17.21" in remediations[0].get("details") assert "express:4.18.0" in remediations[0].get("details") - + def test_handles_sbom_packages_with_matching_and_non_matching_packages(self): """Test handling when SBOM has a matching package and a non-matching package.""" ghsa = CveIntelGhsa( @@ -346,5 +365,22 @@ def test_handles_product_name_without_slash(self): assert "unknown" == product_tree.get("branches")[0].get("name") assert "simpleapp" == product_tree.get("branches")[0].get("branches")[0].get("name") + def test_validation_failure_returns_empty_dict(self, mock_state): + """Test that validation failure returns empty dict.""" + generator = CsafVexGenerator() + + # Mock the enrichment function to inject invalid data (None where string expected) + def mock_enrich(csaf_json, intel_map, final_summaries, justifications): + # Inject invalid note with None text (violates schema) + for v in csaf_json.get("vulnerabilities", []): + v["notes"] = [{"category": "not_allowed_category", "title": "Test", "text": "test text"}] + return csaf_json + + with patch( + "vuln_analysis.utils.vex.implementations.csaf_generator._enrich_vulnerabilities_with_notes", + side_effect=mock_enrich + ): + result = generator.generate(mock_state) + assert result == {} From 075fe57397c87048dbf5e59e1474a96c67f8cf7b Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 18 Jan 2026 10:31:34 +0200 Subject: [PATCH 168/286] chore: remove redundant return statement Signed-off-by: Ilona Shishov --- .../utils/vex/implementations/csaf_generator.py | 2 -- .../utils/vex/tests/test_csaf_generator_integration.py | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index dfd270d4b..d0ebb7d7d 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -146,8 +146,6 @@ def _enrich_vulnerabilities_with_notes( v["notes"] = notes - return csaf_json - class CsafVexGenerator(VexGenerator): """ diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index 64ec70c01..5bca102d3 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -369,12 +369,11 @@ def test_validation_failure_returns_empty_dict(self, mock_state): """Test that validation failure returns empty dict.""" generator = CsafVexGenerator() - # Mock the enrichment function to inject invalid data (None where string expected) + # Mock the enrichment function to inject invalid data (modifies csaf_json in-place) def mock_enrich(csaf_json, intel_map, final_summaries, justifications): - # Inject invalid note with None text (violates schema) + # Inject invalid note category (violates schema) for v in csaf_json.get("vulnerabilities", []): v["notes"] = [{"category": "not_allowed_category", "title": "Test", "text": "test text"}] - return csaf_json with patch( "vuln_analysis.utils.vex.implementations.csaf_generator._enrich_vulnerabilities_with_notes", From 1c9d2777520add0d683f21bf6b1bc00cc4475db0 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 19 Jan 2026 13:01:09 +0200 Subject: [PATCH 169/286] Configure Nginx to use IPv4 only for external connections --- kustomize/base/nginx/nginx_cache.conf | 15 ++----- .../templates/routes/intel.conf.template | 42 +++++++++++++------ .../nginx/templates/routes/nemo.conf.template | 8 +++- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index 21ff562ee..9f4e239ae 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -73,17 +73,10 @@ http { set $http_authorization_present '[REDACTED]'; # Set to '[REDACTED]' when the Authorization header is present } - # Configure a resolver to use for DNS resolution. This uses the Docker DNS resolver - # See https://tenzer.dk/nginx-with-dynamic-upstreams/ for why this is necessary - # When considering what the "base_url" should be, consider the following: - # - The base_url should be the unchangable part of the URL for any request tho that API - # - If the API uses versioning, the version should be included in the base_url - # - If the API is a subpath of a larger API, the base_url should be the path to the API - # - Examples: - # - GET `https://api.first.org/data/v1/epss` => base_url=`https://api.first.org/data/v1` - # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` - - # resolver 127.0.0.11 [::1]:5353 valid=60s; + # Configure a resolver to use for DNS resolution with IPv4-only + # ipv6=off ensures that only IPv4 addresses are used for all external domains + # This prevents "Network is unreachable" errors in environments without IPv6 support + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 ipv6=off; # rewrite_log on; diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index 2e40ccf85..6a742a173 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -1,8 +1,10 @@ ####################### Intel APIs ####################### location /serpapi { + set $backend "serpapi.com"; rewrite ^\/serpapi(\/.*)$ $1 break; - proxy_pass https://serpapi.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -11,8 +13,10 @@ location /serpapi { } location /nvd { + set $backend "services.nvd.nist.gov"; rewrite ^\/nvd(\/.*)$ /rest$1 break; - proxy_pass https://services.nvd.nist.gov; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_valid 200 202 365d; proxy_cache_methods GET; @@ -22,8 +26,10 @@ location /nvd { } location /cve-details { + set $backend "www.cvedetails.com"; rewrite ^\/cve-details(\/.*)$ $1 break; - proxy_pass https://www.cvedetails.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -32,8 +38,10 @@ location /cve-details { } location /cwe-details { + set $backend "cwe.mitre.org"; rewrite ^\/cwe-details(\/.*)$ $1 break; - proxy_pass https://cwe.mitre.org; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -44,8 +52,10 @@ location /cwe-details { } location /first { + set $backend "api.first.org"; rewrite ^\/first(\/.*)$ /data/v1$1 break; - proxy_pass https://api.first.org; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -54,8 +64,10 @@ location /first { } location /recf { + set $backend "api.recordedfuture.com"; rewrite ^\/recf(\/.*)$ /v2$1 break; - proxy_pass https://api.recordedfuture.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -64,8 +76,10 @@ location /recf { } location /ghsa { + set $backend "api.github.com"; rewrite ^\/ghsa(\/.*)$ $1 break; - proxy_pass https://api.github.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -74,8 +88,10 @@ location /ghsa { } location /rhsa { + set $backend "access.redhat.com"; rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; - proxy_pass https://access.redhat.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -84,8 +100,10 @@ location /rhsa { } location /ubuntu { + set $backend "ubuntu.com"; rewrite ^\/ubuntu(\/.*)$ $1 break; - proxy_pass https://ubuntu.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -94,15 +112,15 @@ location /ubuntu { } location /depsdev { + set $backend "api.deps.dev"; rewrite ^ $request_uri; rewrite ^\/depsdev\/(.*) $1 break; return 400; - proxy_pass https://api.deps.dev/$uri; + proxy_pass https://$backend/$uri; + proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; add_header X-Cache-Status $upstream_cache_status; client_body_buffer_size 4m; - - resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 valid=60s; } diff --git a/kustomize/base/nginx/templates/routes/nemo.conf.template b/kustomize/base/nginx/templates/routes/nemo.conf.template index 117dd7b4d..fb05d7bba 100644 --- a/kustomize/base/nginx/templates/routes/nemo.conf.template +++ b/kustomize/base/nginx/templates/routes/nemo.conf.template @@ -1,8 +1,10 @@ location /nemo { location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + set $backend "api.llm.ngc.nvidia.com"; rewrite ^\/nemo(\/.*)$ $1 break; - proxy_pass https://api.llm.ngc.nvidia.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; proxy_set_header Connection ''; proxy_set_header Authorization $nemo_http_authorization; proxy_set_header Organization-ID $nemo_http_organization_id; @@ -14,8 +16,10 @@ location /nemo { } location /nemo/v1 { + set $backend "api.llm.ngc.nvidia.com"; rewrite ^\/nemo(\/.*)$ $1 break; - proxy_pass https://api.llm.ngc.nvidia.com; + proxy_pass https://$backend; + proxy_ssl_server_name on; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } From aaa678d09620c0e2fe9f5c2a55a3cf7e37d30460 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 18 Jan 2026 15:00:58 +0200 Subject: [PATCH 170/286] chore: add vex configuration to kustomize config file Signed-off-by: Ilona Shishov --- kustomize/base/exploit-iq-config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 4cdcf0469..f6bf85bd2 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -129,6 +129,10 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + # vex_format: csaf cve_http_output: _type: cve_http_output url: CALLBACK_URL_PLACEHOLDER @@ -230,6 +234,7 @@ workflow: cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output From 5ab4c1e8d58bbccaa6c16f80de0379b80c3d9407 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 26 Jan 2026 10:02:04 +0200 Subject: [PATCH 171/286] chore: buildah task should get resolved from new ci namespace Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 2 +- .tekton/on-push.yaml | 2 +- .tekton/on-tag.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 9ae42742e..00d714fb7 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -113,7 +113,7 @@ spec: - name: name value: buildah-pvc - name: namespace - value: ruben-morpheus + value: exploit-iq-tests workspaces: - name: source workspace: source diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml index 47fc26807..e597bfacf 100644 --- a/.tekton/on-push.yaml +++ b/.tekton/on-push.yaml @@ -103,7 +103,7 @@ spec: - name: name value: buildah-pvc - name: namespace - value: ruben-morpheus + value: exploit-iq-tests workspaces: - name: source workspace: source diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml index fd27734f6..a820663d3 100644 --- a/.tekton/on-tag.yaml +++ b/.tekton/on-tag.yaml @@ -132,7 +132,7 @@ spec: - name: name value: buildah-pvc - name: namespace - value: ruben-morpheus + value: exploit-iq-tests workspaces: - name: source workspace: source From 32618bd68bb599b74bb0c396f87c27b626eb576d Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 26 Jan 2026 14:46:34 +0200 Subject: [PATCH 172/286] chore: add mlops overlay to kustomize with grafana kustomization Signed-off-by: Ilona Shishov --- kustomize/base/kustomization.yaml | 2 +- kustomize/operators/grafana/subscription.yaml | 11 + .../mlops/grafana/dashboards/batch-info.yaml | 166 ++++++++++++ .../grafana/dashboards/batch-metrics.yaml | 195 +++++++++++++++ .../mlops/grafana/dashboards/evals.yaml | 236 ++++++++++++++++++ .../mlops/grafana/dashboards/jobs.yaml | 236 ++++++++++++++++++ .../mlops/grafana/datasources/infinity.yaml | 20 ++ .../mlops/grafana/folders/batches.yaml | 12 + .../overlays/mlops/grafana/folders/evals.yaml | 13 + .../overlays/mlops/grafana/folders/jobs.yaml | 13 + .../mlops/grafana/instances/grafana.yaml | 30 +++ .../overlays/mlops/grafana/kustomization.yaml | 52 ++++ kustomize/overlays/mlops/kustomization.yaml | 10 + 13 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 kustomize/operators/grafana/subscription.yaml create mode 100644 kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml create mode 100644 kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml create mode 100644 kustomize/overlays/mlops/grafana/dashboards/evals.yaml create mode 100644 kustomize/overlays/mlops/grafana/dashboards/jobs.yaml create mode 100644 kustomize/overlays/mlops/grafana/datasources/infinity.yaml create mode 100644 kustomize/overlays/mlops/grafana/folders/batches.yaml create mode 100644 kustomize/overlays/mlops/grafana/folders/evals.yaml create mode 100644 kustomize/overlays/mlops/grafana/folders/jobs.yaml create mode 100644 kustomize/overlays/mlops/grafana/instances/grafana.yaml create mode 100644 kustomize/overlays/mlops/grafana/kustomization.yaml create mode 100644 kustomize/overlays/mlops/kustomization.yaml diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 504d3924b..1e0afee87 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -77,7 +77,7 @@ patches: images: - name: quay.io/ecosystem-appeng/agent-morpheus-rh - newTag: nat + newTag: latest - name: quay.io/ecosystem-appeng/agent-morpheus-client newTag: latest diff --git a/kustomize/operators/grafana/subscription.yaml b/kustomize/operators/grafana/subscription.yaml new file mode 100644 index 000000000..f6d61a71a --- /dev/null +++ b/kustomize/operators/grafana/subscription.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: grafana-operator + namespace: openshift-operators +spec: + channel: v5 + installPlanApproval: Automatic + name: grafana-operator + source: community-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml new file mode 100644 index 000000000..f89d1861e --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml @@ -0,0 +1,166 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: batch-info +spec: + allowCrossNamespaceImport: false + folder: Batches + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Batch ID the batch was executed by", + "name": "batch_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Batch ID", + "root_selector": "$.batch_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Batch execution start timestamp", + "name": "execution_start", + "type": "textbox" + }, + { + "definition": "Batch language", + "name": "language", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.language)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_batch_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches?batch_id=${batch_id:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batch by Batch ID", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_execution_start", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches?execution_start=${execution_start}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batches by Execution Time Start", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_language", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches?language=${language}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batches by Language", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "Batch info" + } \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml new file mode 100644 index 000000000..bef993b74 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml @@ -0,0 +1,195 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: batch-metrics +spec: + allowCrossNamespaceImport: false + folder: Batches + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "panels": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1 , + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "batch_id": 0, + "language": 1, + "accuracy": 2, + "f1_score": 3, + "precision": 4, + "recall": 5 + } + } + } + ], + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "confusion_matrix", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"language\": $v.language, \"accuracy\": $v.accuracy, \"precision\": $v.precision, \"f1_score\": $v.f1_score, \"recall\": $v.recall} })", + "parser": "backend" + } + ], + "title": "Confusion Matrix", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "batch_id": 0, + "regressive_job_count": 1, + "regression_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "regressive_job_count_by_batch_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"regressive_job_count\": $v.regressive_job_count, \"regression_ratio\": $v.total_jobs_executed != 0 ? $v.regressive_job_count / $v.total_jobs_executed : null} })", + "parser": "backend" + } + ], + "title": "Regressive Job Count by Batch ID", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "language": 0, + "regressive_job_count": 1, + "regression_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "regressive_job_count_by_language", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($distinct($.language), function($lang) { {\"language\": $lang, \"regressive_job_count\": $sum($filter($, function($v) { $v.language = $lang }).regressive_job_count), \"regression_ratio\": $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) != 0 ? $sum($filter($, function($v) { $v.language = $lang }).regressive_job_count) / $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) : null} })", + "parser": "backend" + } + ], + "title": "Regressive Job Count by Language", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "language": 0, + "failure_count": 1, + "failure_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "failure_job_count_by_language", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/batches", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($distinct($.language), function($lang) { {\"language\": $lang, \"failure_count\": $sum($filter($, function($v) { $v.language = $lang }).failure_count), \"failed_ratio\": $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) != 0 ? $sum($filter($, function($v) { $v.language = $lang }).failure_count) / $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) : null} })", + "parser": "backend" + } + ], + "title": "Failure Job Count by Language", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "Batch Metrics" + } \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml new file mode 100644 index 000000000..efb6d032c --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -0,0 +1,236 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: evals +spec: + allowCrossNamespaceImport: false + folder: Evals + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Job request/trace id", + "name": "request_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Request ID", + "root_selector": "$.request_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component", + "name": "component", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.component)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component version", + "name": "component_version", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component Version", + "root_selector": "$distinct($.component_version)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "CVE identifier", + "name": "cve", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "CVE", + "root_selector": "$distinct($.cve)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Metric name", + "name": "metric_name", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Metric Name", + "root_selector": "$distinct($.metric_name)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_request_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals?request_id=${request_id:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Job by Request ID", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_cve_component", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_cve_component_metric", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/evals?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}&metric_name=${metric_name:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version + Metric Name", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "evals" + } diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml new file mode 100644 index 000000000..c9c973814 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -0,0 +1,236 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: jobs +spec: + allowCrossNamespaceImport: false + folder: Jobs + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Batch ID the job was executed by", + "name": "batch_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Batch ID", + "root_selector": "$distinct($.batch_id)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job request/trace id", + "name": "request_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Request ID", + "root_selector": "$.request_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component", + "name": "component", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.component)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component version", + "name": "component_version", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component Version", + "root_selector": "$distinct($.component_version)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "CVE identifier", + "name": "cve", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "CVE", + "root_selector": "$distinct($.cve)", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_request_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs/${request_id:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Job by Request ID", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_batch_id", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs?batch_id=${batch_id:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by Batch ID", + "type": "table" + }, + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "name": "mlops-grafana-datasource" + }, + "format": "table", + "refId": "by_cve_component", + "source": "url", + "type": "json", + "url": "http://mongo-grafana-api:8000/jobs?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "jobs" + } diff --git a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml new file mode 100644 index 000000000..a48b78ea3 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml @@ -0,0 +1,20 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: infinity +spec: + instanceSelector: + matchLabels: + app: exploit-iq + datasource: + access: proxy + isDefault: true + jsonData: + global_queries: [] + url: http://mongo-grafana-api:8000 + name: yesoreyeram-infinity-datasource + type: yesoreyeram-infinity-datasource + plugins: + - name: yesoreyeram-infinity-datasource + version: 3.6.0 + diff --git a/kustomize/overlays/mlops/grafana/folders/batches.yaml b/kustomize/overlays/mlops/grafana/folders/batches.yaml new file mode 100644 index 000000000..1f198b6d7 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/batches.yaml @@ -0,0 +1,12 @@ +# GrafanaFolder Custom Resource +# Organizes dashboards into logical folders in Grafana +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: batches +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Batches" + diff --git a/kustomize/overlays/mlops/grafana/folders/evals.yaml b/kustomize/overlays/mlops/grafana/folders/evals.yaml new file mode 100644 index 000000000..0c1f3f709 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/evals.yaml @@ -0,0 +1,13 @@ +# GrafanaFolder Custom Resource +# Organizes dashboards into logical folders in Grafana +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: evals +spec: + # TODO: Configure your Grafana instance selector + instanceSelector: + matchLabels: + app: exploit-iq + title: "Evals" + diff --git a/kustomize/overlays/mlops/grafana/folders/jobs.yaml b/kustomize/overlays/mlops/grafana/folders/jobs.yaml new file mode 100644 index 000000000..5ead94024 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/jobs.yaml @@ -0,0 +1,13 @@ +# GrafanaFolder Custom Resource +# Organizes dashboards into logical folders in Grafana +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: jobs +spec: + # TODO: Configure your Grafana instance selector + instanceSelector: + matchLabels: + app: exploit-iq + title: "Jobs" + diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml new file mode 100644 index 000000000..9d5efd44c --- /dev/null +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -0,0 +1,30 @@ +kind: Grafana +apiVersion: grafana.integreatly.org/v1beta1 +metadata: + name: grafana +spec: + config: + auth: + disable_login_form: 'false' + log: + mode: console + security: + secret: + name: grafana-secret + keyUser: grafana_user + keyPassword: grafana_password +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: grafana + namespace: mlops +spec: + to: + kind: Service + name: grafana-service + port: + targetPort: 3000 + tls: + termination: edge + wildcardPolicy: None diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml new file mode 100644 index 000000000..1b06b3eb1 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -0,0 +1,52 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - instances/grafana.yaml + + - folders/jobs.yaml + - folders/evals.yaml + - folders/batches.yaml + + - datasources/infinity.yaml + + - dashboards/jobs.yaml + - dashboards/evals.yaml + - dashboards/batch-info.yaml + - dashboards/batch-metrics.yaml + +labels: + - pairs: + app: exploit-iq + +namePrefix: exploit-iq- + +replacements: + - source: + kind: Grafana + name: grafana + fieldPath: metadata.name + targets: + - select: + kind: Route + name: grafana + fieldPaths: + - spec.to.name + options: + delimiter: "-" + index: 0 + - source: + kind: Secret + name: grafana-secret + fieldPath: metadata.name + targets: + - select: + kind: Grafana + name: grafana + fieldPaths: + - spec.config.security.secret.name + +secretGenerator: + - name: grafana-secret + envs: + - secrets.env diff --git a/kustomize/overlays/mlops/kustomization.yaml b/kustomize/overlays/mlops/kustomization.yaml new file mode 100644 index 000000000..27fbe79ea --- /dev/null +++ b/kustomize/overlays/mlops/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../self-hosted-llama3.1-70b-4bit + # Grafana monitoring resources + - grafana + +commonAnnotations: + deployment-specialization: mlops \ No newline at end of file From 78f3f3db6c4c20975ae78162f2ed5ad3c7007f1b Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf <98809100+TamarW0@users.noreply.github.com> Date: Mon, 26 Jan 2026 22:23:52 +0200 Subject: [PATCH 173/286] fix: Python 2 detection regex false positive on tuple exception syntax Co-authored-by: Tamar Weisskopf --- .../python_segmenters_with_classes_methods.py | 2 +- .../tests/test_transitive_code_search.py | 5 +- tests/test_python_segmenter.py | 95 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 tests/test_python_segmenter.py diff --git a/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py index de20d3645..d39d6ce66 100644 --- a/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py +++ b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py @@ -36,7 +36,7 @@ def parse_all_classes_methods(code: str) -> list[str]: # Compiled once for speed PY2_PRINT_REGEX = re.compile(r'^\s*print (?![\(])', re.MULTILINE) -PY2_EXCEPT_REGEX = re.compile(r'^\s*except .+,', re.MULTILINE) # matches: except Exception, e: +PY2_EXCEPT_REGEX = re.compile(r'^\s*except\s+(?:[\w.]+|\([^)]+\))\s*,\s*\w+\s*:', re.MULTILINE) # matches: except Exception, e: PY2_RAW_INPUT_REGEX = re.compile(r'\braw_input\s*\(', re.MULTILINE) # matches: raw_input( PY2_RAISE_REGEX = re.compile(r'^\s*raise\s+\w+\s*,', re.MULTILINE) # matches: raise Exception, diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 80b881eaa..67143f695 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -223,7 +223,7 @@ async def test_python_transitive_search(): set_input_for_next_run( git_repository='https://github.com/TamarW0/example-repo', - git_ref='17af124607ff06fdf95734419b7ecd3af769c2f8', + git_ref='f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0', included_extensions=['**/*.py'], excluded_extensions=['**/*test*.py'] ) @@ -244,7 +244,7 @@ async def test_repeated_runs_of_transitive_search(): set_input_for_next_run( git_repository='https://github.com/TamarW0/example-repo', - git_ref='17af124607ff06fdf95734419b7ecd3af769c2f8', + git_ref='f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0', included_extensions=['**/*.py'], excluded_extensions=['**/*test*.py'] ) @@ -376,6 +376,7 @@ async def test_transitive_search_java_2(): assert 'src/main/java/io/cryostat' in document.metadata['source'] assert 'StringUtils.isBlank(' in document.page_content +@pytest.mark.skip @pytest.mark.asyncio async def test_transitive_search_java_3(): transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() diff --git a/tests/test_python_segmenter.py b/tests/test_python_segmenter.py new file mode 100644 index 000000000..c909d0c04 --- /dev/null +++ b/tests/test_python_segmenter.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from exploit_iq_commons.utils.python_segmenters_with_classes_methods import ( + is_python2_code, + PythonSegmenterWithClassesMethods, +) + +@pytest.mark.parametrize( + "code,expected,description", + [ + # Python 2 patterns (should return True) + # Simple exception with comma syntax + ("except Exception, e:", True, "py2 simple except with comma"), + ("except ValueError, err:", True, "py2 except ValueError with comma"), + (" except Exception, e:", True, "py2 indented except with comma"), + # Dotted module exception names + ("except module.Error, e:", True, "py2 dotted module exception"), + ("except urllib2.URLError, e:", True, "py2 urllib2 exception"), + ("except xml.parsers.expat.ExpatError, e:", True, "py2 deeply nested module exception"), + # Tuple of exceptions with comma syntax (Python 2) + ("except (IOError, OSError), e:", True, "py2 tuple exceptions with comma"), + ("except (IOError, OSError, KeyError), e:", True, "py2 tuple exceptions with comma"), + ("except (ValueError, TypeError), err:", True, "py2 tuple exceptions with variable"), + (" except (KeyError, IndexError), exc:", True, "py2 indented tuple exceptions"), + # Print statement (no parentheses) + ('print "hello"', True, "py2 print statement with double quotes"), + ("print 'hello'", True, "py2 print statement with single quotes"), + ('print "hello", "world"', True, "py2 print statement with multiple args"), + (" print 'indented'", True, "py2 indented print statement"), + # raw_input function + ('raw_input("prompt")', True, "py2 raw_input call"), + ("raw_input('Enter: ')", True, "py2 raw_input with single quotes"), + ("x = raw_input()", True, "py2 raw_input assignment"), + # Raise with comma syntax + ('raise Exception, "error message"', True, "py2 raise with comma and string"), + ("raise ValueError, msg", True, "py2 raise with comma and variable"), + (" raise TypeError, 'error'", True, "py2 indented raise with comma"), + # Shebang with python2 + ("#!/usr/bin/python2\nprint 'hello'", True, "py2 shebang with python2"), + ("#!/usr/bin/env python2\nx = 1", True, "py2 env shebang with python2"), + ("#!/usr/bin/python2.7\npass", True, "py2 shebang with python2.7"), + # Python 3 patterns (should return False) + # Python 3 tuple exceptions (no variable after) + ("except (KeyError, ValueError):", False, "py3 tuple exceptions"), + ("except (IOError, OSError):", False, "py3 IO tuple exceptions"), + (" except (TypeError, AttributeError):", False, "py3 indented tuple exceptions"), + # Python 3 'as' syntax + ("except Exception as e:", False, "py3 except as syntax"), + ("except ValueError as err:", False, "py3 ValueError as syntax"), + (" except KeyError as exc:", False, "py3 indented as syntax"), + # Python 3 dotted exception with 'as' + ("except module.Error as e:", False, "py3 dotted exception as"), + ("except urllib.error.URLError as e:", False, "py3 urllib exception as"), + # Print function + ('print("hello")', False, "py3 print function"), + ("print('hello', 'world')", False, "py3 print function multiple args"), + ("print()", False, "py3 empty print function"), + # Input function (Python 3) + ('input("prompt")', False, "py3 input function"), + ("x = input()", False, "py3 input assignment"), + # Raise with parentheses + ('raise Exception("error")', False, "py3 raise with parentheses"), + ("raise ValueError('message')", False, "py3 raise ValueError with parens"), + # Modern Python 3 code + ("def func() -> int:", False, "py3 type hints"), + ("x: int = 5", False, "py3 variable annotation"), + ("async def coro():", False, "py3 async function"), + ("f'hello {name}'", False, "py3 f-string"), + # Python 3 shebang + ("#!/usr/bin/python3\nprint('hello')", False, "py3 python3 shebang"), + ("#!/usr/bin/env python3\nx = 1", False, "py3 env python3 shebang"), + # Empty or minimal code + ("pass", False, "py3 pass statement"), + ("x = 1", False, "py3 simple assignment"), + ("import os", False, "py3 import statement"), + ], +) +def test_is_python2_code(self, code: str, expected: bool, description: str): + """Test that Python 2/3 patterns are correctly detected.""" + result = is_python2_code(code) + assert result is expected, f"Expected {expected} for {description}, got {result}" \ No newline at end of file From 391ba4052cc96cb331aee8b5ae7db5334b51bc2a Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 26 Jan 2026 10:21:42 +0200 Subject: [PATCH 174/286] chore: improve C/C++ project detection using content verification --- src/exploit_iq_commons/utils/dep_tree.py | 2 +- .../utils/transitive_code_searcher_tool.py | 36 ++++++++++++++++++- tests/test_transitive_detection.py | 28 +++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 tests/test_transitive_detection.py diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index e4f84a41f..a4572c012 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -53,7 +53,7 @@ class Ecosystem(str, Enum): C_CPLUSPLUS_MANIFEST_2 = "meson.build" C_CPLUSPLUS_MANIFEST_3 = "Makefile" C_CPLUSPLUS_MANIFEST_4 = "configure" - +C_CPLUSPLUS_EXTENSIONS = {".c", ".cpp", ".h", ".hpp", ".cc", ".cxx"} MANIFESTS_TO_ECOSYSTEMS = { "go.mod": Ecosystem.GO, "requirements.txt": Ecosystem.PYTHON, diff --git a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py index 8167f4405..1f47de94c 100644 --- a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py +++ b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py @@ -30,6 +30,7 @@ C_CPLUSPLUS_MANIFEST_2, C_CPLUSPLUS_MANIFEST_3, C_CPLUSPLUS_MANIFEST_4, + C_CPLUSPLUS_EXTENSIONS, ) from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE @@ -51,6 +52,35 @@ class TransitiveCodeSearcher: def __init__(self, chain_of_calls_retriever: ChainOfCallsRetrieverBase): self.chain_of_calls_retriever = chain_of_calls_retriever + @staticmethod + def _has_c_cpp_sources(git_repo_path: Path) -> bool: + """Check for the presence of C/C++ source files in the repository. + + This method scans the directory tree rooted at `git_repo_path` to verify if it contains + actual C/C++ source code (files with extensions defined in C_CPLUSPLUS_EXTENSIONS). + This is used to filter out projects that might have a build file (like Makefile) but + no actual C/C++ code. + + The search is optimized to: + - skip common non-source directories (all starting with '.') + - return True immediately upon finding the first matching file. + + Parameters + ---------- + git_repo_path : Path + The root directory of the git repository to scan. + + Returns + ------- + bool + True if at least one C/C++ source file is found, False otherwise. + """ + for root, dirs, files in os.walk(git_repo_path): + dirs[:] = [d for d in dirs if not d.startswith('.')] + if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files): + return True + return False + @staticmethod def download_dependencies(git_repo_path: Path) -> bool: """ Download all dependencies according to manifest file in the Git repository @@ -84,7 +114,11 @@ def download_dependencies(git_repo_path: Path) -> bool: ] found = [p for p in candidates if p.is_file()] if found: - ecosystem = MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] + if TransitiveCodeSearcher._has_c_cpp_sources(git_repo_path): + ecosystem = MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] + else: + logger.info(f"Generic build file found but no C/C++ sources detected in {git_repo_path}. Skipping C ecosystem.") + return False else: logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") return False diff --git a/tests/test_transitive_detection.py b/tests/test_transitive_detection.py new file mode 100644 index 000000000..5a5fa5a0e --- /dev/null +++ b/tests/test_transitive_detection.py @@ -0,0 +1,28 @@ +import pytest +from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher + +C_CPP_DETECTION_SCENARIOS = [ + # Positive cases + (["src/main.c"], True), + (["src/utils.cpp"], True), + (["include/lib.h"], True), + (["src/deep/nested/logic.cxx"], True), + # Negative cases + (["Makefile", "app.py"], False), + (["README.md", "LICENSE"], False), + (["main.cs"], False), + (["style.css"], False), + ([], False), + # Ignored directories + ([".git/objects/obj.c"], False), + ([".venv/lib/site/test.c"], False), + ([".config/settings.h"], False), +] +@pytest.mark.parametrize("file_paths, expected", C_CPP_DETECTION_SCENARIOS) +def test_has_c_cpp_sources_scenarios(tmp_path, file_paths, expected): + for path in file_paths: + full_path = tmp_path / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.touch() + result = TransitiveCodeSearcher._has_c_cpp_sources(tmp_path) + assert result is expected, f"Failed for scenario: {file_paths}" \ No newline at end of file From 2af8010566187f89ebf19e52669d8a89282e719a Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 27 Jan 2026 12:14:24 +0200 Subject: [PATCH 175/286] chore: fix Python integration tests repo ref Signed-off-by: Vladimir Belousov --- ci/it/integration-tests-input.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json index ea8c4f30b..500540be0 100644 --- a/ci/it/integration-tests-input.json +++ b/ci/it/integration-tests-input.json @@ -38,7 +38,7 @@ "vuln_id": "CVE-2024-49767", "git": { "repo": "https://github.com/TamarW0/example-repo", - "ref": "17af124607ff06fdf95734419b7ecd3af769c2f8" + "ref": "f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0" }, "use_sbom": false, "expected_label": "vulnerable", From bec2ca24be7aa41fafc33993f44d10069d36c819 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 21 Jan 2026 13:06:40 +0200 Subject: [PATCH 176/286] chore(nginx): improve API key injection, redaction and caching Signed-off-by: Vladimir Belousov --- kustomize/base/nginx/nginx_cache.conf | 10 +++++----- .../nginx/templates/routes/intel.conf.template | 9 ++++++++- .../variables/template-variables.conf.template | 14 ++++++++++++++ src/vuln_analysis/utils/serp_api_wrapper.py | 2 -- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index 9f4e239ae..7d24b4128 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -14,17 +14,17 @@ http { proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; log_format upstream_time '$remote_addr - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' + '"$request_method $loggable_request_uri $server_protocol" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"' 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; - log_format cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + log_format cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; - log_format no_cache_log '[$time_local] traceId: $http_traceId - (BYPASSED) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + log_format no_cache_log '[$time_local] traceId: $http_traceId - (BYPASSED) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; - log_format mirror_log '[$time_local] traceId: $http_traceId - (MIRROR) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + log_format mirror_log '[$time_local] traceId: $http_traceId - (MIRROR) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; - log_format nvai_cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; + log_format nvai_cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; include /etc/nginx/conf.d/variables/*.conf; diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index 6a742a173..e6225731f 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -2,12 +2,19 @@ location /serpapi { set $backend "serpapi.com"; + + # Inject api_key if not provided by client + set $args "$api_key_prefix$args"; + rewrite ^\/serpapi(\/.*)$ $1 break; proxy_pass https://$backend; proxy_ssl_server_name on; proxy_cache intel_cache; proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; + + # Cache key without api_key (share cache across keys) + proxy_cache_key "$request_method|$request_uri_no_key"; + add_header X-Cache-Status $upstream_cache_status; client_body_buffer_size 4m; } diff --git a/kustomize/base/nginx/templates/variables/template-variables.conf.template b/kustomize/base/nginx/templates/variables/template-variables.conf.template index 4330bc950..a1cf62efb 100644 --- a/kustomize/base/nginx/templates/variables/template-variables.conf.template +++ b/kustomize/base/nginx/templates/variables/template-variables.conf.template @@ -40,3 +40,17 @@ map $http_authorization $openai_http_authorization { default $http_authorization; } +map $arg_api_key $api_key_prefix { + "" "api_key=${SERPAPI_API_KEY}&"; + default ""; +} + +map $request_uri $loggable_request_uri { + default $request_uri; + "~^(?P.*[?&]api_key=)[^&]+(?P.*)$" "$prefix[REDACTED]$suffix"; +} + +map $request_uri $request_uri_no_key { + default $request_uri; + "~^(?P.*)[?&]api_key=[^&]*(?P.*)$" "$prefix$suffix"; +} \ No newline at end of file diff --git a/src/vuln_analysis/utils/serp_api_wrapper.py b/src/vuln_analysis/utils/serp_api_wrapper.py index a412cc82d..c0100386a 100644 --- a/src/vuln_analysis/utils/serp_api_wrapper.py +++ b/src/vuln_analysis/utils/serp_api_wrapper.py @@ -51,8 +51,6 @@ async def aresults(self, query: str) -> dict: def construct_url_and_params() -> tuple[str, dict[str, str]]: params = self.get_params(query) params["source"] = "python" - if self.serpapi_api_key: - params["serp_api_key"] = self.serpapi_api_key params["output"] = "json" # Use the base path for the URL (add a "/" to ensure they get joined) From b1d4589b8a24b01de84d00e9151dbbfceb3b4794 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 21 Jan 2026 18:57:10 +0200 Subject: [PATCH 177/286] feat: implement SerpAPI key rotation for rate limit handling Signed-off-by: Vladimir Belousov --- src/vuln_analysis/utils/async_http_utils.py | 15 +- src/vuln_analysis/utils/serp_api_wrapper.py | 146 ++++++++++++++++--- tests/test_async_http_utils.py | 95 +++++++++++++ tests/test_serp_api_key_rotation.py | 147 ++++++++++++++++++++ 4 files changed, 384 insertions(+), 19 deletions(-) create mode 100644 tests/test_async_http_utils.py create mode 100644 tests/test_serp_api_key_rotation.py diff --git a/src/vuln_analysis/utils/async_http_utils.py b/src/vuln_analysis/utils/async_http_utils.py index 5d2a3c1a3..93d1b5a8d 100644 --- a/src/vuln_analysis/utils/async_http_utils.py +++ b/src/vuln_analysis/utils/async_http_utils.py @@ -78,7 +78,9 @@ async def request_with_retry(session: aiohttp.ClientSession, _P = typing.ParamSpec('_P') -def retry_async(exception_types: type[BaseException] | tuple[type[BaseException], ...] = Exception): +def retry_async(exception_types: type[BaseException] | tuple[type[BaseException], ...] = Exception, + exclude_http_statuses: list[int] | None = None + ): """ Retries an async function with exponential backoff @@ -86,14 +88,23 @@ def retry_async(exception_types: type[BaseException] | tuple[type[BaseException] ---------- exception_types : type[BaseException] | tuple[type[BaseException], ...], optional The types of exceptions to trigger a retry, by default Exception + exclude_http_statuses : list[int] | None, optional + The HTTP statuses to exclude from retries, by default None """ import tenacity + def should_retry(exception: BaseException) -> bool: + if (exclude_http_statuses + and isinstance(exception, aiohttp.ClientResponseError) + and exception.status in exclude_http_statuses): + return False + return isinstance(exception, exception_types) + def inner(func: typing.Callable[_P, typing.Awaitable[_T]]) -> typing.Callable[_P, typing.Awaitable[_T]]: @tenacity.retry(wait=tenacity.wait_exponential_jitter(0.1), stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(exception_types), + retry=tenacity.retry_if_exception(should_retry), reraise=True) async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: return await func(*args, **kwargs) diff --git a/src/vuln_analysis/utils/serp_api_wrapper.py b/src/vuln_analysis/utils/serp_api_wrapper.py index c0100386a..eecac3514 100644 --- a/src/vuln_analysis/utils/serp_api_wrapper.py +++ b/src/vuln_analysis/utils/serp_api_wrapper.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading +from typing import ClassVar + import aiohttp from langchain.utils.env import get_from_env from langchain_community.utilities import SerpAPIWrapper @@ -23,10 +26,38 @@ class MorpheusSerpAPIWrapper(SerpAPIWrapper): - + """Custom SerpAPI wrapper with multi-key rotation support. + + This wrapper extends the standard SerpAPIWrapper to support multiple API keys + for increased request capacity. When a key encounters rate limiting (429) or + payment issues (402), the wrapper automatically rotates to the next available key. + + Key rotation state is shared across all instances to ensure that when one instance + encounters a rate limit, all other instances benefit from the rotation without making + redundant failing requests. + + Attributes: + base_url: Base URL for SerpAPI service + max_retries: Maximum retry attempts for network errors + serp_api_keys: Pool of API keys loaded from SERPAPI_API_KEY env variable + serp_api_key_index: Index of currently active key in the pool + """ + _key_rotation_lock: ClassVar[threading.Lock] = threading.Lock() + _serp_api_keys: ClassVar[list[str]] = [] + _serp_api_key_index: ClassVar[int] = 0 + base_url: str = "https://serpapi.com" max_retries: int = 10 - + + @property + def serp_api_keys(self) -> list[str]: + """Shared API keys pool.""" + return self.__class__._serp_api_keys + + @property + def serp_api_key_index(self) -> int: + """Shared current key index.""" + return self.__class__._serp_api_key_index @model_validator(mode="after") def validate_base_url(self) -> "MorpheusSerpAPIWrapper": """Validate the base URL from the environment.""" @@ -36,17 +67,86 @@ def validate_base_url(self) -> "MorpheusSerpAPIWrapper": # Update the base URL for search_engine self.search_engine.BACKEND = self.base_url return self - - @retry_async() + @model_validator(mode="after") + def validate_serp_api_keys(self) -> "MorpheusSerpAPIWrapper": + """Initialize API keys pool from SERPAPI_API_KEY environment variable. + + Parses comma-separated keys from the serpapi_api_key field (populated by parent class) + and initializes the key pool. Sets the first key as the active key. + + Returns: + Self for method chaining + + Raises: + ValueError: If SERPAPI_API_KEY is empty or not set + """ + if not self.serpapi_api_key: + raise ValueError("SERPAPI_API_KEY must not be empty") + keys = [k.strip() for k in self.serpapi_api_key.split(",") if k.strip()] + + cls = self.__class__ + with cls._key_rotation_lock: + # Initialize class-level keys (first time or if keys changed) + if not cls._serp_api_keys or cls._serp_api_keys != keys: + cls._serp_api_keys = keys + cls._serp_api_key_index = 0 + # Set instance's active key to current class-level key + self.serpapi_api_key = cls._serp_api_keys[cls._serp_api_key_index] + return self + def _rotate_next_key(self) -> str: + """Rotate to the next API key in the pool. + + Advances key index cyclically through the pool and updates the active key. + This method is called when the current key encounters rate limiting (429) or + payment errors (402). All instances share the same rotation state. + + Returns: + The new active API key (returned under lock to ensure atomicity) + """ + cls = self.__class__ + with cls._key_rotation_lock: + cls._serp_api_key_index = (cls._serp_api_key_index + 1) % len(cls._serp_api_keys) + self.serpapi_api_key = cls._serp_api_keys[cls._serp_api_key_index] + return self.serpapi_api_key + @retry_async(exclude_http_statuses=[402, 429]) async def _session_get_with_retry(self, session: aiohttp.ClientSession, url: str, params: dict) -> dict: - + """Execute HTTP GET request with retry logic. + + Uses retry_async decorator to retry on network errors (timeout, 5xx), but excludes + 402 and 429 status codes from retry (these trigger key rotation in aresults). + + Args: + session: Active aiohttp client session + url: Target URL for the request + params: Query parameters including API key + + Returns: + JSON response as dictionary + + Raises: + aiohttp.ClientResponseError: For HTTP errors (including 402/429) + """ async with session.get(url, params=params) as response: - res = await response.json() - return res + response.raise_for_status() + return await response.json() - # Override the method with hardcoded URL async def aresults(self, query: str) -> dict: - """Use aiohttp to run query through SerpAPI and return the results async.""" + """Execute async SerpAPI search with automatic key rotation on rate limits. + + Implements two-level retry strategy: + 1. Network errors (timeout, 5xx) are retried by @retry_async decorator + 2. Rate limit errors (402, 429) trigger rotation to next API key + + Args: + query: Search query string + + Returns: + SerpAPI response as dictionary + + Raises: + Exception: If all API keys are exhausted + aiohttp.ClientResponseError: For other HTTP errors (400, 500, etc.) + """ def construct_url_and_params() -> tuple[str, dict[str, str]]: params = self.get_params(query) @@ -58,14 +158,26 @@ def construct_url_and_params() -> tuple[str, dict[str, str]]: return url, params url, params = construct_url_and_params() - if not self.aiosession: - async with aiohttp.ClientSession() as session: - res = await self._session_get_with_retry(session, url, params) - else: - res = await self._session_get_with_retry(self.aiosession, url, params) - - return res - + session = self.aiosession if self.aiosession else aiohttp.ClientSession() + close_session_after = not self.aiosession + attempts = 0 + max_attempts = len(self.serp_api_keys) + try: + while attempts < max_attempts: + try: + res = await self._session_get_with_retry(session, url, params) + return res + except aiohttp.ClientResponseError as e: + if e.status in [402, 429]: + attempts += 1 + if attempts < max_attempts: + params["api_key"] = self._rotate_next_key() + else: + raise + raise Exception("All API keys exhausted") + finally: + if close_session_after: + await session.close() @staticmethod def _process_response(res: dict) -> str: """Catch the ValueError and return a message if no good search result found.""" diff --git a/tests/test_async_http_utils.py b/tests/test_async_http_utils.py new file mode 100644 index 000000000..828010517 --- /dev/null +++ b/tests/test_async_http_utils.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for retry_async decorator in async_http_utils.py. +""" + +import aiohttp +import pytest +from unittest.mock import Mock + +from vuln_analysis.utils.async_http_utils import retry_async + +# Constants for test data +SUCCESS_RESULT = "Success" +RETRY_THRESHOLD = 3 # Number of attempts before success + + +@pytest.mark.asyncio +async def test_basic_retry_on_timeout(): + """Test that retry_async retries on TimeoutError.""" + call_count = 0 + + @retry_async() + async def failing_function(): + nonlocal call_count + call_count += 1 + # Fail on first 2 attempts, succeed on 3rd + if call_count < RETRY_THRESHOLD: + raise TimeoutError("Network timeout") + return SUCCESS_RESULT + + result = await failing_function() + + assert result == SUCCESS_RESULT + # Verify retry occurred: 2 failures + 1 success = 3 total attempts + assert call_count == RETRY_THRESHOLD + + +@pytest.mark.asyncio +async def test_retry_on_500(): + """Test that retry_async retries on server errors.""" + call_count = 0 + + @retry_async() + async def failing_function(): + nonlocal call_count + call_count += 1 + # Simulate transient server errors that eventually resolve + if call_count < RETRY_THRESHOLD: + raise Exception("Server error 500") + return SUCCESS_RESULT + + result = await failing_function() + + assert result == SUCCESS_RESULT + # Verify retry occurred: function called 3 times before success + assert call_count == RETRY_THRESHOLD + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [402, 429]) +async def test_no_retry_when_excluded(status_code): + """Test that retry_async does NOT retry when status is excluded.""" + call_count = 0 + + @retry_async(exclude_http_statuses=[status_code]) + async def failing_function(): + nonlocal call_count + call_count += 1 + # Always raise excluded status (should NOT be retried) + raise aiohttp.ClientResponseError( + request_info=Mock(), + history=(), + status=status_code + ) + + # Expect immediate exception without retry + with pytest.raises(aiohttp.ClientResponseError): + await failing_function() + + # Verify NO retry: should be called exactly once then exception raised + assert call_count == 1 diff --git a/tests/test_serp_api_key_rotation.py b/tests/test_serp_api_key_rotation.py new file mode 100644 index 000000000..acc8a1108 --- /dev/null +++ b/tests/test_serp_api_key_rotation.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for SerpAPI key rotation in serp_api_wrapper.py. +""" + +import re +import threading + +import pytest +from aioresponses import aioresponses +from aiohttp import ClientResponseError +from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper + +SERPAPI_SEARCH_URL_PATTERN = re.compile(r'https://serpapi\.com/search\?.*') +TEST_PAYLOAD = {"results": ["test"]} +TEST_QUERY = "test" +SINGLE_KEY = "key1" +TWO_KEYS = "key1,key2" + + +@pytest.fixture +def serpapi_wrapper_single_key(): + """Create a wrapper with a single API key.""" + # Reset class-level state before each test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + return MorpheusSerpAPIWrapper(serpapi_api_key=SINGLE_KEY) + + +@pytest.fixture +def serpapi_wrapper_two_keys(): + """Create a wrapper with two API keys.""" + # Reset class-level state before each test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + return MorpheusSerpAPIWrapper(serpapi_api_key=TWO_KEYS) + + +@pytest.mark.asyncio +async def test_single_key_success(serpapi_wrapper_single_key): + """Test that a single key is used successfully.""" + with aioresponses() as mock: + # Mock successful SerpAPI response + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=200, + payload=TEST_PAYLOAD, + repeat=True, + ) + result = await serpapi_wrapper_single_key.aresults(TEST_QUERY) + assert result == TEST_PAYLOAD + + +@pytest.mark.asyncio +async def test_single_key_failure(serpapi_wrapper_single_key): + """Test that single failed key raises exception.""" + with aioresponses() as mock: + # Mock 402 (payment required) response + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=402, + repeat=True, + ) + with pytest.raises(Exception, match="All API keys exhausted"): + await serpapi_wrapper_single_key.aresults(TEST_QUERY) + + +@pytest.mark.asyncio +async def test_all_keys_exhausted(serpapi_wrapper_two_keys): + """Test that exception is raised when all keys are exhausted.""" + with aioresponses() as mock: + # Mock 402 for all requests (both keys will fail) + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=402, + repeat=True, + ) + with pytest.raises(Exception, match="All API keys exhausted"): + await serpapi_wrapper_two_keys.aresults(TEST_QUERY) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_code", [402, 429]) +async def test_key_rotation(error_code, serpapi_wrapper_two_keys): + """Test that key rotation works for rate limit (429) and payment (402) errors.""" + with aioresponses() as mock: + # First request returns error (triggers rotation) + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=error_code, + ) + # Second request succeeds with rotated key + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=200, + payload=TEST_PAYLOAD, + repeat=True, + ) + result = await serpapi_wrapper_two_keys.aresults(TEST_QUERY) + + # Verify successful response + assert result == TEST_PAYLOAD + + # Verify rotation occurred: index advanced from 0 to 1 + assert serpapi_wrapper_two_keys.serp_api_key_index == 1 + + # Verify active key changed from "key1" to "key2" + assert serpapi_wrapper_two_keys.serpapi_api_key == "key2" + +def test_concurrent_rotation(): + """Test that concurrent key rotation is thread-safe.""" + # Reset class-level state before test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + wrapper = MorpheusSerpAPIWrapper(serpapi_api_key="key1,key2,key3") + num_threads = 10 + iterations_per_thread = 5 + + def rotate_many_times(): + for _ in range(iterations_per_thread): + key = wrapper._rotate_next_key() + # Key must always be from the pool + assert key in wrapper.serp_api_keys + print(f"{threading.current_thread().name} rotated to {key}", flush=True) + + threads = [threading.Thread(target=rotate_many_times) for _ in range(num_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + + # After all rotations, index must be valid + assert 0 <= wrapper.serp_api_key_index < len(wrapper.serp_api_keys) From e704e0719def999e948eaf0c9b36e4afda2f8a8c Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 27 Jan 2026 16:40:36 +0200 Subject: [PATCH 178/286] chore: move subscription into mlops overlay Signed-off-by: Ilona Shishov --- kustomize/base/exploit-iq-config.yml | 4 ++-- .../mlops/grafana/dashboards/batch-info.yaml | 12 ++++++------ .../mlops/grafana/dashboards/batch-metrics.yaml | 16 ++++++++-------- .../mlops/grafana/dashboards/evals.yaml | 12 ++++++------ .../overlays/mlops/grafana/dashboards/jobs.yaml | 12 ++++++------ .../mlops/grafana/dashboards/kustomization.yaml | 8 ++++++++ .../mlops/grafana/datasources/infinity.yaml | 3 ++- .../grafana/datasources/kustomization.yaml | 5 +++++ .../mlops/grafana/folders/kustomization.yaml | 7 +++++++ .../mlops/grafana/instances/grafana.yaml | 6 +----- .../mlops/grafana/instances/kustomization.yaml | 5 +++++ .../overlays/mlops/grafana/kustomization.yaml | 17 +++++++---------- .../grafana/operator/grafana-subscription.yaml} | 1 - .../mlops/grafana/operator/kustomization.yaml | 6 ++++++ .../mlops/grafana/operator/operator-group.yaml | 7 +++++++ 15 files changed, 76 insertions(+), 45 deletions(-) create mode 100644 kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml create mode 100644 kustomize/overlays/mlops/grafana/datasources/kustomization.yaml create mode 100644 kustomize/overlays/mlops/grafana/folders/kustomization.yaml create mode 100644 kustomize/overlays/mlops/grafana/instances/kustomization.yaml rename kustomize/{operators/grafana/subscription.yaml => overlays/mlops/grafana/operator/grafana-subscription.yaml} (88%) create mode 100644 kustomize/overlays/mlops/grafana/operator/kustomization.yaml create mode 100644 kustomize/overlays/mlops/grafana/operator/operator-group.yaml diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index f6bf85bd2..0d971b888 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -47,7 +47,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: CALLBACK_URL_PLACEHOLDER/api/v1/vulnerabilities/{vuln_id}/comments + endpoint: https://exploit-iq-client.eiq-test.svc:8443/api/v1/vulnerabilities/{vuln_id}/comments token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt @@ -135,7 +135,7 @@ functions: # vex_format: csaf cve_http_output: _type: cve_http_output - url: CALLBACK_URL_PLACEHOLDER + url: https://exploit-iq-client.eiq-test.svc:8443 endpoint: /api/v1/reports auth_type: bearer token_path: /var/run/secrets/kubernetes.io/serviceaccount/token diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml index f89d1861e..6cd06cda0 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml @@ -72,7 +72,7 @@ spec: "panels": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -84,7 +84,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_batch_id", @@ -101,7 +101,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -113,7 +113,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_execution_start", @@ -130,7 +130,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -142,7 +142,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_language", diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml index bef993b74..571f6eac6 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml @@ -13,7 +13,7 @@ spec: "panels": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -40,7 +40,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "confusion_matrix", @@ -59,7 +59,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -83,7 +83,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "regressive_job_count_by_batch_id", @@ -102,7 +102,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -126,7 +126,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "regressive_job_count_by_language", @@ -145,7 +145,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -169,7 +169,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "failure_job_count_by_language", diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml index efb6d032c..c4f4bc1b8 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -142,7 +142,7 @@ spec: "panels": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -154,7 +154,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_request_id", @@ -171,7 +171,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -183,7 +183,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_cve_component", @@ -200,7 +200,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -212,7 +212,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_cve_component_metric", diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml index c9c973814..1c911822f 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -142,7 +142,7 @@ spec: "panels": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -154,7 +154,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_request_id", @@ -171,7 +171,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -183,7 +183,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_batch_id", @@ -200,7 +200,7 @@ spec: }, { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "gridPos": { "h": 8, @@ -212,7 +212,7 @@ spec: "targets": [ { "datasource": { - "name": "mlops-grafana-datasource" + "uid": "infinity" }, "format": "table", "refId": "by_cve_component", diff --git a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml new file mode 100644 index 000000000..d5920d2a3 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - jobs.yaml + - evals.yaml + - batch-info.yaml + - batch-metrics.yaml diff --git a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml index a48b78ea3..6a4540cf1 100644 --- a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml +++ b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml @@ -12,7 +12,8 @@ spec: jsonData: global_queries: [] url: http://mongo-grafana-api:8000 - name: yesoreyeram-infinity-datasource + name: infinity + uid: infinity type: yesoreyeram-infinity-datasource plugins: - name: yesoreyeram-infinity-datasource diff --git a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml new file mode 100644 index 000000000..a0a2aa20a --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - infinity.yaml diff --git a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml new file mode 100644 index 000000000..675cffd1e --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - jobs.yaml + - evals.yaml + - batches.yaml diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml index 9d5efd44c..4b3553e3d 100644 --- a/kustomize/overlays/mlops/grafana/instances/grafana.yaml +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -9,16 +9,12 @@ spec: log: mode: console security: - secret: - name: grafana-secret - keyUser: grafana_user - keyPassword: grafana_password + secret: grafana-secret --- apiVersion: route.openshift.io/v1 kind: Route metadata: name: grafana - namespace: mlops spec: to: kind: Service diff --git a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml new file mode 100644 index 000000000..7070604e6 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - grafana.yaml diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 1b06b3eb1..74fe3a932 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -2,18 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - instances/grafana.yaml + - operator - - folders/jobs.yaml - - folders/evals.yaml - - folders/batches.yaml + - instances + + - folders - - datasources/infinity.yaml + - datasources - - dashboards/jobs.yaml - - dashboards/evals.yaml - - dashboards/batch-info.yaml - - dashboards/batch-metrics.yaml + - dashboards labels: - pairs: @@ -44,7 +41,7 @@ replacements: kind: Grafana name: grafana fieldPaths: - - spec.config.security.secret.name + - spec.config.security.secret secretGenerator: - name: grafana-secret diff --git a/kustomize/operators/grafana/subscription.yaml b/kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml similarity index 88% rename from kustomize/operators/grafana/subscription.yaml rename to kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml index f6d61a71a..98c4bca4d 100644 --- a/kustomize/operators/grafana/subscription.yaml +++ b/kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml @@ -2,7 +2,6 @@ apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: grafana-operator - namespace: openshift-operators spec: channel: v5 installPlanApproval: Automatic diff --git a/kustomize/overlays/mlops/grafana/operator/kustomization.yaml b/kustomize/overlays/mlops/grafana/operator/kustomization.yaml new file mode 100644 index 000000000..6ff410c89 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/operator/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - grafana-subscription.yaml + - operator-group.yaml diff --git a/kustomize/overlays/mlops/grafana/operator/operator-group.yaml b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml new file mode 100644 index 000000000..eb2a9a338 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: grafana-operators +spec: + targetNamespaces: + - REPLACE_NAMESPACE From 31b6e33426835f64316163d9b3bba4e33799f5ef Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 1 Feb 2026 11:22:47 +0200 Subject: [PATCH 179/286] add tls connection for https route Signed-off-by: Ilona Shishov --- .../mlops/grafana/dashboards/batch-info.yaml | 10 +++---- .../grafana/dashboards/batch-metrics.yaml | 8 +++--- .../mlops/grafana/dashboards/evals.yaml | 16 +++++------ .../mlops/grafana/dashboards/jobs.yaml | 16 +++++------ .../mlops/grafana/datasources/infinity.yaml | 21 +++++++++++++- .../mlops/grafana/instances/grafana.yaml | 16 +++++++++++ .../overlays/mlops/grafana/kustomization.yaml | 28 +++++++++++++++++++ .../grafana/operator/operator-group.yaml | 2 +- 8 files changed, 90 insertions(+), 27 deletions(-) diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml index 6cd06cda0..dde666c82 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml @@ -25,7 +25,7 @@ spec: "root_selector": "$.batch_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" } @@ -55,7 +55,7 @@ spec: "root_selector": "$distinct($.language)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" } @@ -90,7 +90,7 @@ spec: "refId": "by_batch_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches?batch_id=${batch_id:percentencode}", + "url": "/batch/${batch_id:percentencode}", "url_options": { "method": "GET" } @@ -119,7 +119,7 @@ spec: "refId": "by_execution_start", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches?execution_start=${execution_start}", + "url": "/batch/all?execution_start=${execution_start}", "url_options": { "method": "GET" } @@ -148,7 +148,7 @@ spec: "refId": "by_language", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches?language=${language}", + "url": "/batch/all?language=${language}", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml index 571f6eac6..9f1b8b490 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml @@ -46,7 +46,7 @@ spec: "refId": "confusion_matrix", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" }, @@ -89,7 +89,7 @@ spec: "refId": "regressive_job_count_by_batch_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" }, @@ -132,7 +132,7 @@ spec: "refId": "regressive_job_count_by_language", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" }, @@ -175,7 +175,7 @@ spec: "refId": "failure_job_count_by_language", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/batches", + "url": "/batch/all", "url_options": { "method": "GET" }, diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml index c4f4bc1b8..b223ca29e 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -25,7 +25,7 @@ spec: "root_selector": "$.request_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals", + "url": "/eval/all", "url_options": { "method": "GET" } @@ -50,7 +50,7 @@ spec: "root_selector": "$distinct($.component)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals", + "url": "/eval/all", "url_options": { "method": "GET" } @@ -75,7 +75,7 @@ spec: "root_selector": "$distinct($.component_version)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals", + "url": "/eval/all", "url_options": { "method": "GET" } @@ -100,7 +100,7 @@ spec: "root_selector": "$distinct($.cve)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals", + "url": "/eval/all", "url_options": { "method": "GET" } @@ -125,7 +125,7 @@ spec: "root_selector": "$distinct($.metric_name)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals", + "url": "/eval/all", "url_options": { "method": "GET" } @@ -160,7 +160,7 @@ spec: "refId": "by_request_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals?request_id=${request_id:percentencode}", + "url": "/eval/all?request_id=${request_id:percentencode}", "url_options": { "method": "GET" } @@ -189,7 +189,7 @@ spec: "refId": "by_cve_component", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url": "/eval/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", "url_options": { "method": "GET" } @@ -218,7 +218,7 @@ spec: "refId": "by_cve_component_metric", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/evals?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}&metric_name=${metric_name:percentencode}", + "url": "/eval/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}&metric_name=${metric_name:percentencode}", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml index 1c911822f..639a53c6b 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -25,7 +25,7 @@ spec: "root_selector": "$distinct($.batch_id)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs", + "url": "/jobs/all", "url_options": { "method": "GET" } @@ -50,7 +50,7 @@ spec: "root_selector": "$.request_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs", + "url": "/jobs/all", "url_options": { "method": "GET" } @@ -75,7 +75,7 @@ spec: "root_selector": "$distinct($.component)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs", + "url": "/jobs/all", "url_options": { "method": "GET" } @@ -100,7 +100,7 @@ spec: "root_selector": "$distinct($.component_version)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs", + "url": "/jobs/all", "url_options": { "method": "GET" } @@ -125,7 +125,7 @@ spec: "root_selector": "$distinct($.cve)", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs", + "url": "/jobs/all", "url_options": { "method": "GET" } @@ -160,7 +160,7 @@ spec: "refId": "by_request_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs/${request_id:percentencode}", + "url": "/jobs/${request_id:percentencode}", "url_options": { "method": "GET" } @@ -189,7 +189,7 @@ spec: "refId": "by_batch_id", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs?batch_id=${batch_id:percentencode}", + "url": "/jobs/all?batch_id=${batch_id:percentencode}", "url_options": { "method": "GET" } @@ -218,7 +218,7 @@ spec: "refId": "by_cve_component", "source": "url", "type": "json", - "url": "http://mongo-grafana-api:8000/jobs?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url": "/jobs/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml index 6a4540cf1..d1aa1d3af 100644 --- a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml +++ b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml @@ -3,15 +3,34 @@ kind: GrafanaDatasource metadata: name: infinity spec: + valuesFrom: + - targetPath: "secureJsonData.httpHeaderValue1" + valueFrom: + secretKeyRef: + name: "grafana-bearer-token" + key: "token" + - targetPath: "secureJsonData.tlsCACert" + valueFrom: + configMapKeyRef: + name: "openshift-service-ca.crt" + key: "service-ca.crt" instanceSelector: matchLabels: app: exploit-iq datasource: access: proxy isDefault: true + url: https://exploit-iq-client..svc:8443 jsonData: + httpHeaderName1: "Authorization" + tlsSkipVerify: false + tlsAuthWithCACert: true global_queries: [] - url: http://mongo-grafana-api:8000 + allowedHosts: + - "https://exploit-iq-client..svc:8443" + secureJsonData: + httpHeaderValue1: "Bearer ${token}" + tlsCACert: ${service-ca.crt} name: infinity uid: infinity type: yesoreyeram-infinity-datasource diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml index 4b3553e3d..bbc200227 100644 --- a/kustomize/overlays/mlops/grafana/instances/grafana.yaml +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -10,6 +10,22 @@ spec: mode: console security: secret: grafana-secret + deployment: + spec: + template: + spec: + containers: + - name: grafana + readinessProbe: + failureThreshold: 1 + httpGet: + path: /api/health + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 30 --- apiVersion: route.openshift.io/v1 kind: Route diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 74fe3a932..16fd7bf3a 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -1,6 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: REPLACE_NAMESPACE + resources: - operator @@ -15,6 +17,7 @@ resources: labels: - pairs: app: exploit-iq + component: grafana namePrefix: exploit-iq- @@ -42,6 +45,31 @@ replacements: name: grafana fieldPaths: - spec.config.security.secret + - source: + kind: Grafana + name: grafana + fieldPath: metadata.namespace + targets: + - select: + kind: GrafanaDatasource + name: infinity + fieldPaths: + - spec.datasource.url + - spec.datasource.jsonData.allowedHosts.0 + options: + delimiter: "." + index: 1 + - source: + kind: Grafana + name: grafana + fieldPath: metadata.namespace + targets: + - select: + kind: OperatorGroup + name: grafana-operators + fieldPaths: + - spec.targetNamespaces.0 + secretGenerator: - name: grafana-secret diff --git a/kustomize/overlays/mlops/grafana/operator/operator-group.yaml b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml index eb2a9a338..a5902be80 100644 --- a/kustomize/overlays/mlops/grafana/operator/operator-group.yaml +++ b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml @@ -4,4 +4,4 @@ metadata: name: grafana-operators spec: targetNamespaces: - - REPLACE_NAMESPACE + - From 1770aa1b281343d53042bb8103d6cbb284269089 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 1 Feb 2026 12:02:22 +0200 Subject: [PATCH 180/286] add CRDs to mlops overlay to ensure CR creation Signed-off-by: Ilona Shishov --- .../mlops/grafana/crds/grafanadashboards.yaml | 18323 ++++++++++++++++ .../grafana/crds/grafanadatasources.yaml | 348 + .../mlops/grafana/crds/grafanafolders.yaml | 241 + .../overlays/mlops/grafana/crds/grafanas.yaml | 8687 ++++++++ .../mlops/grafana/crds/kustomization.yaml | 8 + .../overlays/mlops/grafana/kustomization.yaml | 3 + 6 files changed, 27610 insertions(+) create mode 100644 kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml create mode 100644 kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml create mode 100644 kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml create mode 100644 kustomize/overlays/mlops/grafana/crds/grafanas.yaml create mode 100644 kustomize/overlays/mlops/grafana/crds/kustomization.yaml diff --git a/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml b/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml new file mode 100644 index 000000000..b5f5bd67d --- /dev/null +++ b/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml @@ -0,0 +1,18323 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanadashboards.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaDashboard + listKind: GrafanaDashboardList + plural: grafanadashboards + singular: grafanadashboard + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaDashboard is the Schema for the grafanadashboards API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas outside the current namespace + type: boolean + configMapRef: + description: model from configmap + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + contentCacheDuration: + description: Cache duration for models fetched from URLs + type: string + datasources: + description: maps required data sources to existing ones + items: + description: |- + GrafanaResourceDatasource is used to set the datasource name of any templated datasources in + content definitions (e.g., dashboard JSON). + properties: + datasourceName: + type: string + inputName: + type: string + required: + - datasourceName + - inputName + type: object + type: array + envFrom: + description: environments variables from secrets or config maps + items: + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: array + envs: + description: environments variables as a map + items: + properties: + name: + type: string + value: + description: Inline env value + type: string + valueFrom: + description: Reference on value source, might be the reference on a secret or config map + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + folder: + description: folder assignment for dashboard + type: string + folderRef: + description: Name of a `GrafanaFolder` resource in the same namespace + type: string + folderUID: + description: UID of the target folder for this dashboard + type: string + grafanaCom: + description: grafana.com/dashboards + properties: + id: + type: integer + revision: + type: integer + required: + - id + type: object + gzipJson: + description: GzipJson the model's JSON compressed with Gzip. Base64-encoded when in YAML. + format: byte + type: string + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + json: + description: model json + type: string + jsonnet: + description: Jsonnet + type: string + jsonnetLib: + description: Jsonnet project build + properties: + fileName: + type: string + gzipJsonnetProject: + format: byte + type: string + jPath: + items: + type: string + type: array + required: + - fileName + - gzipJsonnetProject + type: object + plugins: + description: plugins + items: + properties: + name: + minLength: 1 + type: string + version: + pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ + type: string + required: + - name + - version + type: object + type: array + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator to ignore changes + type: boolean + uid: + description: |- + Manually specify the uid, overwrites uids already present in the json model. + Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40. + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + url: + description: model url + type: string + urlAuthorization: + description: authorization options for model from url + properties: + basicAuth: + properties: + password: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - instanceSelector + type: object + x-kubernetes-validations: + - message: Only one of folderUID or folderRef can be declared at the same time + rule: (has(self.folderUID) && !(has(self.folderRef))) || (has(self.folderRef) && !(has(self.folderUID))) || !(has(self.folderRef) && (has(self.folderUID))) + - message: folder field cannot be set when folderUID or folderRef is already declared + rule: (has(self.folder) && !(has(self.folderRef) || has(self.folderUID))) || !(has(self.folder)) + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' + status: + description: GrafanaDashboardStatus defines the observed state of GrafanaDashboard + properties: + NoMatchingInstances: + description: The dashboard instanceSelector can't find matching grafana instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + contentCache: + format: byte + type: string + contentTimestamp: + format: date-time + type: string + contentUrl: + type: string + hash: + type: string + lastResync: + description: Last time the resource was synchronized with Grafana instances + format: date-time + type: string + uid: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanadatasources.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaDatasource + listKind: GrafanaDatasourceList + plural: grafanadatasources + singular: grafanadatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaDatasource is the Schema for the grafanadatasources API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas outside the current namespace + type: boolean + datasource: + properties: + access: + type: string + basicAuth: + type: boolean + basicAuthUser: + type: string + database: + type: string + editable: + description: Whether to enable/disable editing of the datasource in Grafana UI + type: boolean + isDefault: + type: boolean + jsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + orgId: + description: Deprecated field, it has no effect + format: int64 + type: integer + secureJsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + type: + type: string + uid: + description: Deprecated field, use spec.uid instead + type: string + url: + type: string + user: + type: string + type: object + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + plugins: + description: plugins + items: + properties: + name: + minLength: 1 + type: string + version: + pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ + type: string + required: + - name + - version + type: object + type: array + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator to ignore changes + type: boolean + uid: + description: |- + The UID, for the datasource, fallback to the deprecated spec.datasource.uid + and metadata.uid. Can be any string consisting of alphanumeric characters, + - and _ with a maximum length of 40 +optional + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + valuesFrom: + description: environments variables from secrets or config maps + items: + properties: + targetPath: + type: string + valueFrom: + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: Either configMapKeyRef or secretKeyRef must be set + rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) + required: + - targetPath + - valueFrom + type: object + maxItems: 99 + type: array + required: + - datasource + - instanceSelector + type: object + x-kubernetes-validations: + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' + status: + description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource + properties: + NoMatchingInstances: + description: The datasource instanceSelector can't find matching grafana instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastMessage: + description: 'Deprecated: Check status.conditions or operator logs' + type: string + lastResync: + description: Last time the resource was synchronized with Grafana instances + format: date-time + type: string + uid: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanafolders.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaFolder + listKind: GrafanaFolderList + plural: grafanafolders + singular: grafanafolder + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaFolder is the Schema for the grafanafolders API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaFolderSpec defines the desired state of GrafanaFolder + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas outside the current namespace + type: boolean + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + parentFolderRef: + description: Reference to an existing GrafanaFolder CR in the same namespace + type: string + parentFolderUID: + description: UID of the folder in which the current folder should be created + type: string + permissions: + description: Raw json with folder permissions, potentially exported from Grafana + type: string + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator to ignore changes + type: boolean + title: + description: Display name of the folder in Grafana + type: string + uid: + description: Manually specify the UID the Folder is created with. Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40 + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + required: + - instanceSelector + type: object + x-kubernetes-validations: + - message: Only one of parentFolderUID or parentFolderRef can be set + rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) && (has(self.parentFolderUID))) + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' + status: + description: GrafanaFolderStatus defines the observed state of GrafanaFolder + properties: + NoMatchingInstances: + description: The folder instanceSelector can't find matching grafana instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastResync: + description: Last time the resource was synchronized with Grafana instances + format: date-time + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanas.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: Grafana + listKind: GrafanaList + plural: grafanas + singular: grafana + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.version + name: Version + type: string + - jsonPath: .status.stage + name: Stage + type: string + - jsonPath: .status.stageStatus + name: Stage status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Grafana is the Schema for the grafanas API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaSpec defines the desired state of Grafana + properties: + client: + description: Client defines how the grafana-operator talks to the grafana instance. + properties: + headers: + additionalProperties: + type: string + description: Custom HTTP headers to use when interacting with this Grafana. + type: object + preferIngress: + description: If the operator should send it's request through the grafana instances ingress object instead of through the service. + nullable: true + type: boolean + timeout: + nullable: true + type: integer + tls: + description: TLS Configuration used to talk with the grafana instance. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate information + properties: + name: + description: name is unique within a namespace to reference a secret resource. + type: string + namespace: + description: namespace defines the space within which the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean + type: object + config: + additionalProperties: + additionalProperties: + type: string + type: object + description: Config defines how your grafana ini file should looks like. + type: object + x-kubernetes-preserve-unknown-fields: true + deployment: + description: Deployment sets how the deployment object should look like with your grafana instance, contains a number of defaults. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + paused: + type: boolean + progressDeadlineSeconds: + format: int32 + type: integer + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + os: + properties: + name: + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + default: ext4 + type: string + kind: + type: string + readOnly: + default: false + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + default: default + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + format: int32 + type: integer + signerName: + type: string + required: + - keyType + - signerName + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + default: /etc/ceph/keyring + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + default: xfs + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + default: ThinProvisioned + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + type: object + type: object + disableDefaultAdminSecret: + description: DisableDefaultAdminSecret prevents operator from creating default admin-credentials secret + type: boolean + disableDefaultSecurityContext: + description: DisableDefaultSecurityContext prevents the operator from populating securityContext on deployments + enum: + - Pod + - Container + - All + type: string + external: + description: External enables you to configure external grafana instances that is not managed by the operator. + properties: + adminPassword: + description: AdminPassword key to talk to the external grafana instance. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + adminUser: + description: AdminUser key to talk to the external grafana instance. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + apiKey: + description: The API key to talk to the external grafana instance, you need to define ether apiKey or adminUser/adminPassword. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + description: DEPRECATED, use top level `tls` instead. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate information + properties: + name: + description: name is unique within a namespace to reference a secret resource. + type: string + namespace: + description: namespace defines the space within which the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + url: + description: URL of the external grafana instance you want to manage. + type: string + required: + - url + type: object + httpRoute: + description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. It will not be created if this is not set + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: HTTPRouteSpec defines the desired state of HTTPRoute + properties: + hostnames: + description: |- + Hostnames defines a set of hostnames that should match against the HTTP Host + header to select a HTTPRoute used to process the request. Implementations + MUST ignore any port value specified in the HTTP Host header while + performing a match and (absent of any applicable header modification + configuration) MUST forward this header unmodified to the backend. + + Valid values for Hostnames are determined by RFC 1123 definition of a + hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + If a hostname is specified by both the Listener and HTTPRoute, there + must be at least one intersecting hostname for the HTTPRoute to be + attached to the Listener. For example: + + * A Listener with `test.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames, or have specified at + least one of `test.example.com` or `*.example.com`. + * A Listener with `*.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames or have specified at least + one hostname that matches the Listener hostname. For example, + `*.example.com`, `test.example.com`, and `foo.test.example.com` would + all match. On the other hand, `example.com` and `test.example.net` would + not match. + + Hostnames that are prefixed with a wildcard label (`*.`) are interpreted + as a suffix match. That means that a match for `*.example.com` would match + both `test.example.com`, and `foo.test.example.com`, but not `example.com`. + + If both the Listener and HTTPRoute have specified hostnames, any + HTTPRoute hostnames that do not match the Listener hostname MUST be + ignored. For example, if a Listener specified `*.example.com`, and the + HTTPRoute specified `test.example.com` and `test.example.net`, + `test.example.net` must not be considered for a match. + + If both the Listener and HTTPRoute have specified hostnames, and none + match with the criteria above, then the HTTPRoute is not accepted. The + implementation must raise an 'Accepted' Condition with a status of + `False` in the corresponding RouteParentStatus. + + In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. + overlapping wildcard matching and exact matching hostnames), precedence must + be given to rules from the HTTPRoute with the largest number of: + + * Characters in a matching non-wildcard hostname. + * Characters in a matching hostname. + + If ties exist across multiple Routes, the matching precedence rules for + HTTPRouteMatches takes over. + + Support: Core + items: + description: |- + Hostname is the fully qualified domain name of a network host. This matches + the RFC 1123 definition of a hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + Hostname can be "precise" which is a domain name without the terminating + dot of a network host (e.g. "foo.example.com") or "wildcard", which is a + domain name prefixed with a single wildcard label (e.g. `*.example.com`). + + Note that as per RFC1035 and RFC1123, a *label* must consist of lower case + alphanumeric characters or '-', and must start and end with an alphanumeric + character. No other punctuation is allowed. + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + type: array + parentRefs: + description: |- + ParentRefs references the resources (usually Gateways) that a Route wants + to be attached to. Note that the referenced parent resource needs to + allow this for the attachment to be complete. For Gateways, that means + the Gateway needs to allow attachment from Routes of this kind and + namespace. For Services, that means the Service must either be in the same + namespace for a "producer" route, or the mesh implementation must support + and allow "consumer" routes for the referenced Service. ReferenceGrant is + not applicable for governing ParentRefs to Services - it is not possible to + create a "producer" route for a Service in a different namespace from the + Route. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + ParentRefs must be _distinct_. This means either that: + + * They select different objects. If this is the case, then parentRef + entries are distinct. In terms of fields, this means that the + multi-part key defined by `group`, `kind`, `namespace`, and `name` must + be unique across all parentRef entries in the Route. + * They do not select different objects, but for each optional field used, + each ParentRef that selects the same object must set the same set of + optional fields to different values. If one ParentRef sets a + combination of optional fields, all must set the same combination. + + Some examples: + + * If one ParentRef sets `sectionName`, all ParentRefs referencing the + same object must also set `sectionName`. + * If one ParentRef sets `port`, all ParentRefs referencing the same + object must also set `port`. + * If one ParentRef sets `sectionName` and `port`, all ParentRefs + referencing the same object must also set `sectionName` and `port`. + + It is possible to separately reference multiple distinct objects that may + be collapsed by an implementation. For example, some implementations may + choose to merge compatible Gateway Listeners together. If that is the + case, the list of routes attached to those resources should also be + merged. + + Note that for ParentRefs that cross namespace boundaries, there are specific + rules. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example, + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable other kinds of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + + + + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + rules: + default: + - matches: + - path: + type: PathPrefix + value: / + description: |- + Rules are a list of HTTP matchers, filters and actions. + + + items: + description: |- + HTTPRouteRule defines semantics for matching an HTTP request based on + conditions (matches), processing it (filters), and forwarding the request to + an API object (backendRefs). + properties: + backendRefs: + description: |- + BackendRefs defines the backend(s) where matching requests should be + sent. + + Failure behavior here depends on how many BackendRefs are specified and + how many are invalid. + + If *all* entries in BackendRefs are invalid, and there are also no filters + specified in this route rule, *all* traffic which matches this rule MUST + receive a 500 status code. + + See the HTTPBackendRef definition for the rules about what makes a single + HTTPBackendRef invalid. + + When a HTTPBackendRef is invalid, 500 status codes MUST be returned for + requests that would have otherwise been routed to an invalid backend. If + multiple backends are specified, and some are invalid, the proportion of + requests that would otherwise have been routed to an invalid backend + MUST receive a 500 status code. + + For example, if two backends are specified with equal weights, and one is + invalid, 50 percent of traffic must receive a 500. Implementations may + choose how that 50 percent is determined. + + When a HTTPBackendRef refers to a Service that has no ready endpoints, + implementations SHOULD return a 503 for requests to that backend instead. + If an implementation chooses to do this, all of the above rules for 500 responses + MUST also apply for responses that return a 503. + + Support: Core for Kubernetes Service + + Support: Extended for Kubernetes ServiceImport + + Support: Implementation-specific for any other resource + + Support for weight: Core + items: + description: |- + HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. + + Note that when a namespace different than the local namespace is specified, a + ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + + When the BackendRef points to a Kubernetes Service, implementations SHOULD + honor the appProtocol field if it is set for the target Service Port. + + Implementations supporting appProtocol SHOULD recognize the Kubernetes + Standard Application Protocols defined in KEP-3726. + + If a Service appProtocol isn't specified, an implementation MAY infer the + backend protocol through its own means. Implementations MAY infer the + protocol from the Route type referring to the backend Service. + + If a Route is not able to send traffic to the backend using the specified + protocol then the backend is considered invalid. Implementations MUST set the + "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + + properties: + filters: + description: |- + Filters defined at this level should be executed if and only if the + request is being forwarded to the backend defined here. + + Support: Implementation-specific (For broader support of filters, use the + Filters field in HTTPRouteRule.) + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain '*' alongside other methods + rule: '!(''*'' in self && self.size() > 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type + rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil if the filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' + - message: filter.requestMirror must be specified for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' + - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' + - message: filter.urlRewrite must be specified for URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' + - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' + - message: filter.extensionRef must be specified for ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 + - message: ResponseHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: |- + Weight specifies the proportion of requests forwarded to the referenced + backend. This is computed as weight/(sum of all weights in this + BackendRefs list). For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision an + implementation supports. Weight is not a percentage and the sum of + weights does not need to equal 100. + + If only one backend is specified and it has a weight greater than 0, 100% + of the traffic is forwarded to that backend. If weight is set to 0, no + traffic should be forwarded for this entry. If unspecified, weight + defaults to 1. + + Support for this field varies based on the context where used. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + maxItems: 16 + type: array + filters: + description: |- + Filters define the filters that are applied to requests that match + this rule. + + Wherever possible, implementations SHOULD implement filters in the order + they are specified. + + Implementations MAY choose to implement this ordering strictly, rejecting + any combination or order of filters that cannot be supported. If implementations + choose a strict interpretation of filter ordering, they MUST clearly document + that behavior. + + To reject an invalid combination or order of filters, implementations SHOULD + consider the Route Rules with this configuration invalid. If all Route Rules + in a Route are invalid, the entire Route would be considered invalid. If only + a portion of Route Rules are invalid, implementations MUST set the + "PartiallyInvalid" condition for the Route. + + Conformance-levels at this level are defined based on the type of filter: + + - ALL core filters MUST be supported by all implementations. + - Implementers are encouraged to support extended filters. + - Implementation-specific custom filters have no API guarantees across + implementations. + + Specifying the same filter multiple times is not supported unless explicitly + indicated in the filter. + + All filters are expected to be compatible with each other except for the + URLRewrite and RequestRedirect filters, which may not be combined. If an + implementation cannot support other combinations of filters, they must clearly + document that limitation. In cases where incompatible or unsupported + filters are specified and cause the `Accepted` condition to be set to status + `False`, implementations may use the `IncompatibleFilters` reason to specify + this configuration error. + + Support: Core + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain '*' alongside other methods + rule: '!(''*'' in self && self.size() > 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type + rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil if the filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' + - message: filter.requestMirror must be specified for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' + - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' + - message: filter.urlRewrite must be specified for URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' + - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' + - message: filter.extensionRef must be specified for ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 + - message: ResponseHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 + matches: + default: + - path: + type: PathPrefix + value: / + description: |- + Matches define conditions used for matching the rule against incoming + HTTP requests. Each match is independent, i.e. this rule will be matched + if **any** one of the matches is satisfied. + + For example, take the following matches configuration: + + ``` + matches: + - path: + value: "/foo" + headers: + - name: "version" + value: "v2" + - path: + value: "/v2/foo" + ``` + + For a request to match against this rule, a request must satisfy + EITHER of the two conditions: + + - path prefixed with `/foo` AND contains the header `version: v2` + - path prefix of `/v2/foo` + + See the documentation for HTTPRouteMatch on how to specify multiple + match conditions that should be ANDed together. + + If no matches are specified, the default is a prefix + path match on "/", which has the effect of matching every + HTTP request. + + Proxy or Load Balancer routing configuration generated from HTTPRoutes + MUST prioritize matches based on the following criteria, continuing on + ties. Across all rules specified on applicable Routes, precedence must be + given to the match having: + + * "Exact" path match. + * "Prefix" path match with largest number of characters. + * Method match. + * Largest number of header matches. + * Largest number of query param matches. + + Note: The precedence of RegularExpression path matches are implementation-specific. + + If ties still exist across multiple Routes, matching precedence MUST be + determined in order of the following criteria, continuing on ties: + + * The oldest Route based on creation timestamp. + * The Route appearing first in alphabetical order by + "{namespace}/{name}". + + If ties still exist within an HTTPRoute, matching precedence MUST be granted + to the FIRST matching rule (in list order) with a match meeting the above + criteria. + + When no rules matching a request have been successfully attached to the + parent a request is coming from, a HTTP 404 status code MUST be returned. + items: + description: "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t value \"v1\"\n\n```" + properties: + headers: + description: |- + Headers specifies HTTP request header matchers. Multiple match values are + ANDed together, meaning, a request must match all the specified headers + to select the route. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + + When a header is repeated in an HTTP request, it is + implementation-specific behavior as to how this is represented. + Generally, proxies should follow the guidance from the RFC: + https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding + processing a repeated header, with special handling for "Set-Cookie". + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the header. + + Support: Core (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression HeaderMatchType has implementation-specific + conformance, implementations can support POSIX, PCRE or any other dialects + of regular expressions. Please read the implementation's documentation to + determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + method: + description: |- + Method specifies HTTP method matcher. + When specified, this route will be matched only if the request has the + specified method. + + Support: Extended + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + type: string + path: + default: + type: PathPrefix + value: / + description: |- + Path specifies a HTTP request path matcher. If this field is not + specified, a default prefix match on the "/" path is provided. + properties: + type: + default: PathPrefix + description: |- + Type specifies how to match against the path Value. + + Support: Core (Exact, PathPrefix) + + Support: Implementation-specific (RegularExpression) + enum: + - Exact + - PathPrefix + - RegularExpression + type: string + value: + default: / + description: Value of the HTTP path to match against. + maxLength: 1024 + type: string + type: object + x-kubernetes-validations: + - message: value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.startsWith(''/'') : true' + - message: must not contain '//' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''//'') : true' + - message: must not contain '/./' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/./'') : true' + - message: must not contain '/../' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/../'') : true' + - message: must not contain '%2f' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2f'') : true' + - message: must not contain '%2F' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2F'') : true' + - message: must not contain '#' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''#'') : true' + - message: must not end with '/..' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/..'') : true' + - message: must not end with '/.' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/.'') : true' + - message: type must be one of ['Exact', 'PathPrefix', 'RegularExpression'] + rule: self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression' + - message: must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") : true' + queryParams: + description: |- + QueryParams specifies HTTP query parameter matchers. Multiple match + values are ANDed together, meaning, a request must match all the + specified query parameters to select the route. + + Support: Extended + items: + description: |- + HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP + query parameters. + properties: + name: + description: |- + Name is the name of the HTTP query param to be matched. This must be an + exact string match. (See + https://tools.ietf.org/html/rfc7230#section-2.7.3). + + If multiple entries specify equivalent query param names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent query param name MUST be ignored. + + If a query param is repeated in an HTTP request, the behavior is + purposely left undefined, since different data planes have different + capabilities. However, it is *recommended* that implementations should + match against the first value of the param if the data plane supports it, + as this behavior is expected in other load balancing contexts outside of + the Gateway API. + + Users SHOULD NOT route traffic based on repeated query params to guard + themselves against potential differences in the implementations. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the query parameter. + + Support: Extended (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression QueryParamMatchType has Implementation-specific + conformance, implementations can support POSIX, PCRE or any other + dialects of regular expressions. Please read the implementation's + documentation to determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP query param to be matched. + maxLength: 1024 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + maxItems: 64 + type: array + name: + description: |- + Name is the name of the route rule. This name MUST be unique within a Route if it is set. + + Support: Extended + + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retry: + description: |- + Retry defines the configuration for when to retry an HTTP request. + + Support: Extended + + + properties: + attempts: + description: |- + Attempts specifies the maximum number of times an individual request + from the gateway to a backend should be retried. + + If the maximum number of retries has been attempted without a successful + response from the backend, the Gateway MUST return an error. + + When this field is unspecified, the number of times to attempt to retry + a backend request is implementation-specific. + + Support: Extended + type: integer + backoff: + description: |- + Backoff specifies the minimum duration a Gateway should wait between + retry attempts and is represented in Gateway API Duration formatting. + + For example, setting the `rules[].retry.backoff` field to the value + `100ms` will cause a backend request to first be retried approximately + 100 milliseconds after timing out or receiving a response code configured + to be retryable. + + An implementation MAY use an exponential or alternative backoff strategy + for subsequent retry attempts, MAY cap the maximum backoff duration to + some amount greater than the specified minimum, and MAY add arbitrary + jitter to stagger requests, as long as unsuccessful backend requests are + not retried before the configured minimum duration. + + If a Request timeout (`rules[].timeouts.request`) is configured on the + route, the entire duration of the initial request and any retry attempts + MUST not exceed the Request timeout duration. If any retry attempts are + still in progress when the Request timeout duration has been reached, + these SHOULD be canceled if possible and the Gateway MUST immediately + return a timeout error. + + If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is + configured on the route, any retry attempts which reach the configured + BackendRequest timeout duration without a response SHOULD be canceled if + possible and the Gateway should wait for at least the specified backoff + duration before attempting to retry the backend request again. + + If a BackendRequest timeout is _not_ configured on the route, retry + attempts MAY time out after an implementation default duration, or MAY + remain pending until a configured Request timeout or implementation + default duration for total request time is reached. + + When this field is unspecified, the time to wait between retry attempts + is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + codes: + description: |- + Codes defines the HTTP response status codes for which a backend request + should be retried. + + Support: Extended + items: + description: |- + HTTPRouteRetryStatusCode defines an HTTP response status code for + which a backend request should be retried. + + Implementations MUST support the following status codes as retryable: + + * 500 + * 502 + * 503 + * 504 + + Implementations MAY support specifying additional discrete values in the + 500-599 range. + + Implementations MAY support specifying discrete values in the 400-499 range, + which are often inadvisable to retry. + + + maximum: 599 + minimum: 400 + type: integer + type: array + type: object + sessionPersistence: + description: |- + SessionPersistence defines and configures session persistence + for the route rule. + + Support: Extended + + + properties: + absoluteTimeout: + description: |- + AbsoluteTimeout defines the absolute timeout of the persistent + session. Once the AbsoluteTimeout duration has elapsed, the + session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + cookieConfig: + description: |- + CookieConfig provides configuration settings that are specific + to cookie-based session persistence. + + Support: Core + properties: + lifetimeType: + default: Session + description: |- + LifetimeType specifies whether the cookie has a permanent or + session-based lifetime. A permanent cookie persists until its + specified expiry time, defined by the Expires or Max-Age cookie + attributes, while a session cookie is deleted when the current + session ends. + + When set to "Permanent", AbsoluteTimeout indicates the + cookie's lifetime via the Expires or Max-Age cookie attributes + and is required. + + When set to "Session", AbsoluteTimeout indicates the + absolute lifetime of the cookie tracked by the gateway and + is optional. + + Defaults to "Session". + + Support: Core for "Session" type + + Support: Extended for "Permanent" type + enum: + - Permanent + - Session + type: string + type: object + idleTimeout: + description: |- + IdleTimeout defines the idle timeout of the persistent session. + Once the session has been idle for more than the specified + IdleTimeout duration, the session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + sessionName: + description: |- + SessionName defines the name of the persistent session token + which may be reflected in the cookie or the header. Users + should avoid reusing session names to prevent unintended + consequences, such as rejection or unpredictable behavior. + + Support: Implementation-specific + maxLength: 128 + type: string + type: + default: Cookie + description: |- + Type defines the type of session persistence such as through + the use a header or cookie. Defaults to cookie based session + persistence. + + Support: Core for "Cookie" type + + Support: Extended for "Header" type + enum: + - Cookie + - Header + type: string + type: object + x-kubernetes-validations: + - message: AbsoluteTimeout must be specified when cookie lifetimeType is Permanent + rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' + timeouts: + description: |- + Timeouts defines the timeouts that can be configured for an HTTP request. + + Support: Extended + properties: + backendRequest: + description: |- + BackendRequest specifies a timeout for an individual request from the gateway + to a backend. This covers the time from when the request first starts being + sent from the gateway to when the full response has been received from the backend. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + An entire client HTTP transaction with a gateway, covered by the Request timeout, + may result in more than one call from the gateway to the destination backend, + for example, if automatic retries are supported. + + The value of BackendRequest must be a Gateway API Duration string as defined by + GEP-2257. When this field is unspecified, its behavior is implementation-specific; + when specified, the value of BackendRequest must be no more than the value of the + Request timeout (since the Request timeout encompasses the BackendRequest timeout). + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + request: + description: |- + Request specifies the maximum duration for a gateway to respond to an HTTP request. + If the gateway has not been able to respond before this deadline is met, the gateway + MUST return a timeout error. + + For example, setting the `rules.timeouts.request` field to the value `10s` in an + `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds + to complete. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + This timeout is intended to cover as close to the whole request-response transaction + as possible although an implementation MAY choose to start the timeout after the entire + request stream has been received instead of immediately after the transaction is + initiated by the client. + + The value of Request is a Gateway API Duration string as defined by GEP-2257. When this + field is unspecified, request timeout behavior is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: backendRequest timeout cannot be longer than request timeout + rule: '!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration(''0s'') && duration(self.backendRequest) > duration(self.request))' + type: object + x-kubernetes-validations: + - message: RequestRedirect filter must not be used together with backendRefs + rule: '(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true' + - message: When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less than 128 + rule: '(self.size() > 0 ? self[0].matches.size() : 0) + (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() : 0) + (self.size() > 5 ? self[5].matches.size() : 0) + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() : 0) + (self.size() > 10 ? self[10].matches.size() : 0) + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() > 12 ? self[12].matches.size() : 0) + (self.size() > 13 ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() : 0) + (self.size() > 15 ? self[15].matches.size() : 0) <= 128' + type: object + type: object + ingress: + description: Ingress sets how the ingress object should look like with your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: IngressSpec describes the Ingress the user wishes to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain name of a network host, as defined by RFC 3986.\nNote the following deviations from the \"host\" part of the\nURI as defined in RFC 3986:\n1. IPs are not allowed. Currently an IngressRuleValue can only apply to\n the IP in the Spec of the parent Ingress.\n2. The `:` delimiter is not respected because ports are not allowed.\n\t Currently the port of an Ingress is implicitly :80 for http and\n\t :443 for https.\nBoth these may change in the future.\nIncoming requests are matched against the host before the\nIngressRuleValue. If the host is unspecified, the Ingress routes all\ntraffic based on the specified IngressRuleValue.\n\nhost can be \"precise\" which is a domain name without the terminating dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\", which is a domain name\nprefixed with a single wildcard label (e.g. \"*.foo.com\").\nThe wildcard character '*' must appear by itself as the first DNS label and\nmatches only a single label. You cannot have a wildcard label by itself (e.g. Host == \"*\").\nRequests will be matched against the Host field in the following way:\n1. If host is precise, the request matches this rule if the http host header is equal to Host.\n2. If host is a wildcard, then the request matches this rule if the http host header\nis to equal to the suffix (removing the first label) of the wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + jsonnet: + properties: + libraryLabelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + persistentVolumeClaim: + description: PersistentVolumeClaim creates a PVC if you need to attach one to your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + description: PersistentVolumeMode describes how a volume is intended to be consumed, either Block or Filesystem. + type: string + volumeName: + description: VolumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + type: object + preferences: + description: Preferences holds the Grafana Preferences settings + properties: + homeDashboardUid: + type: string + type: object + route: + description: Route sets how the ingress object should look like with your grafana instance, this only works in Openshift. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + type: array + host: + type: string + path: + type: string + port: + description: RoutePort defines a port mapping from a router to an endpoint in the service endpoints. + properties: + targetPort: + anyOf: + - type: integer + - type: string + description: |- + The target port on pods selected by the service this route points to. + If this is a string, it will be looked up as a named port in the target + endpoints port list. Required + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + type: string + tls: + description: TLSConfig defines config used to secure a route and provide termination + properties: + caCertificate: + description: caCertificate provides the cert authority certificate contents + type: string + certificate: + description: |- + certificate provides certificate contents. This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. + type: string + destinationCACertificate: + description: |- + destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt + termination this file should be provided in order to have routers use it for health checks on the secure connection. + If this field is not specified, the router may provide its own destination CA and perform hostname validation using + the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically + verify. + type: string + externalCertificate: + description: |- + externalCertificate provides certificate contents as a secret reference. + This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. The secret referenced should + be present in the same namespace as that of the Route. + Forbidden when `certificate` is set. + The router service account needs to be granted with read-only access to this secret, + please refer to openshift docs for additional details. + properties: + name: + description: |- + name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + insecureEdgeTerminationPolicy: + description: |- + insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While + each router may make its own decisions on which ports to expose, this is normally port 80. + + If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". + + * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). + + * None - no traffic is allowed on the insecure port (default). + + * Redirect - clients are redirected to the secure port. + enum: + - Allow + - None + - Redirect + - "" + type: string + key: + description: key provides key file contents + type: string + termination: + description: |- + termination indicates termination type. + + * edge - TLS termination is done by the router and http is used to communicate with the backend (default) + * passthrough - Traffic is sent straight to the destination without the router providing TLS termination + * reencrypt - TLS termination is done by the router and https is used to communicate with the backend + + Note: passthrough termination is incompatible with httpHeader actions + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + x-kubernetes-validations: + - message: 'cannot have both spec.tls.termination: passthrough and spec.tls.insecureEdgeTerminationPolicy: Allow' + rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) : true' + to: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + wildcardPolicy: + description: WildcardPolicyType indicates the type of wildcard support needed by routes. + type: string + type: object + type: object + service: + description: Service sets how the service object should look like with your grafana instance, contains a number of defaults. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: ServiceSpec describes the attributes that a user creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic + is distributed to Service endpoints. Implementations can use this field + as a hint, but are not required to guarantee strict adherence. If the + field is not set, the implementation will apply its default routing + strategy. If set to "PreferClose", implementations should prioritize + endpoints that are in the same zone. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + serviceAccount: + description: ServiceAccount sets how the ServiceAccount object should look like with your grafana instance, contains a number of defaults. + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + description: ObjectReference contains enough information to let you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + suspend: + description: Suspend pauses reconciliation of owned resources like deployments, Services, Etc. upon changes + type: boolean + version: + description: |- + Version sets the tag of the default image: docker.io/grafana/grafana. + Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" + default: 12.3.0 + type: string + type: object + status: + description: GrafanaStatus defines the observed state of Grafana + properties: + adminUrl: + type: string + alertRuleGroups: + items: + type: string + type: array + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + contactPoints: + items: + type: string + type: array + dashboards: + items: + type: string + type: array + datasources: + items: + type: string + type: array + folders: + items: + type: string + type: array + lastMessage: + type: string + libraryPanels: + items: + type: string + type: array + muteTimings: + items: + type: string + type: array + notificationTemplates: + items: + type: string + type: array + serviceaccounts: + items: + type: string + type: array + stage: + type: string + stageStatus: + type: string + version: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanadatasources.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaDatasource + listKind: GrafanaDatasourceList + plural: grafanadatasources + singular: grafanadatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaDatasource is the Schema for the grafanadatasources API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas outside the current namespace + type: boolean + datasource: + properties: + access: + type: string + basicAuth: + type: boolean + basicAuthUser: + type: string + database: + type: string + editable: + description: Whether to enable/disable editing of the datasource in Grafana UI + type: boolean + isDefault: + type: boolean + jsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + orgId: + description: Deprecated field, it has no effect + format: int64 + type: integer + secureJsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + type: + type: string + uid: + description: Deprecated field, use spec.uid instead + type: string + url: + type: string + user: + type: string + type: object + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + plugins: + description: plugins + items: + properties: + name: + minLength: 1 + type: string + version: + pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ + type: string + required: + - name + - version + type: object + type: array + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator to ignore changes + type: boolean + uid: + description: |- + The UID, for the datasource, fallback to the deprecated spec.datasource.uid + and metadata.uid. Can be any string consisting of alphanumeric characters, + - and _ with a maximum length of 40 +optional + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + valuesFrom: + description: environments variables from secrets or config maps + items: + properties: + targetPath: + type: string + valueFrom: + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: Either configMapKeyRef or secretKeyRef must be set + rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) + required: + - targetPath + - valueFrom + type: object + maxItems: 99 + type: array + required: + - datasource + - instanceSelector + type: object + x-kubernetes-validations: + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' + status: + description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource + properties: + NoMatchingInstances: + description: The datasource instanceSelector can't find matching grafana instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastMessage: + description: 'Deprecated: Check status.conditions or operator logs' + type: string + lastResync: + description: Last time the resource was synchronized with Grafana instances + format: date-time + type: string + uid: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanafolders.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaFolder + listKind: GrafanaFolderList + plural: grafanafolders + singular: grafanafolder + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaFolder is the Schema for the grafanafolders API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaFolderSpec defines the desired state of GrafanaFolder + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas outside the current namespace + type: boolean + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + parentFolderRef: + description: Reference to an existing GrafanaFolder CR in the same namespace + type: string + parentFolderUID: + description: UID of the folder in which the current folder should be created + type: string + permissions: + description: Raw json with folder permissions, potentially exported from Grafana + type: string + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator to ignore changes + type: boolean + title: + description: Display name of the folder in Grafana + type: string + uid: + description: Manually specify the UID the Folder is created with. Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40 + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + required: + - instanceSelector + type: object + x-kubernetes-validations: + - message: Only one of parentFolderUID or parentFolderRef can be set + rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) && (has(self.parentFolderUID))) + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' + status: + description: GrafanaFolderStatus defines the observed state of GrafanaFolder + properties: + NoMatchingInstances: + description: The folder instanceSelector can't find matching grafana instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastResync: + description: Last time the resource was synchronized with Grafana instances + format: date-time + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanas.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: Grafana + listKind: GrafanaList + plural: grafanas + singular: grafana + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.version + name: Version + type: string + - jsonPath: .status.stage + name: Stage + type: string + - jsonPath: .status.stageStatus + name: Stage status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Grafana is the Schema for the grafanas API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaSpec defines the desired state of Grafana + properties: + client: + description: Client defines how the grafana-operator talks to the grafana instance. + properties: + headers: + additionalProperties: + type: string + description: Custom HTTP headers to use when interacting with this Grafana. + type: object + preferIngress: + description: If the operator should send it's request through the grafana instances ingress object instead of through the service. + nullable: true + type: boolean + timeout: + nullable: true + type: integer + tls: + description: TLS Configuration used to talk with the grafana instance. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate information + properties: + name: + description: name is unique within a namespace to reference a secret resource. + type: string + namespace: + description: namespace defines the space within which the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean + type: object + config: + additionalProperties: + additionalProperties: + type: string + type: object + description: Config defines how your grafana ini file should looks like. + type: object + x-kubernetes-preserve-unknown-fields: true + deployment: + description: Deployment sets how the deployment object should look like with your grafana instance, contains a number of defaults. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + paused: + type: boolean + progressDeadlineSeconds: + format: int32 + type: integer + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + os: + properties: + name: + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + default: ext4 + type: string + kind: + type: string + readOnly: + default: false + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + default: default + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + format: int32 + type: integer + signerName: + type: string + required: + - keyType + - signerName + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + default: /etc/ceph/keyring + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + default: xfs + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + default: ThinProvisioned + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + type: object + type: object + disableDefaultAdminSecret: + description: DisableDefaultAdminSecret prevents operator from creating default admin-credentials secret + type: boolean + disableDefaultSecurityContext: + description: DisableDefaultSecurityContext prevents the operator from populating securityContext on deployments + enum: + - Pod + - Container + - All + type: string + external: + description: External enables you to configure external grafana instances that is not managed by the operator. + properties: + adminPassword: + description: AdminPassword key to talk to the external grafana instance. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + adminUser: + description: AdminUser key to talk to the external grafana instance. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + apiKey: + description: The API key to talk to the external grafana instance, you need to define ether apiKey or adminUser/adminPassword. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + description: DEPRECATED, use top level `tls` instead. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate information + properties: + name: + description: name is unique within a namespace to reference a secret resource. + type: string + namespace: + description: namespace defines the space within which the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + url: + description: URL of the external grafana instance you want to manage. + type: string + required: + - url + type: object + httpRoute: + description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. It will not be created if this is not set + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: HTTPRouteSpec defines the desired state of HTTPRoute + properties: + hostnames: + description: |- + Hostnames defines a set of hostnames that should match against the HTTP Host + header to select a HTTPRoute used to process the request. Implementations + MUST ignore any port value specified in the HTTP Host header while + performing a match and (absent of any applicable header modification + configuration) MUST forward this header unmodified to the backend. + + Valid values for Hostnames are determined by RFC 1123 definition of a + hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + If a hostname is specified by both the Listener and HTTPRoute, there + must be at least one intersecting hostname for the HTTPRoute to be + attached to the Listener. For example: + + * A Listener with `test.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames, or have specified at + least one of `test.example.com` or `*.example.com`. + * A Listener with `*.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames or have specified at least + one hostname that matches the Listener hostname. For example, + `*.example.com`, `test.example.com`, and `foo.test.example.com` would + all match. On the other hand, `example.com` and `test.example.net` would + not match. + + Hostnames that are prefixed with a wildcard label (`*.`) are interpreted + as a suffix match. That means that a match for `*.example.com` would match + both `test.example.com`, and `foo.test.example.com`, but not `example.com`. + + If both the Listener and HTTPRoute have specified hostnames, any + HTTPRoute hostnames that do not match the Listener hostname MUST be + ignored. For example, if a Listener specified `*.example.com`, and the + HTTPRoute specified `test.example.com` and `test.example.net`, + `test.example.net` must not be considered for a match. + + If both the Listener and HTTPRoute have specified hostnames, and none + match with the criteria above, then the HTTPRoute is not accepted. The + implementation must raise an 'Accepted' Condition with a status of + `False` in the corresponding RouteParentStatus. + + In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. + overlapping wildcard matching and exact matching hostnames), precedence must + be given to rules from the HTTPRoute with the largest number of: + + * Characters in a matching non-wildcard hostname. + * Characters in a matching hostname. + + If ties exist across multiple Routes, the matching precedence rules for + HTTPRouteMatches takes over. + + Support: Core + items: + description: |- + Hostname is the fully qualified domain name of a network host. This matches + the RFC 1123 definition of a hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + Hostname can be "precise" which is a domain name without the terminating + dot of a network host (e.g. "foo.example.com") or "wildcard", which is a + domain name prefixed with a single wildcard label (e.g. `*.example.com`). + + Note that as per RFC1035 and RFC1123, a *label* must consist of lower case + alphanumeric characters or '-', and must start and end with an alphanumeric + character. No other punctuation is allowed. + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + type: array + parentRefs: + description: |- + ParentRefs references the resources (usually Gateways) that a Route wants + to be attached to. Note that the referenced parent resource needs to + allow this for the attachment to be complete. For Gateways, that means + the Gateway needs to allow attachment from Routes of this kind and + namespace. For Services, that means the Service must either be in the same + namespace for a "producer" route, or the mesh implementation must support + and allow "consumer" routes for the referenced Service. ReferenceGrant is + not applicable for governing ParentRefs to Services - it is not possible to + create a "producer" route for a Service in a different namespace from the + Route. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + ParentRefs must be _distinct_. This means either that: + + * They select different objects. If this is the case, then parentRef + entries are distinct. In terms of fields, this means that the + multi-part key defined by `group`, `kind`, `namespace`, and `name` must + be unique across all parentRef entries in the Route. + * They do not select different objects, but for each optional field used, + each ParentRef that selects the same object must set the same set of + optional fields to different values. If one ParentRef sets a + combination of optional fields, all must set the same combination. + + Some examples: + + * If one ParentRef sets `sectionName`, all ParentRefs referencing the + same object must also set `sectionName`. + * If one ParentRef sets `port`, all ParentRefs referencing the same + object must also set `port`. + * If one ParentRef sets `sectionName` and `port`, all ParentRefs + referencing the same object must also set `sectionName` and `port`. + + It is possible to separately reference multiple distinct objects that may + be collapsed by an implementation. For example, some implementations may + choose to merge compatible Gateway Listeners together. If that is the + case, the list of routes attached to those resources should also be + merged. + + Note that for ParentRefs that cross namespace boundaries, there are specific + rules. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example, + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable other kinds of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + + + + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + rules: + default: + - matches: + - path: + type: PathPrefix + value: / + description: |- + Rules are a list of HTTP matchers, filters and actions. + + + items: + description: |- + HTTPRouteRule defines semantics for matching an HTTP request based on + conditions (matches), processing it (filters), and forwarding the request to + an API object (backendRefs). + properties: + backendRefs: + description: |- + BackendRefs defines the backend(s) where matching requests should be + sent. + + Failure behavior here depends on how many BackendRefs are specified and + how many are invalid. + + If *all* entries in BackendRefs are invalid, and there are also no filters + specified in this route rule, *all* traffic which matches this rule MUST + receive a 500 status code. + + See the HTTPBackendRef definition for the rules about what makes a single + HTTPBackendRef invalid. + + When a HTTPBackendRef is invalid, 500 status codes MUST be returned for + requests that would have otherwise been routed to an invalid backend. If + multiple backends are specified, and some are invalid, the proportion of + requests that would otherwise have been routed to an invalid backend + MUST receive a 500 status code. + + For example, if two backends are specified with equal weights, and one is + invalid, 50 percent of traffic must receive a 500. Implementations may + choose how that 50 percent is determined. + + When a HTTPBackendRef refers to a Service that has no ready endpoints, + implementations SHOULD return a 503 for requests to that backend instead. + If an implementation chooses to do this, all of the above rules for 500 responses + MUST also apply for responses that return a 503. + + Support: Core for Kubernetes Service + + Support: Extended for Kubernetes ServiceImport + + Support: Implementation-specific for any other resource + + Support for weight: Core + items: + description: |- + HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. + + Note that when a namespace different than the local namespace is specified, a + ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + + When the BackendRef points to a Kubernetes Service, implementations SHOULD + honor the appProtocol field if it is set for the target Service Port. + + Implementations supporting appProtocol SHOULD recognize the Kubernetes + Standard Application Protocols defined in KEP-3726. + + If a Service appProtocol isn't specified, an implementation MAY infer the + backend protocol through its own means. Implementations MAY infer the + protocol from the Route type referring to the backend Service. + + If a Route is not able to send traffic to the backend using the specified + protocol then the backend is considered invalid. Implementations MUST set the + "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + + properties: + filters: + description: |- + Filters defined at this level should be executed if and only if the + request is being forwarded to the backend defined here. + + Support: Implementation-specific (For broader support of filters, use the + Filters field in HTTPRouteRule.) + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain '*' alongside other methods + rule: '!(''*'' in self && self.size() > 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type + rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil if the filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' + - message: filter.requestMirror must be specified for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' + - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' + - message: filter.urlRewrite must be specified for URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' + - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' + - message: filter.extensionRef must be specified for ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 + - message: ResponseHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: |- + Weight specifies the proportion of requests forwarded to the referenced + backend. This is computed as weight/(sum of all weights in this + BackendRefs list). For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision an + implementation supports. Weight is not a percentage and the sum of + weights does not need to equal 100. + + If only one backend is specified and it has a weight greater than 0, 100% + of the traffic is forwarded to that backend. If weight is set to 0, no + traffic should be forwarded for this entry. If unspecified, weight + defaults to 1. + + Support for this field varies based on the context where used. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + maxItems: 16 + type: array + filters: + description: |- + Filters define the filters that are applied to requests that match + this rule. + + Wherever possible, implementations SHOULD implement filters in the order + they are specified. + + Implementations MAY choose to implement this ordering strictly, rejecting + any combination or order of filters that cannot be supported. If implementations + choose a strict interpretation of filter ordering, they MUST clearly document + that behavior. + + To reject an invalid combination or order of filters, implementations SHOULD + consider the Route Rules with this configuration invalid. If all Route Rules + in a Route are invalid, the entire Route would be considered invalid. If only + a portion of Route Rules are invalid, implementations MUST set the + "PartiallyInvalid" condition for the Route. + + Conformance-levels at this level are defined based on the type of filter: + + - ALL core filters MUST be supported by all implementations. + - Implementers are encouraged to support extended filters. + - Implementation-specific custom filters have no API guarantees across + implementations. + + Specifying the same filter multiple times is not supported unless explicitly + indicated in the filter. + + All filters are expected to be compatible with each other except for the + URLRewrite and RequestRedirect filters, which may not be combined. If an + implementation cannot support other combinations of filters, they must clearly + document that limitation. In cases where incompatible or unsupported + filters are specified and cause the `Accepted` condition to be set to status + `False`, implementations may use the `IncompatibleFilters` reason to specify + this configuration error. + + Support: Core + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain '*' alongside other methods + rule: '!(''*'' in self && self.size() > 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type + rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil if the filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' + - message: filter.requestMirror must be specified for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' + - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' + - message: filter.urlRewrite must be specified for URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' + - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' + - message: filter.extensionRef must be specified for ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 + - message: ResponseHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 + matches: + default: + - path: + type: PathPrefix + value: / + description: |- + Matches define conditions used for matching the rule against incoming + HTTP requests. Each match is independent, i.e. this rule will be matched + if **any** one of the matches is satisfied. + + For example, take the following matches configuration: + + ``` + matches: + - path: + value: "/foo" + headers: + - name: "version" + value: "v2" + - path: + value: "/v2/foo" + ``` + + For a request to match against this rule, a request must satisfy + EITHER of the two conditions: + + - path prefixed with `/foo` AND contains the header `version: v2` + - path prefix of `/v2/foo` + + See the documentation for HTTPRouteMatch on how to specify multiple + match conditions that should be ANDed together. + + If no matches are specified, the default is a prefix + path match on "/", which has the effect of matching every + HTTP request. + + Proxy or Load Balancer routing configuration generated from HTTPRoutes + MUST prioritize matches based on the following criteria, continuing on + ties. Across all rules specified on applicable Routes, precedence must be + given to the match having: + + * "Exact" path match. + * "Prefix" path match with largest number of characters. + * Method match. + * Largest number of header matches. + * Largest number of query param matches. + + Note: The precedence of RegularExpression path matches are implementation-specific. + + If ties still exist across multiple Routes, matching precedence MUST be + determined in order of the following criteria, continuing on ties: + + * The oldest Route based on creation timestamp. + * The Route appearing first in alphabetical order by + "{namespace}/{name}". + + If ties still exist within an HTTPRoute, matching precedence MUST be granted + to the FIRST matching rule (in list order) with a match meeting the above + criteria. + + When no rules matching a request have been successfully attached to the + parent a request is coming from, a HTTP 404 status code MUST be returned. + items: + description: "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t value \"v1\"\n\n```" + properties: + headers: + description: |- + Headers specifies HTTP request header matchers. Multiple match values are + ANDed together, meaning, a request must match all the specified headers + to select the route. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + + When a header is repeated in an HTTP request, it is + implementation-specific behavior as to how this is represented. + Generally, proxies should follow the guidance from the RFC: + https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding + processing a repeated header, with special handling for "Set-Cookie". + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the header. + + Support: Core (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression HeaderMatchType has implementation-specific + conformance, implementations can support POSIX, PCRE or any other dialects + of regular expressions. Please read the implementation's documentation to + determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + method: + description: |- + Method specifies HTTP method matcher. + When specified, this route will be matched only if the request has the + specified method. + + Support: Extended + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + type: string + path: + default: + type: PathPrefix + value: / + description: |- + Path specifies a HTTP request path matcher. If this field is not + specified, a default prefix match on the "/" path is provided. + properties: + type: + default: PathPrefix + description: |- + Type specifies how to match against the path Value. + + Support: Core (Exact, PathPrefix) + + Support: Implementation-specific (RegularExpression) + enum: + - Exact + - PathPrefix + - RegularExpression + type: string + value: + default: / + description: Value of the HTTP path to match against. + maxLength: 1024 + type: string + type: object + x-kubernetes-validations: + - message: value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.startsWith(''/'') : true' + - message: must not contain '//' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''//'') : true' + - message: must not contain '/./' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/./'') : true' + - message: must not contain '/../' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/../'') : true' + - message: must not contain '%2f' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2f'') : true' + - message: must not contain '%2F' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2F'') : true' + - message: must not contain '#' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''#'') : true' + - message: must not end with '/..' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/..'') : true' + - message: must not end with '/.' when type one of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/.'') : true' + - message: type must be one of ['Exact', 'PathPrefix', 'RegularExpression'] + rule: self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression' + - message: must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") : true' + queryParams: + description: |- + QueryParams specifies HTTP query parameter matchers. Multiple match + values are ANDed together, meaning, a request must match all the + specified query parameters to select the route. + + Support: Extended + items: + description: |- + HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP + query parameters. + properties: + name: + description: |- + Name is the name of the HTTP query param to be matched. This must be an + exact string match. (See + https://tools.ietf.org/html/rfc7230#section-2.7.3). + + If multiple entries specify equivalent query param names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent query param name MUST be ignored. + + If a query param is repeated in an HTTP request, the behavior is + purposely left undefined, since different data planes have different + capabilities. However, it is *recommended* that implementations should + match against the first value of the param if the data plane supports it, + as this behavior is expected in other load balancing contexts outside of + the Gateway API. + + Users SHOULD NOT route traffic based on repeated query params to guard + themselves against potential differences in the implementations. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the query parameter. + + Support: Extended (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression QueryParamMatchType has Implementation-specific + conformance, implementations can support POSIX, PCRE or any other + dialects of regular expressions. Please read the implementation's + documentation to determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP query param to be matched. + maxLength: 1024 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + maxItems: 64 + type: array + name: + description: |- + Name is the name of the route rule. This name MUST be unique within a Route if it is set. + + Support: Extended + + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retry: + description: |- + Retry defines the configuration for when to retry an HTTP request. + + Support: Extended + + + properties: + attempts: + description: |- + Attempts specifies the maximum number of times an individual request + from the gateway to a backend should be retried. + + If the maximum number of retries has been attempted without a successful + response from the backend, the Gateway MUST return an error. + + When this field is unspecified, the number of times to attempt to retry + a backend request is implementation-specific. + + Support: Extended + type: integer + backoff: + description: |- + Backoff specifies the minimum duration a Gateway should wait between + retry attempts and is represented in Gateway API Duration formatting. + + For example, setting the `rules[].retry.backoff` field to the value + `100ms` will cause a backend request to first be retried approximately + 100 milliseconds after timing out or receiving a response code configured + to be retryable. + + An implementation MAY use an exponential or alternative backoff strategy + for subsequent retry attempts, MAY cap the maximum backoff duration to + some amount greater than the specified minimum, and MAY add arbitrary + jitter to stagger requests, as long as unsuccessful backend requests are + not retried before the configured minimum duration. + + If a Request timeout (`rules[].timeouts.request`) is configured on the + route, the entire duration of the initial request and any retry attempts + MUST not exceed the Request timeout duration. If any retry attempts are + still in progress when the Request timeout duration has been reached, + these SHOULD be canceled if possible and the Gateway MUST immediately + return a timeout error. + + If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is + configured on the route, any retry attempts which reach the configured + BackendRequest timeout duration without a response SHOULD be canceled if + possible and the Gateway should wait for at least the specified backoff + duration before attempting to retry the backend request again. + + If a BackendRequest timeout is _not_ configured on the route, retry + attempts MAY time out after an implementation default duration, or MAY + remain pending until a configured Request timeout or implementation + default duration for total request time is reached. + + When this field is unspecified, the time to wait between retry attempts + is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + codes: + description: |- + Codes defines the HTTP response status codes for which a backend request + should be retried. + + Support: Extended + items: + description: |- + HTTPRouteRetryStatusCode defines an HTTP response status code for + which a backend request should be retried. + + Implementations MUST support the following status codes as retryable: + + * 500 + * 502 + * 503 + * 504 + + Implementations MAY support specifying additional discrete values in the + 500-599 range. + + Implementations MAY support specifying discrete values in the 400-499 range, + which are often inadvisable to retry. + + + maximum: 599 + minimum: 400 + type: integer + type: array + type: object + sessionPersistence: + description: |- + SessionPersistence defines and configures session persistence + for the route rule. + + Support: Extended + + + properties: + absoluteTimeout: + description: |- + AbsoluteTimeout defines the absolute timeout of the persistent + session. Once the AbsoluteTimeout duration has elapsed, the + session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + cookieConfig: + description: |- + CookieConfig provides configuration settings that are specific + to cookie-based session persistence. + + Support: Core + properties: + lifetimeType: + default: Session + description: |- + LifetimeType specifies whether the cookie has a permanent or + session-based lifetime. A permanent cookie persists until its + specified expiry time, defined by the Expires or Max-Age cookie + attributes, while a session cookie is deleted when the current + session ends. + + When set to "Permanent", AbsoluteTimeout indicates the + cookie's lifetime via the Expires or Max-Age cookie attributes + and is required. + + When set to "Session", AbsoluteTimeout indicates the + absolute lifetime of the cookie tracked by the gateway and + is optional. + + Defaults to "Session". + + Support: Core for "Session" type + + Support: Extended for "Permanent" type + enum: + - Permanent + - Session + type: string + type: object + idleTimeout: + description: |- + IdleTimeout defines the idle timeout of the persistent session. + Once the session has been idle for more than the specified + IdleTimeout duration, the session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + sessionName: + description: |- + SessionName defines the name of the persistent session token + which may be reflected in the cookie or the header. Users + should avoid reusing session names to prevent unintended + consequences, such as rejection or unpredictable behavior. + + Support: Implementation-specific + maxLength: 128 + type: string + type: + default: Cookie + description: |- + Type defines the type of session persistence such as through + the use a header or cookie. Defaults to cookie based session + persistence. + + Support: Core for "Cookie" type + + Support: Extended for "Header" type + enum: + - Cookie + - Header + type: string + type: object + x-kubernetes-validations: + - message: AbsoluteTimeout must be specified when cookie lifetimeType is Permanent + rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' + timeouts: + description: |- + Timeouts defines the timeouts that can be configured for an HTTP request. + + Support: Extended + properties: + backendRequest: + description: |- + BackendRequest specifies a timeout for an individual request from the gateway + to a backend. This covers the time from when the request first starts being + sent from the gateway to when the full response has been received from the backend. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + An entire client HTTP transaction with a gateway, covered by the Request timeout, + may result in more than one call from the gateway to the destination backend, + for example, if automatic retries are supported. + + The value of BackendRequest must be a Gateway API Duration string as defined by + GEP-2257. When this field is unspecified, its behavior is implementation-specific; + when specified, the value of BackendRequest must be no more than the value of the + Request timeout (since the Request timeout encompasses the BackendRequest timeout). + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + request: + description: |- + Request specifies the maximum duration for a gateway to respond to an HTTP request. + If the gateway has not been able to respond before this deadline is met, the gateway + MUST return a timeout error. + + For example, setting the `rules.timeouts.request` field to the value `10s` in an + `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds + to complete. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + This timeout is intended to cover as close to the whole request-response transaction + as possible although an implementation MAY choose to start the timeout after the entire + request stream has been received instead of immediately after the transaction is + initiated by the client. + + The value of Request is a Gateway API Duration string as defined by GEP-2257. When this + field is unspecified, request timeout behavior is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: backendRequest timeout cannot be longer than request timeout + rule: '!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration(''0s'') && duration(self.backendRequest) > duration(self.request))' + type: object + x-kubernetes-validations: + - message: RequestRedirect filter must not be used together with backendRefs + rule: '(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true' + - message: When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + - message: Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less than 128 + rule: '(self.size() > 0 ? self[0].matches.size() : 0) + (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() : 0) + (self.size() > 5 ? self[5].matches.size() : 0) + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() : 0) + (self.size() > 10 ? self[10].matches.size() : 0) + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() > 12 ? self[12].matches.size() : 0) + (self.size() > 13 ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() : 0) + (self.size() > 15 ? self[15].matches.size() : 0) <= 128' + type: object + type: object + ingress: + description: Ingress sets how the ingress object should look like with your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: IngressSpec describes the Ingress the user wishes to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain name of a network host, as defined by RFC 3986.\nNote the following deviations from the \"host\" part of the\nURI as defined in RFC 3986:\n1. IPs are not allowed. Currently an IngressRuleValue can only apply to\n the IP in the Spec of the parent Ingress.\n2. The `:` delimiter is not respected because ports are not allowed.\n\t Currently the port of an Ingress is implicitly :80 for http and\n\t :443 for https.\nBoth these may change in the future.\nIncoming requests are matched against the host before the\nIngressRuleValue. If the host is unspecified, the Ingress routes all\ntraffic based on the specified IngressRuleValue.\n\nhost can be \"precise\" which is a domain name without the terminating dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\", which is a domain name\nprefixed with a single wildcard label (e.g. \"*.foo.com\").\nThe wildcard character '*' must appear by itself as the first DNS label and\nmatches only a single label. You cannot have a wildcard label by itself (e.g. Host == \"*\").\nRequests will be matched against the Host field in the following way:\n1. If host is precise, the request matches this rule if the http host header is equal to Host.\n2. If host is a wildcard, then the request matches this rule if the http host header\nis to equal to the suffix (removing the first label) of the wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + jsonnet: + properties: + libraryLabelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + persistentVolumeClaim: + description: PersistentVolumeClaim creates a PVC if you need to attach one to your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + description: PersistentVolumeMode describes how a volume is intended to be consumed, either Block or Filesystem. + type: string + volumeName: + description: VolumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + type: object + preferences: + description: Preferences holds the Grafana Preferences settings + properties: + homeDashboardUid: + type: string + type: object + route: + description: Route sets how the ingress object should look like with your grafana instance, this only works in Openshift. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + type: array + host: + type: string + path: + type: string + port: + description: RoutePort defines a port mapping from a router to an endpoint in the service endpoints. + properties: + targetPort: + anyOf: + - type: integer + - type: string + description: |- + The target port on pods selected by the service this route points to. + If this is a string, it will be looked up as a named port in the target + endpoints port list. Required + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + type: string + tls: + description: TLSConfig defines config used to secure a route and provide termination + properties: + caCertificate: + description: caCertificate provides the cert authority certificate contents + type: string + certificate: + description: |- + certificate provides certificate contents. This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. + type: string + destinationCACertificate: + description: |- + destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt + termination this file should be provided in order to have routers use it for health checks on the secure connection. + If this field is not specified, the router may provide its own destination CA and perform hostname validation using + the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically + verify. + type: string + externalCertificate: + description: |- + externalCertificate provides certificate contents as a secret reference. + This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. The secret referenced should + be present in the same namespace as that of the Route. + Forbidden when `certificate` is set. + The router service account needs to be granted with read-only access to this secret, + please refer to openshift docs for additional details. + properties: + name: + description: |- + name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + insecureEdgeTerminationPolicy: + description: |- + insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While + each router may make its own decisions on which ports to expose, this is normally port 80. + + If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". + + * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). + + * None - no traffic is allowed on the insecure port (default). + + * Redirect - clients are redirected to the secure port. + enum: + - Allow + - None + - Redirect + - "" + type: string + key: + description: key provides key file contents + type: string + termination: + description: |- + termination indicates termination type. + + * edge - TLS termination is done by the router and http is used to communicate with the backend (default) + * passthrough - Traffic is sent straight to the destination without the router providing TLS termination + * reencrypt - TLS termination is done by the router and https is used to communicate with the backend + + Note: passthrough termination is incompatible with httpHeader actions + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + x-kubernetes-validations: + - message: 'cannot have both spec.tls.termination: passthrough and spec.tls.insecureEdgeTerminationPolicy: Allow' + rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) : true' + to: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + wildcardPolicy: + description: WildcardPolicyType indicates the type of wildcard support needed by routes. + type: string + type: object + type: object + service: + description: Service sets how the service object should look like with your grafana instance, contains a number of defaults. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: ServiceSpec describes the attributes that a user creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic + is distributed to Service endpoints. Implementations can use this field + as a hint, but are not required to guarantee strict adherence. If the + field is not set, the implementation will apply its default routing + strategy. If set to "PreferClose", implementations should prioritize + endpoints that are in the same zone. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + serviceAccount: + description: ServiceAccount sets how the ServiceAccount object should look like with your grafana instance, contains a number of defaults. + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + description: ObjectReference contains enough information to let you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + suspend: + description: Suspend pauses reconciliation of owned resources like deployments, Services, Etc. upon changes + type: boolean + version: + description: |- + Version sets the tag of the default image: docker.io/grafana/grafana. + Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" + default: 12.3.0 + type: string + type: object + status: + description: GrafanaStatus defines the observed state of Grafana + properties: + adminUrl: + type: string + alertRuleGroups: + items: + type: string + type: array + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + contactPoints: + items: + type: string + type: array + dashboards: + items: + type: string + type: array + datasources: + items: + type: string + type: array + folders: + items: + type: string + type: array + lastMessage: + type: string + libraryPanels: + items: + type: string + type: array + muteTimings: + items: + type: string + type: array + notificationTemplates: + items: + type: string + type: array + serviceaccounts: + items: + type: string + type: array + stage: + type: string + stageStatus: + type: string + version: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml b/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml new file mode 100644 index 000000000..0106f1e11 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml @@ -0,0 +1,348 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanadatasources.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaDatasource + listKind: GrafanaDatasourceList + plural: grafanadatasources + singular: grafanadatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaDatasource is the Schema for the grafanadatasources API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas + outside the current namespace + type: boolean + datasource: + properties: + access: + type: string + basicAuth: + type: boolean + basicAuthUser: + type: string + database: + type: string + editable: + description: Whether to enable/disable editing of the datasource + in Grafana UI + type: boolean + isDefault: + type: boolean + jsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + orgId: + description: Deprecated field, it has no effect + format: int64 + type: integer + secureJsonData: + type: object + x-kubernetes-preserve-unknown-fields: true + type: + type: string + uid: + description: Deprecated field, use spec.uid instead + type: string + url: + type: string + user: + type: string + type: object + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + plugins: + description: plugins + items: + properties: + name: + minLength: 1 + type: string + version: + pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ + type: string + required: + - name + - version + type: object + type: array + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if + not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator + to ignore changes + type: boolean + uid: + description: |- + The UID, for the datasource, fallback to the deprecated spec.datasource.uid + and metadata.uid. Can be any string consisting of alphanumeric characters, + - and _ with a maximum length of 40 +optional + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + valuesFrom: + description: environments variables from secrets or config maps + items: + properties: + targetPath: + type: string + valueFrom: + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: Either configMapKeyRef or secretKeyRef must be set + rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) + || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) + required: + - targetPath + - valueFrom + type: object + maxItems: 99 + type: array + required: + - datasource + - instanceSelector + type: object + x-kubernetes-validations: + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && + has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate + to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport + && self.allowCrossNamespaceImport)' + status: + description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource + properties: + NoMatchingInstances: + description: The datasource instanceSelector can't find matching grafana + instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastMessage: + description: 'Deprecated: Check status.conditions or operator logs' + type: string + lastResync: + description: Last time the resource was synchronized with Grafana + instances + format: date-time + type: string + uid: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml b/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml new file mode 100644 index 000000000..d1f75184b --- /dev/null +++ b/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml @@ -0,0 +1,241 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanafolders.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: GrafanaFolder + listKind: GrafanaFolderList + plural: grafanafolders + singular: grafanafolder + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.NoMatchingInstances + name: No matching instances + type: boolean + - format: date-time + jsonPath: .status.lastResync + name: Last resync + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: GrafanaFolder is the Schema for the grafanafolders API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaFolderSpec defines the desired state of GrafanaFolder + properties: + allowCrossNamespaceImport: + default: false + description: Allow the Operator to match this resource with Grafanas + outside the current namespace + type: boolean + instanceSelector: + description: Selects Grafana instances for import + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: spec.instanceSelector is immutable + rule: self == oldSelf + parentFolderRef: + description: Reference to an existing GrafanaFolder CR in the same + namespace + type: string + parentFolderUID: + description: UID of the folder in which the current folder should + be created + type: string + permissions: + description: Raw json with folder permissions, potentially exported + from Grafana + type: string + resyncPeriod: + description: How often the resource is synced, defaults to 10m0s if + not set + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + suspend: + description: Suspend pauses synchronizing attempts and tells the operator + to ignore changes + type: boolean + title: + description: Display name of the folder in Grafana + type: string + uid: + description: Manually specify the UID the Folder is created with. + Can be any string consisting of alphanumeric characters, - and _ + with a maximum length of 40 + maxLength: 40 + pattern: ^[a-zA-Z0-9-_]+$ + type: string + x-kubernetes-validations: + - message: spec.uid is immutable + rule: self == oldSelf + required: + - instanceSelector + type: object + x-kubernetes-validations: + - message: Only one of parentFolderUID or parentFolderRef can be set + rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || + (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) + && (has(self.parentFolderUID))) + - message: spec.uid is immutable + rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && + has(self.uid))) + - message: disabling spec.allowCrossNamespaceImport requires a recreate + to ensure desired state + rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport + && self.allowCrossNamespaceImport)' + status: + description: GrafanaFolderStatus defines the observed state of GrafanaFolder + properties: + NoMatchingInstances: + description: The folder instanceSelector can't find matching grafana + instances + type: boolean + conditions: + description: Results when synchonizing resource with Grafana instances + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + hash: + type: string + lastResync: + description: Last time the resource was synchronized with Grafana + instances + format: date-time + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanas.yaml b/kustomize/overlays/mlops/grafana/crds/grafanas.yaml new file mode 100644 index 000000000..13c3357d2 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/crds/grafanas.yaml @@ -0,0 +1,8687 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 + labels: + olm.managed: "true" + operators.coreos.com/grafana-operator.exploit-iq-mlops: "" + name: grafanas.grafana.integreatly.org +spec: + conversion: + strategy: None + group: grafana.integreatly.org + names: + categories: + - grafana-operator + kind: Grafana + listKind: GrafanaList + plural: grafanas + singular: grafana + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.version + name: Version + type: string + - jsonPath: .status.stage + name: Stage + type: string + - jsonPath: .status.stageStatus + name: Stage status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Grafana is the Schema for the grafanas API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GrafanaSpec defines the desired state of Grafana + properties: + client: + description: Client defines how the grafana-operator talks to the + grafana instance. + properties: + headers: + additionalProperties: + type: string + description: Custom HTTP headers to use when interacting with + this Grafana. + type: object + preferIngress: + description: If the operator should send it's request through + the grafana instances ingress object instead of through the + service. + nullable: true + type: boolean + timeout: + nullable: true + type: integer + tls: + description: TLS Configuration used to talk with the grafana instance. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate + information + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set + at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) + || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean + type: object + config: + additionalProperties: + additionalProperties: + type: string + type: object + description: Config defines how your grafana ini file should looks + like. + type: object + x-kubernetes-preserve-unknown-fields: true + deployment: + description: Deployment sets how the deployment object should look + like with your grafana instance, contains a number of defaults. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + paused: + type: boolean + progressDeadlineSeconds: + format: int32 + type: integer + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + os: + properties: + name: + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + default: ext4 + type: string + kind: + type: string + readOnly: + default: false + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + default: default + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + format: int32 + type: integer + signerName: + type: string + required: + - keyType + - signerName + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + default: /etc/ceph/keyring + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + default: xfs + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + default: ThinProvisioned + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + type: object + type: object + disableDefaultAdminSecret: + description: DisableDefaultAdminSecret prevents operator from creating + default admin-credentials secret + type: boolean + disableDefaultSecurityContext: + description: DisableDefaultSecurityContext prevents the operator from + populating securityContext on deployments + enum: + - Pod + - Container + - All + type: string + external: + description: External enables you to configure external grafana instances + that is not managed by the operator. + properties: + adminPassword: + description: AdminPassword key to talk to the external grafana + instance. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + adminUser: + description: AdminUser key to talk to the external grafana instance. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + apiKey: + description: The API key to talk to the external grafana instance, + you need to define ether apiKey or adminUser/adminPassword. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + description: DEPRECATED, use top level `tls` instead. + properties: + certSecretRef: + description: Use a secret as a reference to give TLS Certificate + information + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + description: Disable the CA check of the server + type: boolean + type: object + x-kubernetes-validations: + - message: insecureSkipVerify and certSecretRef cannot be set + at the same time + rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) + || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + url: + description: URL of the external grafana instance you want to + manage. + type: string + required: + - url + type: object + httpRoute: + description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. + It will not be created if this is not set + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: HTTPRouteSpec defines the desired state of HTTPRoute + properties: + hostnames: + description: |- + Hostnames defines a set of hostnames that should match against the HTTP Host + header to select a HTTPRoute used to process the request. Implementations + MUST ignore any port value specified in the HTTP Host header while + performing a match and (absent of any applicable header modification + configuration) MUST forward this header unmodified to the backend. + + Valid values for Hostnames are determined by RFC 1123 definition of a + hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + If a hostname is specified by both the Listener and HTTPRoute, there + must be at least one intersecting hostname for the HTTPRoute to be + attached to the Listener. For example: + + * A Listener with `test.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames, or have specified at + least one of `test.example.com` or `*.example.com`. + * A Listener with `*.example.com` as the hostname matches HTTPRoutes + that have either not specified any hostnames or have specified at least + one hostname that matches the Listener hostname. For example, + `*.example.com`, `test.example.com`, and `foo.test.example.com` would + all match. On the other hand, `example.com` and `test.example.net` would + not match. + + Hostnames that are prefixed with a wildcard label (`*.`) are interpreted + as a suffix match. That means that a match for `*.example.com` would match + both `test.example.com`, and `foo.test.example.com`, but not `example.com`. + + If both the Listener and HTTPRoute have specified hostnames, any + HTTPRoute hostnames that do not match the Listener hostname MUST be + ignored. For example, if a Listener specified `*.example.com`, and the + HTTPRoute specified `test.example.com` and `test.example.net`, + `test.example.net` must not be considered for a match. + + If both the Listener and HTTPRoute have specified hostnames, and none + match with the criteria above, then the HTTPRoute is not accepted. The + implementation must raise an 'Accepted' Condition with a status of + `False` in the corresponding RouteParentStatus. + + In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. + overlapping wildcard matching and exact matching hostnames), precedence must + be given to rules from the HTTPRoute with the largest number of: + + * Characters in a matching non-wildcard hostname. + * Characters in a matching hostname. + + If ties exist across multiple Routes, the matching precedence rules for + HTTPRouteMatches takes over. + + Support: Core + items: + description: |- + Hostname is the fully qualified domain name of a network host. This matches + the RFC 1123 definition of a hostname with 2 notable exceptions: + + 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + label must appear by itself as the first label. + + Hostname can be "precise" which is a domain name without the terminating + dot of a network host (e.g. "foo.example.com") or "wildcard", which is a + domain name prefixed with a single wildcard label (e.g. `*.example.com`). + + Note that as per RFC1035 and RFC1123, a *label* must consist of lower case + alphanumeric characters or '-', and must start and end with an alphanumeric + character. No other punctuation is allowed. + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + type: array + parentRefs: + description: |- + ParentRefs references the resources (usually Gateways) that a Route wants + to be attached to. Note that the referenced parent resource needs to + allow this for the attachment to be complete. For Gateways, that means + the Gateway needs to allow attachment from Routes of this kind and + namespace. For Services, that means the Service must either be in the same + namespace for a "producer" route, or the mesh implementation must support + and allow "consumer" routes for the referenced Service. ReferenceGrant is + not applicable for governing ParentRefs to Services - it is not possible to + create a "producer" route for a Service in a different namespace from the + Route. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + ParentRefs must be _distinct_. This means either that: + + * They select different objects. If this is the case, then parentRef + entries are distinct. In terms of fields, this means that the + multi-part key defined by `group`, `kind`, `namespace`, and `name` must + be unique across all parentRef entries in the Route. + * They do not select different objects, but for each optional field used, + each ParentRef that selects the same object must set the same set of + optional fields to different values. If one ParentRef sets a + combination of optional fields, all must set the same combination. + + Some examples: + + * If one ParentRef sets `sectionName`, all ParentRefs referencing the + same object must also set `sectionName`. + * If one ParentRef sets `port`, all ParentRefs referencing the same + object must also set `port`. + * If one ParentRef sets `sectionName` and `port`, all ParentRefs + referencing the same object must also set `sectionName` and `port`. + + It is possible to separately reference multiple distinct objects that may + be collapsed by an implementation. For example, some implementations may + choose to merge compatible Gateway Listeners together. If that is the + case, the list of routes attached to those resources should also be + merged. + + Note that for ParentRefs that cross namespace boundaries, there are specific + rules. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example, + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable other kinds of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + + + + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + rules: + default: + - matches: + - path: + type: PathPrefix + value: / + description: |- + Rules are a list of HTTP matchers, filters and actions. + + + items: + description: |- + HTTPRouteRule defines semantics for matching an HTTP request based on + conditions (matches), processing it (filters), and forwarding the request to + an API object (backendRefs). + properties: + backendRefs: + description: |- + BackendRefs defines the backend(s) where matching requests should be + sent. + + Failure behavior here depends on how many BackendRefs are specified and + how many are invalid. + + If *all* entries in BackendRefs are invalid, and there are also no filters + specified in this route rule, *all* traffic which matches this rule MUST + receive a 500 status code. + + See the HTTPBackendRef definition for the rules about what makes a single + HTTPBackendRef invalid. + + When a HTTPBackendRef is invalid, 500 status codes MUST be returned for + requests that would have otherwise been routed to an invalid backend. If + multiple backends are specified, and some are invalid, the proportion of + requests that would otherwise have been routed to an invalid backend + MUST receive a 500 status code. + + For example, if two backends are specified with equal weights, and one is + invalid, 50 percent of traffic must receive a 500. Implementations may + choose how that 50 percent is determined. + + When a HTTPBackendRef refers to a Service that has no ready endpoints, + implementations SHOULD return a 503 for requests to that backend instead. + If an implementation chooses to do this, all of the above rules for 500 responses + MUST also apply for responses that return a 503. + + Support: Core for Kubernetes Service + + Support: Extended for Kubernetes ServiceImport + + Support: Implementation-specific for any other resource + + Support for weight: Core + items: + description: |- + HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. + + Note that when a namespace different than the local namespace is specified, a + ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + + When the BackendRef points to a Kubernetes Service, implementations SHOULD + honor the appProtocol field if it is set for the target Service Port. + + Implementations supporting appProtocol SHOULD recognize the Kubernetes + Standard Application Protocols defined in KEP-3726. + + If a Service appProtocol isn't specified, an implementation MAY infer the + backend protocol through its own means. Implementations MAY infer the + protocol from the Route type referring to the backend Service. + + If a Route is not able to send traffic to the backend using the specified + protocol then the backend is considered invalid. Implementations MUST set the + "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + + properties: + filters: + description: |- + Filters defined at this level should be executed if and only if the + request is being forwarded to the backend defined here. + + Support: Implementation-specific (For broader support of filters, use the + Filters field in HTTPRouteRule.) + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain + '*' alongside other methods + rule: '!(''*'' in self && self.size() + > 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents + an HTTP Header name and value as + defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents + an HTTP Header name and value as + defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of + the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service + reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) + : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction + may be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified + when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' + ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' + when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? + self.type == ''ReplaceFullPath'' + : true' + - message: replacePrefixMatch must be + specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' + ? has(self.replacePrefixMatch) : + true' + - message: type must be 'ReplacePrefixMatch' + when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) + ? self.type == ''ReplacePrefixMatch'' + : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents + an HTTP Header name and value as + defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents + an HTTP Header name and value as + defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified + when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' + ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' + when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? + self.type == ''ReplaceFullPath'' + : true' + - message: replacePrefixMatch must be + specified when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' + ? has(self.replacePrefixMatch) : + true' + - message: type must be 'ReplacePrefixMatch' + when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) + ? self.type == ''ReplacePrefixMatch'' + : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must + be nil if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && + self.type != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must + be specified for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) + && self.type == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must + be nil if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) + && self.type != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must + be specified for ResponseHeaderModifier + filter.type + rule: '!(!has(self.responseHeaderModifier) + && self.type == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil + if the filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type + != ''RequestMirror'')' + - message: filter.requestMirror must be specified + for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type + == ''RequestMirror'')' + - message: filter.requestRedirect must be nil + if the filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type + != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified + for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type + == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if + the filter.type is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type + != ''URLRewrite'')' + - message: filter.urlRewrite must be specified + for URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type + == ''URLRewrite'')' + - message: filter.extensionRef must be nil if + the filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type + != ''ExtensionRef'')' + - message: filter.extensionRef must be specified + for ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type + == ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect + or httpRouteFilterRequestRewrite, but not + both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') + && self.exists(f, f.type == ''URLRewrite''))' + - message: May specify either httpRouteFilterRequestRedirect + or httpRouteFilterRequestRewrite, but not + both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') + && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot + be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() + <= 1 + - message: ResponseHeaderModifier filter cannot + be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() + <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() + <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() + <= 1 + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: |- + Weight specifies the proportion of requests forwarded to the referenced + backend. This is computed as weight/(sum of all weights in this + BackendRefs list). For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision an + implementation supports. Weight is not a percentage and the sum of + weights does not need to equal 100. + + If only one backend is specified and it has a weight greater than 0, 100% + of the traffic is forwarded to that backend. If weight is set to 0, no + traffic should be forwarded for this entry. If unspecified, weight + defaults to 1. + + Support for this field varies based on the context where used. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + maxItems: 16 + type: array + filters: + description: |- + Filters define the filters that are applied to requests that match + this rule. + + Wherever possible, implementations SHOULD implement filters in the order + they are specified. + + Implementations MAY choose to implement this ordering strictly, rejecting + any combination or order of filters that cannot be supported. If implementations + choose a strict interpretation of filter ordering, they MUST clearly document + that behavior. + + To reject an invalid combination or order of filters, implementations SHOULD + consider the Route Rules with this configuration invalid. If all Route Rules + in a Route are invalid, the entire Route would be considered invalid. If only + a portion of Route Rules are invalid, implementations MUST set the + "PartiallyInvalid" condition for the Route. + + Conformance-levels at this level are defined based on the type of filter: + + - ALL core filters MUST be supported by all implementations. + - Implementers are encouraged to support extended filters. + - Implementation-specific custom filters have no API guarantees across + implementations. + + Specifying the same filter multiple times is not supported unless explicitly + indicated in the filter. + + All filters are expected to be compatible with each other except for the + URLRewrite and RequestRedirect filters, which may not be combined. If an + implementation cannot support other combinations of filters, they must clearly + document that limitation. In cases where incompatible or unsupported + filters are specified and cause the `Accepted` condition to be set to status + `False`, implementations may use the `IncompatibleFilters` reason to specify + this configuration error. + + Support: Core + items: + description: |- + HTTPRouteFilter defines processing steps that must be completed during the + request or response lifecycle. HTTPRouteFilters are meant as an extension + point to express processing that may be done in Gateway implementations. Some + examples include request or response modification, implementing + authentication strategies, rate-limiting, and traffic shaping. API + guarantee/conformance is defined based on the type of the filter. + + + + properties: + cors: + description: |- + CORS defines a schema for a filter that responds to the + cross-origin request based on HTTP response header. + + Support: Extended + + + properties: + allowCredentials: + description: |- + AllowCredentials indicates whether the actual cross-origin request allows + to include credentials. + + The only valid value for the `Access-Control-Allow-Credentials` response + header is true (case-sensitive). + + If the credentials are not allowed in cross-origin requests, the gateway + will omit the header `Access-Control-Allow-Credentials` entirely rather + than setting its value to false. + + Support: Extended + enum: + - true + type: boolean + allowHeaders: + description: |- + AllowHeaders indicates which HTTP request headers are supported for + accessing the requested resource. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Allow-Headers` + response header are separated by a comma (","). + + When the `AllowHeaders` field is configured with one or more headers, the + gateway must return the `Access-Control-Allow-Headers` response header + which value is present in the `AllowHeaders` field. + + If any header name in the `Access-Control-Request-Headers` request header + is not included in the list of header names specified by the response + header `Access-Control-Allow-Headers`, it will present an error on the + client side. + + If any header name in the `Access-Control-Allow-Headers` response header + does not recognize by the client, it will also occur an error on the + client side. + + A wildcard indicates that the requests with all HTTP headers are allowed. + The `Access-Control-Allow-Headers` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowHeaders` field + specified with the `*` wildcard, the gateway must specify one or more + HTTP headers in the value of the `Access-Control-Allow-Headers` response + header. The value of the header `Access-Control-Allow-Headers` is same as + the `Access-Control-Request-Headers` header provided by the client. If + the header `Access-Control-Request-Headers` is not included in the + request, the gateway will omit the `Access-Control-Allow-Headers` + response header, instead of specifying the `*` wildcard. A Gateway + implementation may choose to add implementation-specific default headers. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + allowMethods: + description: |- + AllowMethods indicates which HTTP methods are supported for accessing the + requested resource. + + Valid values are any method defined by RFC9110, along with the special + value `*`, which represents all HTTP methods are allowed. + + Method names are case sensitive, so these values are also case-sensitive. + (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + + Multiple method names in the value of the `Access-Control-Allow-Methods` + response header are separated by a comma (","). + + A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. + (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The + CORS-safelisted methods are always allowed, regardless of whether they + are specified in the `AllowMethods` field. + + When the `AllowMethods` field is configured with one or more methods, the + gateway must return the `Access-Control-Allow-Methods` response header + which value is present in the `AllowMethods` field. + + If the HTTP method of the `Access-Control-Request-Method` request header + is not included in the list of methods specified by the response header + `Access-Control-Allow-Methods`, it will present an error on the client + side. + + The `Access-Control-Allow-Methods` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowMethods` field + specified with the `*` wildcard, the gateway must specify one HTTP method + in the value of the Access-Control-Allow-Methods response header. The + value of the header `Access-Control-Allow-Methods` is same as the + `Access-Control-Request-Method` header provided by the client. If the + header `Access-Control-Request-Method` is not included in the request, + the gateway will omit the `Access-Control-Allow-Methods` response header, + instead of specifying the `*` wildcard. A Gateway implementation may + choose to add implementation-specific default methods. + + Support: Extended + items: + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + - '*' + type: string + maxItems: 9 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: AllowMethods cannot contain '*' + alongside other methods + rule: '!(''*'' in self && self.size() > + 1)' + allowOrigins: + description: |- + AllowOrigins indicates whether the response can be shared with requested + resource from the given `Origin`. + + The `Origin` consists of a scheme and a host, with an optional port, and + takes the form `://(:)`. + + Valid values for scheme are: `http` and `https`. + + Valid values for port are any integer between 1 and 65535 (the list of + available TCP/UDP ports). Note that, if not included, port `80` is + assumed for `http` scheme origins, and port `443` is assumed for `https` + origins. This may affect origin matching. + + The host part of the origin may contain the wildcard character `*`. These + wildcard characters behave as follows: + + * `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. + * A wildcard by itself matches all hosts. + + An origin value that includes _only_ the `*` character indicates requests + from all `Origin`s are allowed. + + When the `AllowOrigins` field is configured with multiple origins, it + means the server supports clients from multiple origins. If the request + `Origin` matches the configured allowed origins, the gateway must return + the given `Origin` and sets value of the header + `Access-Control-Allow-Origin` same as the `Origin` header provided by the + client. + + The status code of a successful response to a "preflight" request is + always an OK status (i.e., 204 or 200). + + If the request `Origin` does not match the configured allowed origins, + the gateway returns 204/200 response but doesn't set the relevant + cross-origin response headers. Alternatively, the gateway responds with + 403 status to the "preflight" request is denied, coupled with omitting + the CORS headers. The cross-origin request fails on the client side. + Therefore, the client doesn't attempt the actual cross-origin request. + + The `Access-Control-Allow-Origin` response header can only use `*` + wildcard as value when the `AllowCredentials` field is unspecified. + + When the `AllowCredentials` field is specified and `AllowOrigins` field + specified with the `*` wildcard, the gateway must return a single origin + in the value of the `Access-Control-Allow-Origin` response header, + instead of specifying the `*` wildcard. The value of the header + `Access-Control-Allow-Origin` is same as the `Origin` header provided by + the client. + + Support: Extended + items: + description: |- + The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and + encoding rules specified in RFC3986. The AbsoluteURI MUST include both a + scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that + include an authority MUST include a fully qualified domain name or + IP address as the host. + The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. + maxLength: 253 + minLength: 1 + pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + exposeHeaders: + description: |- + ExposeHeaders indicates which HTTP response headers can be exposed + to client-side scripts in response to a cross-origin request. + + A CORS-safelisted response header is an HTTP header in a CORS response + that it is considered safe to expose to the client scripts. + The CORS-safelisted response headers include the following headers: + `Cache-Control` + `Content-Language` + `Content-Length` + `Content-Type` + `Expires` + `Last-Modified` + `Pragma` + (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) + The CORS-safelisted response headers are exposed to client by default. + + When an HTTP header name is specified using the `ExposeHeaders` field, + this additional header will be exposed as part of the response to the + client. + + Header names are not case sensitive. + + Multiple header names in the value of the `Access-Control-Expose-Headers` + response header are separated by a comma (","). + + A wildcard indicates that the responses with all HTTP headers are exposed + to clients. The `Access-Control-Expose-Headers` response header can only + use `*` wildcard as value when the `AllowCredentials` field is + unspecified. + + Support: Extended + items: + description: |- + HTTPHeaderName is the name of an HTTP header. + + Valid values include: + + * "Authorization" + * "Set-Cookie" + + Invalid values include: + + - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo + headers are not currently supported by this type. + - "/invalid" - "/ " is an invalid character + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: set + maxAge: + default: 5 + description: |- + MaxAge indicates the duration (in seconds) for the client to cache the + results of a "preflight" request. + + The information provided by the `Access-Control-Allow-Methods` and + `Access-Control-Allow-Headers` response headers can be cached by the + client until the time specified by `Access-Control-Max-Age` elapses. + + The default value of `Access-Control-Max-Age` response header is 5 + (seconds). + format: int32 + minimum: 1 + type: integer + type: object + extensionRef: + description: |- + ExtensionRef is an optional, implementation-specific extension to the + "filter" behavior. For example, resource "myroutefilter" in group + "networking.example.net"). ExtensionRef MUST NOT be used for core and + extended filters. + + This filter can be used multiple times within the same rule. + + Support: Implementation-specific + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: |- + RequestHeaderModifier defines a schema for a filter that modifies request + headers. + + Support: Core + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: |- + RequestMirror defines a schema for a filter that mirrors requests. + Requests are sent to the specified destination, but responses from + that destination are ignored. + + This filter can be used multiple times within the same rule. Note that + not all implementations will be able to support mirroring to multiple + backends. + + Support: Extended + properties: + backendRef: + description: |- + BackendRef references a resource where mirrored requests are sent. + + Mirrored requests must be sent only to a single destination endpoint + within this BackendRef, irrespective of how many endpoints are present + within this BackendRef. + + If the referent cannot be found, this BackendRef is invalid and must be + dropped from the Gateway. The controller must ensure the "ResolvedRefs" + condition on the Route status is set to `status: False` and not configure + this backend in the underlying implementation. + + If there is a cross-namespace reference to an *existing* object + that is not allowed by a ReferenceGrant, the controller must ensure the + "ResolvedRefs" condition on the Route is set to `status: False`, + with the "RefNotPermitted" reason and not configure this backend in the + underlying implementation. + + In either error case, the Message of the `ResolvedRefs` Condition + should be used to provide more detail about the problem. + + Support: Extended for Kubernetes Service + + Support: Implementation-specific for any other resource + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + fraction: + description: |- + Fraction represents the fraction of requests that should be + mirrored to BackendRef. + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or + equal to denominator + rule: self.numerator <= self.denominator + percent: + description: |- + Percent represents the percentage of requests that should be + mirrored to BackendRef. Its minimum value is 0 (indicating 0% of + requests) and its maximum value is 100 (indicating 100% of requests). + + Only one of Fraction or Percent may be specified. If neither field + is specified, 100% of requests will be mirrored. + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - backendRef + type: object + x-kubernetes-validations: + - message: Only one of percent or fraction may + be specified in HTTPRequestMirrorFilter + rule: '!(has(self.percent) && has(self.fraction))' + requestRedirect: + description: |- + RequestRedirect defines a schema for a filter that responds to the + request with an HTTP redirection. + + Support: Core + properties: + hostname: + description: |- + Hostname is the hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname in the `Host` header of the request is used. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines parameters used to modify the path of the incoming request. + The modified path is then used to construct the `Location` header. When + empty, the request path is used as-is. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified + when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' + ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' + when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type + == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified + when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' + ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' + when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type + == ''ReplacePrefixMatch'' : true' + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + + If no port is specified, the redirect port MUST be derived using the + following rules: + + * If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. + * If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + + Implementations SHOULD NOT add the port number in the 'Location' + header in the following cases: + + * A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. + * A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` header in + the response. When empty, the scheme of the request is used. + + Scheme redirects can affect the port of the redirect, for more information, + refer to the documentation for the port field of this filter. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Extended + enum: + - http + - https + type: string + statusCode: + default: 302 + description: |- + StatusCode is the HTTP status code to be used in response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + Support: Core + enum: + - 301 + - 302 + type: integer + type: object + responseHeaderModifier: + description: |- + ResponseHeaderModifier defines a schema for a filter that modifies response + headers. + + Support: Extended + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: + description: |- + Type identifies the type of filter to apply. As with other API fields, + types are classified into three conformance levels: + + - Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + + - Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + + - Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + + Implementers are encouraged to define custom implementation types to + extend the core API with implementation-specific behavior. + + If a reference to a custom filter type cannot be resolved, the filter + MUST NOT be skipped. Instead, requests that would have been processed by + that filter MUST receive a HTTP error response. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + + + enum: + - RequestHeaderModifier + - ResponseHeaderModifier + - RequestMirror + - RequestRedirect + - URLRewrite + - ExtensionRef + type: string + urlRewrite: + description: |- + URLRewrite defines a schema for a filter that modifies a request during forwarding. + + Support: Extended + properties: + hostname: + description: |- + Hostname is the value to be used to replace the Host header value during + forwarding. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path defines a path rewrite. + + Support: Extended + properties: + replaceFullPath: + description: |- + ReplaceFullPath specifies the value with which to replace the full path + of a request during a rewrite or redirect. + maxLength: 1024 + type: string + replacePrefixMatch: + description: |- + ReplacePrefixMatch specifies the value with which to replace the prefix + match of a request during a rewrite or redirect. For example, a request + to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch + of "/xyz" would be modified to "/xyz/bar". + + Note that this matches the behavior of the PathPrefix match type. This + matches full path elements. A path element refers to the list of labels + in the path split by the `/` separator. When specified, a trailing `/` is + ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all + match the prefix `/abc`, but the path `/abcd` would not. + + ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. + Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in + the implementation setting the Accepted Condition for the Route to `status: False`. + + Request Path | Prefix Match | Replace Prefix | Modified Path + maxLength: 1024 + type: string + type: + description: |- + Type defines the type of path modifier. Additional types may be + added in a future release of the API. + + Note that values may be added to this enum, implementations + must ensure that unknown values will not cause a crash. + + Unknown values here must result in the implementation setting the + Accepted Condition for the Route to `status: False`, with a + Reason of `UnsupportedValue`. + enum: + - ReplaceFullPath + - ReplacePrefixMatch + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: replaceFullPath must be specified + when type is set to 'ReplaceFullPath' + rule: 'self.type == ''ReplaceFullPath'' + ? has(self.replaceFullPath) : true' + - message: type must be 'ReplaceFullPath' + when replaceFullPath is set + rule: 'has(self.replaceFullPath) ? self.type + == ''ReplaceFullPath'' : true' + - message: replacePrefixMatch must be specified + when type is set to 'ReplacePrefixMatch' + rule: 'self.type == ''ReplacePrefixMatch'' + ? has(self.replacePrefixMatch) : true' + - message: type must be 'ReplacePrefixMatch' + when replacePrefixMatch is set + rule: 'has(self.replacePrefixMatch) ? self.type + == ''ReplacePrefixMatch'' : true' + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: filter.requestHeaderModifier must be nil + if the filter.type is not RequestHeaderModifier + rule: '!(has(self.requestHeaderModifier) && self.type + != ''RequestHeaderModifier'')' + - message: filter.requestHeaderModifier must be specified + for RequestHeaderModifier filter.type + rule: '!(!has(self.requestHeaderModifier) && self.type + == ''RequestHeaderModifier'')' + - message: filter.responseHeaderModifier must be nil + if the filter.type is not ResponseHeaderModifier + rule: '!(has(self.responseHeaderModifier) && self.type + != ''ResponseHeaderModifier'')' + - message: filter.responseHeaderModifier must be specified + for ResponseHeaderModifier filter.type + rule: '!(!has(self.responseHeaderModifier) && self.type + == ''ResponseHeaderModifier'')' + - message: filter.requestMirror must be nil if the + filter.type is not RequestMirror + rule: '!(has(self.requestMirror) && self.type != + ''RequestMirror'')' + - message: filter.requestMirror must be specified + for RequestMirror filter.type + rule: '!(!has(self.requestMirror) && self.type == + ''RequestMirror'')' + - message: filter.requestRedirect must be nil if the + filter.type is not RequestRedirect + rule: '!(has(self.requestRedirect) && self.type + != ''RequestRedirect'')' + - message: filter.requestRedirect must be specified + for RequestRedirect filter.type + rule: '!(!has(self.requestRedirect) && self.type + == ''RequestRedirect'')' + - message: filter.urlRewrite must be nil if the filter.type + is not URLRewrite + rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' + - message: filter.urlRewrite must be specified for + URLRewrite filter.type + rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' + - message: filter.extensionRef must be nil if the + filter.type is not ExtensionRef + rule: '!(has(self.extensionRef) && self.type != + ''ExtensionRef'')' + - message: filter.extensionRef must be specified for + ExtensionRef filter.type + rule: '!(!has(self.extensionRef) && self.type == + ''ExtensionRef'')' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: May specify either httpRouteFilterRequestRedirect + or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') + && self.exists(f, f.type == ''URLRewrite''))' + - message: RequestHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'RequestHeaderModifier').size() + <= 1 + - message: ResponseHeaderModifier filter cannot be repeated + rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() + <= 1 + - message: RequestRedirect filter cannot be repeated + rule: self.filter(f, f.type == 'RequestRedirect').size() + <= 1 + - message: URLRewrite filter cannot be repeated + rule: self.filter(f, f.type == 'URLRewrite').size() + <= 1 + matches: + default: + - path: + type: PathPrefix + value: / + description: |- + Matches define conditions used for matching the rule against incoming + HTTP requests. Each match is independent, i.e. this rule will be matched + if **any** one of the matches is satisfied. + + For example, take the following matches configuration: + + ``` + matches: + - path: + value: "/foo" + headers: + - name: "version" + value: "v2" + - path: + value: "/v2/foo" + ``` + + For a request to match against this rule, a request must satisfy + EITHER of the two conditions: + + - path prefixed with `/foo` AND contains the header `version: v2` + - path prefix of `/v2/foo` + + See the documentation for HTTPRouteMatch on how to specify multiple + match conditions that should be ANDed together. + + If no matches are specified, the default is a prefix + path match on "/", which has the effect of matching every + HTTP request. + + Proxy or Load Balancer routing configuration generated from HTTPRoutes + MUST prioritize matches based on the following criteria, continuing on + ties. Across all rules specified on applicable Routes, precedence must be + given to the match having: + + * "Exact" path match. + * "Prefix" path match with largest number of characters. + * Method match. + * Largest number of header matches. + * Largest number of query param matches. + + Note: The precedence of RegularExpression path matches are implementation-specific. + + If ties still exist across multiple Routes, matching precedence MUST be + determined in order of the following criteria, continuing on ties: + + * The oldest Route based on creation timestamp. + * The Route appearing first in alphabetical order by + "{namespace}/{name}". + + If ties still exist within an HTTPRoute, matching precedence MUST be granted + to the FIRST matching rule (in list order) with a match meeting the above + criteria. + + When no rules matching a request have been successfully attached to the + parent a request is coming from, a HTTP 404 status code MUST be returned. + items: + description: "HTTPRouteMatch defines the predicate + used to match requests to a given\naction. Multiple + match types are ANDed together, i.e. the match will\nevaluate + to true only if all conditions are satisfied.\n\nFor + example, the match below will match a HTTP request + only if its path\nstarts with `/foo` AND it contains + the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t + \ value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t + \ value \"v1\"\n\n```" + properties: + headers: + description: |- + Headers specifies HTTP request header matchers. Multiple match values are + ANDed together, meaning, a request must match all the specified headers + to select the route. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + + When a header is repeated in an HTTP request, it is + implementation-specific behavior as to how this is represented. + Generally, proxies should follow the guidance from the RFC: + https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding + processing a repeated header, with special handling for "Set-Cookie". + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the header. + + Support: Core (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression HeaderMatchType has implementation-specific + conformance, implementations can support POSIX, PCRE or any other dialects + of regular expressions. Please read the implementation's documentation to + determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + method: + description: |- + Method specifies HTTP method matcher. + When specified, this route will be matched only if the request has the + specified method. + + Support: Extended + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + type: string + path: + default: + type: PathPrefix + value: / + description: |- + Path specifies a HTTP request path matcher. If this field is not + specified, a default prefix match on the "/" path is provided. + properties: + type: + default: PathPrefix + description: |- + Type specifies how to match against the path Value. + + Support: Core (Exact, PathPrefix) + + Support: Implementation-specific (RegularExpression) + enum: + - Exact + - PathPrefix + - RegularExpression + type: string + value: + default: / + description: Value of the HTTP path to match + against. + maxLength: 1024 + type: string + type: object + x-kubernetes-validations: + - message: value must be an absolute path and + start with '/' when type one of ['Exact', + 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? self.value.startsWith(''/'') : true' + - message: must not contain '//' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''//'') : true' + - message: must not contain '/./' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''/./'') : true' + - message: must not contain '/../' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''/../'') : true' + - message: must not contain '%2f' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''%2f'') : true' + - message: must not contain '%2F' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''%2F'') : true' + - message: must not contain '#' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.contains(''#'') : true' + - message: must not end with '/..' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.endsWith(''/..'') : true' + - message: must not end with '/.' when type one + of ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? !self.value.endsWith(''/.'') : true' + - message: type must be one of ['Exact', 'PathPrefix', + 'RegularExpression'] + rule: self.type in ['Exact','PathPrefix'] || + self.type == 'RegularExpression' + - message: must only contain valid characters + (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) + for types ['Exact', 'PathPrefix'] + rule: '(self.type in [''Exact'',''PathPrefix'']) + ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") + : true' + queryParams: + description: |- + QueryParams specifies HTTP query parameter matchers. Multiple match + values are ANDed together, meaning, a request must match all the + specified query parameters to select the route. + + Support: Extended + items: + description: |- + HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP + query parameters. + properties: + name: + description: |- + Name is the name of the HTTP query param to be matched. This must be an + exact string match. (See + https://tools.ietf.org/html/rfc7230#section-2.7.3). + + If multiple entries specify equivalent query param names, only the first + entry with an equivalent name MUST be considered for a match. Subsequent + entries with an equivalent query param name MUST be ignored. + + If a query param is repeated in an HTTP request, the behavior is + purposely left undefined, since different data planes have different + capabilities. However, it is *recommended* that implementations should + match against the first value of the param if the data plane supports it, + as this behavior is expected in other load balancing contexts outside of + the Gateway API. + + Users SHOULD NOT route traffic based on repeated query params to guard + themselves against potential differences in the implementations. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: |- + Type specifies how to match against the value of the query parameter. + + Support: Extended (Exact) + + Support: Implementation-specific (RegularExpression) + + Since RegularExpression QueryParamMatchType has Implementation-specific + conformance, implementations can support POSIX, PCRE or any other + dialects of regular expressions. Please read the implementation's + documentation to determine the supported dialect. + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP + query param to be matched. + maxLength: 1024 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + maxItems: 64 + type: array + name: + description: |- + Name is the name of the route rule. This name MUST be unique within a Route if it is set. + + Support: Extended + + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retry: + description: |- + Retry defines the configuration for when to retry an HTTP request. + + Support: Extended + + + properties: + attempts: + description: |- + Attempts specifies the maximum number of times an individual request + from the gateway to a backend should be retried. + + If the maximum number of retries has been attempted without a successful + response from the backend, the Gateway MUST return an error. + + When this field is unspecified, the number of times to attempt to retry + a backend request is implementation-specific. + + Support: Extended + type: integer + backoff: + description: |- + Backoff specifies the minimum duration a Gateway should wait between + retry attempts and is represented in Gateway API Duration formatting. + + For example, setting the `rules[].retry.backoff` field to the value + `100ms` will cause a backend request to first be retried approximately + 100 milliseconds after timing out or receiving a response code configured + to be retryable. + + An implementation MAY use an exponential or alternative backoff strategy + for subsequent retry attempts, MAY cap the maximum backoff duration to + some amount greater than the specified minimum, and MAY add arbitrary + jitter to stagger requests, as long as unsuccessful backend requests are + not retried before the configured minimum duration. + + If a Request timeout (`rules[].timeouts.request`) is configured on the + route, the entire duration of the initial request and any retry attempts + MUST not exceed the Request timeout duration. If any retry attempts are + still in progress when the Request timeout duration has been reached, + these SHOULD be canceled if possible and the Gateway MUST immediately + return a timeout error. + + If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is + configured on the route, any retry attempts which reach the configured + BackendRequest timeout duration without a response SHOULD be canceled if + possible and the Gateway should wait for at least the specified backoff + duration before attempting to retry the backend request again. + + If a BackendRequest timeout is _not_ configured on the route, retry + attempts MAY time out after an implementation default duration, or MAY + remain pending until a configured Request timeout or implementation + default duration for total request time is reached. + + When this field is unspecified, the time to wait between retry attempts + is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + codes: + description: |- + Codes defines the HTTP response status codes for which a backend request + should be retried. + + Support: Extended + items: + description: |- + HTTPRouteRetryStatusCode defines an HTTP response status code for + which a backend request should be retried. + + Implementations MUST support the following status codes as retryable: + + * 500 + * 502 + * 503 + * 504 + + Implementations MAY support specifying additional discrete values in the + 500-599 range. + + Implementations MAY support specifying discrete values in the 400-499 range, + which are often inadvisable to retry. + + + maximum: 599 + minimum: 400 + type: integer + type: array + type: object + sessionPersistence: + description: |- + SessionPersistence defines and configures session persistence + for the route rule. + + Support: Extended + + + properties: + absoluteTimeout: + description: |- + AbsoluteTimeout defines the absolute timeout of the persistent + session. Once the AbsoluteTimeout duration has elapsed, the + session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + cookieConfig: + description: |- + CookieConfig provides configuration settings that are specific + to cookie-based session persistence. + + Support: Core + properties: + lifetimeType: + default: Session + description: |- + LifetimeType specifies whether the cookie has a permanent or + session-based lifetime. A permanent cookie persists until its + specified expiry time, defined by the Expires or Max-Age cookie + attributes, while a session cookie is deleted when the current + session ends. + + When set to "Permanent", AbsoluteTimeout indicates the + cookie's lifetime via the Expires or Max-Age cookie attributes + and is required. + + When set to "Session", AbsoluteTimeout indicates the + absolute lifetime of the cookie tracked by the gateway and + is optional. + + Defaults to "Session". + + Support: Core for "Session" type + + Support: Extended for "Permanent" type + enum: + - Permanent + - Session + type: string + type: object + idleTimeout: + description: |- + IdleTimeout defines the idle timeout of the persistent session. + Once the session has been idle for more than the specified + IdleTimeout duration, the session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + sessionName: + description: |- + SessionName defines the name of the persistent session token + which may be reflected in the cookie or the header. Users + should avoid reusing session names to prevent unintended + consequences, such as rejection or unpredictable behavior. + + Support: Implementation-specific + maxLength: 128 + type: string + type: + default: Cookie + description: |- + Type defines the type of session persistence such as through + the use a header or cookie. Defaults to cookie based session + persistence. + + Support: Core for "Cookie" type + + Support: Extended for "Header" type + enum: + - Cookie + - Header + type: string + type: object + x-kubernetes-validations: + - message: AbsoluteTimeout must be specified when cookie + lifetimeType is Permanent + rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) + || self.cookieConfig.lifetimeType != ''Permanent'' + || has(self.absoluteTimeout)' + timeouts: + description: |- + Timeouts defines the timeouts that can be configured for an HTTP request. + + Support: Extended + properties: + backendRequest: + description: |- + BackendRequest specifies a timeout for an individual request from the gateway + to a backend. This covers the time from when the request first starts being + sent from the gateway to when the full response has been received from the backend. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + An entire client HTTP transaction with a gateway, covered by the Request timeout, + may result in more than one call from the gateway to the destination backend, + for example, if automatic retries are supported. + + The value of BackendRequest must be a Gateway API Duration string as defined by + GEP-2257. When this field is unspecified, its behavior is implementation-specific; + when specified, the value of BackendRequest must be no more than the value of the + Request timeout (since the Request timeout encompasses the BackendRequest timeout). + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + request: + description: |- + Request specifies the maximum duration for a gateway to respond to an HTTP request. + If the gateway has not been able to respond before this deadline is met, the gateway + MUST return a timeout error. + + For example, setting the `rules.timeouts.request` field to the value `10s` in an + `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds + to complete. + + Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout + completely. Implementations that cannot completely disable the timeout MUST + instead interpret the zero duration as the longest possible value to which + the timeout can be set. + + This timeout is intended to cover as close to the whole request-response transaction + as possible although an implementation MAY choose to start the timeout after the entire + request stream has been received instead of immediately after the transaction is + initiated by the client. + + The value of Request is a Gateway API Duration string as defined by GEP-2257. When this + field is unspecified, request timeout behavior is implementation-specific. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: backendRequest timeout cannot be longer than + request timeout + rule: '!(has(self.request) && has(self.backendRequest) + && duration(self.request) != duration(''0s'') && + duration(self.backendRequest) > duration(self.request))' + type: object + x-kubernetes-validations: + - message: RequestRedirect filter must not be used together + with backendRefs + rule: '(has(self.backendRefs) && size(self.backendRefs) + > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): + true' + - message: When using RequestRedirect filter with path.replacePrefixMatch, + exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, + has(f.requestRedirect) && has(f.requestRedirect.path) + && f.requestRedirect.path.type == ''ReplacePrefixMatch'' + && has(f.requestRedirect.path.replacePrefixMatch))) + ? ((size(self.matches) != 1 || !has(self.matches[0].path) + || self.matches[0].path.type != ''PathPrefix'') ? false + : true) : true' + - message: When using URLRewrite filter with path.replacePrefixMatch, + exactly one PathPrefix match must be specified + rule: '(has(self.filters) && self.filters.exists_one(f, + has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type + == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) + ? ((size(self.matches) != 1 || !has(self.matches[0].path) + || self.matches[0].path.type != ''PathPrefix'') ? false + : true) : true' + - message: Within backendRefs, when using RequestRedirect + filter with path.replacePrefixMatch, exactly one PathPrefix + match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, + (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) + && has(f.requestRedirect.path) && f.requestRedirect.path.type + == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) + )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) + || self.matches[0].path.type != ''PathPrefix'') ? false + : true) : true' + - message: Within backendRefs, When using URLRewrite filter + with path.replacePrefixMatch, exactly one PathPrefix + match must be specified + rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, + (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) + && has(f.urlRewrite.path) && f.urlRewrite.path.type + == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) + )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) + || self.matches[0].path.type != ''PathPrefix'') ? false + : true) : true' + maxItems: 16 + type: array + x-kubernetes-validations: + - message: While 16 rules and 64 matches per rule are allowed, + the total number of matches across all rules in a route + must be less than 128 + rule: '(self.size() > 0 ? self[0].matches.size() : 0) + + (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() + > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? + self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() + : 0) + (self.size() > 5 ? self[5].matches.size() : 0) + + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() + > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? + self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() + : 0) + (self.size() > 10 ? self[10].matches.size() : 0) + + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() + > 12 ? self[12].matches.size() : 0) + (self.size() > 13 + ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() + : 0) + (self.size() > 15 ? self[15].matches.size() : 0) + <= 128' + type: object + type: object + ingress: + description: Ingress sets how the ingress object should look like + with your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain name + of a network host, as defined by RFC 3986.\nNote the + following deviations from the \"host\" part of the\nURI + as defined in RFC 3986:\n1. IPs are not allowed. Currently + an IngressRuleValue can only apply to\n the IP in + the Spec of the parent Ingress.\n2. The `:` delimiter + is not respected because ports are not allowed.\n\t + \ Currently the port of an Ingress is implicitly :80 + for http and\n\t :443 for https.\nBoth these may + change in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. If + the host is unspecified, the Ingress routes all\ntraffic + based on the specified IngressRuleValue.\n\nhost can + be \"precise\" which is a domain name without the + terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as the + first DNS label and\nmatches only a single label. + You cannot have a wildcard label by itself (e.g. Host + == \"*\").\nRequests will be matched against the Host + field in the following way:\n1. If host is precise, + the request matches this rule if the http host header + is equal to Host.\n2. If host is a wildcard, then + the request matches this rule if the http host header\nis + to equal to the suffix (removing the first label) + of the wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths that + map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer security + associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + jsonnet: + properties: + libraryLabelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + persistentVolumeClaim: + description: PersistentVolumeClaim creates a PVC if you need to attach + one to your grafana instance. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + TypedLocalObjectReference contains enough information to let you locate the + typed referenced object inside the same namespace. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + description: PersistentVolumeMode describes how a volume is + intended to be consumed, either Block or Filesystem. + type: string + volumeName: + description: VolumeName is the binding reference to the PersistentVolume + backing this claim. + type: string + type: object + type: object + preferences: + description: Preferences holds the Grafana Preferences settings + properties: + homeDashboardUid: + type: string + type: object + route: + description: Route sets how the ingress object should look like with + your grafana instance, this only works in Openshift. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring + to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being + referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + type: array + host: + type: string + path: + type: string + port: + description: RoutePort defines a port mapping from a router + to an endpoint in the service endpoints. + properties: + targetPort: + anyOf: + - type: integer + - type: string + description: |- + The target port on pods selected by the service this route points to. + If this is a string, it will be looked up as a named port in the target + endpoints port list. Required + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + type: string + tls: + description: TLSConfig defines config used to secure a route + and provide termination + properties: + caCertificate: + description: caCertificate provides the cert authority + certificate contents + type: string + certificate: + description: |- + certificate provides certificate contents. This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. + type: string + destinationCACertificate: + description: |- + destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt + termination this file should be provided in order to have routers use it for health checks on the secure connection. + If this field is not specified, the router may provide its own destination CA and perform hostname validation using + the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically + verify. + type: string + externalCertificate: + description: |- + externalCertificate provides certificate contents as a secret reference. + This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. The secret referenced should + be present in the same namespace as that of the Route. + Forbidden when `certificate` is set. + The router service account needs to be granted with read-only access to this secret, + please refer to openshift docs for additional details. + properties: + name: + description: |- + name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + insecureEdgeTerminationPolicy: + description: |- + insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While + each router may make its own decisions on which ports to expose, this is normally port 80. + + If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". + + * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). + + * None - no traffic is allowed on the insecure port (default). + + * Redirect - clients are redirected to the secure port. + enum: + - Allow + - None + - Redirect + - "" + type: string + key: + description: key provides key file contents + type: string + termination: + description: |- + termination indicates termination type. + + * edge - TLS termination is done by the router and http is used to communicate with the backend (default) + * passthrough - Traffic is sent straight to the destination without the router providing TLS termination + * reencrypt - TLS termination is done by the router and https is used to communicate with the backend + + Note: passthrough termination is incompatible with httpHeader actions + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + x-kubernetes-validations: + - message: 'cannot have both spec.tls.termination: passthrough + and spec.tls.insecureEdgeTerminationPolicy: Allow' + rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) + ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) + : true' + to: + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. + properties: + kind: + default: Service + description: The kind of target that the route is referring + to. Currently, only 'Service' is allowed + enum: + - Service + - "" + type: string + name: + description: name of the service/target that is being + referred to. e.g. name of the service + minLength: 1 + type: string + weight: + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. + format: int32 + maximum: 256 + minimum: 0 + type: integer + required: + - kind + - name + type: object + wildcardPolicy: + description: WildcardPolicyType indicates the type of wildcard + support needed by routes. + type: string + type: object + type: object + service: + description: Service sets how the service object should look like + with your grafana instance, contains a number of defaults. + properties: + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: ServiceSpec describes the attributes that a user + creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client + IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic + is distributed to Service endpoints. Implementations can use this field + as a hint, but are not required to guarantee strict adherence. If the + field is not set, the implementation will apply its default routing + strategy. If set to "PreferClose", implementations should prioritize + endpoints that are in the same zone. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + serviceAccount: + description: ServiceAccount sets how the ServiceAccount object should + look like with your grafana instance, contains a number of defaults. + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + description: ObjectMeta contains only a [subset of the fields + included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + description: ObjectReference contains enough information to + let you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + suspend: + description: Suspend pauses reconciliation of owned resources like + deployments, Services, Etc. upon changes + type: boolean + version: + description: |- + Version sets the tag of the default image: docker.io/grafana/grafana. + Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" + default: 12.3.0 + type: string + type: object + status: + description: GrafanaStatus defines the observed state of Grafana + properties: + adminUrl: + type: string + alertRuleGroups: + items: + type: string + type: array + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + contactPoints: + items: + type: string + type: array + dashboards: + items: + type: string + type: array + datasources: + items: + type: string + type: array + folders: + items: + type: string + type: array + lastMessage: + type: string + libraryPanels: + items: + type: string + type: array + muteTimings: + items: + type: string + type: array + notificationTemplates: + items: + type: string + type: array + serviceaccounts: + items: + type: string + type: array + stage: + type: string + stageStatus: + type: string + version: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/kustomization.yaml b/kustomize/overlays/mlops/grafana/crds/kustomization.yaml new file mode 100644 index 000000000..f40befc67 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/crds/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - grafanas.yaml + - grafanadatasources.yaml + - grafanafolders.yaml + - grafanadashboards.yaml diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 16fd7bf3a..94cd43aee 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -3,6 +3,9 @@ kind: Kustomization namespace: REPLACE_NAMESPACE +crds: + - crds + resources: - operator From 62f95e57e84d676e20ea2044e794ae233abef400 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 3 Feb 2026 14:32:48 +0200 Subject: [PATCH 181/286] remove CRDs from mlops overlay, add grafana login via openshift IDP, update nginx env vars Signed-off-by: Ilona Shishov --- kustomize/README.md | 17 +- kustomize/base/nginx.yaml | 2 +- .../mlops/grafana/crds/grafanadashboards.yaml | 18323 ---------------- .../grafana/crds/grafanadatasources.yaml | 348 - .../mlops/grafana/crds/grafanafolders.yaml | 241 - .../overlays/mlops/grafana/crds/grafanas.yaml | 8687 -------- .../mlops/grafana/crds/kustomization.yaml | 8 - .../mlops/grafana/instances/grafana.yaml | 68 +- .../grafana/instances/kustomization.yaml | 1 + .../grafana/instances/serviceAccount.yaml | 6 + .../overlays/mlops/grafana/kustomization.yaml | 34 +- .../nginx-patch.yaml | 4 +- 12 files changed, 78 insertions(+), 27661 deletions(-) delete mode 100644 kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml delete mode 100644 kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml delete mode 100644 kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml delete mode 100644 kustomize/overlays/mlops/grafana/crds/grafanas.yaml delete mode 100644 kustomize/overlays/mlops/grafana/crds/kustomization.yaml create mode 100644 kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 0e837506f..e35a39c03 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -149,10 +149,14 @@ Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique s export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" ``` +```shell +export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) +``` ```shell cat > base/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET +session-secret=$OAUTH_SESSION_SECRET openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` @@ -175,7 +179,7 @@ export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` >[!IMPORTANT] -You should only run one of 9,10 steps, depends on if you want to run the service with a self hosted LLM or not. +You should only run one of the steps 9,10 or 11, depending on if you want to run the service with a self hosted LLM, self hosted LLM with MLOps or Nvidia remote NIM. 9. To deploy `ExploitIQ` with a self-hosted LLM , run: ```shell @@ -184,6 +188,17 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ``` +10. To deploy `ExploitIQ` with a self-hosted LLM and MLOps, run: + +```shell +# Patch overlay kustomization yaml with deployment namespace value +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomize.yaml + +# Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM and MLOps +oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME + +``` + 10. Alternatively, to deploy `ExploitIQ` with a fully remote nim LLM, run: ```shell # Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index 68e9922bb..865347357 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -61,7 +61,7 @@ spec: - name: NGINX_UPSTREAM_NIM_LLM value: https://integrate.api.nvidia.com - name: NGINX_UPSTREAM_NIM_EMBED - value: http://nim-embed.agent-morpheus-models.svc.cluster.local:8000 + value: http://nim-embed.exploit-iq-models.svc.cluster.local:8000 - name: NGINX_UPSTREAM_OPENAI value: https://integrate.api.nvidia.com volumeMounts: diff --git a/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml b/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml deleted file mode 100644 index b5f5bd67d..000000000 --- a/kustomize/overlays/mlops/grafana/crds/grafanadashboards.yaml +++ /dev/null @@ -1,18323 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanadashboards.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaDashboard - listKind: GrafanaDashboardList - plural: grafanadashboards - singular: grafanadashboard - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaDashboard is the Schema for the grafanadashboards API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas outside the current namespace - type: boolean - configMapRef: - description: model from configmap - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - contentCacheDuration: - description: Cache duration for models fetched from URLs - type: string - datasources: - description: maps required data sources to existing ones - items: - description: |- - GrafanaResourceDatasource is used to set the datasource name of any templated datasources in - content definitions (e.g., dashboard JSON). - properties: - datasourceName: - type: string - inputName: - type: string - required: - - datasourceName - - inputName - type: object - type: array - envFrom: - description: environments variables from secrets or config maps - items: - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - type: array - envs: - description: environments variables as a map - items: - properties: - name: - type: string - value: - description: Inline env value - type: string - valueFrom: - description: Reference on value source, might be the reference on a secret or config map - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - folder: - description: folder assignment for dashboard - type: string - folderRef: - description: Name of a `GrafanaFolder` resource in the same namespace - type: string - folderUID: - description: UID of the target folder for this dashboard - type: string - grafanaCom: - description: grafana.com/dashboards - properties: - id: - type: integer - revision: - type: integer - required: - - id - type: object - gzipJson: - description: GzipJson the model's JSON compressed with Gzip. Base64-encoded when in YAML. - format: byte - type: string - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - json: - description: model json - type: string - jsonnet: - description: Jsonnet - type: string - jsonnetLib: - description: Jsonnet project build - properties: - fileName: - type: string - gzipJsonnetProject: - format: byte - type: string - jPath: - items: - type: string - type: array - required: - - fileName - - gzipJsonnetProject - type: object - plugins: - description: plugins - items: - properties: - name: - minLength: 1 - type: string - version: - pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ - type: string - required: - - name - - version - type: object - type: array - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator to ignore changes - type: boolean - uid: - description: |- - Manually specify the uid, overwrites uids already present in the json model. - Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40. - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - url: - description: model url - type: string - urlAuthorization: - description: authorization options for model from url - properties: - basicAuth: - properties: - password: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - type: object - required: - - instanceSelector - type: object - x-kubernetes-validations: - - message: Only one of folderUID or folderRef can be declared at the same time - rule: (has(self.folderUID) && !(has(self.folderRef))) || (has(self.folderRef) && !(has(self.folderUID))) || !(has(self.folderRef) && (has(self.folderUID))) - - message: folder field cannot be set when folderUID or folderRef is already declared - rule: (has(self.folder) && !(has(self.folderRef) || has(self.folderUID))) || !(has(self.folder)) - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' - status: - description: GrafanaDashboardStatus defines the observed state of GrafanaDashboard - properties: - NoMatchingInstances: - description: The dashboard instanceSelector can't find matching grafana instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - contentCache: - format: byte - type: string - contentTimestamp: - format: date-time - type: string - contentUrl: - type: string - hash: - type: string - lastResync: - description: Last time the resource was synchronized with Grafana instances - format: date-time - type: string - uid: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanadatasources.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaDatasource - listKind: GrafanaDatasourceList - plural: grafanadatasources - singular: grafanadatasource - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaDatasource is the Schema for the grafanadatasources API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas outside the current namespace - type: boolean - datasource: - properties: - access: - type: string - basicAuth: - type: boolean - basicAuthUser: - type: string - database: - type: string - editable: - description: Whether to enable/disable editing of the datasource in Grafana UI - type: boolean - isDefault: - type: boolean - jsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - name: - type: string - orgId: - description: Deprecated field, it has no effect - format: int64 - type: integer - secureJsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - type: - type: string - uid: - description: Deprecated field, use spec.uid instead - type: string - url: - type: string - user: - type: string - type: object - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - plugins: - description: plugins - items: - properties: - name: - minLength: 1 - type: string - version: - pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ - type: string - required: - - name - - version - type: object - type: array - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator to ignore changes - type: boolean - uid: - description: |- - The UID, for the datasource, fallback to the deprecated spec.datasource.uid - and metadata.uid. Can be any string consisting of alphanumeric characters, - - and _ with a maximum length of 40 +optional - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - valuesFrom: - description: environments variables from secrets or config maps - items: - properties: - targetPath: - type: string - valueFrom: - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: Either configMapKeyRef or secretKeyRef must be set - rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) - required: - - targetPath - - valueFrom - type: object - maxItems: 99 - type: array - required: - - datasource - - instanceSelector - type: object - x-kubernetes-validations: - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' - status: - description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource - properties: - NoMatchingInstances: - description: The datasource instanceSelector can't find matching grafana instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastMessage: - description: 'Deprecated: Check status.conditions or operator logs' - type: string - lastResync: - description: Last time the resource was synchronized with Grafana instances - format: date-time - type: string - uid: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanafolders.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaFolder - listKind: GrafanaFolderList - plural: grafanafolders - singular: grafanafolder - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaFolder is the Schema for the grafanafolders API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaFolderSpec defines the desired state of GrafanaFolder - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas outside the current namespace - type: boolean - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - parentFolderRef: - description: Reference to an existing GrafanaFolder CR in the same namespace - type: string - parentFolderUID: - description: UID of the folder in which the current folder should be created - type: string - permissions: - description: Raw json with folder permissions, potentially exported from Grafana - type: string - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator to ignore changes - type: boolean - title: - description: Display name of the folder in Grafana - type: string - uid: - description: Manually specify the UID the Folder is created with. Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40 - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - required: - - instanceSelector - type: object - x-kubernetes-validations: - - message: Only one of parentFolderUID or parentFolderRef can be set - rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) && (has(self.parentFolderUID))) - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' - status: - description: GrafanaFolderStatus defines the observed state of GrafanaFolder - properties: - NoMatchingInstances: - description: The folder instanceSelector can't find matching grafana instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastResync: - description: Last time the resource was synchronized with Grafana instances - format: date-time - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanas.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: Grafana - listKind: GrafanaList - plural: grafanas - singular: grafana - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.version - name: Version - type: string - - jsonPath: .status.stage - name: Stage - type: string - - jsonPath: .status.stageStatus - name: Stage status - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Grafana is the Schema for the grafanas API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaSpec defines the desired state of Grafana - properties: - client: - description: Client defines how the grafana-operator talks to the grafana instance. - properties: - headers: - additionalProperties: - type: string - description: Custom HTTP headers to use when interacting with this Grafana. - type: object - preferIngress: - description: If the operator should send it's request through the grafana instances ingress object instead of through the service. - nullable: true - type: boolean - timeout: - nullable: true - type: integer - tls: - description: TLS Configuration used to talk with the grafana instance. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate information - properties: - name: - description: name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: namespace defines the space within which the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - useKubeAuth: - description: |- - Use Kubernetes Serviceaccount as authentication - Requires configuring [auth.jwt] in the instance - type: boolean - type: object - config: - additionalProperties: - additionalProperties: - type: string - type: object - description: Config defines how your grafana ini file should looks like. - type: object - x-kubernetes-preserve-unknown-fields: true - deployment: - description: Deployment sets how the deployment object should look like with your grafana instance, contains a number of defaults. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - minReadySeconds: - format: int32 - type: integer - paused: - type: boolean - progressDeadlineSeconds: - format: int32 - type: integer - replicas: - format: int32 - type: integer - revisionHistoryLimit: - format: int32 - type: integer - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - strategy: - properties: - rollingUpdate: - properties: - maxSurge: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: - type: string - type: object - template: - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - activeDeadlineSeconds: - format: int64 - type: integer - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - type: boolean - containers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - type: string - enableServiceLinks: - type: boolean - ephemeralContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - targetContainerName: - type: string - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - type: string - required: - - ip - type: object - type: array - hostIPC: - type: boolean - hostNetwork: - type: boolean - hostPID: - type: boolean - hostUsers: - type: boolean - hostname: - type: string - imagePullSecrets: - items: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - type: array - initContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - nodeName: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - x-kubernetes-map-type: atomic - os: - properties: - name: - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - preemptionPolicy: - type: string - priority: - format: int32 - type: integer - priorityClassName: - type: string - readinessGates: - items: - properties: - conditionType: - type: string - required: - - conditionType - type: object - type: array - restartPolicy: - type: string - runtimeClassName: - type: string - schedulerName: - type: string - securityContext: - properties: - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - fsGroup: - format: int64 - type: integer - fsGroupChangePolicy: - type: string - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxChangePolicy: - type: string - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - type: string - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - serviceAccountName: - type: string - setHostnameAsFQDN: - type: boolean - shareProcessNamespace: - type: boolean - subdomain: - type: string - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - topologySpreadConstraints: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - format: int32 - type: integer - minDomains: - format: int32 - type: integer - nodeAffinityPolicy: - type: string - nodeTaintsPolicy: - type: string - topologyKey: - type: string - whenUnsatisfiable: - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - default: ext4 - type: string - kind: - type: string - readOnly: - default: false - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - type: boolean - volumeAttributes: - additionalProperties: - type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - image: - properties: - pullPolicy: - type: string - reference: - type: string - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - default: default - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - clusterTrustBundle: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - name: - type: string - optional: - type: boolean - path: - type: string - signerName: - type: string - required: - - path - type: object - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podCertificate: - properties: - certificateChainPath: - type: string - credentialBundlePath: - type: string - keyPath: - type: string - keyType: - type: string - maxExpirationSeconds: - format: int32 - type: integer - signerName: - type: string - required: - - keyType - - signerName - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - default: /etc/ceph/keyring - type: string - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - default: xfs - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - type: boolean - storageMode: - default: ThinProvisioned - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - type: object - type: object - type: object - type: object - disableDefaultAdminSecret: - description: DisableDefaultAdminSecret prevents operator from creating default admin-credentials secret - type: boolean - disableDefaultSecurityContext: - description: DisableDefaultSecurityContext prevents the operator from populating securityContext on deployments - enum: - - Pod - - Container - - All - type: string - external: - description: External enables you to configure external grafana instances that is not managed by the operator. - properties: - adminPassword: - description: AdminPassword key to talk to the external grafana instance. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - adminUser: - description: AdminUser key to talk to the external grafana instance. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - apiKey: - description: The API key to talk to the external grafana instance, you need to define ether apiKey or adminUser/adminPassword. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - description: DEPRECATED, use top level `tls` instead. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate information - properties: - name: - description: name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: namespace defines the space within which the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - url: - description: URL of the external grafana instance you want to manage. - type: string - required: - - url - type: object - httpRoute: - description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. It will not be created if this is not set - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: HTTPRouteSpec defines the desired state of HTTPRoute - properties: - hostnames: - description: |- - Hostnames defines a set of hostnames that should match against the HTTP Host - header to select a HTTPRoute used to process the request. Implementations - MUST ignore any port value specified in the HTTP Host header while - performing a match and (absent of any applicable header modification - configuration) MUST forward this header unmodified to the backend. - - Valid values for Hostnames are determined by RFC 1123 definition of a - hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - If a hostname is specified by both the Listener and HTTPRoute, there - must be at least one intersecting hostname for the HTTPRoute to be - attached to the Listener. For example: - - * A Listener with `test.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames, or have specified at - least one of `test.example.com` or `*.example.com`. - * A Listener with `*.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames or have specified at least - one hostname that matches the Listener hostname. For example, - `*.example.com`, `test.example.com`, and `foo.test.example.com` would - all match. On the other hand, `example.com` and `test.example.net` would - not match. - - Hostnames that are prefixed with a wildcard label (`*.`) are interpreted - as a suffix match. That means that a match for `*.example.com` would match - both `test.example.com`, and `foo.test.example.com`, but not `example.com`. - - If both the Listener and HTTPRoute have specified hostnames, any - HTTPRoute hostnames that do not match the Listener hostname MUST be - ignored. For example, if a Listener specified `*.example.com`, and the - HTTPRoute specified `test.example.com` and `test.example.net`, - `test.example.net` must not be considered for a match. - - If both the Listener and HTTPRoute have specified hostnames, and none - match with the criteria above, then the HTTPRoute is not accepted. The - implementation must raise an 'Accepted' Condition with a status of - `False` in the corresponding RouteParentStatus. - - In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. - overlapping wildcard matching and exact matching hostnames), precedence must - be given to rules from the HTTPRoute with the largest number of: - - * Characters in a matching non-wildcard hostname. - * Characters in a matching hostname. - - If ties exist across multiple Routes, the matching precedence rules for - HTTPRouteMatches takes over. - - Support: Core - items: - description: |- - Hostname is the fully qualified domain name of a network host. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - Hostname can be "precise" which is a domain name without the terminating - dot of a network host (e.g. "foo.example.com") or "wildcard", which is a - domain name prefixed with a single wildcard label (e.g. `*.example.com`). - - Note that as per RFC1035 and RFC1123, a *label* must consist of lower case - alphanumeric characters or '-', and must start and end with an alphanumeric - character. No other punctuation is allowed. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - maxItems: 16 - type: array - parentRefs: - description: |- - ParentRefs references the resources (usually Gateways) that a Route wants - to be attached to. Note that the referenced parent resource needs to - allow this for the attachment to be complete. For Gateways, that means - the Gateway needs to allow attachment from Routes of this kind and - namespace. For Services, that means the Service must either be in the same - namespace for a "producer" route, or the mesh implementation must support - and allow "consumer" routes for the referenced Service. ReferenceGrant is - not applicable for governing ParentRefs to Services - it is not possible to - create a "producer" route for a Service in a different namespace from the - Route. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - ParentRefs must be _distinct_. This means either that: - - * They select different objects. If this is the case, then parentRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, `namespace`, and `name` must - be unique across all parentRef entries in the Route. - * They do not select different objects, but for each optional field used, - each ParentRef that selects the same object must set the same set of - optional fields to different values. If one ParentRef sets a - combination of optional fields, all must set the same combination. - - Some examples: - - * If one ParentRef sets `sectionName`, all ParentRefs referencing the - same object must also set `sectionName`. - * If one ParentRef sets `port`, all ParentRefs referencing the same - object must also set `port`. - * If one ParentRef sets `sectionName` and `port`, all ParentRefs - referencing the same object must also set `sectionName` and `port`. - - It is possible to separately reference multiple distinct objects that may - be collapsed by an implementation. For example, some implementations may - choose to merge compatible Gateway Listeners together. If that is the - case, the list of routes attached to those resources should also be - merged. - - Note that for ParentRefs that cross namespace boundaries, there are specific - rules. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example, - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable other kinds of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - - - - - items: - description: |- - ParentReference identifies an API object (usually a Gateway) that can be considered - a parent of this resource (usually a route). There are two kinds of parent resources - with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - maxItems: 32 - type: array - rules: - default: - - matches: - - path: - type: PathPrefix - value: / - description: |- - Rules are a list of HTTP matchers, filters and actions. - - - items: - description: |- - HTTPRouteRule defines semantics for matching an HTTP request based on - conditions (matches), processing it (filters), and forwarding the request to - an API object (backendRefs). - properties: - backendRefs: - description: |- - BackendRefs defines the backend(s) where matching requests should be - sent. - - Failure behavior here depends on how many BackendRefs are specified and - how many are invalid. - - If *all* entries in BackendRefs are invalid, and there are also no filters - specified in this route rule, *all* traffic which matches this rule MUST - receive a 500 status code. - - See the HTTPBackendRef definition for the rules about what makes a single - HTTPBackendRef invalid. - - When a HTTPBackendRef is invalid, 500 status codes MUST be returned for - requests that would have otherwise been routed to an invalid backend. If - multiple backends are specified, and some are invalid, the proportion of - requests that would otherwise have been routed to an invalid backend - MUST receive a 500 status code. - - For example, if two backends are specified with equal weights, and one is - invalid, 50 percent of traffic must receive a 500. Implementations may - choose how that 50 percent is determined. - - When a HTTPBackendRef refers to a Service that has no ready endpoints, - implementations SHOULD return a 503 for requests to that backend instead. - If an implementation chooses to do this, all of the above rules for 500 responses - MUST also apply for responses that return a 503. - - Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Core - items: - description: |- - HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. - - Note that when a namespace different than the local namespace is specified, a - ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - - - When the BackendRef points to a Kubernetes Service, implementations SHOULD - honor the appProtocol field if it is set for the target Service Port. - - Implementations supporting appProtocol SHOULD recognize the Kubernetes - Standard Application Protocols defined in KEP-3726. - - If a Service appProtocol isn't specified, an implementation MAY infer the - backend protocol through its own means. Implementations MAY infer the - protocol from the Route type referring to the backend Service. - - If a Route is not able to send traffic to the backend using the specified - protocol then the backend is considered invalid. Implementations MUST set the - "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. - - - properties: - filters: - description: |- - Filters defined at this level should be executed if and only if the - request is being forwarded to the backend defined here. - - Support: Implementation-specific (For broader support of filters, use the - Filters field in HTTPRouteRule.) - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than or equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type - rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil if the filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' - - message: filter.requestMirror must be specified for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' - - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' - - message: filter.urlRewrite must be specified for URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' - - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' - - message: filter.extensionRef must be specified for ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 - - message: ResponseHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - weight: - default: 1 - description: |- - Weight specifies the proportion of requests forwarded to the referenced - backend. This is computed as weight/(sum of all weights in this - BackendRefs list). For non-zero values, there may be some epsilon from - the exact proportion defined here depending on the precision an - implementation supports. Weight is not a percentage and the sum of - weights does not need to equal 100. - - If only one backend is specified and it has a weight greater than 0, 100% - of the traffic is forwarded to that backend. If weight is set to 0, no - traffic should be forwarded for this entry. If unspecified, weight - defaults to 1. - - Support for this field varies based on the context where used. - format: int32 - maximum: 1000000 - minimum: 0 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - maxItems: 16 - type: array - filters: - description: |- - Filters define the filters that are applied to requests that match - this rule. - - Wherever possible, implementations SHOULD implement filters in the order - they are specified. - - Implementations MAY choose to implement this ordering strictly, rejecting - any combination or order of filters that cannot be supported. If implementations - choose a strict interpretation of filter ordering, they MUST clearly document - that behavior. - - To reject an invalid combination or order of filters, implementations SHOULD - consider the Route Rules with this configuration invalid. If all Route Rules - in a Route are invalid, the entire Route would be considered invalid. If only - a portion of Route Rules are invalid, implementations MUST set the - "PartiallyInvalid" condition for the Route. - - Conformance-levels at this level are defined based on the type of filter: - - - ALL core filters MUST be supported by all implementations. - - Implementers are encouraged to support extended filters. - - Implementation-specific custom filters have no API guarantees across - implementations. - - Specifying the same filter multiple times is not supported unless explicitly - indicated in the filter. - - All filters are expected to be compatible with each other except for the - URLRewrite and RequestRedirect filters, which may not be combined. If an - implementation cannot support other combinations of filters, they must clearly - document that limitation. In cases where incompatible or unsupported - filters are specified and cause the `Accepted` condition to be set to status - `False`, implementations may use the `IncompatibleFilters` reason to specify - this configuration error. - - Support: Core - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than or equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type - rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil if the filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' - - message: filter.requestMirror must be specified for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' - - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' - - message: filter.urlRewrite must be specified for URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' - - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' - - message: filter.extensionRef must be specified for ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 - - message: ResponseHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 - matches: - default: - - path: - type: PathPrefix - value: / - description: |- - Matches define conditions used for matching the rule against incoming - HTTP requests. Each match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. - - For example, take the following matches configuration: - - ``` - matches: - - path: - value: "/foo" - headers: - - name: "version" - value: "v2" - - path: - value: "/v2/foo" - ``` - - For a request to match against this rule, a request must satisfy - EITHER of the two conditions: - - - path prefixed with `/foo` AND contains the header `version: v2` - - path prefix of `/v2/foo` - - See the documentation for HTTPRouteMatch on how to specify multiple - match conditions that should be ANDed together. - - If no matches are specified, the default is a prefix - path match on "/", which has the effect of matching every - HTTP request. - - Proxy or Load Balancer routing configuration generated from HTTPRoutes - MUST prioritize matches based on the following criteria, continuing on - ties. Across all rules specified on applicable Routes, precedence must be - given to the match having: - - * "Exact" path match. - * "Prefix" path match with largest number of characters. - * Method match. - * Largest number of header matches. - * Largest number of query param matches. - - Note: The precedence of RegularExpression path matches are implementation-specific. - - If ties still exist across multiple Routes, matching precedence MUST be - determined in order of the following criteria, continuing on ties: - - * The oldest Route based on creation timestamp. - * The Route appearing first in alphabetical order by - "{namespace}/{name}". - - If ties still exist within an HTTPRoute, matching precedence MUST be granted - to the FIRST matching rule (in list order) with a match meeting the above - criteria. - - When no rules matching a request have been successfully attached to the - parent a request is coming from, a HTTP 404 status code MUST be returned. - items: - description: "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t value \"v1\"\n\n```" - properties: - headers: - description: |- - Headers specifies HTTP request header matchers. Multiple match values are - ANDed together, meaning, a request must match all the specified headers - to select the route. - items: - description: |- - HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request - headers. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - - When a header is repeated in an HTTP request, it is - implementation-specific behavior as to how this is represented. - Generally, proxies should follow the guidance from the RFC: - https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding - processing a repeated header, with special handling for "Set-Cookie". - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the header. - - Support: Core (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression HeaderMatchType has implementation-specific - conformance, implementations can support POSIX, PCRE or any other dialects - of regular expressions. Please read the implementation's documentation to - determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - method: - description: |- - Method specifies HTTP method matcher. - When specified, this route will be matched only if the request has the - specified method. - - Support: Extended - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - type: string - path: - default: - type: PathPrefix - value: / - description: |- - Path specifies a HTTP request path matcher. If this field is not - specified, a default prefix match on the "/" path is provided. - properties: - type: - default: PathPrefix - description: |- - Type specifies how to match against the path Value. - - Support: Core (Exact, PathPrefix) - - Support: Implementation-specific (RegularExpression) - enum: - - Exact - - PathPrefix - - RegularExpression - type: string - value: - default: / - description: Value of the HTTP path to match against. - maxLength: 1024 - type: string - type: object - x-kubernetes-validations: - - message: value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.startsWith(''/'') : true' - - message: must not contain '//' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''//'') : true' - - message: must not contain '/./' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/./'') : true' - - message: must not contain '/../' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/../'') : true' - - message: must not contain '%2f' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2f'') : true' - - message: must not contain '%2F' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2F'') : true' - - message: must not contain '#' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''#'') : true' - - message: must not end with '/..' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/..'') : true' - - message: must not end with '/.' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/.'') : true' - - message: type must be one of ['Exact', 'PathPrefix', 'RegularExpression'] - rule: self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression' - - message: must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") : true' - queryParams: - description: |- - QueryParams specifies HTTP query parameter matchers. Multiple match - values are ANDed together, meaning, a request must match all the - specified query parameters to select the route. - - Support: Extended - items: - description: |- - HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP - query parameters. - properties: - name: - description: |- - Name is the name of the HTTP query param to be matched. This must be an - exact string match. (See - https://tools.ietf.org/html/rfc7230#section-2.7.3). - - If multiple entries specify equivalent query param names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent query param name MUST be ignored. - - If a query param is repeated in an HTTP request, the behavior is - purposely left undefined, since different data planes have different - capabilities. However, it is *recommended* that implementations should - match against the first value of the param if the data plane supports it, - as this behavior is expected in other load balancing contexts outside of - the Gateway API. - - Users SHOULD NOT route traffic based on repeated query params to guard - themselves against potential differences in the implementations. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the query parameter. - - Support: Extended (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression QueryParamMatchType has Implementation-specific - conformance, implementations can support POSIX, PCRE or any other - dialects of regular expressions. Please read the implementation's - documentation to determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP query param to be matched. - maxLength: 1024 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - maxItems: 64 - type: array - name: - description: |- - Name is the name of the route rule. This name MUST be unique within a Route if it is set. - - Support: Extended - - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retry: - description: |- - Retry defines the configuration for when to retry an HTTP request. - - Support: Extended - - - properties: - attempts: - description: |- - Attempts specifies the maximum number of times an individual request - from the gateway to a backend should be retried. - - If the maximum number of retries has been attempted without a successful - response from the backend, the Gateway MUST return an error. - - When this field is unspecified, the number of times to attempt to retry - a backend request is implementation-specific. - - Support: Extended - type: integer - backoff: - description: |- - Backoff specifies the minimum duration a Gateway should wait between - retry attempts and is represented in Gateway API Duration formatting. - - For example, setting the `rules[].retry.backoff` field to the value - `100ms` will cause a backend request to first be retried approximately - 100 milliseconds after timing out or receiving a response code configured - to be retryable. - - An implementation MAY use an exponential or alternative backoff strategy - for subsequent retry attempts, MAY cap the maximum backoff duration to - some amount greater than the specified minimum, and MAY add arbitrary - jitter to stagger requests, as long as unsuccessful backend requests are - not retried before the configured minimum duration. - - If a Request timeout (`rules[].timeouts.request`) is configured on the - route, the entire duration of the initial request and any retry attempts - MUST not exceed the Request timeout duration. If any retry attempts are - still in progress when the Request timeout duration has been reached, - these SHOULD be canceled if possible and the Gateway MUST immediately - return a timeout error. - - If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is - configured on the route, any retry attempts which reach the configured - BackendRequest timeout duration without a response SHOULD be canceled if - possible and the Gateway should wait for at least the specified backoff - duration before attempting to retry the backend request again. - - If a BackendRequest timeout is _not_ configured on the route, retry - attempts MAY time out after an implementation default duration, or MAY - remain pending until a configured Request timeout or implementation - default duration for total request time is reached. - - When this field is unspecified, the time to wait between retry attempts - is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - codes: - description: |- - Codes defines the HTTP response status codes for which a backend request - should be retried. - - Support: Extended - items: - description: |- - HTTPRouteRetryStatusCode defines an HTTP response status code for - which a backend request should be retried. - - Implementations MUST support the following status codes as retryable: - - * 500 - * 502 - * 503 - * 504 - - Implementations MAY support specifying additional discrete values in the - 500-599 range. - - Implementations MAY support specifying discrete values in the 400-499 range, - which are often inadvisable to retry. - - - maximum: 599 - minimum: 400 - type: integer - type: array - type: object - sessionPersistence: - description: |- - SessionPersistence defines and configures session persistence - for the route rule. - - Support: Extended - - - properties: - absoluteTimeout: - description: |- - AbsoluteTimeout defines the absolute timeout of the persistent - session. Once the AbsoluteTimeout duration has elapsed, the - session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - cookieConfig: - description: |- - CookieConfig provides configuration settings that are specific - to cookie-based session persistence. - - Support: Core - properties: - lifetimeType: - default: Session - description: |- - LifetimeType specifies whether the cookie has a permanent or - session-based lifetime. A permanent cookie persists until its - specified expiry time, defined by the Expires or Max-Age cookie - attributes, while a session cookie is deleted when the current - session ends. - - When set to "Permanent", AbsoluteTimeout indicates the - cookie's lifetime via the Expires or Max-Age cookie attributes - and is required. - - When set to "Session", AbsoluteTimeout indicates the - absolute lifetime of the cookie tracked by the gateway and - is optional. - - Defaults to "Session". - - Support: Core for "Session" type - - Support: Extended for "Permanent" type - enum: - - Permanent - - Session - type: string - type: object - idleTimeout: - description: |- - IdleTimeout defines the idle timeout of the persistent session. - Once the session has been idle for more than the specified - IdleTimeout duration, the session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - sessionName: - description: |- - SessionName defines the name of the persistent session token - which may be reflected in the cookie or the header. Users - should avoid reusing session names to prevent unintended - consequences, such as rejection or unpredictable behavior. - - Support: Implementation-specific - maxLength: 128 - type: string - type: - default: Cookie - description: |- - Type defines the type of session persistence such as through - the use a header or cookie. Defaults to cookie based session - persistence. - - Support: Core for "Cookie" type - - Support: Extended for "Header" type - enum: - - Cookie - - Header - type: string - type: object - x-kubernetes-validations: - - message: AbsoluteTimeout must be specified when cookie lifetimeType is Permanent - rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - timeouts: - description: |- - Timeouts defines the timeouts that can be configured for an HTTP request. - - Support: Extended - properties: - backendRequest: - description: |- - BackendRequest specifies a timeout for an individual request from the gateway - to a backend. This covers the time from when the request first starts being - sent from the gateway to when the full response has been received from the backend. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - An entire client HTTP transaction with a gateway, covered by the Request timeout, - may result in more than one call from the gateway to the destination backend, - for example, if automatic retries are supported. - - The value of BackendRequest must be a Gateway API Duration string as defined by - GEP-2257. When this field is unspecified, its behavior is implementation-specific; - when specified, the value of BackendRequest must be no more than the value of the - Request timeout (since the Request timeout encompasses the BackendRequest timeout). - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - request: - description: |- - Request specifies the maximum duration for a gateway to respond to an HTTP request. - If the gateway has not been able to respond before this deadline is met, the gateway - MUST return a timeout error. - - For example, setting the `rules.timeouts.request` field to the value `10s` in an - `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds - to complete. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - This timeout is intended to cover as close to the whole request-response transaction - as possible although an implementation MAY choose to start the timeout after the entire - request stream has been received instead of immediately after the transaction is - initiated by the client. - - The value of Request is a Gateway API Duration string as defined by GEP-2257. When this - field is unspecified, request timeout behavior is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - type: object - x-kubernetes-validations: - - message: backendRequest timeout cannot be longer than request timeout - rule: '!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration(''0s'') && duration(self.backendRequest) > duration(self.request))' - type: object - x-kubernetes-validations: - - message: RequestRedirect filter must not be used together with backendRefs - rule: '(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true' - - message: When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less than 128 - rule: '(self.size() > 0 ? self[0].matches.size() : 0) + (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() : 0) + (self.size() > 5 ? self[5].matches.size() : 0) + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() : 0) + (self.size() > 10 ? self[10].matches.size() : 0) + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() > 12 ? self[12].matches.size() : 0) + (self.size() > 13 ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() : 0) + (self.size() > 15 ? self[15].matches.size() : 0) <= 128' - type: object - type: object - ingress: - description: Ingress sets how the ingress object should look like with your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: IngressSpec describes the Ingress the user wishes to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain name of a network host, as defined by RFC 3986.\nNote the following deviations from the \"host\" part of the\nURI as defined in RFC 3986:\n1. IPs are not allowed. Currently an IngressRuleValue can only apply to\n the IP in the Spec of the parent Ingress.\n2. The `:` delimiter is not respected because ports are not allowed.\n\t Currently the port of an Ingress is implicitly :80 for http and\n\t :443 for https.\nBoth these may change in the future.\nIncoming requests are matched against the host before the\nIngressRuleValue. If the host is unspecified, the Ingress routes all\ntraffic based on the specified IngressRuleValue.\n\nhost can be \"precise\" which is a domain name without the terminating dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\", which is a domain name\nprefixed with a single wildcard label (e.g. \"*.foo.com\").\nThe wildcard character '*' must appear by itself as the first DNS label and\nmatches only a single label. You cannot have a wildcard label by itself (e.g. Host == \"*\").\nRequests will be matched against the Host field in the following way:\n1. If host is precise, the request matches this rule if the http host header is equal to Host.\n2. If host is a wildcard, then the request matches this rule if the http host header\nis to equal to the suffix (removing the first label) of the wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths that map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer security associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - jsonnet: - properties: - libraryLabelSelector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - type: object - persistentVolumeClaim: - description: PersistentVolumeClaim creates a PVC if you need to attach one to your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeMode: - description: PersistentVolumeMode describes how a volume is intended to be consumed, either Block or Filesystem. - type: string - volumeName: - description: VolumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - type: object - preferences: - description: Preferences holds the Grafana Preferences settings - properties: - homeDashboardUid: - type: string - type: object - route: - description: Route sets how the ingress object should look like with your grafana instance, this only works in Openshift. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - alternateBackends: - items: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - type: array - host: - type: string - path: - type: string - port: - description: RoutePort defines a port mapping from a router to an endpoint in the service endpoints. - properties: - targetPort: - anyOf: - - type: integer - - type: string - description: |- - The target port on pods selected by the service this route points to. - If this is a string, it will be looked up as a named port in the target - endpoints port list. Required - x-kubernetes-int-or-string: true - required: - - targetPort - type: object - subdomain: - type: string - tls: - description: TLSConfig defines config used to secure a route and provide termination - properties: - caCertificate: - description: caCertificate provides the cert authority certificate contents - type: string - certificate: - description: |- - certificate provides certificate contents. This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. - type: string - destinationCACertificate: - description: |- - destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt - termination this file should be provided in order to have routers use it for health checks on the secure connection. - If this field is not specified, the router may provide its own destination CA and perform hostname validation using - the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically - verify. - type: string - externalCertificate: - description: |- - externalCertificate provides certificate contents as a secret reference. - This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. The secret referenced should - be present in the same namespace as that of the Route. - Forbidden when `certificate` is set. - The router service account needs to be granted with read-only access to this secret, - please refer to openshift docs for additional details. - properties: - name: - description: |- - name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - insecureEdgeTerminationPolicy: - description: |- - insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While - each router may make its own decisions on which ports to expose, this is normally port 80. - - If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". - - * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). - - * None - no traffic is allowed on the insecure port (default). - - * Redirect - clients are redirected to the secure port. - enum: - - Allow - - None - - Redirect - - "" - type: string - key: - description: key provides key file contents - type: string - termination: - description: |- - termination indicates termination type. - - * edge - TLS termination is done by the router and http is used to communicate with the backend (default) - * passthrough - Traffic is sent straight to the destination without the router providing TLS termination - * reencrypt - TLS termination is done by the router and https is used to communicate with the backend - - Note: passthrough termination is incompatible with httpHeader actions - enum: - - edge - - reencrypt - - passthrough - type: string - required: - - termination - type: object - x-kubernetes-validations: - - message: 'cannot have both spec.tls.termination: passthrough and spec.tls.insecureEdgeTerminationPolicy: Allow' - rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) : true' - to: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - wildcardPolicy: - description: WildcardPolicyType indicates the type of wildcard support needed by routes. - type: string - type: object - type: object - service: - description: Service sets how the service object should look like with your grafana instance, contains a number of defaults. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: ServiceSpec describes the attributes that a user creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: |- - allocateLoadBalancerNodePorts defines if NodePorts will be automatically - allocated for services with type LoadBalancer. Default is "true". It - may be set to "false" if the cluster load-balancer does not rely on - NodePorts. If the caller requests specific NodePorts (by specifying a - value), those requests will be respected, regardless of this field. - This field may only be set for services with type LoadBalancer and will - be cleared if the type is changed to any other type. - type: boolean - clusterIP: - description: |- - clusterIP is the IP address of the service and is usually assigned - randomly. If an address is specified manually, is in-range (as per - system configuration), and is not in use, it will be allocated to the - service; otherwise creation of the service will fail. This field may not - be changed through updates unless the type field is also being changed - to ExternalName (which requires this field to be blank) or the type - field is being changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values are "None", - empty string (""), or a valid IP address. Setting this to "None" makes a - "headless service" (no virtual IP), which is useful when direct endpoint - connections are preferred and proxying is not required. Only applies to - types ClusterIP, NodePort, and LoadBalancer. If this field is specified - when creating a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - clusterIPs: - description: |- - ClusterIPs is a list of IP addresses assigned to this service, and are - usually assigned randomly. If an address is specified manually, is - in-range (as per system configuration), and is not in use, it will be - allocated to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the type field is - also being changed to ExternalName (which requires this field to be - empty) or the type field is being changed from ExternalName (in which - case this field may optionally be specified, as describe above). Valid - values are "None", empty string (""), or a valid IP address. Setting - this to "None" makes a "headless service" (no virtual IP), which is - useful when direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, and - LoadBalancer. If this field is specified when creating a Service of type - ExternalName, creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, it will - be initialized from the clusterIP field. If this field is specified, - clients must ensure that clusterIPs[0] and clusterIP have the same - value. - - This field may hold a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies field. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: |- - externalIPs is a list of IP addresses for which nodes in the cluster - will also accept traffic for this service. These IPs are not managed by - Kubernetes. The user is responsible for ensuring that traffic arrives - at a node with this IP. A common example is external load-balancers - that are not part of the Kubernetes system. - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalName: - description: |- - externalName is the external reference that discovery mechanisms will - return as an alias for this service (e.g. a DNS CNAME record). No - proxying will be involved. Must be a lowercase RFC-1123 hostname - (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: |- - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure - the service in a way that assumes that external load balancers will take care - of balancing the service traffic between nodes, and so each node will deliver - traffic only to the node-local endpoints of the service, without masquerading - the client source IP. (Traffic mistakenly sent to a node with no endpoints will - be dropped.) The default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology and other - features). Note that traffic sent to an External IP or LoadBalancer IP from - within the cluster will always get "Cluster" semantics, but clients sending to - a NodePort from within the cluster may need to take traffic policy into account - when picking a node. - type: string - healthCheckNodePort: - description: |- - healthCheckNodePort specifies the healthcheck nodePort for the service. - This only applies when type is set to LoadBalancer and - externalTrafficPolicy is set to Local. If a value is specified, is - in-range, and is not in use, it will be used. If not specified, a value - will be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints for this - service or not. If this field is specified when creating a Service - which does not need it, creation will fail. This field will be wiped - when updating a Service to no longer need it (e.g. changing type). - This field cannot be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: |- - InternalTrafficPolicy describes how nodes distribute service traffic they - receive on the ClusterIP. If set to "Local", the proxy will assume that pods - only want to talk to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The default value, - "Cluster", uses the standard behavior of routing to all endpoints evenly - (possibly modified by topology and other features). - type: string - ipFamilies: - description: |- - IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this - service. This field is usually assigned automatically based on cluster - configuration and the ipFamilyPolicy field. If this field is specified - manually, the requested family is available in the cluster, - and ipFamilyPolicy allows it, it will be used; otherwise creation of - the service will fail. This field is conditionally mutable: it allows - for adding or removing a secondary IP family, but it does not allow - changing the primary IP family of the Service. Valid values are "IPv4" - and "IPv6". This field only applies to Services of types ClusterIP, - NodePort, and LoadBalancer, and does apply to "headless" services. - This field will be wiped when updating a Service to type ExternalName. - - This field may hold a maximum of two entries (dual-stack families, in - either order). These families must correspond to the values of the - clusterIPs field, if specified. Both clusterIPs and ipFamilies are - governed by the ipFamilyPolicy field. - items: - description: |- - IPFamily represents the IP Family (IPv4 or IPv6). This type is used - to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: |- - IPFamilyPolicy represents the dual-stack-ness requested or required by - this Service. If there is no value provided, then this field will be set - to SingleStack. Services can be "SingleStack" (a single IP family), - "PreferDualStack" (two IP families on dual-stack configured clusters or - a single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise fail). The - ipFamilies and clusterIPs fields depend on the value of this field. This - field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: |- - loadBalancerClass is the class of the load balancer implementation this Service belongs to. - If specified, the value of this field must be a label-style identifier, with an optional prefix, - e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. - This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load - balancer implementation is used, today this is typically done through the cloud provider integration, - but should apply for any default implementation. If set, it is assumed that a load balancer - implementation is watching for Services with a matching class. Any default load balancer - implementation (e.g. cloud providers) should ignore Services that set this field. - This field can only be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: |- - Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider supports specifying - the loadBalancerIP when a load balancer is created. - This field will be ignored if the cloud-provider does not support the feature. - Deprecated: This field was under-specified and its meaning varies across implementations. - Using it is non-portable and it may not support dual-stack. - Users are encouraged to use implementation-specific annotations when available. - type: string - loadBalancerSourceRanges: - description: |- - If specified and supported by the platform, this will restrict traffic through the cloud-provider - load-balancer will be restricted to the specified client IPs. This field will be ignored if the - cloud-provider does not support the feature." - More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ - items: - type: string - type: array - x-kubernetes-list-type: atomic - ports: - description: |- - The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - description: ServicePort contains information on service's port. - properties: - appProtocol: - description: |- - The application protocol for this port. - This is used as a hint for implementations to offer richer behavior for protocols that they understand. - This field follows standard Kubernetes label syntax. - Valid values are either: - - * Un-prefixed protocol names - reserved for IANA standard service names (as per - RFC-6335 and https://www.iana.org/assignments/service-names). - - * Kubernetes-defined prefixed names: - * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- - * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 - * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 - - * Other protocols should use implementation-defined prefixed names such as - mycompany.com/my-custom-protocol. - type: string - name: - description: |- - The name of this port within the service. This must be a DNS_LABEL. - All ports within a ServiceSpec must have unique names. When considering - the endpoints for a Service, this must match the 'name' field in the - EndpointPort. - Optional if only one ServicePort is defined on this service. - type: string - nodePort: - description: |- - The port on each node on which this service is exposed when type is - NodePort or LoadBalancer. Usually assigned by the system. If a value is - specified, in-range, and not in use it will be used, otherwise the - operation will fail. If not specified, a port will be allocated if this - Service requires one. If this field is specified when creating a - Service which does not need it, creation will fail. This field will be - wiped when updating a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). - More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: |- - The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". - Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the pods targeted by the service. - Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named port in the - target Pod's container ports. If this is not specified, the value - of the 'port' field is used (an identity map). - This field is ignored for services with clusterIP=None, and should be - omitted or set equal to the 'port' field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: |- - publishNotReadyAddresses indicates that any agent which deals with endpoints for this - Service should disregard any indications of ready/not-ready. - The primary use case for setting this field is for a StatefulSet's Headless Service to - propagate SRV DNS records for its Pods for the purpose of peer discovery. - The Kubernetes controllers that generate Endpoints and EndpointSlice resources for - Services interpret this to mean that all endpoints are considered "ready" even if the - Pods themselves are not. Agents which consume only Kubernetes generated endpoints - through the Endpoints or EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: |- - Route service traffic to pods with label keys and values matching this - selector. If empty or not present, the service is assumed to have an - external process managing its endpoints, which Kubernetes will not - modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. - Ignored if type is ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: |- - Supports "ClientIP" and "None". Used to maintain session affinity. - Enable client IP based session affinity. - Must be ClientIP or None. - Defaults to None. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client IP based session affinity. - properties: - timeoutSeconds: - description: |- - timeoutSeconds specifies the seconds of ClientIP type session sticky time. - The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". - Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - trafficDistribution: - description: |- - TrafficDistribution offers a way to express preferences for how traffic - is distributed to Service endpoints. Implementations can use this field - as a hint, but are not required to guarantee strict adherence. If the - field is not set, the implementation will apply its default routing - strategy. If set to "PreferClose", implementations should prioritize - endpoints that are in the same zone. - type: string - type: - description: |- - type determines how the Service is exposed. Defaults to ClusterIP. Valid - options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - "ClusterIP" allocates a cluster-internal IP address for load-balancing - to endpoints. Endpoints are determined by the selector or if that is not - specified, by manual construction of an Endpoints object or - EndpointSlice objects. If clusterIP is "None", no virtual IP is - allocated and the endpoints are published as a set of endpoints rather - than a virtual IP. - "NodePort" builds on ClusterIP and allocates a port on every node which - routes to the same endpoints as the clusterIP. - "LoadBalancer" builds on NodePort and creates an external load-balancer - (if supported in the current cloud) which routes to the same endpoints - as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: string - type: object - type: object - serviceAccount: - description: ServiceAccount sets how the ServiceAccount object should look like with your grafana instance, contains a number of defaults. - properties: - automountServiceAccountToken: - type: boolean - imagePullSecrets: - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - secrets: - items: - description: ObjectReference contains enough information to let you inspect or modify the referred object. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: array - type: object - suspend: - description: Suspend pauses reconciliation of owned resources like deployments, Services, Etc. upon changes - type: boolean - version: - description: |- - Version sets the tag of the default image: docker.io/grafana/grafana. - Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" - default: 12.3.0 - type: string - type: object - status: - description: GrafanaStatus defines the observed state of Grafana - properties: - adminUrl: - type: string - alertRuleGroups: - items: - type: string - type: array - conditions: - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - contactPoints: - items: - type: string - type: array - dashboards: - items: - type: string - type: array - datasources: - items: - type: string - type: array - folders: - items: - type: string - type: array - lastMessage: - type: string - libraryPanels: - items: - type: string - type: array - muteTimings: - items: - type: string - type: array - notificationTemplates: - items: - type: string - type: array - serviceaccounts: - items: - type: string - type: array - stage: - type: string - stageStatus: - type: string - version: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanadatasources.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaDatasource - listKind: GrafanaDatasourceList - plural: grafanadatasources - singular: grafanadatasource - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaDatasource is the Schema for the grafanadatasources API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas outside the current namespace - type: boolean - datasource: - properties: - access: - type: string - basicAuth: - type: boolean - basicAuthUser: - type: string - database: - type: string - editable: - description: Whether to enable/disable editing of the datasource in Grafana UI - type: boolean - isDefault: - type: boolean - jsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - name: - type: string - orgId: - description: Deprecated field, it has no effect - format: int64 - type: integer - secureJsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - type: - type: string - uid: - description: Deprecated field, use spec.uid instead - type: string - url: - type: string - user: - type: string - type: object - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - plugins: - description: plugins - items: - properties: - name: - minLength: 1 - type: string - version: - pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ - type: string - required: - - name - - version - type: object - type: array - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator to ignore changes - type: boolean - uid: - description: |- - The UID, for the datasource, fallback to the deprecated spec.datasource.uid - and metadata.uid. Can be any string consisting of alphanumeric characters, - - and _ with a maximum length of 40 +optional - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - valuesFrom: - description: environments variables from secrets or config maps - items: - properties: - targetPath: - type: string - valueFrom: - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: Either configMapKeyRef or secretKeyRef must be set - rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) - required: - - targetPath - - valueFrom - type: object - maxItems: 99 - type: array - required: - - datasource - - instanceSelector - type: object - x-kubernetes-validations: - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' - status: - description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource - properties: - NoMatchingInstances: - description: The datasource instanceSelector can't find matching grafana instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastMessage: - description: 'Deprecated: Check status.conditions or operator logs' - type: string - lastResync: - description: Last time the resource was synchronized with Grafana instances - format: date-time - type: string - uid: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanafolders.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaFolder - listKind: GrafanaFolderList - plural: grafanafolders - singular: grafanafolder - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaFolder is the Schema for the grafanafolders API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaFolderSpec defines the desired state of GrafanaFolder - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas outside the current namespace - type: boolean - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - parentFolderRef: - description: Reference to an existing GrafanaFolder CR in the same namespace - type: string - parentFolderUID: - description: UID of the folder in which the current folder should be created - type: string - permissions: - description: Raw json with folder permissions, potentially exported from Grafana - type: string - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator to ignore changes - type: boolean - title: - description: Display name of the folder in Grafana - type: string - uid: - description: Manually specify the UID the Folder is created with. Can be any string consisting of alphanumeric characters, - and _ with a maximum length of 40 - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - required: - - instanceSelector - type: object - x-kubernetes-validations: - - message: Only one of parentFolderUID or parentFolderRef can be set - rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) && (has(self.parentFolderUID))) - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport && self.allowCrossNamespaceImport)' - status: - description: GrafanaFolderStatus defines the observed state of GrafanaFolder - properties: - NoMatchingInstances: - description: The folder instanceSelector can't find matching grafana instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastResync: - description: Last time the resource was synchronized with Grafana instances - format: date-time - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanas.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: Grafana - listKind: GrafanaList - plural: grafanas - singular: grafana - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.version - name: Version - type: string - - jsonPath: .status.stage - name: Stage - type: string - - jsonPath: .status.stageStatus - name: Stage status - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Grafana is the Schema for the grafanas API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaSpec defines the desired state of Grafana - properties: - client: - description: Client defines how the grafana-operator talks to the grafana instance. - properties: - headers: - additionalProperties: - type: string - description: Custom HTTP headers to use when interacting with this Grafana. - type: object - preferIngress: - description: If the operator should send it's request through the grafana instances ingress object instead of through the service. - nullable: true - type: boolean - timeout: - nullable: true - type: integer - tls: - description: TLS Configuration used to talk with the grafana instance. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate information - properties: - name: - description: name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: namespace defines the space within which the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - useKubeAuth: - description: |- - Use Kubernetes Serviceaccount as authentication - Requires configuring [auth.jwt] in the instance - type: boolean - type: object - config: - additionalProperties: - additionalProperties: - type: string - type: object - description: Config defines how your grafana ini file should looks like. - type: object - x-kubernetes-preserve-unknown-fields: true - deployment: - description: Deployment sets how the deployment object should look like with your grafana instance, contains a number of defaults. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - minReadySeconds: - format: int32 - type: integer - paused: - type: boolean - progressDeadlineSeconds: - format: int32 - type: integer - replicas: - format: int32 - type: integer - revisionHistoryLimit: - format: int32 - type: integer - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - strategy: - properties: - rollingUpdate: - properties: - maxSurge: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: - type: string - type: object - template: - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - activeDeadlineSeconds: - format: int64 - type: integer - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - type: boolean - containers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - type: string - enableServiceLinks: - type: boolean - ephemeralContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - targetContainerName: - type: string - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - type: string - required: - - ip - type: object - type: array - hostIPC: - type: boolean - hostNetwork: - type: boolean - hostPID: - type: boolean - hostUsers: - type: boolean - hostname: - type: string - imagePullSecrets: - items: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - type: array - initContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - nodeName: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - x-kubernetes-map-type: atomic - os: - properties: - name: - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - preemptionPolicy: - type: string - priority: - format: int32 - type: integer - priorityClassName: - type: string - readinessGates: - items: - properties: - conditionType: - type: string - required: - - conditionType - type: object - type: array - restartPolicy: - type: string - runtimeClassName: - type: string - schedulerName: - type: string - securityContext: - properties: - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - fsGroup: - format: int64 - type: integer - fsGroupChangePolicy: - type: string - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxChangePolicy: - type: string - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - type: string - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - serviceAccountName: - type: string - setHostnameAsFQDN: - type: boolean - shareProcessNamespace: - type: boolean - subdomain: - type: string - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - topologySpreadConstraints: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - format: int32 - type: integer - minDomains: - format: int32 - type: integer - nodeAffinityPolicy: - type: string - nodeTaintsPolicy: - type: string - topologyKey: - type: string - whenUnsatisfiable: - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - default: ext4 - type: string - kind: - type: string - readOnly: - default: false - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - type: boolean - volumeAttributes: - additionalProperties: - type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - image: - properties: - pullPolicy: - type: string - reference: - type: string - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - default: default - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - clusterTrustBundle: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - name: - type: string - optional: - type: boolean - path: - type: string - signerName: - type: string - required: - - path - type: object - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podCertificate: - properties: - certificateChainPath: - type: string - credentialBundlePath: - type: string - keyPath: - type: string - keyType: - type: string - maxExpirationSeconds: - format: int32 - type: integer - signerName: - type: string - required: - - keyType - - signerName - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - default: /etc/ceph/keyring - type: string - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - default: xfs - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - type: boolean - storageMode: - default: ThinProvisioned - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - type: object - type: object - type: object - type: object - disableDefaultAdminSecret: - description: DisableDefaultAdminSecret prevents operator from creating default admin-credentials secret - type: boolean - disableDefaultSecurityContext: - description: DisableDefaultSecurityContext prevents the operator from populating securityContext on deployments - enum: - - Pod - - Container - - All - type: string - external: - description: External enables you to configure external grafana instances that is not managed by the operator. - properties: - adminPassword: - description: AdminPassword key to talk to the external grafana instance. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - adminUser: - description: AdminUser key to talk to the external grafana instance. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - apiKey: - description: The API key to talk to the external grafana instance, you need to define ether apiKey or adminUser/adminPassword. - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - description: DEPRECATED, use top level `tls` instead. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate information - properties: - name: - description: name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: namespace defines the space within which the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - url: - description: URL of the external grafana instance you want to manage. - type: string - required: - - url - type: object - httpRoute: - description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. It will not be created if this is not set - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: HTTPRouteSpec defines the desired state of HTTPRoute - properties: - hostnames: - description: |- - Hostnames defines a set of hostnames that should match against the HTTP Host - header to select a HTTPRoute used to process the request. Implementations - MUST ignore any port value specified in the HTTP Host header while - performing a match and (absent of any applicable header modification - configuration) MUST forward this header unmodified to the backend. - - Valid values for Hostnames are determined by RFC 1123 definition of a - hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - If a hostname is specified by both the Listener and HTTPRoute, there - must be at least one intersecting hostname for the HTTPRoute to be - attached to the Listener. For example: - - * A Listener with `test.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames, or have specified at - least one of `test.example.com` or `*.example.com`. - * A Listener with `*.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames or have specified at least - one hostname that matches the Listener hostname. For example, - `*.example.com`, `test.example.com`, and `foo.test.example.com` would - all match. On the other hand, `example.com` and `test.example.net` would - not match. - - Hostnames that are prefixed with a wildcard label (`*.`) are interpreted - as a suffix match. That means that a match for `*.example.com` would match - both `test.example.com`, and `foo.test.example.com`, but not `example.com`. - - If both the Listener and HTTPRoute have specified hostnames, any - HTTPRoute hostnames that do not match the Listener hostname MUST be - ignored. For example, if a Listener specified `*.example.com`, and the - HTTPRoute specified `test.example.com` and `test.example.net`, - `test.example.net` must not be considered for a match. - - If both the Listener and HTTPRoute have specified hostnames, and none - match with the criteria above, then the HTTPRoute is not accepted. The - implementation must raise an 'Accepted' Condition with a status of - `False` in the corresponding RouteParentStatus. - - In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. - overlapping wildcard matching and exact matching hostnames), precedence must - be given to rules from the HTTPRoute with the largest number of: - - * Characters in a matching non-wildcard hostname. - * Characters in a matching hostname. - - If ties exist across multiple Routes, the matching precedence rules for - HTTPRouteMatches takes over. - - Support: Core - items: - description: |- - Hostname is the fully qualified domain name of a network host. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - Hostname can be "precise" which is a domain name without the terminating - dot of a network host (e.g. "foo.example.com") or "wildcard", which is a - domain name prefixed with a single wildcard label (e.g. `*.example.com`). - - Note that as per RFC1035 and RFC1123, a *label* must consist of lower case - alphanumeric characters or '-', and must start and end with an alphanumeric - character. No other punctuation is allowed. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - maxItems: 16 - type: array - parentRefs: - description: |- - ParentRefs references the resources (usually Gateways) that a Route wants - to be attached to. Note that the referenced parent resource needs to - allow this for the attachment to be complete. For Gateways, that means - the Gateway needs to allow attachment from Routes of this kind and - namespace. For Services, that means the Service must either be in the same - namespace for a "producer" route, or the mesh implementation must support - and allow "consumer" routes for the referenced Service. ReferenceGrant is - not applicable for governing ParentRefs to Services - it is not possible to - create a "producer" route for a Service in a different namespace from the - Route. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - ParentRefs must be _distinct_. This means either that: - - * They select different objects. If this is the case, then parentRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, `namespace`, and `name` must - be unique across all parentRef entries in the Route. - * They do not select different objects, but for each optional field used, - each ParentRef that selects the same object must set the same set of - optional fields to different values. If one ParentRef sets a - combination of optional fields, all must set the same combination. - - Some examples: - - * If one ParentRef sets `sectionName`, all ParentRefs referencing the - same object must also set `sectionName`. - * If one ParentRef sets `port`, all ParentRefs referencing the same - object must also set `port`. - * If one ParentRef sets `sectionName` and `port`, all ParentRefs - referencing the same object must also set `sectionName` and `port`. - - It is possible to separately reference multiple distinct objects that may - be collapsed by an implementation. For example, some implementations may - choose to merge compatible Gateway Listeners together. If that is the - case, the list of routes attached to those resources should also be - merged. - - Note that for ParentRefs that cross namespace boundaries, there are specific - rules. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example, - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable other kinds of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - - - - - items: - description: |- - ParentReference identifies an API object (usually a Gateway) that can be considered - a parent of this resource (usually a route). There are two kinds of parent resources - with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - maxItems: 32 - type: array - rules: - default: - - matches: - - path: - type: PathPrefix - value: / - description: |- - Rules are a list of HTTP matchers, filters and actions. - - - items: - description: |- - HTTPRouteRule defines semantics for matching an HTTP request based on - conditions (matches), processing it (filters), and forwarding the request to - an API object (backendRefs). - properties: - backendRefs: - description: |- - BackendRefs defines the backend(s) where matching requests should be - sent. - - Failure behavior here depends on how many BackendRefs are specified and - how many are invalid. - - If *all* entries in BackendRefs are invalid, and there are also no filters - specified in this route rule, *all* traffic which matches this rule MUST - receive a 500 status code. - - See the HTTPBackendRef definition for the rules about what makes a single - HTTPBackendRef invalid. - - When a HTTPBackendRef is invalid, 500 status codes MUST be returned for - requests that would have otherwise been routed to an invalid backend. If - multiple backends are specified, and some are invalid, the proportion of - requests that would otherwise have been routed to an invalid backend - MUST receive a 500 status code. - - For example, if two backends are specified with equal weights, and one is - invalid, 50 percent of traffic must receive a 500. Implementations may - choose how that 50 percent is determined. - - When a HTTPBackendRef refers to a Service that has no ready endpoints, - implementations SHOULD return a 503 for requests to that backend instead. - If an implementation chooses to do this, all of the above rules for 500 responses - MUST also apply for responses that return a 503. - - Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Core - items: - description: |- - HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. - - Note that when a namespace different than the local namespace is specified, a - ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - - - When the BackendRef points to a Kubernetes Service, implementations SHOULD - honor the appProtocol field if it is set for the target Service Port. - - Implementations supporting appProtocol SHOULD recognize the Kubernetes - Standard Application Protocols defined in KEP-3726. - - If a Service appProtocol isn't specified, an implementation MAY infer the - backend protocol through its own means. Implementations MAY infer the - protocol from the Route type referring to the backend Service. - - If a Route is not able to send traffic to the backend using the specified - protocol then the backend is considered invalid. Implementations MUST set the - "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. - - - properties: - filters: - description: |- - Filters defined at this level should be executed if and only if the - request is being forwarded to the backend defined here. - - Support: Implementation-specific (For broader support of filters, use the - Filters field in HTTPRouteRule.) - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than or equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type - rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil if the filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' - - message: filter.requestMirror must be specified for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' - - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' - - message: filter.urlRewrite must be specified for URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' - - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' - - message: filter.extensionRef must be specified for ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 - - message: ResponseHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - weight: - default: 1 - description: |- - Weight specifies the proportion of requests forwarded to the referenced - backend. This is computed as weight/(sum of all weights in this - BackendRefs list). For non-zero values, there may be some epsilon from - the exact proportion defined here depending on the precision an - implementation supports. Weight is not a percentage and the sum of - weights does not need to equal 100. - - If only one backend is specified and it has a weight greater than 0, 100% - of the traffic is forwarded to that backend. If weight is set to 0, no - traffic should be forwarded for this entry. If unspecified, weight - defaults to 1. - - Support for this field varies based on the context where used. - format: int32 - maximum: 1000000 - minimum: 0 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - maxItems: 16 - type: array - filters: - description: |- - Filters define the filters that are applied to requests that match - this rule. - - Wherever possible, implementations SHOULD implement filters in the order - they are specified. - - Implementations MAY choose to implement this ordering strictly, rejecting - any combination or order of filters that cannot be supported. If implementations - choose a strict interpretation of filter ordering, they MUST clearly document - that behavior. - - To reject an invalid combination or order of filters, implementations SHOULD - consider the Route Rules with this configuration invalid. If all Route Rules - in a Route are invalid, the entire Route would be considered invalid. If only - a portion of Route Rules are invalid, implementations MUST set the - "PartiallyInvalid" condition for the Route. - - Conformance-levels at this level are defined based on the type of filter: - - - ALL core filters MUST be supported by all implementations. - - Implementers are encouraged to support extended filters. - - Implementation-specific custom filters have no API guarantees across - implementations. - - Specifying the same filter multiple times is not supported unless explicitly - indicated in the filter. - - All filters are expected to be compatible with each other except for the - URLRewrite and RequestRedirect filters, which may not be combined. If an - implementation cannot support other combinations of filters, they must clearly - document that limitation. In cases where incompatible or unsupported - filters are specified and cause the `Accepted` condition to be set to status - `False`, implementations may use the `IncompatibleFilters` reason to specify - this configuration error. - - Support: Core - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than or equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction may be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type == ''ReplacePrefixMatch'' : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && self.type != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) && self.type == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) && self.type != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type - rule: '!(!has(self.responseHeaderModifier) && self.type == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil if the filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type != ''RequestMirror'')' - - message: filter.requestMirror must be specified for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type == ''RequestMirror'')' - - message: filter.requestRedirect must be nil if the filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if the filter.type is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' - - message: filter.urlRewrite must be specified for URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' - - message: filter.extensionRef must be nil if the filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type != ''ExtensionRef'')' - - message: filter.extensionRef must be specified for ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 - - message: ResponseHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() <= 1 - matches: - default: - - path: - type: PathPrefix - value: / - description: |- - Matches define conditions used for matching the rule against incoming - HTTP requests. Each match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. - - For example, take the following matches configuration: - - ``` - matches: - - path: - value: "/foo" - headers: - - name: "version" - value: "v2" - - path: - value: "/v2/foo" - ``` - - For a request to match against this rule, a request must satisfy - EITHER of the two conditions: - - - path prefixed with `/foo` AND contains the header `version: v2` - - path prefix of `/v2/foo` - - See the documentation for HTTPRouteMatch on how to specify multiple - match conditions that should be ANDed together. - - If no matches are specified, the default is a prefix - path match on "/", which has the effect of matching every - HTTP request. - - Proxy or Load Balancer routing configuration generated from HTTPRoutes - MUST prioritize matches based on the following criteria, continuing on - ties. Across all rules specified on applicable Routes, precedence must be - given to the match having: - - * "Exact" path match. - * "Prefix" path match with largest number of characters. - * Method match. - * Largest number of header matches. - * Largest number of query param matches. - - Note: The precedence of RegularExpression path matches are implementation-specific. - - If ties still exist across multiple Routes, matching precedence MUST be - determined in order of the following criteria, continuing on ties: - - * The oldest Route based on creation timestamp. - * The Route appearing first in alphabetical order by - "{namespace}/{name}". - - If ties still exist within an HTTPRoute, matching precedence MUST be granted - to the FIRST matching rule (in list order) with a match meeting the above - criteria. - - When no rules matching a request have been successfully attached to the - parent a request is coming from, a HTTP 404 status code MUST be returned. - items: - description: "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t value \"v1\"\n\n```" - properties: - headers: - description: |- - Headers specifies HTTP request header matchers. Multiple match values are - ANDed together, meaning, a request must match all the specified headers - to select the route. - items: - description: |- - HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request - headers. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - - When a header is repeated in an HTTP request, it is - implementation-specific behavior as to how this is represented. - Generally, proxies should follow the guidance from the RFC: - https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding - processing a repeated header, with special handling for "Set-Cookie". - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the header. - - Support: Core (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression HeaderMatchType has implementation-specific - conformance, implementations can support POSIX, PCRE or any other dialects - of regular expressions. Please read the implementation's documentation to - determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - method: - description: |- - Method specifies HTTP method matcher. - When specified, this route will be matched only if the request has the - specified method. - - Support: Extended - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - type: string - path: - default: - type: PathPrefix - value: / - description: |- - Path specifies a HTTP request path matcher. If this field is not - specified, a default prefix match on the "/" path is provided. - properties: - type: - default: PathPrefix - description: |- - Type specifies how to match against the path Value. - - Support: Core (Exact, PathPrefix) - - Support: Implementation-specific (RegularExpression) - enum: - - Exact - - PathPrefix - - RegularExpression - type: string - value: - default: / - description: Value of the HTTP path to match against. - maxLength: 1024 - type: string - type: object - x-kubernetes-validations: - - message: value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.startsWith(''/'') : true' - - message: must not contain '//' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''//'') : true' - - message: must not contain '/./' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/./'') : true' - - message: must not contain '/../' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''/../'') : true' - - message: must not contain '%2f' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2f'') : true' - - message: must not contain '%2F' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''%2F'') : true' - - message: must not contain '#' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.contains(''#'') : true' - - message: must not end with '/..' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/..'') : true' - - message: must not end with '/.' when type one of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? !self.value.endsWith(''/.'') : true' - - message: type must be one of ['Exact', 'PathPrefix', 'RegularExpression'] - rule: self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression' - - message: must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") : true' - queryParams: - description: |- - QueryParams specifies HTTP query parameter matchers. Multiple match - values are ANDed together, meaning, a request must match all the - specified query parameters to select the route. - - Support: Extended - items: - description: |- - HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP - query parameters. - properties: - name: - description: |- - Name is the name of the HTTP query param to be matched. This must be an - exact string match. (See - https://tools.ietf.org/html/rfc7230#section-2.7.3). - - If multiple entries specify equivalent query param names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent query param name MUST be ignored. - - If a query param is repeated in an HTTP request, the behavior is - purposely left undefined, since different data planes have different - capabilities. However, it is *recommended* that implementations should - match against the first value of the param if the data plane supports it, - as this behavior is expected in other load balancing contexts outside of - the Gateway API. - - Users SHOULD NOT route traffic based on repeated query params to guard - themselves against potential differences in the implementations. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the query parameter. - - Support: Extended (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression QueryParamMatchType has Implementation-specific - conformance, implementations can support POSIX, PCRE or any other - dialects of regular expressions. Please read the implementation's - documentation to determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP query param to be matched. - maxLength: 1024 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - maxItems: 64 - type: array - name: - description: |- - Name is the name of the route rule. This name MUST be unique within a Route if it is set. - - Support: Extended - - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retry: - description: |- - Retry defines the configuration for when to retry an HTTP request. - - Support: Extended - - - properties: - attempts: - description: |- - Attempts specifies the maximum number of times an individual request - from the gateway to a backend should be retried. - - If the maximum number of retries has been attempted without a successful - response from the backend, the Gateway MUST return an error. - - When this field is unspecified, the number of times to attempt to retry - a backend request is implementation-specific. - - Support: Extended - type: integer - backoff: - description: |- - Backoff specifies the minimum duration a Gateway should wait between - retry attempts and is represented in Gateway API Duration formatting. - - For example, setting the `rules[].retry.backoff` field to the value - `100ms` will cause a backend request to first be retried approximately - 100 milliseconds after timing out or receiving a response code configured - to be retryable. - - An implementation MAY use an exponential or alternative backoff strategy - for subsequent retry attempts, MAY cap the maximum backoff duration to - some amount greater than the specified minimum, and MAY add arbitrary - jitter to stagger requests, as long as unsuccessful backend requests are - not retried before the configured minimum duration. - - If a Request timeout (`rules[].timeouts.request`) is configured on the - route, the entire duration of the initial request and any retry attempts - MUST not exceed the Request timeout duration. If any retry attempts are - still in progress when the Request timeout duration has been reached, - these SHOULD be canceled if possible and the Gateway MUST immediately - return a timeout error. - - If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is - configured on the route, any retry attempts which reach the configured - BackendRequest timeout duration without a response SHOULD be canceled if - possible and the Gateway should wait for at least the specified backoff - duration before attempting to retry the backend request again. - - If a BackendRequest timeout is _not_ configured on the route, retry - attempts MAY time out after an implementation default duration, or MAY - remain pending until a configured Request timeout or implementation - default duration for total request time is reached. - - When this field is unspecified, the time to wait between retry attempts - is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - codes: - description: |- - Codes defines the HTTP response status codes for which a backend request - should be retried. - - Support: Extended - items: - description: |- - HTTPRouteRetryStatusCode defines an HTTP response status code for - which a backend request should be retried. - - Implementations MUST support the following status codes as retryable: - - * 500 - * 502 - * 503 - * 504 - - Implementations MAY support specifying additional discrete values in the - 500-599 range. - - Implementations MAY support specifying discrete values in the 400-499 range, - which are often inadvisable to retry. - - - maximum: 599 - minimum: 400 - type: integer - type: array - type: object - sessionPersistence: - description: |- - SessionPersistence defines and configures session persistence - for the route rule. - - Support: Extended - - - properties: - absoluteTimeout: - description: |- - AbsoluteTimeout defines the absolute timeout of the persistent - session. Once the AbsoluteTimeout duration has elapsed, the - session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - cookieConfig: - description: |- - CookieConfig provides configuration settings that are specific - to cookie-based session persistence. - - Support: Core - properties: - lifetimeType: - default: Session - description: |- - LifetimeType specifies whether the cookie has a permanent or - session-based lifetime. A permanent cookie persists until its - specified expiry time, defined by the Expires or Max-Age cookie - attributes, while a session cookie is deleted when the current - session ends. - - When set to "Permanent", AbsoluteTimeout indicates the - cookie's lifetime via the Expires or Max-Age cookie attributes - and is required. - - When set to "Session", AbsoluteTimeout indicates the - absolute lifetime of the cookie tracked by the gateway and - is optional. - - Defaults to "Session". - - Support: Core for "Session" type - - Support: Extended for "Permanent" type - enum: - - Permanent - - Session - type: string - type: object - idleTimeout: - description: |- - IdleTimeout defines the idle timeout of the persistent session. - Once the session has been idle for more than the specified - IdleTimeout duration, the session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - sessionName: - description: |- - SessionName defines the name of the persistent session token - which may be reflected in the cookie or the header. Users - should avoid reusing session names to prevent unintended - consequences, such as rejection or unpredictable behavior. - - Support: Implementation-specific - maxLength: 128 - type: string - type: - default: Cookie - description: |- - Type defines the type of session persistence such as through - the use a header or cookie. Defaults to cookie based session - persistence. - - Support: Core for "Cookie" type - - Support: Extended for "Header" type - enum: - - Cookie - - Header - type: string - type: object - x-kubernetes-validations: - - message: AbsoluteTimeout must be specified when cookie lifetimeType is Permanent - rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - timeouts: - description: |- - Timeouts defines the timeouts that can be configured for an HTTP request. - - Support: Extended - properties: - backendRequest: - description: |- - BackendRequest specifies a timeout for an individual request from the gateway - to a backend. This covers the time from when the request first starts being - sent from the gateway to when the full response has been received from the backend. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - An entire client HTTP transaction with a gateway, covered by the Request timeout, - may result in more than one call from the gateway to the destination backend, - for example, if automatic retries are supported. - - The value of BackendRequest must be a Gateway API Duration string as defined by - GEP-2257. When this field is unspecified, its behavior is implementation-specific; - when specified, the value of BackendRequest must be no more than the value of the - Request timeout (since the Request timeout encompasses the BackendRequest timeout). - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - request: - description: |- - Request specifies the maximum duration for a gateway to respond to an HTTP request. - If the gateway has not been able to respond before this deadline is met, the gateway - MUST return a timeout error. - - For example, setting the `rules.timeouts.request` field to the value `10s` in an - `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds - to complete. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - This timeout is intended to cover as close to the whole request-response transaction - as possible although an implementation MAY choose to start the timeout after the entire - request stream has been received instead of immediately after the transaction is - initiated by the client. - - The value of Request is a Gateway API Duration string as defined by GEP-2257. When this - field is unspecified, request timeout behavior is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - type: object - x-kubernetes-validations: - - message: backendRequest timeout cannot be longer than request timeout - rule: '!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration(''0s'') && duration(self.backendRequest) > duration(self.request))' - type: object - x-kubernetes-validations: - - message: RequestRedirect filter must not be used together with backendRefs - rule: '(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true' - - message: When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - - message: Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less than 128 - rule: '(self.size() > 0 ? self[0].matches.size() : 0) + (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() : 0) + (self.size() > 5 ? self[5].matches.size() : 0) + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() : 0) + (self.size() > 10 ? self[10].matches.size() : 0) + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() > 12 ? self[12].matches.size() : 0) + (self.size() > 13 ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() : 0) + (self.size() > 15 ? self[15].matches.size() : 0) <= 128' - type: object - type: object - ingress: - description: Ingress sets how the ingress object should look like with your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: IngressSpec describes the Ingress the user wishes to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain name of a network host, as defined by RFC 3986.\nNote the following deviations from the \"host\" part of the\nURI as defined in RFC 3986:\n1. IPs are not allowed. Currently an IngressRuleValue can only apply to\n the IP in the Spec of the parent Ingress.\n2. The `:` delimiter is not respected because ports are not allowed.\n\t Currently the port of an Ingress is implicitly :80 for http and\n\t :443 for https.\nBoth these may change in the future.\nIncoming requests are matched against the host before the\nIngressRuleValue. If the host is unspecified, the Ingress routes all\ntraffic based on the specified IngressRuleValue.\n\nhost can be \"precise\" which is a domain name without the terminating dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\", which is a domain name\nprefixed with a single wildcard label (e.g. \"*.foo.com\").\nThe wildcard character '*' must appear by itself as the first DNS label and\nmatches only a single label. You cannot have a wildcard label by itself (e.g. Host == \"*\").\nRequests will be matched against the Host field in the following way:\n1. If host is precise, the request matches this rule if the http host header is equal to Host.\n2. If host is a wildcard, then the request matches this rule if the http host header\nis to equal to the suffix (removing the first label) of the wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths that map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer security associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - jsonnet: - properties: - libraryLabelSelector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - type: object - persistentVolumeClaim: - description: PersistentVolumeClaim creates a PVC if you need to attach one to your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeMode: - description: PersistentVolumeMode describes how a volume is intended to be consumed, either Block or Filesystem. - type: string - volumeName: - description: VolumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - type: object - preferences: - description: Preferences holds the Grafana Preferences settings - properties: - homeDashboardUid: - type: string - type: object - route: - description: Route sets how the ingress object should look like with your grafana instance, this only works in Openshift. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - alternateBackends: - items: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - type: array - host: - type: string - path: - type: string - port: - description: RoutePort defines a port mapping from a router to an endpoint in the service endpoints. - properties: - targetPort: - anyOf: - - type: integer - - type: string - description: |- - The target port on pods selected by the service this route points to. - If this is a string, it will be looked up as a named port in the target - endpoints port list. Required - x-kubernetes-int-or-string: true - required: - - targetPort - type: object - subdomain: - type: string - tls: - description: TLSConfig defines config used to secure a route and provide termination - properties: - caCertificate: - description: caCertificate provides the cert authority certificate contents - type: string - certificate: - description: |- - certificate provides certificate contents. This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. - type: string - destinationCACertificate: - description: |- - destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt - termination this file should be provided in order to have routers use it for health checks on the secure connection. - If this field is not specified, the router may provide its own destination CA and perform hostname validation using - the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically - verify. - type: string - externalCertificate: - description: |- - externalCertificate provides certificate contents as a secret reference. - This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. The secret referenced should - be present in the same namespace as that of the Route. - Forbidden when `certificate` is set. - The router service account needs to be granted with read-only access to this secret, - please refer to openshift docs for additional details. - properties: - name: - description: |- - name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - insecureEdgeTerminationPolicy: - description: |- - insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While - each router may make its own decisions on which ports to expose, this is normally port 80. - - If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". - - * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). - - * None - no traffic is allowed on the insecure port (default). - - * Redirect - clients are redirected to the secure port. - enum: - - Allow - - None - - Redirect - - "" - type: string - key: - description: key provides key file contents - type: string - termination: - description: |- - termination indicates termination type. - - * edge - TLS termination is done by the router and http is used to communicate with the backend (default) - * passthrough - Traffic is sent straight to the destination without the router providing TLS termination - * reencrypt - TLS termination is done by the router and https is used to communicate with the backend - - Note: passthrough termination is incompatible with httpHeader actions - enum: - - edge - - reencrypt - - passthrough - type: string - required: - - termination - type: object - x-kubernetes-validations: - - message: 'cannot have both spec.tls.termination: passthrough and spec.tls.insecureEdgeTerminationPolicy: Allow' - rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) : true' - to: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - wildcardPolicy: - description: WildcardPolicyType indicates the type of wildcard support needed by routes. - type: string - type: object - type: object - service: - description: Service sets how the service object should look like with your grafana instance, contains a number of defaults. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: ServiceSpec describes the attributes that a user creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: |- - allocateLoadBalancerNodePorts defines if NodePorts will be automatically - allocated for services with type LoadBalancer. Default is "true". It - may be set to "false" if the cluster load-balancer does not rely on - NodePorts. If the caller requests specific NodePorts (by specifying a - value), those requests will be respected, regardless of this field. - This field may only be set for services with type LoadBalancer and will - be cleared if the type is changed to any other type. - type: boolean - clusterIP: - description: |- - clusterIP is the IP address of the service and is usually assigned - randomly. If an address is specified manually, is in-range (as per - system configuration), and is not in use, it will be allocated to the - service; otherwise creation of the service will fail. This field may not - be changed through updates unless the type field is also being changed - to ExternalName (which requires this field to be blank) or the type - field is being changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values are "None", - empty string (""), or a valid IP address. Setting this to "None" makes a - "headless service" (no virtual IP), which is useful when direct endpoint - connections are preferred and proxying is not required. Only applies to - types ClusterIP, NodePort, and LoadBalancer. If this field is specified - when creating a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - clusterIPs: - description: |- - ClusterIPs is a list of IP addresses assigned to this service, and are - usually assigned randomly. If an address is specified manually, is - in-range (as per system configuration), and is not in use, it will be - allocated to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the type field is - also being changed to ExternalName (which requires this field to be - empty) or the type field is being changed from ExternalName (in which - case this field may optionally be specified, as describe above). Valid - values are "None", empty string (""), or a valid IP address. Setting - this to "None" makes a "headless service" (no virtual IP), which is - useful when direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, and - LoadBalancer. If this field is specified when creating a Service of type - ExternalName, creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, it will - be initialized from the clusterIP field. If this field is specified, - clients must ensure that clusterIPs[0] and clusterIP have the same - value. - - This field may hold a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies field. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: |- - externalIPs is a list of IP addresses for which nodes in the cluster - will also accept traffic for this service. These IPs are not managed by - Kubernetes. The user is responsible for ensuring that traffic arrives - at a node with this IP. A common example is external load-balancers - that are not part of the Kubernetes system. - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalName: - description: |- - externalName is the external reference that discovery mechanisms will - return as an alias for this service (e.g. a DNS CNAME record). No - proxying will be involved. Must be a lowercase RFC-1123 hostname - (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: |- - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure - the service in a way that assumes that external load balancers will take care - of balancing the service traffic between nodes, and so each node will deliver - traffic only to the node-local endpoints of the service, without masquerading - the client source IP. (Traffic mistakenly sent to a node with no endpoints will - be dropped.) The default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology and other - features). Note that traffic sent to an External IP or LoadBalancer IP from - within the cluster will always get "Cluster" semantics, but clients sending to - a NodePort from within the cluster may need to take traffic policy into account - when picking a node. - type: string - healthCheckNodePort: - description: |- - healthCheckNodePort specifies the healthcheck nodePort for the service. - This only applies when type is set to LoadBalancer and - externalTrafficPolicy is set to Local. If a value is specified, is - in-range, and is not in use, it will be used. If not specified, a value - will be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints for this - service or not. If this field is specified when creating a Service - which does not need it, creation will fail. This field will be wiped - when updating a Service to no longer need it (e.g. changing type). - This field cannot be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: |- - InternalTrafficPolicy describes how nodes distribute service traffic they - receive on the ClusterIP. If set to "Local", the proxy will assume that pods - only want to talk to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The default value, - "Cluster", uses the standard behavior of routing to all endpoints evenly - (possibly modified by topology and other features). - type: string - ipFamilies: - description: |- - IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this - service. This field is usually assigned automatically based on cluster - configuration and the ipFamilyPolicy field. If this field is specified - manually, the requested family is available in the cluster, - and ipFamilyPolicy allows it, it will be used; otherwise creation of - the service will fail. This field is conditionally mutable: it allows - for adding or removing a secondary IP family, but it does not allow - changing the primary IP family of the Service. Valid values are "IPv4" - and "IPv6". This field only applies to Services of types ClusterIP, - NodePort, and LoadBalancer, and does apply to "headless" services. - This field will be wiped when updating a Service to type ExternalName. - - This field may hold a maximum of two entries (dual-stack families, in - either order). These families must correspond to the values of the - clusterIPs field, if specified. Both clusterIPs and ipFamilies are - governed by the ipFamilyPolicy field. - items: - description: |- - IPFamily represents the IP Family (IPv4 or IPv6). This type is used - to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: |- - IPFamilyPolicy represents the dual-stack-ness requested or required by - this Service. If there is no value provided, then this field will be set - to SingleStack. Services can be "SingleStack" (a single IP family), - "PreferDualStack" (two IP families on dual-stack configured clusters or - a single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise fail). The - ipFamilies and clusterIPs fields depend on the value of this field. This - field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: |- - loadBalancerClass is the class of the load balancer implementation this Service belongs to. - If specified, the value of this field must be a label-style identifier, with an optional prefix, - e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. - This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load - balancer implementation is used, today this is typically done through the cloud provider integration, - but should apply for any default implementation. If set, it is assumed that a load balancer - implementation is watching for Services with a matching class. Any default load balancer - implementation (e.g. cloud providers) should ignore Services that set this field. - This field can only be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: |- - Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider supports specifying - the loadBalancerIP when a load balancer is created. - This field will be ignored if the cloud-provider does not support the feature. - Deprecated: This field was under-specified and its meaning varies across implementations. - Using it is non-portable and it may not support dual-stack. - Users are encouraged to use implementation-specific annotations when available. - type: string - loadBalancerSourceRanges: - description: |- - If specified and supported by the platform, this will restrict traffic through the cloud-provider - load-balancer will be restricted to the specified client IPs. This field will be ignored if the - cloud-provider does not support the feature." - More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ - items: - type: string - type: array - x-kubernetes-list-type: atomic - ports: - description: |- - The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - description: ServicePort contains information on service's port. - properties: - appProtocol: - description: |- - The application protocol for this port. - This is used as a hint for implementations to offer richer behavior for protocols that they understand. - This field follows standard Kubernetes label syntax. - Valid values are either: - - * Un-prefixed protocol names - reserved for IANA standard service names (as per - RFC-6335 and https://www.iana.org/assignments/service-names). - - * Kubernetes-defined prefixed names: - * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- - * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 - * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 - - * Other protocols should use implementation-defined prefixed names such as - mycompany.com/my-custom-protocol. - type: string - name: - description: |- - The name of this port within the service. This must be a DNS_LABEL. - All ports within a ServiceSpec must have unique names. When considering - the endpoints for a Service, this must match the 'name' field in the - EndpointPort. - Optional if only one ServicePort is defined on this service. - type: string - nodePort: - description: |- - The port on each node on which this service is exposed when type is - NodePort or LoadBalancer. Usually assigned by the system. If a value is - specified, in-range, and not in use it will be used, otherwise the - operation will fail. If not specified, a port will be allocated if this - Service requires one. If this field is specified when creating a - Service which does not need it, creation will fail. This field will be - wiped when updating a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). - More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: |- - The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". - Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the pods targeted by the service. - Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named port in the - target Pod's container ports. If this is not specified, the value - of the 'port' field is used (an identity map). - This field is ignored for services with clusterIP=None, and should be - omitted or set equal to the 'port' field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: |- - publishNotReadyAddresses indicates that any agent which deals with endpoints for this - Service should disregard any indications of ready/not-ready. - The primary use case for setting this field is for a StatefulSet's Headless Service to - propagate SRV DNS records for its Pods for the purpose of peer discovery. - The Kubernetes controllers that generate Endpoints and EndpointSlice resources for - Services interpret this to mean that all endpoints are considered "ready" even if the - Pods themselves are not. Agents which consume only Kubernetes generated endpoints - through the Endpoints or EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: |- - Route service traffic to pods with label keys and values matching this - selector. If empty or not present, the service is assumed to have an - external process managing its endpoints, which Kubernetes will not - modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. - Ignored if type is ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: |- - Supports "ClientIP" and "None". Used to maintain session affinity. - Enable client IP based session affinity. - Must be ClientIP or None. - Defaults to None. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client IP based session affinity. - properties: - timeoutSeconds: - description: |- - timeoutSeconds specifies the seconds of ClientIP type session sticky time. - The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". - Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - trafficDistribution: - description: |- - TrafficDistribution offers a way to express preferences for how traffic - is distributed to Service endpoints. Implementations can use this field - as a hint, but are not required to guarantee strict adherence. If the - field is not set, the implementation will apply its default routing - strategy. If set to "PreferClose", implementations should prioritize - endpoints that are in the same zone. - type: string - type: - description: |- - type determines how the Service is exposed. Defaults to ClusterIP. Valid - options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - "ClusterIP" allocates a cluster-internal IP address for load-balancing - to endpoints. Endpoints are determined by the selector or if that is not - specified, by manual construction of an Endpoints object or - EndpointSlice objects. If clusterIP is "None", no virtual IP is - allocated and the endpoints are published as a set of endpoints rather - than a virtual IP. - "NodePort" builds on ClusterIP and allocates a port on every node which - routes to the same endpoints as the clusterIP. - "LoadBalancer" builds on NodePort and creates an external load-balancer - (if supported in the current cloud) which routes to the same endpoints - as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: string - type: object - type: object - serviceAccount: - description: ServiceAccount sets how the ServiceAccount object should look like with your grafana instance, contains a number of defaults. - properties: - automountServiceAccountToken: - type: boolean - imagePullSecrets: - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - metadata: - description: ObjectMeta contains only a [subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - secrets: - items: - description: ObjectReference contains enough information to let you inspect or modify the referred object. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: array - type: object - suspend: - description: Suspend pauses reconciliation of owned resources like deployments, Services, Etc. upon changes - type: boolean - version: - description: |- - Version sets the tag of the default image: docker.io/grafana/grafana. - Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" - default: 12.3.0 - type: string - type: object - status: - description: GrafanaStatus defines the observed state of Grafana - properties: - adminUrl: - type: string - alertRuleGroups: - items: - type: string - type: array - conditions: - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - contactPoints: - items: - type: string - type: array - dashboards: - items: - type: string - type: array - datasources: - items: - type: string - type: array - folders: - items: - type: string - type: array - lastMessage: - type: string - libraryPanels: - items: - type: string - type: array - muteTimings: - items: - type: string - type: array - notificationTemplates: - items: - type: string - type: array - serviceaccounts: - items: - type: string - type: array - stage: - type: string - stageStatus: - type: string - version: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml b/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml deleted file mode 100644 index 0106f1e11..000000000 --- a/kustomize/overlays/mlops/grafana/crds/grafanadatasources.yaml +++ /dev/null @@ -1,348 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanadatasources.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaDatasource - listKind: GrafanaDatasourceList - plural: grafanadatasources - singular: grafanadatasource - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaDatasource is the Schema for the grafanadatasources API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaDatasourceSpec defines the desired state of GrafanaDatasource - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas - outside the current namespace - type: boolean - datasource: - properties: - access: - type: string - basicAuth: - type: boolean - basicAuthUser: - type: string - database: - type: string - editable: - description: Whether to enable/disable editing of the datasource - in Grafana UI - type: boolean - isDefault: - type: boolean - jsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - name: - type: string - orgId: - description: Deprecated field, it has no effect - format: int64 - type: integer - secureJsonData: - type: object - x-kubernetes-preserve-unknown-fields: true - type: - type: string - uid: - description: Deprecated field, use spec.uid instead - type: string - url: - type: string - user: - type: string - type: object - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - plugins: - description: plugins - items: - properties: - name: - minLength: 1 - type: string - version: - pattern: ^((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest)$ - type: string - required: - - name - - version - type: object - type: array - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if - not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator - to ignore changes - type: boolean - uid: - description: |- - The UID, for the datasource, fallback to the deprecated spec.datasource.uid - and metadata.uid. Can be any string consisting of alphanumeric characters, - - and _ with a maximum length of 40 +optional - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - valuesFrom: - description: environments variables from secrets or config maps - items: - properties: - targetPath: - type: string - valueFrom: - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: Either configMapKeyRef or secretKeyRef must be set - rule: (has(self.configMapKeyRef) && !has(self.secretKeyRef)) - || (!has(self.configMapKeyRef) && has(self.secretKeyRef)) - required: - - targetPath - - valueFrom - type: object - maxItems: 99 - type: array - required: - - datasource - - instanceSelector - type: object - x-kubernetes-validations: - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && - has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate - to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport - && self.allowCrossNamespaceImport)' - status: - description: GrafanaDatasourceStatus defines the observed state of GrafanaDatasource - properties: - NoMatchingInstances: - description: The datasource instanceSelector can't find matching grafana - instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastMessage: - description: 'Deprecated: Check status.conditions or operator logs' - type: string - lastResync: - description: Last time the resource was synchronized with Grafana - instances - format: date-time - type: string - uid: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml b/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml deleted file mode 100644 index d1f75184b..000000000 --- a/kustomize/overlays/mlops/grafana/crds/grafanafolders.yaml +++ /dev/null @@ -1,241 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanafolders.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: GrafanaFolder - listKind: GrafanaFolderList - plural: grafanafolders - singular: grafanafolder - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.NoMatchingInstances - name: No matching instances - type: boolean - - format: date-time - jsonPath: .status.lastResync - name: Last resync - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: GrafanaFolder is the Schema for the grafanafolders API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaFolderSpec defines the desired state of GrafanaFolder - properties: - allowCrossNamespaceImport: - default: false - description: Allow the Operator to match this resource with Grafanas - outside the current namespace - type: boolean - instanceSelector: - description: Selects Grafana instances for import - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: spec.instanceSelector is immutable - rule: self == oldSelf - parentFolderRef: - description: Reference to an existing GrafanaFolder CR in the same - namespace - type: string - parentFolderUID: - description: UID of the folder in which the current folder should - be created - type: string - permissions: - description: Raw json with folder permissions, potentially exported - from Grafana - type: string - resyncPeriod: - description: How often the resource is synced, defaults to 10m0s if - not set - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ - type: string - suspend: - description: Suspend pauses synchronizing attempts and tells the operator - to ignore changes - type: boolean - title: - description: Display name of the folder in Grafana - type: string - uid: - description: Manually specify the UID the Folder is created with. - Can be any string consisting of alphanumeric characters, - and _ - with a maximum length of 40 - maxLength: 40 - pattern: ^[a-zA-Z0-9-_]+$ - type: string - x-kubernetes-validations: - - message: spec.uid is immutable - rule: self == oldSelf - required: - - instanceSelector - type: object - x-kubernetes-validations: - - message: Only one of parentFolderUID or parentFolderRef can be set - rule: (has(self.parentFolderUID) && !(has(self.parentFolderRef))) || - (has(self.parentFolderRef) && !(has(self.parentFolderUID))) || !(has(self.parentFolderRef) - && (has(self.parentFolderUID))) - - message: spec.uid is immutable - rule: ((!has(oldSelf.uid) && !has(self.uid)) || (has(oldSelf.uid) && - has(self.uid))) - - message: disabling spec.allowCrossNamespaceImport requires a recreate - to ensure desired state - rule: '!oldSelf.allowCrossNamespaceImport || (oldSelf.allowCrossNamespaceImport - && self.allowCrossNamespaceImport)' - status: - description: GrafanaFolderStatus defines the observed state of GrafanaFolder - properties: - NoMatchingInstances: - description: The folder instanceSelector can't find matching grafana - instances - type: boolean - conditions: - description: Results when synchonizing resource with Grafana instances - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - hash: - type: string - lastResync: - description: Last time the resource was synchronized with Grafana - instances - format: date-time - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/grafanas.yaml b/kustomize/overlays/mlops/grafana/crds/grafanas.yaml deleted file mode 100644 index 13c3357d2..000000000 --- a/kustomize/overlays/mlops/grafana/crds/grafanas.yaml +++ /dev/null @@ -1,8687 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - operatorframework.io/installed-alongside-1b9468b77f87bb71: exploit-iq-mlops/grafana-operator.v5.21.2 - labels: - olm.managed: "true" - operators.coreos.com/grafana-operator.exploit-iq-mlops: "" - name: grafanas.grafana.integreatly.org -spec: - conversion: - strategy: None - group: grafana.integreatly.org - names: - categories: - - grafana-operator - kind: Grafana - listKind: GrafanaList - plural: grafanas - singular: grafana - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.version - name: Version - type: string - - jsonPath: .status.stage - name: Stage - type: string - - jsonPath: .status.stageStatus - name: Stage status - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Grafana is the Schema for the grafanas API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: GrafanaSpec defines the desired state of Grafana - properties: - client: - description: Client defines how the grafana-operator talks to the - grafana instance. - properties: - headers: - additionalProperties: - type: string - description: Custom HTTP headers to use when interacting with - this Grafana. - type: object - preferIngress: - description: If the operator should send it's request through - the grafana instances ingress object instead of through the - service. - nullable: true - type: boolean - timeout: - nullable: true - type: integer - tls: - description: TLS Configuration used to talk with the grafana instance. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate - information - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set - at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) - || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - useKubeAuth: - description: |- - Use Kubernetes Serviceaccount as authentication - Requires configuring [auth.jwt] in the instance - type: boolean - type: object - config: - additionalProperties: - additionalProperties: - type: string - type: object - description: Config defines how your grafana ini file should looks - like. - type: object - x-kubernetes-preserve-unknown-fields: true - deployment: - description: Deployment sets how the deployment object should look - like with your grafana instance, contains a number of defaults. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - minReadySeconds: - format: int32 - type: integer - paused: - type: boolean - progressDeadlineSeconds: - format: int32 - type: integer - replicas: - format: int32 - type: integer - revisionHistoryLimit: - format: int32 - type: integer - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - strategy: - properties: - rollingUpdate: - properties: - maxSurge: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: - type: string - type: object - template: - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - activeDeadlineSeconds: - format: int64 - type: integer - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - type: boolean - containers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - type: string - enableServiceLinks: - type: boolean - ephemeralContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - targetContainerName: - type: string - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - type: string - required: - - ip - type: object - type: array - hostIPC: - type: boolean - hostNetwork: - type: boolean - hostPID: - type: boolean - hostUsers: - type: boolean - hostname: - type: string - imagePullSecrets: - items: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - type: array - initContainers: - items: - properties: - args: - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - properties: - key: - type: string - optional: - default: false - type: boolean - path: - type: string - volumeName: - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - items: - properties: - configMapRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - type: string - secretRef: - properties: - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - sleep: - properties: - seconds: - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - type: string - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - default: TCP - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - resizePolicy: - items: - properties: - resourceName: - type: string - restartPolicy: - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - restartPolicy: - type: string - restartPolicyRules: - items: - properties: - action: - type: string - exitCodes: - properties: - operator: - type: string - values: - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - capabilities: - properties: - add: - items: - type: string - type: array - x-kubernetes-list-type: atomic - drop: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - recursiveReadOnly: - type: string - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - type: string - required: - - name - type: object - type: array - nodeName: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - x-kubernetes-map-type: atomic - os: - properties: - name: - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - preemptionPolicy: - type: string - priority: - format: int32 - type: integer - priorityClassName: - type: string - readinessGates: - items: - properties: - conditionType: - type: string - required: - - conditionType - type: object - type: array - restartPolicy: - type: string - runtimeClassName: - type: string - schedulerName: - type: string - securityContext: - properties: - appArmorProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - fsGroup: - format: int64 - type: integer - fsGroupChangePolicy: - type: string - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxChangePolicy: - type: string - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - seccompProfile: - properties: - localhostProfile: - type: string - type: - type: string - required: - - type - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - type: string - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - hostProcess: - type: boolean - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - serviceAccountName: - type: string - setHostnameAsFQDN: - type: boolean - shareProcessNamespace: - type: boolean - subdomain: - type: string - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - topologySpreadConstraints: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - format: int32 - type: integer - minDomains: - format: int32 - type: integer - nodeAffinityPolicy: - type: string - nodeTaintsPolicy: - type: string - topologyKey: - type: string - whenUnsatisfiable: - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - default: ext4 - type: string - kind: - type: string - readOnly: - default: false - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - type: boolean - volumeAttributes: - additionalProperties: - type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - image: - properties: - pullPolicy: - type: string - reference: - type: string - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - default: default - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - clusterTrustBundle: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - name: - type: string - optional: - type: boolean - path: - type: string - signerName: - type: string - required: - - path - type: object - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podCertificate: - properties: - certificateChainPath: - type: string - credentialBundlePath: - type: string - keyPath: - type: string - keyType: - type: string - maxExpirationSeconds: - format: int32 - type: integer - signerName: - type: string - required: - - keyType - - signerName - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - type: string - optional: - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - default: /etc/ceph/keyring - type: string - monitors: - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - default: xfs - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - type: boolean - storageMode: - default: ThinProvisioned - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - default: "" - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - type: object - type: object - type: object - type: object - disableDefaultAdminSecret: - description: DisableDefaultAdminSecret prevents operator from creating - default admin-credentials secret - type: boolean - disableDefaultSecurityContext: - description: DisableDefaultSecurityContext prevents the operator from - populating securityContext on deployments - enum: - - Pod - - Container - - All - type: string - external: - description: External enables you to configure external grafana instances - that is not managed by the operator. - properties: - adminPassword: - description: AdminPassword key to talk to the external grafana - instance. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - adminUser: - description: AdminUser key to talk to the external grafana instance. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - apiKey: - description: The API key to talk to the external grafana instance, - you need to define ether apiKey or adminUser/adminPassword. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - description: DEPRECATED, use top level `tls` instead. - properties: - certSecretRef: - description: Use a secret as a reference to give TLS Certificate - information - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - insecureSkipVerify: - description: Disable the CA check of the server - type: boolean - type: object - x-kubernetes-validations: - - message: insecureSkipVerify and certSecretRef cannot be set - at the same time - rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) - || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) - url: - description: URL of the external grafana instance you want to - manage. - type: string - required: - - url - type: object - httpRoute: - description: HTTPRoute customizes the GatewayAPI HTTPRoute Object. - It will not be created if this is not set - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: HTTPRouteSpec defines the desired state of HTTPRoute - properties: - hostnames: - description: |- - Hostnames defines a set of hostnames that should match against the HTTP Host - header to select a HTTPRoute used to process the request. Implementations - MUST ignore any port value specified in the HTTP Host header while - performing a match and (absent of any applicable header modification - configuration) MUST forward this header unmodified to the backend. - - Valid values for Hostnames are determined by RFC 1123 definition of a - hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - If a hostname is specified by both the Listener and HTTPRoute, there - must be at least one intersecting hostname for the HTTPRoute to be - attached to the Listener. For example: - - * A Listener with `test.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames, or have specified at - least one of `test.example.com` or `*.example.com`. - * A Listener with `*.example.com` as the hostname matches HTTPRoutes - that have either not specified any hostnames or have specified at least - one hostname that matches the Listener hostname. For example, - `*.example.com`, `test.example.com`, and `foo.test.example.com` would - all match. On the other hand, `example.com` and `test.example.net` would - not match. - - Hostnames that are prefixed with a wildcard label (`*.`) are interpreted - as a suffix match. That means that a match for `*.example.com` would match - both `test.example.com`, and `foo.test.example.com`, but not `example.com`. - - If both the Listener and HTTPRoute have specified hostnames, any - HTTPRoute hostnames that do not match the Listener hostname MUST be - ignored. For example, if a Listener specified `*.example.com`, and the - HTTPRoute specified `test.example.com` and `test.example.net`, - `test.example.net` must not be considered for a match. - - If both the Listener and HTTPRoute have specified hostnames, and none - match with the criteria above, then the HTTPRoute is not accepted. The - implementation must raise an 'Accepted' Condition with a status of - `False` in the corresponding RouteParentStatus. - - In the event that multiple HTTPRoutes specify intersecting hostnames (e.g. - overlapping wildcard matching and exact matching hostnames), precedence must - be given to rules from the HTTPRoute with the largest number of: - - * Characters in a matching non-wildcard hostname. - * Characters in a matching hostname. - - If ties exist across multiple Routes, the matching precedence rules for - HTTPRouteMatches takes over. - - Support: Core - items: - description: |- - Hostname is the fully qualified domain name of a network host. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - Hostname can be "precise" which is a domain name without the terminating - dot of a network host (e.g. "foo.example.com") or "wildcard", which is a - domain name prefixed with a single wildcard label (e.g. `*.example.com`). - - Note that as per RFC1035 and RFC1123, a *label* must consist of lower case - alphanumeric characters or '-', and must start and end with an alphanumeric - character. No other punctuation is allowed. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - maxItems: 16 - type: array - parentRefs: - description: |- - ParentRefs references the resources (usually Gateways) that a Route wants - to be attached to. Note that the referenced parent resource needs to - allow this for the attachment to be complete. For Gateways, that means - the Gateway needs to allow attachment from Routes of this kind and - namespace. For Services, that means the Service must either be in the same - namespace for a "producer" route, or the mesh implementation must support - and allow "consumer" routes for the referenced Service. ReferenceGrant is - not applicable for governing ParentRefs to Services - it is not possible to - create a "producer" route for a Service in a different namespace from the - Route. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - ParentRefs must be _distinct_. This means either that: - - * They select different objects. If this is the case, then parentRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, `namespace`, and `name` must - be unique across all parentRef entries in the Route. - * They do not select different objects, but for each optional field used, - each ParentRef that selects the same object must set the same set of - optional fields to different values. If one ParentRef sets a - combination of optional fields, all must set the same combination. - - Some examples: - - * If one ParentRef sets `sectionName`, all ParentRefs referencing the - same object must also set `sectionName`. - * If one ParentRef sets `port`, all ParentRefs referencing the same - object must also set `port`. - * If one ParentRef sets `sectionName` and `port`, all ParentRefs - referencing the same object must also set `sectionName` and `port`. - - It is possible to separately reference multiple distinct objects that may - be collapsed by an implementation. For example, some implementations may - choose to merge compatible Gateway Listeners together. If that is the - case, the list of routes attached to those resources should also be - merged. - - Note that for ParentRefs that cross namespace boundaries, there are specific - rules. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example, - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable other kinds of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - - - - - items: - description: |- - ParentReference identifies an API object (usually a Gateway) that can be considered - a parent of this resource (usually a route). There are two kinds of parent resources - with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - maxItems: 32 - type: array - rules: - default: - - matches: - - path: - type: PathPrefix - value: / - description: |- - Rules are a list of HTTP matchers, filters and actions. - - - items: - description: |- - HTTPRouteRule defines semantics for matching an HTTP request based on - conditions (matches), processing it (filters), and forwarding the request to - an API object (backendRefs). - properties: - backendRefs: - description: |- - BackendRefs defines the backend(s) where matching requests should be - sent. - - Failure behavior here depends on how many BackendRefs are specified and - how many are invalid. - - If *all* entries in BackendRefs are invalid, and there are also no filters - specified in this route rule, *all* traffic which matches this rule MUST - receive a 500 status code. - - See the HTTPBackendRef definition for the rules about what makes a single - HTTPBackendRef invalid. - - When a HTTPBackendRef is invalid, 500 status codes MUST be returned for - requests that would have otherwise been routed to an invalid backend. If - multiple backends are specified, and some are invalid, the proportion of - requests that would otherwise have been routed to an invalid backend - MUST receive a 500 status code. - - For example, if two backends are specified with equal weights, and one is - invalid, 50 percent of traffic must receive a 500. Implementations may - choose how that 50 percent is determined. - - When a HTTPBackendRef refers to a Service that has no ready endpoints, - implementations SHOULD return a 503 for requests to that backend instead. - If an implementation chooses to do this, all of the above rules for 500 responses - MUST also apply for responses that return a 503. - - Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Core - items: - description: |- - HTTPBackendRef defines how a HTTPRoute forwards a HTTP request. - - Note that when a namespace different than the local namespace is specified, a - ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - - - When the BackendRef points to a Kubernetes Service, implementations SHOULD - honor the appProtocol field if it is set for the target Service Port. - - Implementations supporting appProtocol SHOULD recognize the Kubernetes - Standard Application Protocols defined in KEP-3726. - - If a Service appProtocol isn't specified, an implementation MAY infer the - backend protocol through its own means. Implementations MAY infer the - protocol from the Route type referring to the backend Service. - - If a Route is not able to send traffic to the backend using the specified - protocol then the backend is considered invalid. Implementations MUST set the - "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. - - - properties: - filters: - description: |- - Filters defined at this level should be executed if and only if the - request is being forwarded to the backend defined here. - - Support: Implementation-specific (For broader support of filters, use the - Filters field in HTTPRouteRule.) - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain - '*' alongside other methods - rule: '!(''*'' in self && self.size() - > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. - For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the - referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents - an HTTP Header name and value as - defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value - of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents - an HTTP Header name and value as - defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value - of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of - the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service - reference - rule: '(size(self.group) == 0 && self.kind - == ''Service'') ? has(self.port) - : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than - or equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction - may be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified - when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' - ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' - when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? - self.type == ''ReplaceFullPath'' - : true' - - message: replacePrefixMatch must be - specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' - ? has(self.replacePrefixMatch) : - true' - - message: type must be 'ReplacePrefixMatch' - when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) - ? self.type == ''ReplacePrefixMatch'' - : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents - an HTTP Header name and value as - defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value - of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents - an HTTP Header name and value as - defined by RFC 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value - of HTTP Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified - when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' - ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' - when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? - self.type == ''ReplaceFullPath'' - : true' - - message: replacePrefixMatch must be - specified when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' - ? has(self.replacePrefixMatch) : - true' - - message: type must be 'ReplacePrefixMatch' - when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) - ? self.type == ''ReplacePrefixMatch'' - : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must - be nil if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && - self.type != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must - be specified for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) - && self.type == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must - be nil if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) - && self.type != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must - be specified for ResponseHeaderModifier - filter.type - rule: '!(!has(self.responseHeaderModifier) - && self.type == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil - if the filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type - != ''RequestMirror'')' - - message: filter.requestMirror must be specified - for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type - == ''RequestMirror'')' - - message: filter.requestRedirect must be nil - if the filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type - != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified - for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type - == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if - the filter.type is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type - != ''URLRewrite'')' - - message: filter.urlRewrite must be specified - for URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type - == ''URLRewrite'')' - - message: filter.extensionRef must be nil if - the filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type - != ''ExtensionRef'')' - - message: filter.extensionRef must be specified - for ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type - == ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect - or httpRouteFilterRequestRewrite, but not - both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') - && self.exists(f, f.type == ''URLRewrite''))' - - message: May specify either httpRouteFilterRequestRedirect - or httpRouteFilterRequestRewrite, but not - both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') - && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot - be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() - <= 1 - - message: ResponseHeaderModifier filter cannot - be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() - <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() - <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() - <= 1 - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - weight: - default: 1 - description: |- - Weight specifies the proportion of requests forwarded to the referenced - backend. This is computed as weight/(sum of all weights in this - BackendRefs list). For non-zero values, there may be some epsilon from - the exact proportion defined here depending on the precision an - implementation supports. Weight is not a percentage and the sum of - weights does not need to equal 100. - - If only one backend is specified and it has a weight greater than 0, 100% - of the traffic is forwarded to that backend. If weight is set to 0, no - traffic should be forwarded for this entry. If unspecified, weight - defaults to 1. - - Support for this field varies based on the context where used. - format: int32 - maximum: 1000000 - minimum: 0 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') - ? has(self.port) : true' - maxItems: 16 - type: array - filters: - description: |- - Filters define the filters that are applied to requests that match - this rule. - - Wherever possible, implementations SHOULD implement filters in the order - they are specified. - - Implementations MAY choose to implement this ordering strictly, rejecting - any combination or order of filters that cannot be supported. If implementations - choose a strict interpretation of filter ordering, they MUST clearly document - that behavior. - - To reject an invalid combination or order of filters, implementations SHOULD - consider the Route Rules with this configuration invalid. If all Route Rules - in a Route are invalid, the entire Route would be considered invalid. If only - a portion of Route Rules are invalid, implementations MUST set the - "PartiallyInvalid" condition for the Route. - - Conformance-levels at this level are defined based on the type of filter: - - - ALL core filters MUST be supported by all implementations. - - Implementers are encouraged to support extended filters. - - Implementation-specific custom filters have no API guarantees across - implementations. - - Specifying the same filter multiple times is not supported unless explicitly - indicated in the filter. - - All filters are expected to be compatible with each other except for the - URLRewrite and RequestRedirect filters, which may not be combined. If an - implementation cannot support other combinations of filters, they must clearly - document that limitation. In cases where incompatible or unsupported - filters are specified and cause the `Accepted` condition to be set to status - `False`, implementations may use the `IncompatibleFilters` reason to specify - this configuration error. - - Support: Core - items: - description: |- - HTTPRouteFilter defines processing steps that must be completed during the - request or response lifecycle. HTTPRouteFilters are meant as an extension - point to express processing that may be done in Gateway implementations. Some - examples include request or response modification, implementing - authentication strategies, rate-limiting, and traffic shaping. API - guarantee/conformance is defined based on the type of the filter. - - - - properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - - - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - The only valid value for the `Access-Control-Allow-Credentials` response - header is true (case-sensitive). - - If the credentials are not allowed in cross-origin requests, the gateway - will omit the header `Access-Control-Allow-Credentials` entirely rather - than setting its value to false. - - Support: Extended - enum: - - true - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - The `Access-Control-Allow-Headers` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowHeaders` field - specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. A Gateway - implementation may choose to add implementation-specific default headers. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - The `Access-Control-Allow-Methods` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. A Gateway implementation may - choose to add implementation-specific default methods. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' - alongside other methods - rule: '!(''*'' in self && self.size() > - 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - The `Access-Control-Allow-Origin` response header can only use `*` - wildcard as value when the `AllowCredentials` field is unspecified. - - When the `AllowCredentials` field is specified and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The AbsoluteURI MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The AbsoluteURI MUST include both a - scheme (e.g., "http" or "spiffe") and a scheme-specific-part. URIs that - include an authority MUST include a fully qualified domain name or - IP address as the host. - The below regex is taken from the regex section in RFC 3986 with a slight modification to enforce a full URI and not relative. - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the `AllowCredentials` field is - unspecified. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - format: int32 - minimum: 1 - type: integer - type: object - extensionRef: - description: |- - ExtensionRef is an optional, implementation-specific extension to the - "filter" behavior. For example, resource "myroutefilter" in group - "networking.example.net"). ExtensionRef MUST NOT be used for core and - extended filters. - - This filter can be used multiple times within the same rule. - - Support: Implementation-specific - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. - For example "HTTPRoute" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - requestHeaderModifier: - description: |- - RequestHeaderModifier defines a schema for a filter that modifies request - headers. - - Support: Core - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP - Header name and value as defined by RFC - 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP - Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP - Header name and value as defined by RFC - 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP - Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - requestMirror: - description: |- - RequestMirror defines a schema for a filter that mirrors requests. - Requests are sent to the specified destination, but responses from - that destination are ignored. - - This filter can be used multiple times within the same rule. Note that - not all implementations will be able to support mirroring to multiple - backends. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef references a resource where mirrored requests are sent. - - Mirrored requests must be sent only to a single destination endpoint - within this BackendRef, irrespective of how many endpoints are present - within this BackendRef. - - If the referent cannot be found, this BackendRef is invalid and must be - dropped from the Gateway. The controller must ensure the "ResolvedRefs" - condition on the Route status is set to `status: False` and not configure - this backend in the underlying implementation. - - If there is a cross-namespace reference to an *existing* object - that is not allowed by a ReferenceGrant, the controller must ensure the - "ResolvedRefs" condition on the Route is set to `status: False`, - with the "RefNotPermitted" reason and not configure this backend in the - underlying implementation. - - In either error case, the Message of the `ResolvedRefs` Condition - should be used to provide more detail about the problem. - - Support: Extended for Kubernetes Service - - Support: Implementation-specific for any other resource - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind - == ''Service'') ? has(self.port) : true' - fraction: - description: |- - Fraction represents the fraction of requests that should be - mirrored to BackendRef. - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - properties: - denominator: - default: 100 - format: int32 - minimum: 1 - type: integer - numerator: - format: int32 - minimum: 0 - type: integer - required: - - numerator - type: object - x-kubernetes-validations: - - message: numerator must be less than or - equal to denominator - rule: self.numerator <= self.denominator - percent: - description: |- - Percent represents the percentage of requests that should be - mirrored to BackendRef. Its minimum value is 0 (indicating 0% of - requests) and its maximum value is 100 (indicating 100% of requests). - - Only one of Fraction or Percent may be specified. If neither field - is specified, 100% of requests will be mirrored. - format: int32 - maximum: 100 - minimum: 0 - type: integer - required: - - backendRef - type: object - x-kubernetes-validations: - - message: Only one of percent or fraction may - be specified in HTTPRequestMirrorFilter - rule: '!(has(self.percent) && has(self.fraction))' - requestRedirect: - description: |- - RequestRedirect defines a schema for a filter that responds to the - request with an HTTP redirection. - - Support: Core - properties: - hostname: - description: |- - Hostname is the hostname to be used in the value of the `Location` - header in the response. - When empty, the hostname in the `Host` header of the request is used. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines parameters used to modify the path of the incoming request. - The modified path is then used to construct the `Location` header. When - empty, the request path is used as-is. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified - when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' - ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' - when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type - == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified - when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' - ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' - when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type - == ''ReplacePrefixMatch'' : true' - port: - description: |- - Port is the port to be used in the value of the `Location` - header in the response. - - If no port is specified, the redirect port MUST be derived using the - following rules: - - * If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. - * If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. - - Implementations SHOULD NOT add the port number in the 'Location' - header in the following cases: - - * A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. - * A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - scheme: - description: |- - Scheme is the scheme to be used in the value of the `Location` header in - the response. When empty, the scheme of the request is used. - - Scheme redirects can affect the port of the redirect, for more information, - refer to the documentation for the port field of this filter. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Extended - enum: - - http - - https - type: string - statusCode: - default: 302 - description: |- - StatusCode is the HTTP status code to be used in response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - Support: Core - enum: - - 301 - - 302 - type: integer - type: object - responseHeaderModifier: - description: |- - ResponseHeaderModifier defines a schema for a filter that modifies response - headers. - - Support: Extended - properties: - add: - description: |- - Add adds the given header(s) (name, value) to the request - before the action. It appends to any existing values associated - with the header name. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - add: - - name: "my-header" - value: "bar,baz" - - Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz - items: - description: HTTPHeader represents an HTTP - Header name and value as defined by RFC - 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP - Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - remove: - description: |- - Remove the given header(s) from the HTTP request before the action. The - value of Remove is a list of HTTP header names. Note that the header - names are case-insensitive (see - https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). - - Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz - - Config: - remove: ["my-header1", "my-header3"] - - Output: - GET /foo HTTP/1.1 - my-header2: bar - items: - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - set: - description: |- - Set overwrites the request with the given header (name, value) - before the action. - - Input: - GET /foo HTTP/1.1 - my-header: foo - - Config: - set: - - name: "my-header" - value: "bar" - - Output: - GET /foo HTTP/1.1 - my-header: bar - items: - description: HTTPHeader represents an HTTP - Header name and value as defined by RFC - 7230. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, the first entry with - an equivalent name MUST be considered for a match. Subsequent entries - with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - value: - description: Value is the value of HTTP - Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: - description: |- - Type identifies the type of filter to apply. As with other API fields, - types are classified into three conformance levels: - - - Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - - - Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - - - Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - - Implementers are encouraged to define custom implementation types to - extend the core API with implementation-specific behavior. - - If a reference to a custom filter type cannot be resolved, the filter - MUST NOT be skipped. Instead, requests that would have been processed by - that filter MUST receive a HTTP error response. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - - - enum: - - RequestHeaderModifier - - ResponseHeaderModifier - - RequestMirror - - RequestRedirect - - URLRewrite - - ExtensionRef - type: string - urlRewrite: - description: |- - URLRewrite defines a schema for a filter that modifies a request during forwarding. - - Support: Extended - properties: - hostname: - description: |- - Hostname is the value to be used to replace the Host header value during - forwarding. - - Support: Extended - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - path: - description: |- - Path defines a path rewrite. - - Support: Extended - properties: - replaceFullPath: - description: |- - ReplaceFullPath specifies the value with which to replace the full path - of a request during a rewrite or redirect. - maxLength: 1024 - type: string - replacePrefixMatch: - description: |- - ReplacePrefixMatch specifies the value with which to replace the prefix - match of a request during a rewrite or redirect. For example, a request - to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch - of "/xyz" would be modified to "/xyz/bar". - - Note that this matches the behavior of the PathPrefix match type. This - matches full path elements. A path element refers to the list of labels - in the path split by the `/` separator. When specified, a trailing `/` is - ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all - match the prefix `/abc`, but the path `/abcd` would not. - - ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. - Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in - the implementation setting the Accepted Condition for the Route to `status: False`. - - Request Path | Prefix Match | Replace Prefix | Modified Path - maxLength: 1024 - type: string - type: - description: |- - Type defines the type of path modifier. Additional types may be - added in a future release of the API. - - Note that values may be added to this enum, implementations - must ensure that unknown values will not cause a crash. - - Unknown values here must result in the implementation setting the - Accepted Condition for the Route to `status: False`, with a - Reason of `UnsupportedValue`. - enum: - - ReplaceFullPath - - ReplacePrefixMatch - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: replaceFullPath must be specified - when type is set to 'ReplaceFullPath' - rule: 'self.type == ''ReplaceFullPath'' - ? has(self.replaceFullPath) : true' - - message: type must be 'ReplaceFullPath' - when replaceFullPath is set - rule: 'has(self.replaceFullPath) ? self.type - == ''ReplaceFullPath'' : true' - - message: replacePrefixMatch must be specified - when type is set to 'ReplacePrefixMatch' - rule: 'self.type == ''ReplacePrefixMatch'' - ? has(self.replacePrefixMatch) : true' - - message: type must be 'ReplacePrefixMatch' - when replacePrefixMatch is set - rule: 'has(self.replacePrefixMatch) ? self.type - == ''ReplacePrefixMatch'' : true' - type: object - required: - - type - type: object - x-kubernetes-validations: - - message: filter.requestHeaderModifier must be nil - if the filter.type is not RequestHeaderModifier - rule: '!(has(self.requestHeaderModifier) && self.type - != ''RequestHeaderModifier'')' - - message: filter.requestHeaderModifier must be specified - for RequestHeaderModifier filter.type - rule: '!(!has(self.requestHeaderModifier) && self.type - == ''RequestHeaderModifier'')' - - message: filter.responseHeaderModifier must be nil - if the filter.type is not ResponseHeaderModifier - rule: '!(has(self.responseHeaderModifier) && self.type - != ''ResponseHeaderModifier'')' - - message: filter.responseHeaderModifier must be specified - for ResponseHeaderModifier filter.type - rule: '!(!has(self.responseHeaderModifier) && self.type - == ''ResponseHeaderModifier'')' - - message: filter.requestMirror must be nil if the - filter.type is not RequestMirror - rule: '!(has(self.requestMirror) && self.type != - ''RequestMirror'')' - - message: filter.requestMirror must be specified - for RequestMirror filter.type - rule: '!(!has(self.requestMirror) && self.type == - ''RequestMirror'')' - - message: filter.requestRedirect must be nil if the - filter.type is not RequestRedirect - rule: '!(has(self.requestRedirect) && self.type - != ''RequestRedirect'')' - - message: filter.requestRedirect must be specified - for RequestRedirect filter.type - rule: '!(!has(self.requestRedirect) && self.type - == ''RequestRedirect'')' - - message: filter.urlRewrite must be nil if the filter.type - is not URLRewrite - rule: '!(has(self.urlRewrite) && self.type != ''URLRewrite'')' - - message: filter.urlRewrite must be specified for - URLRewrite filter.type - rule: '!(!has(self.urlRewrite) && self.type == ''URLRewrite'')' - - message: filter.extensionRef must be nil if the - filter.type is not ExtensionRef - rule: '!(has(self.extensionRef) && self.type != - ''ExtensionRef'')' - - message: filter.extensionRef must be specified for - ExtensionRef filter.type - rule: '!(!has(self.extensionRef) && self.type == - ''ExtensionRef'')' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: May specify either httpRouteFilterRequestRedirect - or httpRouteFilterRequestRewrite, but not both - rule: '!(self.exists(f, f.type == ''RequestRedirect'') - && self.exists(f, f.type == ''URLRewrite''))' - - message: RequestHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'RequestHeaderModifier').size() - <= 1 - - message: ResponseHeaderModifier filter cannot be repeated - rule: self.filter(f, f.type == 'ResponseHeaderModifier').size() - <= 1 - - message: RequestRedirect filter cannot be repeated - rule: self.filter(f, f.type == 'RequestRedirect').size() - <= 1 - - message: URLRewrite filter cannot be repeated - rule: self.filter(f, f.type == 'URLRewrite').size() - <= 1 - matches: - default: - - path: - type: PathPrefix - value: / - description: |- - Matches define conditions used for matching the rule against incoming - HTTP requests. Each match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. - - For example, take the following matches configuration: - - ``` - matches: - - path: - value: "/foo" - headers: - - name: "version" - value: "v2" - - path: - value: "/v2/foo" - ``` - - For a request to match against this rule, a request must satisfy - EITHER of the two conditions: - - - path prefixed with `/foo` AND contains the header `version: v2` - - path prefix of `/v2/foo` - - See the documentation for HTTPRouteMatch on how to specify multiple - match conditions that should be ANDed together. - - If no matches are specified, the default is a prefix - path match on "/", which has the effect of matching every - HTTP request. - - Proxy or Load Balancer routing configuration generated from HTTPRoutes - MUST prioritize matches based on the following criteria, continuing on - ties. Across all rules specified on applicable Routes, precedence must be - given to the match having: - - * "Exact" path match. - * "Prefix" path match with largest number of characters. - * Method match. - * Largest number of header matches. - * Largest number of query param matches. - - Note: The precedence of RegularExpression path matches are implementation-specific. - - If ties still exist across multiple Routes, matching precedence MUST be - determined in order of the following criteria, continuing on ties: - - * The oldest Route based on creation timestamp. - * The Route appearing first in alphabetical order by - "{namespace}/{name}". - - If ties still exist within an HTTPRoute, matching precedence MUST be granted - to the FIRST matching rule (in list order) with a match meeting the above - criteria. - - When no rules matching a request have been successfully attached to the - parent a request is coming from, a HTTP 404 status code MUST be returned. - items: - description: "HTTPRouteMatch defines the predicate - used to match requests to a given\naction. Multiple - match types are ANDed together, i.e. the match will\nevaluate - to true only if all conditions are satisfied.\n\nFor - example, the match below will match a HTTP request - only if its path\nstarts with `/foo` AND it contains - the `version: v1` header:\n\n```\nmatch:\n\n\tpath:\n\t - \ value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t - \ value \"v1\"\n\n```" - properties: - headers: - description: |- - Headers specifies HTTP request header matchers. Multiple match values are - ANDed together, meaning, a request must match all the specified headers - to select the route. - items: - description: |- - HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request - headers. - properties: - name: - description: |- - Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - - If multiple entries specify equivalent header names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent header name MUST be ignored. Due to the - case-insensitivity of header names, "foo" and "Foo" are considered - equivalent. - - When a header is repeated in an HTTP request, it is - implementation-specific behavior as to how this is represented. - Generally, proxies should follow the guidance from the RFC: - https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding - processing a repeated header, with special handling for "Set-Cookie". - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the header. - - Support: Core (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression HeaderMatchType has implementation-specific - conformance, implementations can support POSIX, PCRE or any other dialects - of regular expressions. Please read the implementation's documentation to - determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP - Header to be matched. - maxLength: 4096 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - method: - description: |- - Method specifies HTTP method matcher. - When specified, this route will be matched only if the request has the - specified method. - - Support: Extended - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - type: string - path: - default: - type: PathPrefix - value: / - description: |- - Path specifies a HTTP request path matcher. If this field is not - specified, a default prefix match on the "/" path is provided. - properties: - type: - default: PathPrefix - description: |- - Type specifies how to match against the path Value. - - Support: Core (Exact, PathPrefix) - - Support: Implementation-specific (RegularExpression) - enum: - - Exact - - PathPrefix - - RegularExpression - type: string - value: - default: / - description: Value of the HTTP path to match - against. - maxLength: 1024 - type: string - type: object - x-kubernetes-validations: - - message: value must be an absolute path and - start with '/' when type one of ['Exact', - 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? self.value.startsWith(''/'') : true' - - message: must not contain '//' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''//'') : true' - - message: must not contain '/./' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''/./'') : true' - - message: must not contain '/../' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''/../'') : true' - - message: must not contain '%2f' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''%2f'') : true' - - message: must not contain '%2F' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''%2F'') : true' - - message: must not contain '#' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.contains(''#'') : true' - - message: must not end with '/..' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.endsWith(''/..'') : true' - - message: must not end with '/.' when type one - of ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? !self.value.endsWith(''/.'') : true' - - message: type must be one of ['Exact', 'PathPrefix', - 'RegularExpression'] - rule: self.type in ['Exact','PathPrefix'] || - self.type == 'RegularExpression' - - message: must only contain valid characters - (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) - for types ['Exact', 'PathPrefix'] - rule: '(self.type in [''Exact'',''PathPrefix'']) - ? self.value.matches(r"""^(?:[-A-Za-z0-9/._~!$&''()*+,;=:@]|[%][0-9a-fA-F]{2})+$""") - : true' - queryParams: - description: |- - QueryParams specifies HTTP query parameter matchers. Multiple match - values are ANDed together, meaning, a request must match all the - specified query parameters to select the route. - - Support: Extended - items: - description: |- - HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP - query parameters. - properties: - name: - description: |- - Name is the name of the HTTP query param to be matched. This must be an - exact string match. (See - https://tools.ietf.org/html/rfc7230#section-2.7.3). - - If multiple entries specify equivalent query param names, only the first - entry with an equivalent name MUST be considered for a match. Subsequent - entries with an equivalent query param name MUST be ignored. - - If a query param is repeated in an HTTP request, the behavior is - purposely left undefined, since different data planes have different - capabilities. However, it is *recommended* that implementations should - match against the first value of the param if the data plane supports it, - as this behavior is expected in other load balancing contexts outside of - the Gateway API. - - Users SHOULD NOT route traffic based on repeated query params to guard - themselves against potential differences in the implementations. - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - type: - default: Exact - description: |- - Type specifies how to match against the value of the query parameter. - - Support: Extended (Exact) - - Support: Implementation-specific (RegularExpression) - - Since RegularExpression QueryParamMatchType has Implementation-specific - conformance, implementations can support POSIX, PCRE or any other - dialects of regular expressions. Please read the implementation's - documentation to determine the supported dialect. - enum: - - Exact - - RegularExpression - type: string - value: - description: Value is the value of HTTP - query param to be matched. - maxLength: 1024 - minLength: 1 - type: string - required: - - name - - value - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - maxItems: 64 - type: array - name: - description: |- - Name is the name of the route rule. This name MUST be unique within a Route if it is set. - - Support: Extended - - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retry: - description: |- - Retry defines the configuration for when to retry an HTTP request. - - Support: Extended - - - properties: - attempts: - description: |- - Attempts specifies the maximum number of times an individual request - from the gateway to a backend should be retried. - - If the maximum number of retries has been attempted without a successful - response from the backend, the Gateway MUST return an error. - - When this field is unspecified, the number of times to attempt to retry - a backend request is implementation-specific. - - Support: Extended - type: integer - backoff: - description: |- - Backoff specifies the minimum duration a Gateway should wait between - retry attempts and is represented in Gateway API Duration formatting. - - For example, setting the `rules[].retry.backoff` field to the value - `100ms` will cause a backend request to first be retried approximately - 100 milliseconds after timing out or receiving a response code configured - to be retryable. - - An implementation MAY use an exponential or alternative backoff strategy - for subsequent retry attempts, MAY cap the maximum backoff duration to - some amount greater than the specified minimum, and MAY add arbitrary - jitter to stagger requests, as long as unsuccessful backend requests are - not retried before the configured minimum duration. - - If a Request timeout (`rules[].timeouts.request`) is configured on the - route, the entire duration of the initial request and any retry attempts - MUST not exceed the Request timeout duration. If any retry attempts are - still in progress when the Request timeout duration has been reached, - these SHOULD be canceled if possible and the Gateway MUST immediately - return a timeout error. - - If a BackendRequest timeout (`rules[].timeouts.backendRequest`) is - configured on the route, any retry attempts which reach the configured - BackendRequest timeout duration without a response SHOULD be canceled if - possible and the Gateway should wait for at least the specified backoff - duration before attempting to retry the backend request again. - - If a BackendRequest timeout is _not_ configured on the route, retry - attempts MAY time out after an implementation default duration, or MAY - remain pending until a configured Request timeout or implementation - default duration for total request time is reached. - - When this field is unspecified, the time to wait between retry attempts - is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - codes: - description: |- - Codes defines the HTTP response status codes for which a backend request - should be retried. - - Support: Extended - items: - description: |- - HTTPRouteRetryStatusCode defines an HTTP response status code for - which a backend request should be retried. - - Implementations MUST support the following status codes as retryable: - - * 500 - * 502 - * 503 - * 504 - - Implementations MAY support specifying additional discrete values in the - 500-599 range. - - Implementations MAY support specifying discrete values in the 400-499 range, - which are often inadvisable to retry. - - - maximum: 599 - minimum: 400 - type: integer - type: array - type: object - sessionPersistence: - description: |- - SessionPersistence defines and configures session persistence - for the route rule. - - Support: Extended - - - properties: - absoluteTimeout: - description: |- - AbsoluteTimeout defines the absolute timeout of the persistent - session. Once the AbsoluteTimeout duration has elapsed, the - session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - cookieConfig: - description: |- - CookieConfig provides configuration settings that are specific - to cookie-based session persistence. - - Support: Core - properties: - lifetimeType: - default: Session - description: |- - LifetimeType specifies whether the cookie has a permanent or - session-based lifetime. A permanent cookie persists until its - specified expiry time, defined by the Expires or Max-Age cookie - attributes, while a session cookie is deleted when the current - session ends. - - When set to "Permanent", AbsoluteTimeout indicates the - cookie's lifetime via the Expires or Max-Age cookie attributes - and is required. - - When set to "Session", AbsoluteTimeout indicates the - absolute lifetime of the cookie tracked by the gateway and - is optional. - - Defaults to "Session". - - Support: Core for "Session" type - - Support: Extended for "Permanent" type - enum: - - Permanent - - Session - type: string - type: object - idleTimeout: - description: |- - IdleTimeout defines the idle timeout of the persistent session. - Once the session has been idle for more than the specified - IdleTimeout duration, the session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - sessionName: - description: |- - SessionName defines the name of the persistent session token - which may be reflected in the cookie or the header. Users - should avoid reusing session names to prevent unintended - consequences, such as rejection or unpredictable behavior. - - Support: Implementation-specific - maxLength: 128 - type: string - type: - default: Cookie - description: |- - Type defines the type of session persistence such as through - the use a header or cookie. Defaults to cookie based session - persistence. - - Support: Core for "Cookie" type - - Support: Extended for "Header" type - enum: - - Cookie - - Header - type: string - type: object - x-kubernetes-validations: - - message: AbsoluteTimeout must be specified when cookie - lifetimeType is Permanent - rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) - || self.cookieConfig.lifetimeType != ''Permanent'' - || has(self.absoluteTimeout)' - timeouts: - description: |- - Timeouts defines the timeouts that can be configured for an HTTP request. - - Support: Extended - properties: - backendRequest: - description: |- - BackendRequest specifies a timeout for an individual request from the gateway - to a backend. This covers the time from when the request first starts being - sent from the gateway to when the full response has been received from the backend. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - An entire client HTTP transaction with a gateway, covered by the Request timeout, - may result in more than one call from the gateway to the destination backend, - for example, if automatic retries are supported. - - The value of BackendRequest must be a Gateway API Duration string as defined by - GEP-2257. When this field is unspecified, its behavior is implementation-specific; - when specified, the value of BackendRequest must be no more than the value of the - Request timeout (since the Request timeout encompasses the BackendRequest timeout). - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - request: - description: |- - Request specifies the maximum duration for a gateway to respond to an HTTP request. - If the gateway has not been able to respond before this deadline is met, the gateway - MUST return a timeout error. - - For example, setting the `rules.timeouts.request` field to the value `10s` in an - `HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds - to complete. - - Setting a timeout to the zero duration (e.g. "0s") SHOULD disable the timeout - completely. Implementations that cannot completely disable the timeout MUST - instead interpret the zero duration as the longest possible value to which - the timeout can be set. - - This timeout is intended to cover as close to the whole request-response transaction - as possible although an implementation MAY choose to start the timeout after the entire - request stream has been received instead of immediately after the transaction is - initiated by the client. - - The value of Request is a Gateway API Duration string as defined by GEP-2257. When this - field is unspecified, request timeout behavior is implementation-specific. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - type: object - x-kubernetes-validations: - - message: backendRequest timeout cannot be longer than - request timeout - rule: '!(has(self.request) && has(self.backendRequest) - && duration(self.request) != duration(''0s'') && - duration(self.backendRequest) > duration(self.request))' - type: object - x-kubernetes-validations: - - message: RequestRedirect filter must not be used together - with backendRefs - rule: '(has(self.backendRefs) && size(self.backendRefs) - > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): - true' - - message: When using RequestRedirect filter with path.replacePrefixMatch, - exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, - has(f.requestRedirect) && has(f.requestRedirect.path) - && f.requestRedirect.path.type == ''ReplacePrefixMatch'' - && has(f.requestRedirect.path.replacePrefixMatch))) - ? ((size(self.matches) != 1 || !has(self.matches[0].path) - || self.matches[0].path.type != ''PathPrefix'') ? false - : true) : true' - - message: When using URLRewrite filter with path.replacePrefixMatch, - exactly one PathPrefix match must be specified - rule: '(has(self.filters) && self.filters.exists_one(f, - has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type - == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) - ? ((size(self.matches) != 1 || !has(self.matches[0].path) - || self.matches[0].path.type != ''PathPrefix'') ? false - : true) : true' - - message: Within backendRefs, when using RequestRedirect - filter with path.replacePrefixMatch, exactly one PathPrefix - match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, - (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) - && has(f.requestRedirect.path) && f.requestRedirect.path.type - == ''ReplacePrefixMatch'' && has(f.requestRedirect.path.replacePrefixMatch))) - )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) - || self.matches[0].path.type != ''PathPrefix'') ? false - : true) : true' - - message: Within backendRefs, When using URLRewrite filter - with path.replacePrefixMatch, exactly one PathPrefix - match must be specified - rule: '(has(self.backendRefs) && self.backendRefs.exists_one(b, - (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) - && has(f.urlRewrite.path) && f.urlRewrite.path.type - == ''ReplacePrefixMatch'' && has(f.urlRewrite.path.replacePrefixMatch))) - )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) - || self.matches[0].path.type != ''PathPrefix'') ? false - : true) : true' - maxItems: 16 - type: array - x-kubernetes-validations: - - message: While 16 rules and 64 matches per rule are allowed, - the total number of matches across all rules in a route - must be less than 128 - rule: '(self.size() > 0 ? self[0].matches.size() : 0) + - (self.size() > 1 ? self[1].matches.size() : 0) + (self.size() - > 2 ? self[2].matches.size() : 0) + (self.size() > 3 ? - self[3].matches.size() : 0) + (self.size() > 4 ? self[4].matches.size() - : 0) + (self.size() > 5 ? self[5].matches.size() : 0) - + (self.size() > 6 ? self[6].matches.size() : 0) + (self.size() - > 7 ? self[7].matches.size() : 0) + (self.size() > 8 ? - self[8].matches.size() : 0) + (self.size() > 9 ? self[9].matches.size() - : 0) + (self.size() > 10 ? self[10].matches.size() : 0) - + (self.size() > 11 ? self[11].matches.size() : 0) + (self.size() - > 12 ? self[12].matches.size() : 0) + (self.size() > 13 - ? self[13].matches.size() : 0) + (self.size() > 14 ? self[14].matches.size() - : 0) + (self.size() > 15 ? self[15].matches.size() : 0) - <= 128' - type: object - type: object - ingress: - description: Ingress sets how the ingress object should look like - with your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: IngressSpec describes the Ingress the user wishes - to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain name - of a network host, as defined by RFC 3986.\nNote the - following deviations from the \"host\" part of the\nURI - as defined in RFC 3986:\n1. IPs are not allowed. Currently - an IngressRuleValue can only apply to\n the IP in - the Spec of the parent Ingress.\n2. The `:` delimiter - is not respected because ports are not allowed.\n\t - \ Currently the port of an Ingress is implicitly :80 - for http and\n\t :443 for https.\nBoth these may - change in the future.\nIncoming requests are matched - against the host before the\nIngressRuleValue. If - the host is unspecified, the Ingress routes all\ntraffic - based on the specified IngressRuleValue.\n\nhost can - be \"precise\" which is a domain name without the - terminating dot of\na network host (e.g. \"foo.bar.com\") - or \"wildcard\", which is a domain name\nprefixed - with a single wildcard label (e.g. \"*.foo.com\").\nThe - wildcard character '*' must appear by itself as the - first DNS label and\nmatches only a single label. - You cannot have a wildcard label by itself (e.g. Host - == \"*\").\nRequests will be matched against the Host - field in the following way:\n1. If host is precise, - the request matches this rule if the http host header - is equal to Host.\n2. If host is a wildcard, then - the request matches this rule if the http host header\nis - to equal to the suffix (removing the first label) - of the wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths that - map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer security - associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - jsonnet: - properties: - libraryLabelSelector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - type: object - persistentVolumeClaim: - description: PersistentVolumeClaim creates a PVC if you need to attach - one to your grafana instance. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - TypedLocalObjectReference contains enough information to let you locate the - typed referenced object inside the same namespace. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - resources: - description: ResourceRequirements describes the compute resource - requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: |- - A label selector is a label query over a set of resources. The result of matchLabels and - matchExpressions are ANDed. An empty label selector matches all objects. A null - label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeMode: - description: PersistentVolumeMode describes how a volume is - intended to be consumed, either Block or Filesystem. - type: string - volumeName: - description: VolumeName is the binding reference to the PersistentVolume - backing this claim. - type: string - type: object - type: object - preferences: - description: Preferences holds the Grafana Preferences settings - properties: - homeDashboardUid: - type: string - type: object - route: - description: Route sets how the ingress object should look like with - your grafana instance, this only works in Openshift. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - properties: - alternateBackends: - items: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring - to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being - referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - type: array - host: - type: string - path: - type: string - port: - description: RoutePort defines a port mapping from a router - to an endpoint in the service endpoints. - properties: - targetPort: - anyOf: - - type: integer - - type: string - description: |- - The target port on pods selected by the service this route points to. - If this is a string, it will be looked up as a named port in the target - endpoints port list. Required - x-kubernetes-int-or-string: true - required: - - targetPort - type: object - subdomain: - type: string - tls: - description: TLSConfig defines config used to secure a route - and provide termination - properties: - caCertificate: - description: caCertificate provides the cert authority - certificate contents - type: string - certificate: - description: |- - certificate provides certificate contents. This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. - type: string - destinationCACertificate: - description: |- - destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt - termination this file should be provided in order to have routers use it for health checks on the secure connection. - If this field is not specified, the router may provide its own destination CA and perform hostname validation using - the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically - verify. - type: string - externalCertificate: - description: |- - externalCertificate provides certificate contents as a secret reference. - This should be a single serving certificate, not a certificate - chain. Do not include a CA certificate. The secret referenced should - be present in the same namespace as that of the Route. - Forbidden when `certificate` is set. - The router service account needs to be granted with read-only access to this secret, - please refer to openshift docs for additional details. - properties: - name: - description: |- - name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - insecureEdgeTerminationPolicy: - description: |- - insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While - each router may make its own decisions on which ports to expose, this is normally port 80. - - If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". - - * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). - - * None - no traffic is allowed on the insecure port (default). - - * Redirect - clients are redirected to the secure port. - enum: - - Allow - - None - - Redirect - - "" - type: string - key: - description: key provides key file contents - type: string - termination: - description: |- - termination indicates termination type. - - * edge - TLS termination is done by the router and http is used to communicate with the backend (default) - * passthrough - Traffic is sent straight to the destination without the router providing TLS termination - * reencrypt - TLS termination is done by the router and https is used to communicate with the backend - - Note: passthrough termination is incompatible with httpHeader actions - enum: - - edge - - reencrypt - - passthrough - type: string - required: - - termination - type: object - x-kubernetes-validations: - - message: 'cannot have both spec.tls.termination: passthrough - and spec.tls.insecureEdgeTerminationPolicy: Allow' - rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) - ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) - : true' - to: - description: |- - RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' - kind is allowed. Use 'weight' field to emphasize one over others. - properties: - kind: - default: Service - description: The kind of target that the route is referring - to. Currently, only 'Service' is allowed - enum: - - Service - - "" - type: string - name: - description: name of the service/target that is being - referred to. e.g. name of the service - minLength: 1 - type: string - weight: - default: 100 - description: |- - weight as an integer between 0 and 256, default 100, that specifies the target's relative weight - against other target reference objects. 0 suppresses requests to this backend. - format: int32 - maximum: 256 - minimum: 0 - type: integer - required: - - kind - - name - type: object - wildcardPolicy: - description: WildcardPolicyType indicates the type of wildcard - support needed by routes. - type: string - type: object - type: object - service: - description: Service sets how the service object should look like - with your grafana instance, contains a number of defaults. - properties: - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - spec: - description: ServiceSpec describes the attributes that a user - creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: |- - allocateLoadBalancerNodePorts defines if NodePorts will be automatically - allocated for services with type LoadBalancer. Default is "true". It - may be set to "false" if the cluster load-balancer does not rely on - NodePorts. If the caller requests specific NodePorts (by specifying a - value), those requests will be respected, regardless of this field. - This field may only be set for services with type LoadBalancer and will - be cleared if the type is changed to any other type. - type: boolean - clusterIP: - description: |- - clusterIP is the IP address of the service and is usually assigned - randomly. If an address is specified manually, is in-range (as per - system configuration), and is not in use, it will be allocated to the - service; otherwise creation of the service will fail. This field may not - be changed through updates unless the type field is also being changed - to ExternalName (which requires this field to be blank) or the type - field is being changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values are "None", - empty string (""), or a valid IP address. Setting this to "None" makes a - "headless service" (no virtual IP), which is useful when direct endpoint - connections are preferred and proxying is not required. Only applies to - types ClusterIP, NodePort, and LoadBalancer. If this field is specified - when creating a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - clusterIPs: - description: |- - ClusterIPs is a list of IP addresses assigned to this service, and are - usually assigned randomly. If an address is specified manually, is - in-range (as per system configuration), and is not in use, it will be - allocated to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the type field is - also being changed to ExternalName (which requires this field to be - empty) or the type field is being changed from ExternalName (in which - case this field may optionally be specified, as describe above). Valid - values are "None", empty string (""), or a valid IP address. Setting - this to "None" makes a "headless service" (no virtual IP), which is - useful when direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, and - LoadBalancer. If this field is specified when creating a Service of type - ExternalName, creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, it will - be initialized from the clusterIP field. If this field is specified, - clients must ensure that clusterIPs[0] and clusterIP have the same - value. - - This field may hold a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies field. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: |- - externalIPs is a list of IP addresses for which nodes in the cluster - will also accept traffic for this service. These IPs are not managed by - Kubernetes. The user is responsible for ensuring that traffic arrives - at a node with this IP. A common example is external load-balancers - that are not part of the Kubernetes system. - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalName: - description: |- - externalName is the external reference that discovery mechanisms will - return as an alias for this service (e.g. a DNS CNAME record). No - proxying will be involved. Must be a lowercase RFC-1123 hostname - (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: |- - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure - the service in a way that assumes that external load balancers will take care - of balancing the service traffic between nodes, and so each node will deliver - traffic only to the node-local endpoints of the service, without masquerading - the client source IP. (Traffic mistakenly sent to a node with no endpoints will - be dropped.) The default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology and other - features). Note that traffic sent to an External IP or LoadBalancer IP from - within the cluster will always get "Cluster" semantics, but clients sending to - a NodePort from within the cluster may need to take traffic policy into account - when picking a node. - type: string - healthCheckNodePort: - description: |- - healthCheckNodePort specifies the healthcheck nodePort for the service. - This only applies when type is set to LoadBalancer and - externalTrafficPolicy is set to Local. If a value is specified, is - in-range, and is not in use, it will be used. If not specified, a value - will be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints for this - service or not. If this field is specified when creating a Service - which does not need it, creation will fail. This field will be wiped - when updating a Service to no longer need it (e.g. changing type). - This field cannot be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: |- - InternalTrafficPolicy describes how nodes distribute service traffic they - receive on the ClusterIP. If set to "Local", the proxy will assume that pods - only want to talk to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The default value, - "Cluster", uses the standard behavior of routing to all endpoints evenly - (possibly modified by topology and other features). - type: string - ipFamilies: - description: |- - IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this - service. This field is usually assigned automatically based on cluster - configuration and the ipFamilyPolicy field. If this field is specified - manually, the requested family is available in the cluster, - and ipFamilyPolicy allows it, it will be used; otherwise creation of - the service will fail. This field is conditionally mutable: it allows - for adding or removing a secondary IP family, but it does not allow - changing the primary IP family of the Service. Valid values are "IPv4" - and "IPv6". This field only applies to Services of types ClusterIP, - NodePort, and LoadBalancer, and does apply to "headless" services. - This field will be wiped when updating a Service to type ExternalName. - - This field may hold a maximum of two entries (dual-stack families, in - either order). These families must correspond to the values of the - clusterIPs field, if specified. Both clusterIPs and ipFamilies are - governed by the ipFamilyPolicy field. - items: - description: |- - IPFamily represents the IP Family (IPv4 or IPv6). This type is used - to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: |- - IPFamilyPolicy represents the dual-stack-ness requested or required by - this Service. If there is no value provided, then this field will be set - to SingleStack. Services can be "SingleStack" (a single IP family), - "PreferDualStack" (two IP families on dual-stack configured clusters or - a single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise fail). The - ipFamilies and clusterIPs fields depend on the value of this field. This - field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: |- - loadBalancerClass is the class of the load balancer implementation this Service belongs to. - If specified, the value of this field must be a label-style identifier, with an optional prefix, - e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. - This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load - balancer implementation is used, today this is typically done through the cloud provider integration, - but should apply for any default implementation. If set, it is assumed that a load balancer - implementation is watching for Services with a matching class. Any default load balancer - implementation (e.g. cloud providers) should ignore Services that set this field. - This field can only be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: |- - Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider supports specifying - the loadBalancerIP when a load balancer is created. - This field will be ignored if the cloud-provider does not support the feature. - Deprecated: This field was under-specified and its meaning varies across implementations. - Using it is non-portable and it may not support dual-stack. - Users are encouraged to use implementation-specific annotations when available. - type: string - loadBalancerSourceRanges: - description: |- - If specified and supported by the platform, this will restrict traffic through the cloud-provider - load-balancer will be restricted to the specified client IPs. This field will be ignored if the - cloud-provider does not support the feature." - More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ - items: - type: string - type: array - x-kubernetes-list-type: atomic - ports: - description: |- - The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - items: - description: ServicePort contains information on service's - port. - properties: - appProtocol: - description: |- - The application protocol for this port. - This is used as a hint for implementations to offer richer behavior for protocols that they understand. - This field follows standard Kubernetes label syntax. - Valid values are either: - - * Un-prefixed protocol names - reserved for IANA standard service names (as per - RFC-6335 and https://www.iana.org/assignments/service-names). - - * Kubernetes-defined prefixed names: - * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- - * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 - * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 - - * Other protocols should use implementation-defined prefixed names such as - mycompany.com/my-custom-protocol. - type: string - name: - description: |- - The name of this port within the service. This must be a DNS_LABEL. - All ports within a ServiceSpec must have unique names. When considering - the endpoints for a Service, this must match the 'name' field in the - EndpointPort. - Optional if only one ServicePort is defined on this service. - type: string - nodePort: - description: |- - The port on each node on which this service is exposed when type is - NodePort or LoadBalancer. Usually assigned by the system. If a value is - specified, in-range, and not in use it will be used, otherwise the - operation will fail. If not specified, a port will be allocated if this - Service requires one. If this field is specified when creating a - Service which does not need it, creation will fail. This field will be - wiped when updating a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). - More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: |- - The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". - Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the pods targeted by the service. - Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named port in the - target Pod's container ports. If this is not specified, the value - of the 'port' field is used (an identity map). - This field is ignored for services with clusterIP=None, and should be - omitted or set equal to the 'port' field. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: |- - publishNotReadyAddresses indicates that any agent which deals with endpoints for this - Service should disregard any indications of ready/not-ready. - The primary use case for setting this field is for a StatefulSet's Headless Service to - propagate SRV DNS records for its Pods for the purpose of peer discovery. - The Kubernetes controllers that generate Endpoints and EndpointSlice resources for - Services interpret this to mean that all endpoints are considered "ready" even if the - Pods themselves are not. Agents which consume only Kubernetes generated endpoints - through the Endpoints or EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: |- - Route service traffic to pods with label keys and values matching this - selector. If empty or not present, the service is assumed to have an - external process managing its endpoints, which Kubernetes will not - modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. - Ignored if type is ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: |- - Supports "ClientIP" and "None". Used to maintain session affinity. - Enable client IP based session affinity. - Must be ClientIP or None. - Defaults to None. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client - IP based session affinity. - properties: - timeoutSeconds: - description: |- - timeoutSeconds specifies the seconds of ClientIP type session sticky time. - The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". - Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - trafficDistribution: - description: |- - TrafficDistribution offers a way to express preferences for how traffic - is distributed to Service endpoints. Implementations can use this field - as a hint, but are not required to guarantee strict adherence. If the - field is not set, the implementation will apply its default routing - strategy. If set to "PreferClose", implementations should prioritize - endpoints that are in the same zone. - type: string - type: - description: |- - type determines how the Service is exposed. Defaults to ClusterIP. Valid - options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - "ClusterIP" allocates a cluster-internal IP address for load-balancing - to endpoints. Endpoints are determined by the selector or if that is not - specified, by manual construction of an Endpoints object or - EndpointSlice objects. If clusterIP is "None", no virtual IP is - allocated and the endpoints are published as a set of endpoints rather - than a virtual IP. - "NodePort" builds on ClusterIP and allocates a port on every node which - routes to the same endpoints as the clusterIP. - "LoadBalancer" builds on NodePort and creates an external load-balancer - (if supported in the current cloud) which routes to the same endpoints - as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: string - type: object - type: object - serviceAccount: - description: ServiceAccount sets how the ServiceAccount object should - look like with your grafana instance, contains a number of defaults. - properties: - automountServiceAccountToken: - type: boolean - imagePullSecrets: - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - metadata: - description: ObjectMeta contains only a [subset of the fields - included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta). - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - secrets: - items: - description: ObjectReference contains enough information to - let you inspect or modify the referred object. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: array - type: object - suspend: - description: Suspend pauses reconciliation of owned resources like - deployments, Services, Etc. upon changes - type: boolean - version: - description: |- - Version sets the tag of the default image: docker.io/grafana/grafana. - Allows full image refs with/without sha256checksum: "registry/repo/image:tag@sha" - default: 12.3.0 - type: string - type: object - status: - description: GrafanaStatus defines the observed state of Grafana - properties: - adminUrl: - type: string - alertRuleGroups: - items: - type: string - type: array - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - contactPoints: - items: - type: string - type: array - dashboards: - items: - type: string - type: array - datasources: - items: - type: string - type: array - folders: - items: - type: string - type: array - lastMessage: - type: string - libraryPanels: - items: - type: string - type: array - muteTimings: - items: - type: string - type: array - notificationTemplates: - items: - type: string - type: array - serviceaccounts: - items: - type: string - type: array - stage: - type: string - stageStatus: - type: string - version: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/kustomize/overlays/mlops/grafana/crds/kustomization.yaml b/kustomize/overlays/mlops/grafana/crds/kustomization.yaml deleted file mode 100644 index f40befc67..000000000 --- a/kustomize/overlays/mlops/grafana/crds/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - grafanas.yaml - - grafanadatasources.yaml - - grafanafolders.yaml - - grafanadashboards.yaml diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml index bbc200227..5f3b0a49e 100644 --- a/kustomize/overlays/mlops/grafana/instances/grafana.yaml +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -5,16 +5,40 @@ metadata: spec: config: auth: - disable_login_form: 'false' + disable_login_form: 'true' + auth.proxy: + auto_sign_up: 'true' + enabled: 'true' + header_name: X-Forwarded-User + header_property: username log: mode: console - security: - secret: grafana-secret deployment: spec: template: spec: containers: + - args: + - '-provider=openshift' + - '-https-address=:9091' + - '-tls-cert=/etc/proxy/tls/tls.crt' + - '-tls-key=/etc/proxy/tls/tls.key' + - '-upstream=http://localhost:3000' + - '-openshift-service-account=exploit-iq-grafana-sa' + - '-cookie-secret=/etc/proxy/secrets/session-secret' + image: 'quay.io/openshift/origin-oauth-proxy:latest' + name: grafana-oauth-proxy + ports: + - containerPort: 9091 + name: https + protocol: TCP + volumeMounts: + - mountPath: /etc/proxy/secrets + name: oauth-proxy-secret + readOnly: true + - mountPath: /etc/proxy/tls + name: oauth-proxy-tls + readOnly: true - name: grafana readinessProbe: failureThreshold: 1 @@ -26,17 +50,27 @@ spec: periodSeconds: 20 successThreshold: 1 timeoutSeconds: 30 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: grafana -spec: - to: - kind: Service - name: grafana-service - port: - targetPort: 3000 - tls: - termination: edge - wildcardPolicy: None + volumes: + - name: oauth-proxy-tls + secret: + secretName: exploit-iq-grafana-tls + - name: oauth-proxy-secret + secret: + secretName: oauth-client-secret-86gfcb49d4 + route: + spec: + port: + targetPort: https + tls: + insecureEdgeTerminationPolicy: Redirect + termination: reencrypt + service: + metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: exploit-iq-grafana-tls + spec: + ports: + - name: https + port: 9091 + protocol: TCP + targetPort: 9091 diff --git a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml index 7070604e6..db8d3e21a 100644 --- a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - grafana.yaml + - serviceAccount.yaml diff --git a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml new file mode 100644 index 000000000..27be895d0 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-sa + annotations: + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"exploit-iq-grafana"}}' \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 94cd43aee..38d1fdbd9 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -3,9 +3,6 @@ kind: Kustomization namespace: REPLACE_NAMESPACE -crds: - - crds - resources: - operator @@ -25,29 +22,6 @@ labels: namePrefix: exploit-iq- replacements: - - source: - kind: Grafana - name: grafana - fieldPath: metadata.name - targets: - - select: - kind: Route - name: grafana - fieldPaths: - - spec.to.name - options: - delimiter: "-" - index: 0 - - source: - kind: Secret - name: grafana-secret - fieldPath: metadata.name - targets: - - select: - kind: Grafana - name: grafana - fieldPaths: - - spec.config.security.secret - source: kind: Grafana name: grafana @@ -71,10 +45,4 @@ replacements: kind: OperatorGroup name: grafana-operators fieldPaths: - - spec.targetNamespaces.0 - - -secretGenerator: - - name: grafana-secret - envs: - - secrets.env + - spec.targetNamespaces.0 \ No newline at end of file diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml index 59aff2fe7..a814d02f3 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml @@ -9,6 +9,6 @@ spec: - name: nginx env: - name: NGINX_UPSTREAM_NIM_LLM - value: http://llama3-1-70b-instruct-4bit.agent-morpheus-models.svc.cluster.local:8000 + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 - name: NGINX_UPSTREAM_OPENAI - value: http://llama3-1-70b-instruct-4bit.agent-morpheus-models.svc.cluster.local:8000 + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 From b077014ce3b8340e608ca2731c443451789d25b9 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 3 Feb 2026 16:26:15 +0200 Subject: [PATCH 182/286] docs: update kustomize README with mlops overlay instructions Signed-off-by: Ilona Shishov --- kustomize/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kustomize/README.md b/kustomize/README.md index e35a39c03..0247e58f9 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -193,7 +193,14 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ```shell # Patch overlay kustomization yaml with deployment namespace value sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomize.yaml +``` +```shell +# replace EXPLOIT_IQ_SA_TOKEN with ExploitIQ SA Token from bitwarden vault (1 year expiration date) +oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_SA_TOKEN' +``` + +```shell # Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM and MLOps oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME From c089130a5ee24c33b459441429c0867f12ddea4c Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:14:18 +0200 Subject: [PATCH 183/286] Appeng 4284 telemetry add tekton support for passing commands for telemetry --- .tekton/on-cm-runner.yaml | 37 ++++++++++++++++++- .tekton/on-pull-request.yaml | 36 ++++++++++++++++-- src/vuln_analysis/functions/cve_agent.py | 2 +- .../tests/test_transitive_code_search.py | 8 ++-- 4 files changed, 74 insertions(+), 9 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 08e308c74..066f1e96d 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -162,6 +162,8 @@ spec: value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd - name: RHSA_BASE_URL value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) script: | #!/bin/sh @@ -206,6 +208,24 @@ spec: echo "Cache link created successfully." ls -ld "${CACHE_DIR_TARGET}" + + # Copy with verbose error if it fails + echo "--- Exporting Telemetry Config ---" + cp -v configs/config-no-tracing.yml $DATA_DIR/config-no-tracing.yml || echo "Failed to copy config" + + echo "--- Exporting Env Vars ---" + # Explicit env var names to never export (e.g. from integration-tests, server-model-config, redhat-app-creds / .env) + ENV_BLACKLIST="GHSA_API_KEY NVD_API_KEY NVIDIA_API_KEY SERPAPI_API_KEY" + # Regex (case-insensitive) for key names to exclude: .password., .token., .api-key., .api_key., .secret. + ENV_KEY_PATTERN='password|token|api.key|api_key|api-key|secret|credential|private-key|private_key|auth|pwd|pass_|_pass|connection_string|connection-string|pat|certificate' + + : > $DATA_DIR/env.txt + env | while IFS= read -r line; do + key="${line%%=*}" + case " $ENV_BLACKLIST " in *" $key "*) continue ;; esac + echo "$key" | grep -qiE "$ENV_KEY_PATTERN" && continue + echo "$line" >> $DATA_DIR/env.txt + done echo "--- Starting 'nat' Server ---" exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 @@ -305,6 +325,8 @@ spec: secretKeyRef: name: google-sheets-secrets key: output_sheet_id + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) script: | #!/bin/bash @@ -318,10 +340,21 @@ spec: echo "Generated Tag: $TAG_NAME" echo "--- DEBUG: Current Directory ---" pwd - + # prepare the mlops telemetry data + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + #copy the token to src/config/token.txt + echo $TOKEN > /app/src/config/token.txt + + if [ -f "$DATA_DIR/config-no-tracing.yml" ]; then + echo "--- Config file found in $DATA_DIR/config-no-tracing.yml ---" + else + echo "--- Config file not found in $DATA_DIR/config-no-tracing.yml ---" + exit 1 + fi + # 2. Run Python script with the tag cd /app/ - python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" + python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them echo "--- Copying Reports to Shared Workspace ---" diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 00d714fb7..f15b2cd6c 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -313,6 +313,8 @@ spec: value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd - name: RHSA_BASE_URL value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) script: | #!/bin/sh @@ -323,7 +325,7 @@ spec: #3. Making sure that Internet search tool is enabled ( part of the tests are relying on that). echo "--- Generating Config without Tracing ---" python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = True;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" - + CACHE_DIR_TARGET=".cache/am_cache" CACHE_PVC_SOURCE="/exploit-iq-data" @@ -344,6 +346,25 @@ spec: echo "Cache link created successfully." ls -ld "${CACHE_DIR_TARGET}" + + # Copy with verbose error if it fails + + echo "--- Exporting Telemetry Config ---" + cp -v configs/config-no-tracing.yml $DATA_DIR/config-no-tracing.yml || echo "Failed to copy config" + + # 4. Safer Env Dump (blacklist only: explicit names + regex on key) + echo "--- Exporting Env Vars ---" + # Explicit env var names to never export (e.g. from integration-tests, server-model-config, redhat-app-creds / .env) + ENV_BLACKLIST="GHSA_API_KEY NVD_API_KEY NVIDIA_API_KEY SERPAPI_API_KEY" + # Regex (case-insensitive) for key names to exclude: .password., .token., .api-key., .api_key., .secret. + ENV_KEY_PATTERN='password|token|api.key|api_key|api-key|secret|credential|private-key|private_key|auth|pwd|pass_|_pass|connection_string|connection-string|pat|certificate' + : > $DATA_DIR/env.txt + env | while IFS= read -r line; do + key="${line%%=*}" + case " $ENV_BLACKLIST " in *" $key "*) continue ;; esac + echo "$key" | grep -qiE "$ENV_KEY_PATTERN" && continue + echo "$line" >> $DATA_DIR/env.txt + done echo "--- Starting 'nat' Server ---" exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 @@ -362,7 +383,9 @@ spec: image: quay.io/ecosystem-appeng/auto-cm-testing:latest imagePullPolicy: Always workingDir: $(workspaces.source.path) - + env: + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) script: | #!/bin/bash set -e @@ -374,8 +397,15 @@ spec: # 2. Prepares the IT input for test cp -f ci/it/integration-tests-input.json /app/src/input/scan_it.json cd /app/ + # prepare the mlops telemetry data + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + #copy the token to src/config/token.txt + echo $TOKEN > src/config/token.txt + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + # 3. Run IT Script - python3 src/main_integration_tests.py + python3 src/main_integration_tests.py --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index f4a3f62b9..4f1e168d1 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -106,7 +106,7 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, 'tool_selection_strategy': tool_guidance if tool_guidance else "Use available tools as appropriate." } ) - + # using langchain create_react_agent to create the agent agent = create_react_agent(llm=llm, tools=tools, prompt=prompt, diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 67143f695..00e8f9222 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -1,5 +1,5 @@ import logging - +from contextlib import aclosing import pytest from unittest.mock import patch, MagicMock @@ -127,8 +127,10 @@ def set_input_for_next_run(git_repository: str, git_ref: str, included_extension async def get_transitive_code_runner_function(): transitive_code_search = transitive_search(config=TransitiveCodeSearchToolConfig(), builder=None) - async for function in transitive_code_search.gen: - return function.single_fn + # aclosing automatically calls gen.aclose() + async with aclosing(transitive_code_search.gen) as gen: + async for function in gen: + return function.single_fn python_dependency_tree_mock_output = ( 'deptree==0.0.12 # deptree\n' From 7629f8cfac01f5afc07c0f48661f32e8914f80c3 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf <98809100+TamarW0@users.noreply.github.com> Date: Wed, 4 Feb 2026 15:09:33 +0200 Subject: [PATCH 184/286] feat: Transitive dependencies for javascript (#171) Co-authored-by: Tamar Weisskopf --- Dockerfile | 14 + ci/it/integration-tests-input.json | 31 +- kustomize/base/excludes.json | 1 - kustomize/base/includes.json | 3 +- .../utils/chain_of_calls_retriever.py | 7 +- .../utils/chain_of_calls_retriever_base.py | 7 +- src/exploit_iq_commons/utils/dep_tree.py | 50 + .../utils/document_embedding.py | 22 +- .../c_lang_function_parsers.py | 54 +- .../golang_functions_parsers.py | 2 +- .../java_functions_parsers.py | 4 +- .../javascript_functions_parser.py | 986 +++++++ .../lang_functions_parsers.py | 4 +- .../lang_functions_parsers_factory.py | 3 + .../python_functions_parser.py | 21 +- .../utils/javascript_extended_segmenter.py | 379 +++ .../utils/js_extended_parser.py | 103 - .../tests/test_transitive_code_search.py | 626 +++- tests/test_java_script_extended.py | 140 - tests/test_java_script_extended_segmenter.py | 593 ++++ tests/test_javascript_functions_parser.py | 2617 +++++++++++++++++ 21 files changed, 5356 insertions(+), 311 deletions(-) create mode 100644 src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py create mode 100644 src/exploit_iq_commons/utils/javascript_extended_segmenter.py delete mode 100644 src/exploit_iq_commons/utils/js_extended_parser.py delete mode 100644 tests/test_java_script_extended.py create mode 100644 tests/test_java_script_extended_segmenter.py create mode 100644 tests/test_javascript_functions_parser.py diff --git a/Dockerfile b/Dockerfile index 72ea1eefe..681ced2a1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,8 @@ RUN apt-get update && apt-get install -y \ wget \ skopeo \ libarchive-tools \ + xz-utils \ + libatomic1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates @@ -46,6 +48,18 @@ RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24. ENV GOTOOLCHAIN=auto +# --- Node.js 25.2.0 and npm 11.6.2 --- +ARG NODE_VERSION=25.2.0 +RUN curl -fsSL -o /tmp/node.tar.xz \ + "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + && mkdir -p /opt/nodejs \ + && tar -C /opt/nodejs -xJf /tmp/node.tar.xz --strip-components=1 \ + && rm -f /tmp/node.tar.xz +ENV PATH="/opt/nodejs/bin:${PATH}" + +# Verify Node.js and npm installation +RUN node --version && npm --version + # --- Temurin JDK 22 (amd64/x86_64) --- ARG JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz" ARG JDK_DIR="jdk-22.0.2+9" diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json index 500540be0..6d01cb7e3 100644 --- a/ci/it/integration-tests-input.json +++ b/ci/it/integration-tests-input.json @@ -59,7 +59,6 @@ "allowed_deviation_labels" : ["vulnerable"], "skip": false }, - { "language": "go", "vuln_id": "CVE-2024-28180", @@ -73,8 +72,6 @@ "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","protected_by_mitigating_control"], "skip": true }, - - { "language": "java", "vuln_id": "CVE-2025-48734", @@ -87,8 +84,32 @@ "expected_result": "Not Exploitable", "allowed_deviation_labels" : ["code_not_present","code_not_reachable"], "skip": true + }, + { + "language": "javascript", + "vuln_id": "CVE-2021-23369", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "c0223de250a6c06b5bab660f3c7057a53df8921a" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "javascript", + "vuln_id": "CVE-2019-10744", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "aec80a8feef62f25ff8ff200b0d1ad2244b2a1e0" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], + "skip": false } - - ] } diff --git a/kustomize/base/excludes.json b/kustomize/base/excludes.json index f3fac3cb6..9876ce1de 100644 --- a/kustomize/base/excludes.json +++ b/kustomize/base/excludes.json @@ -27,7 +27,6 @@ "**/docs/**/*" ], "JavaScript": [ - "node_modules/**/*", "dist/**/*", "build/**/*", "test/**/*", diff --git a/kustomize/base/includes.json b/kustomize/base/includes.json index 7f1df85f8..3b96ed8b3 100644 --- a/kustomize/base/includes.json +++ b/kustomize/base/includes.json @@ -15,7 +15,8 @@ ], "JavaScript": [ "**/*.js", - "**/*.jsx", + "**/*.mjs", + "**/*.cjs", "webpack.config.js", "rollup.config.js", "babel.config.js", diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index 1b86a476a..277ac4398 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -71,7 +71,7 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat logger.debug("Creating Chain of Calls Retriever") logger.debug("Starting building Chain of Calls Retriever") self.ecosystem = ecosystem - logger.debug("Chain of Calls Retriever - creating dependency tree") + logger.debug("Chain of Calls Retriever - creating dependency tree") # Build dependency tree based on the parameter programming language/package manager. self.dependency_tree = DependencyTree(ecosystem=ecosystem) logger.debug("Chain of Calls Retriever - get language parser") @@ -120,7 +120,7 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat else: self.documents = filtered_documents self.documents_of_functions = [doc for doc in self.documents - if doc.page_content.startswith(self.language_parser.get_function_reserved_word())] + if self.language_parser.is_function(doc)] logger.debug(f"self.documents len : {len(self.documents)}") logger.debug("Chain of Calls Retriever - retaining only types/classes docs " @@ -327,7 +327,8 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack if last_visited == 0: found = self.language_parser.search_for_called_function( caller_function=doc, - callee_function=function_name_to_search, + callee_function_name=function_name_to_search, + callee_function=document_function, callee_function_package=function_package, code_documents=self.documents_of_full_sources, type_documents=self.documents_of_types, diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py index 53488164a..2900e59cb 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py @@ -24,8 +24,9 @@ def get_extension_of_file(file_path: str): extension_start = file_path.rfind(".") return file_path[extension_start:] -def is_function_callable(document: Document, language_parser: LanguageFunctionsParser, callee_function_file_name: str) -> bool: - return (language_parser.is_exported_function(document) or +def is_function_callable(document: Document, language_parser: LanguageFunctionsParser, callee_function_file_name: str, + documents_of_full_sources: dict[str, Document]) -> bool: + return (language_parser.is_exported_function(document, documents_of_full_sources) or document.metadata['source'].lower() == callee_function_file_name.lower()) class ChainOfCallsRetrieverBase(ABC): @@ -88,7 +89,7 @@ def get_functions_for_package(self, package_name: str, documents: list[Document] if (not self.language_parser.is_root_package(document) and document.metadata.get('content_type') == 'functions_classes' and self.language_parser.is_function(document) and - is_function_callable(document, self.language_parser, callee_function_file_name) and + is_function_callable(document, self.language_parser, callee_function_file_name, self.documents_of_full_sources) and self.language_parser.is_supported_file_extensions(doc_extension) and self.document_belongs_to_package(document, package_name) and self.function_called_from_caller_body(document, function_to_search)): diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index a4572c012..b84a20519 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1162,6 +1162,54 @@ def install_dependency(self, dependency, repo_path): if not res: logger.warning(f'Failed to install dependency {dependency}') +class JavaScriptDependencyTreeBuilder(DependencyTreeBuilder): + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + + tree = defaultdict(set) + ROOT_PROJECT = 'root_project' + tree[ROOT_PROJECT] = [ROOT_LEVEL_SENTINEL] + package_json_path = manifest_path / 'package.json' + if not package_json_path.exists(): + logger.error(f"package.json not found at {package_json_path}") + raise FileNotFoundError(f"package.json not found at {package_json_path}") + + result = run_command(f'cd {manifest_path} && npm ls --json --all') + try: + npm_tree = json.loads(result) + except json.JSONDecodeError as e: + raise RuntimeError(f"Failed to parse npm ls JSON output: {e}") + self._parse_npm_tree(npm_tree, tree, ROOT_PROJECT) + + return {k: list(v) for k, v in tree.items()} + + def _parse_npm_tree(self, node: dict, tree: dict, parent: str): + """ + Recursively parse npm ls JSON output to build child->parents mapping. + + Args: + node: Current node in the npm tree (has 'name', 'version', 'dependencies') + tree: The dependency tree being built (child -> set of parents) + parent: Name of the parent package + """ + dependencies = node.get('dependencies', {}) + + for dep_name, dep_info in dependencies.items(): + tree[dep_name].add(parent) + if isinstance(dep_info, dict): + self._parse_npm_tree(dep_info, tree, dep_name) + + def install_dependencies(self, manifest_path: Path): + logger.info(f"Installing JavaScript dependencies in {manifest_path}") + cmd = f'cd {manifest_path} && npm install' + res = run_command(cmd) + if not res: + logger.error(f"Failed to install JavaScript dependencies") + else: + logger.info("JavaScript dependencies installed successfully") + + + def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "") -> DependencyTreeBuilder: """ A Factory method function that gets an programming language as argument, @@ -1176,6 +1224,8 @@ def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "" return GoDependencyTreeBuilder() elif programming_language.value == Ecosystem.PYTHON.value: return PythonDependencyTreeBuilder() + elif programming_language.value == Ecosystem.JAVASCRIPT.value: + return JavaScriptDependencyTreeBuilder() elif programming_language.value == Ecosystem.JAVA.value: return JavaDependencyTreeBuilder(query=query) elif programming_language.value == Ecosystem.C_CPP.value: diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index 9d92fc763..242a9d667 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -40,7 +40,7 @@ from exploit_iq_commons.utils.go_segmenters_with_methods import GoSegmenterWithMethods from exploit_iq_commons.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods from exploit_iq_commons.utils.java_segmenters_with_methods import JavaSegmenterWithMethods -from exploit_iq_commons.utils.js_extended_parser import ExtendedJavaScriptSegmenter +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -105,18 +105,14 @@ class ExtendedLanguageParser(LanguageParser): "hpp": "cpp", } - additional_segmenters: dict[str, type[CodeSegmenter]] = {} - - if os.environ.get('ENABLE_EXTENDED_JS_PARSERS'): - additional_segmenters = { - "javascript": ExtendedJavaScriptSegmenter, - "js": ExtendedJavaScriptSegmenter, - } - additional_segmenters["go"] = GoSegmenterWithMethods - additional_segmenters["python"] = PythonSegmenterWithClassesMethods - additional_segmenters["java"] = JavaSegmenterWithMethods - additional_segmenters["c"] = CSegmenterExtended - + additional_segmenters = { + "go": GoSegmenterWithMethods, + "python": PythonSegmenterWithClassesMethods, + "javascript": ExtendedJavaScriptSegmenter, + "js": ExtendedJavaScriptSegmenter, + "java": JavaSegmenterWithMethods, + "c": CSegmenterExtended +} LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, **additional_segmenters, diff --git a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py index b8522eabb..5e2402aa6 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py @@ -515,61 +515,63 @@ def get_function_name(self, function: Document) -> str: def search_for_called_function( - self, - caller_function: Document, - callee_function: str, - callee_function_package: str, # For C, this is usually the file or library name - code_documents: list[Document], - type_documents: list[Document], - callee_function_file_name: str, - fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict], - documents_of_functions: list[Document]=None, - type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None + self, + caller_function: Document, + callee_function_name: str, + callee_function_package: str, # For C, this is usually the file or library name + code_documents: list[Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document] = None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] = None, + callee_function:Document = None ) -> bool: """ Returns True if caller_function calls callee_function (directly or via struct/function pointer). """ # 1. Extract function body (remove comments for easier parsing) function_body = _remove_c_comments(caller_function.page_content) - # 2. Direct call: callee_function( - direct_call_pattern = re.compile(r'\b' + re.escape(callee_function) + r'\s*\(') + # 2. Direct call: callee_function_name( + direct_call_pattern = re.compile(r'\b' + re.escape(callee_function_name) + r'\s*\(') if direct_call_pattern.search(function_body): return True - # 3. Struct member or pointer call: obj->callee_function( or obj.callee_function( - member_call_pattern = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(->|\.)\s*' + re.escape(callee_function) + r'\s*\(') + # 3. Struct member or pointer call: obj->callee_function_name( or obj.callee_function_name( + member_call_pattern = re.compile( + r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(->|\.)\s*' + re.escape(callee_function_name) + r'\s*\(') for match in member_call_pattern.finditer(function_body): var_name = match.group(1) # Look up var_name in local variables and parameters func_key = self._optimize_func_key( - self.get_function_name(caller_function), + self.get_function_name(caller_function), caller_function.metadata.get('source', 'unknown_source.c') ) - + local_vars = functions_local_variables_index.get(func_key, {}) var_type = local_vars.get(var_name, {}).get("type", "") - # Check if var_type is a struct with a function pointer named callee_function + # Check if var_type is a struct with a function pointer named callee_function_name for (struct_name, _), fields in fields_of_types.items(): if struct_name == var_type: for field_name, field_type in fields: - if field_name == callee_function and "(*" in field_type: + if field_name == callee_function_name and "(*" in field_type: return True # 4. Function pointer variable: foo_ptr( for match in self.fp_var_pattern.finditer(function_body): var_name = match.group(1) - if var_name == callee_function: + if var_name == callee_function_name: continue # Already checked direct call - # Is this variable a function pointer to callee_function? + # Is this variable a function pointer to callee_function_name? func_key = self._optimize_func_key( - self.get_function_name(caller_function), + self.get_function_name(caller_function), caller_function.metadata.get('source', 'unknown_source.c') ) local_vars = functions_local_variables_index.get(func_key, {}) var_type = local_vars.get(var_name, {}).get("type", "") - # Optionally, check if the function pointer is assigned to callee_function somewhere - assignment_pattern = re.compile(rf'{re.escape(var_name)}\s*=\s*&?{re.escape(callee_function)}\s*;') + # Optionally, check if the function pointer is assigned to callee_function_name somewhere + assignment_pattern = re.compile(rf'{re.escape(var_name)}\s*=\s*&?{re.escape(callee_function_name)}\s*;') if assignment_pattern.search(function_body): return True @@ -605,7 +607,7 @@ def is_comment_line(self, line: str) -> bool: def get_comment_line_notation(self) -> str: return "//" - def is_exported_function(self, function: Document) -> bool: + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: # In C, functions have external linkage by default unless declared as static #however internal functions might be called by exported functions #The analysis needs to understand the complete call graph @@ -679,7 +681,7 @@ def filter_docs_by_func_pkg_name(self,function_name:str ,package_name : str, doc return relevant_docs - def document_imports_package(self,documents:list[Document],package_name:str) -> list[Document]: + def document_imports_package(self,documents: dict[str, Document],package_name:str) -> list[Document]: importing_docs = [value for (file, value) in documents.items() if re.search( rf"(include {package_name}|include\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index b31644ba3..fad0ee8f5 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -343,7 +343,7 @@ def get_comment_line_notation() -> str: def dir_name_for_3rd_party_packages(self) -> str: return "vendor" - def is_exported_function(self, function: Document) -> bool: + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: function_name = self.get_function_name(function) return re.search("[A-Z][a-z0-9-]*", function_name) diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index 27ffa2c11..ac6b50f18 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -205,7 +205,7 @@ def get_comment_line_notation(self) -> str: def dir_name_for_3rd_party_packages(self) -> str: return "dependencies-sources" - def is_exported_function(self, function: Document) -> bool: + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: return True def get_function_name(self, function: Document) -> str: @@ -1566,7 +1566,7 @@ def __trace_down_package(self, expression: str, type_documents: list[Document], result = True return result - def document_imports_package(self, documents: list[Document], package_name: str) -> list[Document]: + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: importing_docs = [value for (file, value) in documents.items() if self.is_package_imported(value.page_content, package_name) or package_name.rsplit('.', 1)[-1] in value.page_content] return importing_docs diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py new file mode 100644 index 000000000..a9e3252d6 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -0,0 +1,986 @@ +import re +from typing import List, Tuple + +from langchain_core.documents import Document + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +logger = LoggingFactory.get_agent_logger(__name__) + + +class JavaScriptFunctionsParser(LanguageFunctionsParser): + + def get_function_name(self, function: Document) -> str: + """ + Extract function name from JavaScript code. + + Handles various JavaScript function patterns: + - function myFunc() {...} + - const myFunc = () => {...} + - async function myFunc() {...} + - class methods + - arrow functions without parentheses: const name = param => ... + - computed property methods: [Symbol.iterator]() {...} + """ + content = function.page_content + + if not self.is_function(function): + raise ValueError('Only function document is supported') + + # Try to match function declarations: function name(...) or async function name(...) + match = re.search(r'(?:async\s+)?function\s+(\w+)\s*\(', content) + if match: + return match.group(1) + + # Try to match arrow functions with parentheses: const name = (...) => or const name = async (...) => + match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>', content) + if match: + return match.group(1) + + # Try to match arrow functions without parentheses (single parameter): const name = param => ... + # This pattern matches: const name = identifier => + match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\w+\s*=>', content) + if match: + return match.group(1) + + # Try to match function expressions: const name = function(...) { + match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function\s*\(', content) + if match: + return match.group(1) + + # Try to match computed property methods: [Symbol.iterator]() { or [expr]() { + # Return the computed expression inside brackets + match = re.search(r'^\s*\[([^\]]+)\]\s*\([^)]*\)\s*\{', content, re.MULTILINE) + if match: + return match.group(1) + + # Try to match class methods: methodName(...) { or async methodName(...) { + match = re.search(r'(?:async\s+)?(\w+)\s*\([^)]*\)\s*\{', content) + if match: + return match.group(1) + + logger.warning(f"Could not extract function name from: {content[:100]}") + return '' + + def get_package_names(self, function: Document) -> list[str]: + source = function.metadata.get('source') + if self.dir_name_for_3rd_party_packages() in source: + parts = source.split(self.dir_name_for_3rd_party_packages()) + if len(parts) > 1: + if len(parts) > 2: + parts = parts[-2:] + pkg_parts = parts[1].split('/') + # Handle scoped packages: @babel/core -> @babel/core + if pkg_parts[1].startswith('@') and len(pkg_parts) > 1: + return [f"{pkg_parts[1]}/{pkg_parts[2]}"] + return [pkg_parts[1]] + return ['root_project'] + + def is_root_package(self, function: Document) -> bool: + source = function.metadata.get('source', '') + return self.dir_name_for_3rd_party_packages() not in source + + + def is_function(self, function: Document) -> bool: + content_type = function.metadata.get('content_type', '') + if content_type != 'functions_classes': + return False + + content = function.page_content.strip() + + if re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\s+\w+', content): + return False + + return True + + def supported_files_extensions(self) -> list[str]: + return ['.js', '.mjs', '.cjs'] + + def _get_function_calls(self, caller_function: Document, callee_function_name: str, code_documents: dict[str, Document] = None) -> list[str]: + """ + Extract all function calls matching the given callee_function_name from caller_function. + + Detects both: + - Direct function calls: functionName() or obj.functionName() + - Callback references: setTimeout(functionName, ...) or arr.forEach(functionName) + + Also handles aliased imports in JavaScript: + - ES6: import { originalName as alias } from 'module' + - CommonJS: const { originalName: alias } = require('module') + + Args: + caller_function: Document containing the caller function code + callee_function_name: Name of the function to find calls to + code_documents: Optional dict mapping source paths to full file Documents (for alias resolution) + + Returns: + List of function call patterns found (e.g., ['func', 'obj.func']) + """ + content = caller_function.page_content + content = '\n'.join([line if not self.is_comment_line(line) else ''for line in content.splitlines()]) + + direct_call_pattern = rf'((?:[\w.()]+\.)?(? bool: + + calls = self._get_function_calls(caller_function, callee_function_name) + + if not calls: + return False + + caller_source = caller_function.metadata.get('source') + caller_document = code_documents.get(caller_source) + caller_content = caller_document.page_content if caller_document else caller_function.page_content + + for call in calls: + parts = call.split('.') + + # Direct function call (no qualifier) + if len(parts) == 1: + identifier = parts[0].rstrip('(') + + caller_pkg = self.get_package_names(caller_function)[0] + callee_pkg = self.get_package_names(callee_function)[0] + + if caller_pkg == callee_pkg and identifier == callee_function_name: + return True + + # Check if package is imported and identifier matches + if self.is_package_imported(caller_content, identifier, callee_function_package): + return True + + # Qualified call (obj.func or module.func) + else: + if parts[-1].rstrip('(') == callee_function_name: + identifier = parts[-2] + + if identifier == 'this': + callee_class_name = self.get_class_name_from_class_function(callee_function) + caller_class_name = self.get_class_name_from_class_function(caller_function) + if callee_class_name and callee_class_name == caller_class_name: + return True + + if callee_class_name := self.get_class_name_from_class_function(callee_function): + caller_func_name = self.get_function_name(caller_function) + func_key = f"{caller_func_name}@{caller_source}" + local_vars = functions_local_variables_index.get(func_key, {}) + var_info = local_vars.get(identifier, {}) + var_type = var_info.get('type') + if var_type == callee_class_name or self._is_subclass_of(var_type, callee_class_name, code_documents): + return True + + if self.is_package_imported(caller_content, identifier, callee_function_package): + return True + + return False + + def _is_subclass_of(self, child_class: str, parent_class: str, code_documents: dict[str, Document], + visited: set = None) -> bool: + """ + Check if child_class extends parent_class by searching simplified_code documents. + Supports: + - Simple: class X extends Y + - Mixin: class X extends Mixin(Y) + - Chained: class X extends Mixin1(Mixin2(Y)) + - Transitive: class A extends B, class B extends C -> A is subclass of C + - Prototype-based: util.inherits(Child, Parent), Child.prototype = Object.create(Parent.prototype) + """ + if not child_class or not parent_class: + return False + + if visited is None: + visited = set() + + if child_class in visited: + return False # Prevent infinite recursion on circular references + visited.add(child_class) + + es6_class_pattern = rf'class\s+{re.escape(child_class)}\s+extends\s+([^{{]+)' + + for doc in code_documents.values(): + if match := re.search(es6_class_pattern, doc.page_content): + extends_clause = match.group(1).strip() + # Direct match in extends clause (handles mixins too) + if re.search(rf'\b{re.escape(parent_class)}\b', extends_clause): + return True + + immediate_parent = self._get_parent(child_class, code_documents) + if immediate_parent: + # Direct match: immediate parent is the target + if immediate_parent == parent_class: + return True + # Transitive check: follow the chain + return self._is_subclass_of(immediate_parent, parent_class, code_documents, visited) + + return False + + def _get_direct_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + """ + Extract the immediate parent class name from ES6 class extends syntax. + + Handles: + - Simple: class X extends Y -> returns Y + - Mixin: class X extends Mixin(Y) -> returns Y (the innermost/base class) + - Chained: class X extends Mixin1(Mixin2(Y)) -> returns Y + """ + class_pattern = rf'class\s+{re.escape(child_class)}\s+extends\s+([^{{]+)' + + for doc in code_documents.values(): + if match := re.search(class_pattern, doc.page_content): + extends_clause = match.group(1).strip() + # Find the last identifier before closing parentheses + if innermost_match := re.search(r'(\w+)\s*\)+\s*$', extends_clause): + return innermost_match.group(1) + # Simple case: class X extends Y (no parentheses) + if parent_match := re.match(r'^(\w+)\s*$', extends_clause): + return parent_match.group(1) + return None + + def _get_prototype_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + """ + Extract parent class from prototype-based inheritance patterns. + + Handles: + - util.inherits(Child, Parent) + - inherits(Child, Parent) + - Child.prototype = Object.create(Parent.prototype) + - Object.setPrototypeOf(Child.prototype, Parent.prototype) + """ + patterns = [ + # util.inherits(Child, Parent) or inherits(Child, Parent) + rf'(?:util\.)?inherits\s*\(\s*{re.escape(child_class)}\s*,\s*(\w+)\s*\)', + # Child.prototype = Object.create(Parent.prototype) + rf'{re.escape(child_class)}\.prototype\s*=\s*Object\.create\s*\(\s*(\w+)\.prototype\s*\)', + # Object.setPrototypeOf(Child.prototype, Parent.prototype) + rf'Object\.setPrototypeOf\s*\(\s*{re.escape(child_class)}\.prototype\s*,\s*(\w+)\.prototype\s*\)', + ] + + for doc in code_documents.values(): + for pattern in patterns: + if match := re.search(pattern, doc.page_content): + return match.group(1) + return None + + def _get_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + return self._get_direct_parent(child_class, code_documents) or \ + self._get_prototype_parent(child_class, code_documents) + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = {} + + for func_method in functions_methods_documents: + func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + content = func_method.page_content + + all_vars = {} + + # Extract parameters from function signature + param_match = re.search(r'\(([^)]*)\)', content) + if param_match: + all_vars.update(self._parse_declarations(param_match.group(1), is_param=True)) + + all_vars['return_types'] = [] + + # Extract local variables from function body + if param_match: + first_brace = content.find('{', param_match.end()) + else: + first_brace = content.find('{') + + last_brace = content.rfind('}') + if first_brace != -1 and last_brace != -1: + body = content[first_brace + 1:last_brace] + for statement in self._split_into_statements(body): + if not self.is_comment_line(statement): + if match := re.match(r'(const|let|var)\s+(.+)', statement): + all_vars.update(self._parse_declarations(match.group(2), is_param=False)) + + # Add 'this' reference for class/object methods + if class_name := self.get_class_name_from_class_function(func_method): + all_vars['this'] = {"value": f'{class_name}()', "type": class_name} + + mappings[func_key] = all_vars + + return mappings + + @staticmethod + def _split_into_statements(text: str) -> list[str]: + """ + Split text by semicolons and newlines, respecting string boundaries. + + Handles: + - Statements ending with semicolons + - Comments (// and /* */) + - Function/class declarations (including multiline parameter lists) + - Braces { and } + """ + def normalize(line: str) -> str: + """Normalize whitespace: collapse multiple spaces/newlines into single space and strip.""" + return re.sub(r'\s+', ' ', line).strip() + + parts = [] + current = '' + in_string = False + string_char = None + in_block_comment = False + in_line_comment = False + paren_depth = 0 + just_closed_paren = False + just_closed_block_comment = False + + for i, char in enumerate(text): + if not in_string and not in_line_comment: + if char == '/' and i + 1 < len(text) and text[i + 1] == '*': + in_block_comment = True + elif char == '*' and i + 1 < len(text) and text[i + 1] == '/' and in_block_comment: + in_block_comment = False + just_closed_block_comment = True + elif char == '/' and i + 1 < len(text) and text[i + 1] == '/' and not in_block_comment: + in_line_comment = True + + if not in_string and not in_block_comment and not in_line_comment: + if char == '(': + paren_depth += 1 + just_closed_paren = False + elif char == ')': + paren_depth -= 1 + if paren_depth == 0: + just_closed_paren = True + elif char not in (' ', '\t', '\n', '\r', '=', '>'): + # Reset flag if we see a non-whitespace character that's not part of => or { + if char != '{': + just_closed_paren = False + + if (char in ('"', "'", '`') and (i == 0 or text[i-1] != '\\') + and not in_block_comment and not in_line_comment): + if not in_string: + in_string = True + string_char = char + elif char == string_char: + in_string = False + string_char = None + + current += char + + if just_closed_block_comment and char == '/': + normalized = normalize(current) + if normalized.startswith('/*') and normalized.endswith('*/'): + parts.append(normalized) + current = '' + just_closed_block_comment = False + + if char == '\n' and in_line_comment: + in_line_comment = False + parts.append(normalize(current)) + current = '' + + elif char == ';' and not in_string and not in_block_comment and not in_line_comment: + parts.append(normalize(current)) + current = '' + + # Split on opening brace { for function/method body + elif char == '{' and not in_string and not in_block_comment and not in_line_comment: + normalized = normalize(current) + if just_closed_paren or any(normalized.startswith(kw) for kw in ['class ', 'export class ']): + if normalized: + parts.append(normalized) + current = '' + just_closed_paren = False + + elif char == '\n' and not in_string and not in_block_comment: + normalized = normalize(current) + if normalized: + if normalized == '}': + parts.append(normalized) + current = '' + + # Add remaining content + if current: + normalized = normalize(current) + if normalized: + parts.append(normalized) + + return parts + + def _parse_declarations(self, declaration_str: str, is_param: bool = False, is_multiline: bool = False) -> dict[str, dict]: + """ + Parse JavaScript variable/parameter declarations into a dict. + + Handles: rest params (...x), destructuring ({a,b}, [x,y]), defaults (a=1), multi-declarations (x=1,y=2) + """ + result = {} + + # Split by comma respecting nesting + for item in self._split_by_comma(declaration_str): + item = item.strip() + if not item: + continue + + # Rest parameter: ...rest + if item.startswith('...'): + result[item[3:].strip()] = {"value": "parameter", "type": "Array"} + + # Object/Array destructuring: {a, b} or [x, y] + elif item.startswith(('{', '[')): + closing_char = '}' if item[0] == '{' else ']' + if (end := item.find(closing_char)) != -1: + value = self._extract_value_after_pattern(item, closing_char) + for elem in item[1:end].split(','): + # Handle {oldName: newName} and {prop = default} + name = elem.split(':')[-1].split('=')[0].strip() + if name: + result[name] = {"value": value if not is_param else "parameter", "type": ""} + + # Regular declaration with or without value + else: + name, _, value = item.partition('=') + name = name.strip().rstrip(';') + value = value.strip().rstrip(';') if value else ("parameter" if is_param else "") + + # Extract type from 'new ClassName(...)' pattern + var_type = "" + if new_match := re.match(r'new\s+(\w+)\s*\(', value): + var_type = new_match.group(1) + + result[name] = {"value": value, "type": var_type} + + return result + + def _split_by_comma(self, text: str) -> list[str]: + """Split by comma, respecting nested (), [], {}.""" + items, current, depth = [], [], 0 + for char in text: + if char in '([{': + depth += 1 + elif char in ')]}': + depth -= 1 + if char == ',' and depth == 0: + items.append(''.join(current)) + current = [] + continue + current.append(char) + if current: + items.append(''.join(current)) + return items + + @staticmethod + def _extract_value_after_pattern(text: str, closing_char: str) -> str: + """Extract value after destructuring pattern, e.g., '} = obj' -> 'obj'.""" + if match := re.search(rf'{re.escape(closing_char)}\s*=\s*(.+?)(?:;|$)', text): + return match.group(1).strip() + return "" + + def parse_all_type_struct_class_to_fields(self, types: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] = {}) -> dict[tuple, list[tuple]]: + """ + Parse JavaScript class documents and extract field declarations. + + Extracts: + - Class field declarations: class MyClass { myField = value; } + - Constructor property assignments: constructor() { this.field = value; } + + Returns: + Dict mapping (class_name, source_path) -> list of (field_name, field_type) tuples + """ + types_mapping = {} + for type_doc in types: + class_name = self._extract_class_name(type_doc.page_content) + if not class_name: + continue + fields_list = self._extract_class_fields(type_doc.page_content) + source_path = type_doc.metadata.get('source') + types_mapping[(class_name, source_path)] = fields_list + return types_mapping + + @staticmethod + def _extract_class_name(class_code: str) -> str: + """Extract class name from class definition.""" + match = re.search(r'class\s+(\w+)', class_code) + if match: + return match.group(1) + return '' + + def _extract_class_fields(self, class_code: str) -> list[tuple]: + """ + Extract field declarations from a JavaScript class. + + Returns: + List of (field_name, field_type) tuples. field_type is 'any' for JavaScript. + """ + fields = set() + field_pattern = r'^\s*#?(\w+)\s*[;=]' + this_pattern = r'this\.(\w+)\s*=' + in_constructor = False + in_method = False + brace_depth = 0 + class_body_started = False + + for line in class_code.split('\n'): + line = line.strip() + if self.is_comment_line(line): + continue + if not class_body_started and '{' in line: + class_body_started = True + continue + + if class_body_started: + if re.match(r'^\s*constructor\s*\(', line): + in_constructor = True + in_method = True + brace_depth = 0 + elif re.match(r'^\s*(?:async\s+)?(?:static\s+)?(?:get\s+)?(?:set\s+)?(\w+)\s*\(', line) and not in_method: + in_method = True + in_constructor = False + brace_depth = 0 + + if in_method: + brace_depth += line.count('{') - line.count('}') + if in_constructor: + match = re.search(this_pattern, line) + if match: + field_name = match.group(1) + fields.add((field_name, 'any')) + + if brace_depth == 0 and ('{' in line or '}' in line): + in_method = False + in_constructor = False + + elif not in_method: + match = re.search(field_pattern, line) + if match: + field_name = match.group(1) + if field_name not in ['constructor', 'function', 'class', 'return', 'const', 'let', 'var', 'if', 'for', 'while']: + fields.add((field_name, 'any')) + + return list(fields) + + @staticmethod + def get_comment_line_notation() -> str: + """Return JavaScript comment notation.""" + return '//' + + def get_function_reserved_word(self) -> str: + """Return JavaScript function keyword.""" + return 'function' + + def get_dummy_function(self, function_name): + """Create a dummy JavaScript function for testing.""" + return f"{self.get_function_reserved_word()} {function_name}() {{}}" + + def get_type_reserved_word(self) -> str: + """Return JavaScript class keyword.""" + return "class" + + def is_searchable_file_name(self, function: Document) -> bool: + """Check if file should be searched (exclude test files).""" + file_path = function.metadata['source'] + # Exclude test files (.test.js, .spec.js, __tests__ directory, etc.) + return re.search(r"(\.test\.|\.spec\.|__tests__|/test/|/tests/)", file_path) is None + + def dir_name_for_3rd_party_packages(self) -> str: + return 'node_modules' + + @classmethod + def is_comment_line(cls, line: str) -> bool: + stripped = line.strip() + return stripped.startswith('//') or stripped.startswith('/*') + + def is_doc_type(self, doc: Document) -> bool: + if doc.metadata.get('content_type') != 'functions_classes': + return False + content = doc.page_content.strip() + return re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\s+\w+', content) is not None + + @staticmethod + def is_script_language(): + return True + + @staticmethod + def get_constructor_method_name() -> str: + return 'constructor' + + def get_class_name_from_class_function(self, func: Document) -> str: + pattern = r'//\(class:\s*([\w_]+)\)$' + match = re.search(pattern, func.page_content) + class_name = match.group(1) if match else None + return class_name + + @staticmethod + def _is_position_inside_string_literal(line: str, position: int) -> bool: + """ + Check if a character position in a line is inside a string literal. + + Uses a simple heuristic: if there's an odd number of quotes before the position, + it's likely inside a string literal. + + Args: + line: The line of code to check + position: The character position to check + + Returns: + True if the position appears to be inside a string literal, False otherwise + """ + before_position = line[:position] + + single_quotes = before_position.count("'") - before_position.count("\\'") + double_quotes = before_position.count('"') - before_position.count('\\"') + backticks = before_position.count('`') + + return single_quotes % 2 != 0 or double_quotes % 2 != 0 or backticks % 2 != 0 + + def _trace_variable_to_value(self, variable_name: str, lines: list[str], depth: int = 0, visited: set = None) -> str: + + # Prevent infinite recursion and circular references + MAX_DEPTH = 10 + if depth > MAX_DEPTH: + return '' + + if visited is None: + visited = set() + + if variable_name in visited: + return '' + + visited.add(variable_name) + string_pattern = rf'(?:const|let|var)\s+{re.escape(variable_name)}\s*=\s*([\'"`])([^\1]*?)\1' + var_pattern = rf'(?:const|let|var)\s+{re.escape(variable_name)}\s*=\s*(\w+)' + + for line in lines: + if not self.is_comment_line(line): + if match := re.search(string_pattern, line): + return match.group(2) + + if match := re.search(var_pattern, line): + source_var = match.group(1) + return self._trace_variable_to_value(source_var, lines, depth + 1, visited) + return '' + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + + if callee_package and callee_package not in code_content: + return False + + lines = self._split_into_statements(code_content) + + for line in lines: + if self.is_comment_line(line): + continue + + # Check for ES6 imports + if 'import' in line and 'from' in line: + # import ... from 'package' + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + if identifier == callee_package: + if f'import {identifier}' in line: + return True + else: + # import { template } from 'lodash' + # import * as lodash from 'lodash' + before_from = line.split('from')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_from): + return True + else: + return True + + # Check for CommonJS require + if 'require(' in line: + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + # const template = require('lodash').template + # const { template } = require('lodash') + # const lodash = require('lodash') + before_require = line.split('require')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_require): + return True + else: + return True + + # Check for re-exports: export { func } from 'package' + if 'export' in line and 'from' in line: + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + before_from = line.split('from')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_from): + return True + else: + return True + + if dynamic_import_match := re.search(r'\bimport\s*\(', line): + import_pos = dynamic_import_match.start() + + if self._is_position_inside_string_literal(line, import_pos): + continue + + if string_literal_match := re.search(r'import\s*\(\s*([\'"`])([^\1]*?)\1\s*\)', line): + imported_module = string_literal_match.group(2) + + if callee_package and callee_package in imported_module: + if identifier: + assignment_part = line[:import_pos] + if identifier in assignment_part: + return True + # Pattern: .then(({ identifier }) or .then(({ identifier, ... + after_import = line[string_literal_match.end():] + then_match = re.search(r'\.then\s*\(\s*\(\s*\{[^}]*\b' + re.escape(identifier) + r'\b[^}]*\}', after_import) + if then_match: + return True + else: + return True + + elif var_match := re.search(r'import\s*\(\s*(\w+)\s*\)', line): + var_name = var_match.group(1) + + # TODO: Future enhancement - support object property access for traceable constants + # Currently skips ALL dots, but could support cases like: + # const MODULES = { UTILS: './utils.js' }; + # import(MODULES.UTILS); <- This is statically determinable + # Would need to: + # 1. Update regex to allow dots: r'import\s*\(\s*([\w.]+)\s*\)' + # 2. Create _trace_property_to_value() to parse object literals + # 3. Distinguish local objects (traceable) from runtime objects like process.env (not traceable) + + if traced_value := self._trace_variable_to_value(var_name, lines): + if callee_package and callee_package in traced_value: + if identifier: + assignment_part = line[:import_pos] + if identifier in assignment_part: + return True + else: + return True + return False + + def is_same_package(self, package_name_from_input: str, package_name_from_tree: str) -> bool: + return package_name_from_input.lower() == package_name_from_tree.lower() + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + + """ + Check if a function is explicitly exportable + + This method checks the actual export status of a function by looking for: + - ES6 export syntax: export function name() or export default + - ES6 named exports: export { functionName } + - CommonJS module.exports or exports assignments + - For class methods: check if the class itself is exported + - Re-exports: Functions exported from package index.js files + """ + func_content = function.page_content.strip() + source_file = function.metadata.get('source') + + if source_file not in documents_of_full_sources: + return False + + full_file_content = documents_of_full_sources[source_file].page_content + + # Check for ES6 export syntax in the function content itself + # Pattern: export function name() or export default function + if re.match(r'^\s*export\s+(?:default\s+)?(?:async\s+)?function\s+', func_content): + return True + + # Pattern: export default name (where name is an arrow or regular function) + if re.match(r'^\s*export\s+default\s+', func_content): + return True + + # Check if this is a class/object method + class_name = self.get_class_name_from_class_function(function) + + if class_name: + # Check both class export patterns and object/variable export patterns + if self._is_exportable_class(class_name, full_file_content): + return True + if self._is_exportable_function(class_name, full_file_content): + return True + return False + + # This is a standalone function - check if the function itself is exported + function_name = self.get_function_name(function) + if function_name and source_file: + # Check if exported from the same file + if self._is_exportable_function(function_name, full_file_content): + return True + + # Check if function is re-exported from package index.js + # This handles the pattern where functions are defined in subdirectories + # but exported through the package's main entry point + if self.dir_name_for_3rd_party_packages() in source_file: + return self._check_package_reexport(function_name, source_file, documents_of_full_sources) + + return False + + def _check_package_reexport(self, function_name: str, source_file: str, documents_of_full_sources: dict[str, Document]) -> bool: + """ + Check if a function is re-exported from the package's index.js or main entry point. + + For example: + - Function defined in: node_modules/my-pkg/lib/utils.js + - Exported from: node_modules/my-pkg/index.js + """ + parts = source_file.split(self.dir_name_for_3rd_party_packages()) + if len(parts) < 2: + return False + + after_node_modules = parts[1] + pkg_parts = after_node_modules.split('/') + + # Handle scoped packages: @babel/core + if pkg_parts[1].startswith('@') and len(pkg_parts) > 1: + pkg_path = f"{self.dir_name_for_3rd_party_packages()}/{pkg_parts[1]}/{pkg_parts[2]}" + else: + pkg_path = f"{self.dir_name_for_3rd_party_packages()}/{pkg_parts[1]}" + + entry_points = [ + f"{pkg_path}/index.js", + f"{pkg_path}/index.mjs", + f"{pkg_path}/index.cjs" + ] + + for entry_point in entry_points: + entry_doc = documents_of_full_sources.get(entry_point) + if entry_doc: + if self._is_exportable_function(function_name, entry_doc.page_content): + return True + + return False + + @staticmethod + def _is_exportable_class(class_name: str, full_file_content: str) -> bool: + """ + Check if a class is exported using any export syntax. + + Handles: + - ES6: export class ClassName + - ES6: export default ClassName or export default class ClassName + - ES6: export { ClassName } + - CommonJS: module.exports = ClassName + - CommonJS: module.exports = { ClassName } + - CommonJS: exports.ClassName = ClassName + """ + # ES6: export class ClassName or export default class ClassName + if re.search(rf'export\s+(?:default\s+)?class\s+{re.escape(class_name)}\b', full_file_content): + return True + + # ES6: export default ClassName (after class definition) + if re.search(rf'export\s+default\s+{re.escape(class_name)}\b', full_file_content): + return True + + # ES6: export { ClassName } + export_pattern = rf'export\s+\{{[^}}]*\b{re.escape(class_name)}\b[^}}]*\}}' + if re.search(export_pattern, full_file_content): + return True + + # CommonJS: module.exports = ClassName + if re.search(rf'module\.exports\s*=\s*{re.escape(class_name)}\b', full_file_content): + return True + + # CommonJS: module.exports = { ClassName } or module.exports = { ClassName: ClassName } + if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{re.escape(class_name)}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports.ClassName = ClassName + if re.search(rf'module\.exports\.{re.escape(class_name)}\s*=', full_file_content): + return True + + # CommonJS: exports.ClassName = ClassName + if re.search(rf'exports\.{re.escape(class_name)}\s*=', full_file_content): + return True + + return False + + @staticmethod + def _is_exportable_function(function_name: str, full_file_content: str) -> bool: + """ + Check if a standalone function is exported using any export syntax. + + Handles: + - ES6: export function functionName + - ES6: export default functionName + - ES6: export { functionName } + - CommonJS: module.exports = functionName + - CommonJS: module.exports = { functionName } + - CommonJS: module.exports.functionName = functionName + - CommonJS: exports.functionName = functionName + """ + # ES6: export { functionName } or export { func as functionName } + export_pattern = rf'export\s+\{{[^}}]*\b{re.escape(function_name)}\b[^}}]*\}}' + if re.search(export_pattern, full_file_content): + return True + + # ES6: export default functionName + if re.search(rf'export\s+default\s+{re.escape(function_name)}\b', full_file_content): + return True + + # CommonJS: module.exports = functionName + if re.search(rf'module\.exports\s*=\s*{re.escape(function_name)}\b', full_file_content): + return True + + # CommonJS: module.exports = { functionName } or module.exports = { functionName: functionName } + if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{re.escape(function_name)}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports.functionName = ... + if re.search(rf'module\.exports\.{re.escape(function_name)}\s*=', full_file_content): + return True + + # CommonJS: exports.functionName = ... + if re.search(rf'exports\.{re.escape(function_name)}\s*=', full_file_content): + return True + + return False + + + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: + """Find documents that import a specific package.""" + importing_docs = [] + for doc in documents.values(): + content = doc.page_content + imports_syntaxes = [f'import {package_name}', + f"from '{package_name}'", + f'from "{package_name}"', + f"require('{package_name}')", + f'require("{package_name}")'] + + if any([import_syntax in content for import_syntax in imports_syntaxes]): + importing_docs.append(doc) + return importing_docs + + def is_a_package(self, package_name: str, doc: Document) -> bool: + return package_name in self.get_package_names(doc) + diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py index c816ebfac..0c311919f 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -71,7 +71,7 @@ def get_comment_line_notation() -> str: # This method gets a function document as an argument, and returns True if this contained function is exported to be # Used in other files/packages ( returns True only if it can be called from outside the source file) @abstractmethod - def is_exported_function(self, function: Document) -> bool: + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: pass # This method get a source document, and returns True only if the document type @@ -141,7 +141,7 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package # This method gets a list of documents and a package name as arguments, and returns a list of # documents that contain import statements for the specified package. It searches for both # single import statements and grouped import statements that include the package name. - def document_imports_package(self, documents: list[Document], package_name: str) -> list[Document]: + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: importing_docs = [value for (file, value) in documents.items() if re.search( rf"(import {package_name}|import\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py index 32445b170..3514e1f90 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py @@ -2,6 +2,7 @@ from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from exploit_iq_commons.utils.functions_parsers.java_functions_parsers import JavaLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser from exploit_iq_commons.utils.functions_parsers.c_lang_function_parsers import CLanguageFunctionsParser @@ -19,6 +20,8 @@ def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | No return GoLanguageFunctionsParser() elif ecosystem == Ecosystem.PYTHON: return PythonLanguageFunctionsParser() + elif ecosystem == Ecosystem.JAVASCRIPT: + return JavaScriptFunctionsParser() elif ecosystem == Ecosystem.C_CPP: parser_obj = CLanguageFunctionsParser() if tree is not None: diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index 59b14f1a3..bef207440 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -110,7 +110,7 @@ def dir_name_for_3rd_party_packages(self) -> str: def is_function(self, function: Document) -> bool: return function.page_content.startswith("def") - def is_exported_function(self, function: Document) -> bool: + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: return True @staticmethod @@ -292,17 +292,18 @@ def __prepare_package_lookup(self, expression, variables_mappings): else: return None, None, None - def _get_function_calls(self, caller_function: Document, callee_function: str, code_documets: list[Document] = None): + def _get_function_calls(self, caller_function: Document, callee_function: str, code_documents: list[Document] = None): caller_function_body = self.get_function_body_from_document(caller_function) - regex = fr'\b([a-zA-Z0-9_\[\]\(\).]+\.)?{callee_function}\(' + regex = fr'\b([a-zA-Z0-9_\[\]\(\).]+\.)?{re.escape(callee_function)}\(' calls = [matching.group(0) for matching in re.finditer(regex, caller_function_body, re.MULTILINE)] - if code_documets: - code_documet = code_documets[caller_function.metadata['source']] - for line in code_documet.page_content.split(os.linesep): - if all(word in line for word in ['import', callee_function, 'as']): - splitted_identifier = line.split(f'{callee_function} as') - identifier = splitted_identifier[-1].split()[0].rstrip(',') - calls.extend(self._get_function_calls(caller_function, identifier)) + if code_documents: + code_document = code_documents[caller_function.metadata['source']] + for line in code_document.page_content.split(os.linesep): + if not self.is_comment_line(line): + if all(word in line for word in ['import', callee_function, 'as']): + splitted_identifier = line.split(f'{callee_function} as') + identifier = splitted_identifier[-1].split()[0].rstrip(',') + calls.extend(self._get_function_calls(caller_function, identifier)) return list(set(calls)) def get_function_body_from_document(self, function: Document): diff --git a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py new file mode 100644 index 000000000..eb4fc9ab3 --- /dev/null +++ b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py @@ -0,0 +1,379 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from typing import List +from typing import Tuple + +import esprima +from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + + +class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): + """Extended JavaScript segmenter that handles shebang and ES optional chaining.""" + + def __init__(self, code: str): + """Initialize the segmenter with preprocessed code.""" + super().__init__(code) + # Skip files with shebang (#!) line since they typically contain non-standard JavaScript syntax + if code.startswith("#!"): + logger.warning("File contains a shebang line. Skipping parsing.") + self.skip_file = True + # esprima-python parser limitation: cannot handle JavaScript optional chaining syntax, + # fallback to regular property access + else: + self.skip_file = False + self.code = self.code.replace("?.", ".") + + def _parse_with_fallback(self) -> Any: + """Try to parse code as script first, then as module if that fails.""" + try: + logger.debug("Attempting to parse as a script...") + return esprima.parseScript(self.code, loc=True) + except esprima.Error: + logger.debug("Script parsing failed. Trying module parsing...") + try: + return esprima.parseModule(self.code, loc=True) + except esprima.Error as e: + logger.error("Module parsing failed: %s", str(e)) + print(f'TW Module parsing error: {self.code=} {e=}') + return None + + def extract_functions_classes(self) -> List[str]: + """ + Extract functions, classes, and individual class methods from JavaScript code. + + Handles various JavaScript function patterns: + - Regular function declarations + - Arrow functions (const func = () => {}) + - Async functions + - Generator functions + - Class declarations and their methods + - Exported functions and classes + + Returns: + List of code strings representing all extracted functions and classes + """ + if self.skip_file: + return [] + + tree = self._parse_with_fallback() + if tree is None: + return [] + + functions_classes = [] + for node in tree.body: + # Handle direct function/class declarations (including async and generator) + if isinstance(node, (esprima.nodes.FunctionDeclaration, + esprima.nodes.AsyncFunctionDeclaration, + esprima.nodes.ClassDeclaration)): + functions_classes.append(self._extract_code(node)) + + # Handle variable declarations that might contain arrow functions or function expressions + elif isinstance(node, esprima.nodes.VariableDeclaration): + arrow_funcs = self._extract_arrow_functions(node) + functions_classes.extend(arrow_funcs) + + # Handle exported declarations + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, + esprima.nodes.AsyncFunctionDeclaration, + esprima.nodes.ClassDeclaration)): + functions_classes.append(self._extract_code(node)) + elif isinstance(node.declaration, esprima.nodes.VariableDeclaration): + arrow_funcs = self._extract_arrow_functions(node.declaration) + functions_classes.extend(arrow_funcs) + + # Handle default exports + elif isinstance(node, esprima.nodes.ExportDefaultDeclaration): + if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, + esprima.nodes.AsyncFunctionDeclaration, + esprima.nodes.ClassDeclaration)): + functions_classes.append(self._extract_code(node)) + + # Extract individual class methods + class_methods = self._extract_all_class_methods() + functions_classes.extend(class_methods) + + # Extract individual object methods + object_methods = self._extract_all_object_methods() + functions_classes.extend(object_methods) + + return functions_classes + + def simplify_code(self) -> str: + """Simplify the code by replacing function/class bodies with comments.""" + if self.skip_file: + return self.code + + tree = self._parse_with_fallback() + if tree is None: + return self.code + + simplified_lines = self.source_lines[:] + indices_to_del: List[Tuple[int, int]] = [] + + for node in tree.body: + if isinstance(node, (esprima.nodes.FunctionDeclaration, + esprima.nodes.AsyncFunctionDeclaration, + esprima.nodes.ClassDeclaration)): + start, end = node.loc.start.line - 1, node.loc.end.line + simplified_lines[start] = f"// Code for: {simplified_lines[start]}" + indices_to_del.append((start + 1, end)) + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, + esprima.nodes.AsyncFunctionDeclaration, + esprima.nodes.ClassDeclaration)): + start, end = node.loc.start.line - 1, node.loc.end.line + simplified_lines[start] = f"// Code for: {simplified_lines[start]}" + indices_to_del.append((start + 1, end)) + + for start, end in reversed(indices_to_del): + del simplified_lines[start:end] + + return "\n".join(line for line in simplified_lines) + + def _extract_arrow_functions(self, var_node: esprima.nodes.VariableDeclaration) -> List[str]: + """ + Extract arrow functions, function expressions, and object literals with methods + from variable declarations. + + Handles patterns like: + - const func = () => {} + - const func = async () => {} + - const func = function() {} + - const func = async function() {} + - const obj = { method() {}, prop: function() {} } + + Args: + var_node: VariableDeclaration node that might contain function expressions or objects + + Returns: + List of function/object code strings + """ + functions = [] + + for declarator in var_node.declarations: + if declarator.init is None: + continue + + # Check if this is an arrow function or function expression + is_function_expr = isinstance(declarator.init, ( + esprima.nodes.ArrowFunctionExpression, + esprima.nodes.FunctionExpression + )) + + if is_function_expr: + # Extract the entire variable declaration including the assignment + code = self._extract_code(var_node) + functions.append(code) + break # Only extract once per VariableDeclaration + + # Check if this is an object literal with methods + if isinstance(declarator.init, esprima.nodes.ObjectExpression): + if self._object_has_methods(declarator.init): + code = self._extract_code(var_node) + functions.append(code) + break # Only extract once per VariableDeclaration + + return functions + + def _object_has_methods(self, obj_node: esprima.nodes.ObjectExpression) -> bool: + """ + Check if an object literal contains any method properties. + + Args: + obj_node: ObjectExpression AST node + + Returns: + True if the object contains at least one method property + """ + for prop in obj_node.properties: + if isinstance(prop, esprima.nodes.Property): + # Check for shorthand method syntax: { method() {} } + if prop.method: + return True + # Check for function expression: { prop: function() {} } + # or arrow function: { prop: () => {} } + if isinstance(prop.value, ( + esprima.nodes.FunctionExpression, + esprima.nodes.ArrowFunctionExpression + )): + return True + return False + + def _extract_all_class_methods(self) -> List[str]: + """ + Extract all methods from all classes in the code. + + Each method is extracted as standalone code with a //class: ClassName annotation + appended at the end, enabling later identification of which class the method belongs to. + + Returns: + List of method code strings with class annotations + """ + tree = self._parse_with_fallback() + if tree is None: + return [] + + methods = [] + + for node in tree.body: + if isinstance(node, esprima.nodes.ClassDeclaration): + class_methods = self._extract_class_methods(node) + methods.extend(class_methods) + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, esprima.nodes.ClassDeclaration): + class_methods = self._extract_class_methods(node.declaration) + methods.extend(class_methods) + + return methods + + def _extract_class_methods(self, class_node: esprima.nodes.ClassDeclaration) -> List[str]: + """ + Extract all methods from a single class node. + + Args: + class_node: AST node representing a class declaration + + Returns: + List of method code strings, each annotated with //class: ClassName + """ + class_name = class_node.id.name if class_node.id else "AnonymousClass" + methods = [] + + for method_node in class_node.body.body: + if isinstance(method_node, esprima.nodes.MethodDefinition): + method_code = self._extract_method_code(method_node) + annotated_method = f"{method_code}\n//(class: {class_name})" + methods.append(annotated_method) + + return methods + + def _extract_method_code(self, method_node: esprima.nodes.MethodDefinition) -> str: + """ + Extract the source code for a single method. + + Args: + method_node: AST node representing a method definition + + Returns: + Source code string for the method + """ + if not hasattr(method_node, 'loc') or method_node.loc is None: + logger.warning("Method node has no location information") + return "" + + start_line = method_node.loc.start.line - 1 # Convert to 0-indexed + end_line = method_node.loc.end.line + + method_lines = self.source_lines[start_line:end_line] + return "\n".join(method_lines) + + def _extract_all_object_methods(self) -> List[str]: + """ + Extract all methods from all object literals in the code. + + Each method is extracted as standalone code with a //(object: objectName) annotation + appended at the end, enabling later identification of which object the method belongs to. + + Returns: + List of method code strings with object annotations + """ + tree = self._parse_with_fallback() + if tree is None: + return [] + + methods = [] + + for node in tree.body: + if isinstance(node, esprima.nodes.VariableDeclaration): + for declarator in node.declarations: + if declarator.init is None: + continue + if isinstance(declarator.init, esprima.nodes.ObjectExpression): + if self._object_has_methods(declarator.init): + object_name = declarator.id.name if hasattr(declarator.id, 'name') else "AnonymousObject" + object_methods = self._extract_object_methods(declarator.init, object_name) + methods.extend(object_methods) + + # Handle exported object literals + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, esprima.nodes.VariableDeclaration): + for declarator in node.declaration.declarations: + if declarator.init is None: + continue + if isinstance(declarator.init, esprima.nodes.ObjectExpression): + if self._object_has_methods(declarator.init): + object_name = declarator.id.name if hasattr(declarator.id, 'name') else "AnonymousObject" + object_methods = self._extract_object_methods(declarator.init, object_name) + methods.extend(object_methods) + + return methods + + def _extract_object_methods(self, obj_node: esprima.nodes.ObjectExpression, object_name: str) -> List[str]: + """ + Extract all methods from a single object literal. + + Args: + obj_node: AST node representing an object expression + object_name: Name of the variable holding the object + + Returns: + List of method code strings, each annotated with //(object: objectName) + """ + methods = [] + + for prop in obj_node.properties: + if not isinstance(prop, esprima.nodes.Property): + continue + + # Check if this property is a method + is_method = prop.method or isinstance(prop.value, ( + esprima.nodes.FunctionExpression, + esprima.nodes.ArrowFunctionExpression + )) + + if is_method: + method_code = self._extract_property_method_code(prop) + if method_code: + # Use same annotation pattern as classes for consistency + annotated_method = f"{method_code}\n//(class: {object_name})" + methods.append(annotated_method) + + return methods + + def _extract_property_method_code(self, prop_node: esprima.nodes.Property) -> str: + """ + Extract the source code for a single object property method. + + Args: + prop_node: AST node representing a property with a method value + + Returns: + Source code string for the method + """ + if not hasattr(prop_node, 'loc') or prop_node.loc is None: + logger.warning("Property node has no location information") + return "" + + start_line = prop_node.loc.start.line - 1 # Convert to 0-indexed + end_line = prop_node.loc.end.line + + method_lines = self.source_lines[start_line:end_line] + return "\n".join(method_lines) diff --git a/src/exploit_iq_commons/utils/js_extended_parser.py b/src/exploit_iq_commons/utils/js_extended_parser.py deleted file mode 100644 index a8967ff7b..000000000 --- a/src/exploit_iq_commons/utils/js_extended_parser.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any -from typing import List -from typing import Tuple - -import esprima -from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter - -from exploit_iq_commons.logging.loggers_factory import LoggingFactory -logger = LoggingFactory.get_agent_logger(__name__) - - -class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): - """Extended JavaScript segmenter that handles shebang and ES optional chaining.""" - - def __init__(self, code: str): - """Initialize the segmenter with preprocessed code.""" - super().__init__(code) - # Skip files with shebang (#!) line since they typically contain non-standard JavaScript syntax - if code.startswith("#!"): - logger.warning("File contains a shebang line. Skipping parsing.") - self.skip_file = True - # esprima-python parser limitation: cannot handle JavaScript optional chaining syntax, - # fallback to regular property access - else: - self.skip_file = False - self.code = self.code.replace("?.", ".") - - def _parse_with_fallback(self) -> Any: - """Try to parse code as script first, then as module if that fails.""" - try: - logger.debug("Attempting to parse as a script...") - return esprima.parseScript(self.code, loc=True) - except esprima.Error: - logger.debug("Script parsing failed. Trying module parsing...") - try: - return esprima.parseModule(self.code, loc=True) - except esprima.Error as e: - logger.error("Module parsing failed: %s", str(e)) - return None - - def extract_functions_classes(self) -> List[str]: - """Extract functions, classes and exports from the code.""" - if self.skip_file: - return [] - - tree = self._parse_with_fallback() - if tree is None: - return [] - - functions_classes = [] - for node in tree.body: - # Handle direct function/class declarations - if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - # Handle exported declarations - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - - return functions_classes - - def simplify_code(self) -> str: - """Simplify the code by replacing function/class bodies with comments.""" - if self.skip_file: - return self.code - - tree = self._parse_with_fallback() - if tree is None: - return self.code - - simplified_lines = self.source_lines[:] - indices_to_del: List[Tuple[int, int]] = [] - - for node in tree.body: - if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - - for start, end in reversed(indices_to_del): - del simplified_lines[start:end] - - return "\n".join(line for line in simplified_lines) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 00e8f9222..766dbf3f3 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -2,6 +2,7 @@ from contextlib import aclosing import pytest from unittest.mock import patch, MagicMock +from langchain_core.documents import Document from exploit_iq_commons.data_models.common import AnalysisType from vuln_analysis.data_models.state import AgentMorpheusEngineState @@ -148,6 +149,30 @@ async def get_transitive_code_runner_function(): ' markupsafe==3.0.3 # MarkupSafe>=2.1.1\n' ' mock-package==1.1.1' ) +java_script_dependency_tree_mock_output = '''{ + "name": "root_project", + "version": "1.0.0", + "dependencies": { + "vulnerable-pkg": { + "version": "1.0.0" + }, + "another-pkg": { + "version": "2.0.0" + }, + "deep-pkg": { + "version": "1.0.0" + }, + "middle-pkg": { + "version": "1.0.0", + "dependencies": { + "deep-pkg": { + "version": "1.0.0" + } + } + } + } +}''' + def mock_file_open(*args, **kwargs): file_path = args[0] if args else kwargs.get('file', '') @@ -451,4 +476,603 @@ async def test_transitive_search_java_5(): (path_found, list_path) = result print(result) assert path_found is False - assert len(list_path) is 1 \ No newline at end of file + assert len(list_path) is 1 + +@pytest.mark.asyncio +async def test_java_script_transitive_search_1(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository="https://github.com/trustification/exhort-javascript-api", + git_ref='b951581c8f1bf77721537ea135a546f39e1d7a59', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + for input in ["PackageURL.constructor", "PackageURL", "PackageURL.fromString"]: + result = await transitive_code_search_runner_coroutine(f"packageurl-js,{input}") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 + + +@pytest.mark.asyncio +async def test_java_script_transitive_search_2(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository="https://github.com/openshift/monitoring-plugin", + git_ref='87571978825e63392a56f6920e068af3a71cda6e', + included_extensions=['**/*.js', '**/*.ts', '**/*.tsx'], + excluded_extensions=['**/*test*.js', '**/*test*.ts', '**/*test*.tsx', '**/*.spec.*'] + ) + + result = await transitive_code_search_runner_coroutine("lodash-es,filter") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 + + +def mock_javascript_file_open(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "JAVASCRIPT" + return mock_file + + +def mock_javascript_path_exists(self): + path_str = str(self) + exists_files = ['package.json', 'ecosystem_data.txt', 'node_modules'] + if any([path in path_str for path in exists_files]): + return True + return False + + +@pytest.mark.asyncio +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_basic_reachable(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-basic-call', + git_ref='abc123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # Application function that calls the vulnerable package + Document( + page_content="function processUserInput(userInput) {\n const result = dangerousFunction(userInput);\n return result;\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Vulnerable package function + Document( + page_content="function dangerousFunction(input) {\n console.log('Executing: ' + input);\n return eval(input);\n}", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js (for import resolution and showing calls) + Document( + page_content="const { dangerousFunction, safeFunction } = require('vulnerable-pkg');\n\nfunction processUserInput(userInput) {\n const result = dangerousFunction(userInput);\n // ... rest of code\n}\n\nfunction processData(data) {\n return safeFunction(data);\n // ... rest of code\n}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for vulnerable-pkg/index.js + Document( + page_content="function dangerousFunction(input) {\n return eval(input);\n}\n\nfunction safeFunction(data) {\n return data.toUpperCase();\n}\n\nmodule.exports = { dangerousFunction, safeFunction };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,dangerousFunction") + + (path_found, list_path) = result + assert path_found == True + assert len(list_path) >= 2 + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_unreachable(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-unreachable', + git_ref='def456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # Application function that only calls publicAPI + Document( + page_content="function main() {\n return publicAPI();\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Package's public API that doesn't call vulnerableFunc + Document( + page_content="function publicAPI() {\n return helperFunc();\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Helper function + Document( + page_content="function helperFunc() {\n return 'helper result';\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Vulnerable function that is NOT called by anyone + Document( + page_content="function vulnerableFunc(cmd) {\n return require('child_process').execSync(cmd).toString();\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { publicAPI } = require('another-pkg');\n// Code for: function main() {...}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for another-pkg/index.js + Document( + page_content="function publicAPI() {\n return helperFunc();\n}\n\nfunction helperFunc() {\n return 'helper result';\n}\n\nfunction vulnerableFunc(cmd) {\n return require('child_process').execSync(cmd).toString();\n}\n\nmodule.exports = { publicAPI, helperFunc, vulnerableFunc };", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ) + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("another-pkg,vulnerableFunc") + + (path_found, list_path) = result + assert path_found == False, "vulnerableFunc should NOT be reachable from application code" + assert len(list_path) == 1, "Should have only the target function (not reachable)" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_multi_file_chain(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-multi-file', + git_ref='ghi789', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # App level + Document( + page_content="function appMain() {\n return processRequest();\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Middleware level + Document( + page_content="function processRequest() {\n return vulnerableDeepFunc();\n}", + metadata={'source': 'node_modules/middle-pkg/middleware.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Deep package level + Document( + page_content="function vulnerableDeepFunc() {\n return eval(arguments[0]);\n}", + metadata={'source': 'node_modules/deep-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="import { processRequest } from 'middle-pkg';\nfunction appMain() {\n return processRequest();\n // ... rest of code\n}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for middleware.js + Document( + page_content="const { vulnerableDeepFunc } = require('deep-pkg');\n\nfunction processRequest() {\n return vulnerableDeepFunc();\n // ... rest of code\n}", + metadata={'source': 'node_modules/middle-pkg/middleware.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for middle-pkg/index.js (re-exports from middleware) + Document( + page_content="const { processRequest } = require('./middleware');\n\nmodule.exports = { processRequest };", + metadata={'source': 'node_modules/middle-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for deep-pkg/index.js + Document( + page_content="export function vulnerableDeepFunc() {\n return eval(arguments[0]);\n}\n\nmodule.exports = { vulnerableDeepFunc };", + metadata={'source': 'node_modules/deep-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ) + ] + + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("deep-pkg,vulnerableDeepFunc") + + (path_found, list_path) = result + # Expected call chain: vulnerableDeepFunc -> processRequest -> appMain (3 functions) + assert path_found == True + assert len(list_path) == 3, f"Should have complete call chain of 3 functions, got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_class_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript class methods where: + - A dependency package contains a class with two methods: + - func_a: vulnerable function + - func_b: triggers func_a + - Source code imports the class and calls func_b + + Expected: The vulnerable func_a should be reachable via the call chain: + app.js -> VulnerableClass.func_b -> VulnerableClass.func_a + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-class-transitive', + git_ref='class123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # func_a: vulnerable function (class method) + Document( + page_content="func_a(input) {\n" + " // This is the vulnerable function\n" + " return eval(input);\n" + "}\n" + "//(class: VulnerableClass)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # func_b: triggers func_a (class method) + Document( + page_content="func_b(data) {\n" + " // This method triggers the vulnerable func_a\n" + " const processed = data.toString();\n" + " return this.func_a(processed);\n" + "}\n" + "//(class: VulnerableClass)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that imports and uses the class + Document( + page_content="function processUserData(userData) {\n" + " const handler = new VulnerableClass();\n" + " return handler.func_b(userData);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package + Document( + page_content="class VulnerableClass {\n" + " func_a(input) {\n" + " return eval(input);\n" + " }\n" + " \n" + " func_b(data) {\n" + " const processed = data.toString();\n" + " return this.func_a(processed);\n" + " }\n" + "}\n" + "\n" + "module.exports = { VulnerableClass };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { VulnerableClass } = require('vulnerable-pkg');\n" + "\n" + "function processUserData(userData) {\n" + " const handler = new VulnerableClass();\n" + " return handler.func_b(userData);\n" + "}\n" + "\n" + "module.exports = { processUserData };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,VulnerableClass.func_a") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: func_a -> func_b -> processUserData (3 functions) + assert path_found == True, "func_a should be reachable through func_b -> processUserData" + assert len(list_path) == 3, f"Should have call chain of 3 functions (func_a -> func_b -> processUserData), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_class_inheritance_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript class inheritance where: + - A dependency package contains a base class with a vulnerable method: + - VulnerableBase.executeCommand: vulnerable function using eval + - Source code has a child class that extends the base class: + - CustomHandler extends VulnerableBase + - Another source file creates an instance of child class and calls the inherited method + + Expected: The vulnerable executeCommand should be reachable via the call chain: + app.js -> processRequest -> CustomHandler (instance) -> VulnerableBase.executeCommand + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-inheritance', + git_ref='inherit456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # executeCommand: vulnerable function in base class (dependency) + Document( + page_content="executeCommand(cmd) {\n" + " // Vulnerable: directly evaluates user input\n" + " return eval(cmd);\n" + "}\n" + "//(class: VulnerableBase)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # CustomHandler: child class that extends VulnerableBase (source code) + # Note: Inheritance is shown in simplified_code, not in functions_classes + # processRequest: function that creates instance and calls inherited method + Document( + page_content="function processRequest(userInput) {\n" + " const handler = new CustomHandler();\n" + " return handler.executeCommand(userInput);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (base class) + Document( + page_content="class VulnerableBase {\n" + " executeCommand(cmd) {\n" + " return eval(cmd);\n" + " }\n" + "}\n" + "\n" + "module.exports = { VulnerableBase };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for CustomHandler class (child class) + Document( + page_content="const { VulnerableBase } = require('vulnerable-pkg');\n" + "\n" + "class CustomHandler extends VulnerableBase {\n" + " constructor() {\n" + " super();\n" + " }\n" + " \n" + " // Inherits executeCommand from VulnerableBase\n" + "}\n" + "\n" + "module.exports = { CustomHandler };", + metadata={'source': 'src/handlers/CustomHandler.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js (usage) + Document( + page_content="const { CustomHandler } = require('./handlers/CustomHandler');\n" + "\n" + "function processRequest(userInput) {\n" + " const handler = new CustomHandler();\n" + " return handler.executeCommand(userInput);\n" + "}\n" + "\n" + "module.exports = { processRequest };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,VulnerableBase.executeCommand") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: executeCommand (from VulnerableBase) -> processRequest (2 functions) + # Note: The child class CustomHandler doesn't redefine the method, so it's inherited + assert path_found == True, "executeCommand should be reachable through CustomHandler instance in processRequest" + assert len(list_path) == 2, f"Should have call chain of 2 functions (executeCommand -> processRequest), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_object_literal_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript object literal methods where: + - A dependency package contains an object literal with two methods: + - vulnerableMethod: vulnerable function using eval + - wrapperMethod: triggers vulnerableMethod via this.vulnerableMethod + - Source code imports the object and calls wrapperMethod + + Expected: The vulnerable method should be reachable via the call chain: + app.js -> dangerousUtils.wrapperMethod -> dangerousUtils.vulnerableMethod + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-object-transitive', + git_ref='object123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # vulnerableMethod: vulnerable function (object method with unified class annotation) + Document( + page_content="vulnerableMethod(input) {\n" + " // This is the vulnerable function\n" + " return eval(input);\n" + "}\n" + "//(class: dangerousUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # wrapperMethod: triggers vulnerableMethod (object method with unified class annotation) + Document( + page_content="wrapperMethod(data) {\n" + " // This method triggers the vulnerable method\n" + " const processed = data.toString();\n" + " return this.vulnerableMethod(processed);\n" + "}\n" + "//(class: dangerousUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that imports and uses the object + Document( + page_content="function processUserInput(userData) {\n" + " return dangerousUtils.wrapperMethod(userData);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (object literal) + Document( + page_content="const dangerousUtils = {\n" + " vulnerableMethod(input) {\n" + " return eval(input);\n" + " },\n" + " \n" + " wrapperMethod(data) {\n" + " const processed = data.toString();\n" + " return this.vulnerableMethod(processed);\n" + " }\n" + "};\n" + "\n" + "module.exports = { dangerousUtils };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { dangerousUtils } = require('vulnerable-pkg');\n" + "\n" + "function processUserInput(userData) {\n" + " return dangerousUtils.wrapperMethod(userData);\n" + "}\n" + "\n" + "module.exports = { processUserInput };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,dangerousUtils.vulnerableMethod") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: vulnerableMethod -> wrapperMethod -> processUserInput (3 functions) + assert path_found == True, "vulnerableMethod should be reachable through wrapperMethod -> processUserInput" + assert len(list_path) == 3, f"Should have call chain of 3 functions (vulnerableMethod -> wrapperMethod -> processUserInput), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_object_method_direct_call(mock_open, mock_npm_ls): + """ + Test transitive search for direct JavaScript object method call where: + - A dependency package contains an object literal with a vulnerable method + - Source code directly calls the vulnerable method + + Expected: The vulnerable method should be reachable directly from source: + app.js -> apiUtils.unsafeEval + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-object-direct', + git_ref='direct456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # unsafeEval: vulnerable function (object method with unified class annotation) + Document( + page_content="unsafeEval(code) {\n" + " return eval(code);\n" + "}\n" + "//(class: apiUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that directly calls the vulnerable method + Document( + page_content="function executeCode(userCode) {\n" + " return apiUtils.unsafeEval(userCode);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (object literal) + Document( + page_content="const apiUtils = {\n" + " unsafeEval(code) {\n" + " return eval(code);\n" + " }\n" + "};\n" + "\n" + "module.exports = { apiUtils };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { apiUtils } = require('vulnerable-pkg');\n" + "\n" + "function executeCode(userCode) {\n" + " return apiUtils.unsafeEval(userCode);\n" + "}\n" + "\n" + "module.exports = { executeCode };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,apiUtils.unsafeEval") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: unsafeEval -> executeCode (2 functions) + assert path_found == True, "unsafeEval should be reachable directly from executeCode" + assert len(list_path) == 2, f"Should have call chain of 2 functions (unsafeEval -> executeCode), got {len(list_path)}" diff --git a/tests/test_java_script_extended.py b/tests/test_java_script_extended.py deleted file mode 100644 index 51f535687..000000000 --- a/tests/test_java_script_extended.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from exploit_iq_commons.utils.js_extended_parser import ExtendedJavaScriptSegmenter - -TEST_CASES = [ - { - "name": - "regular_script", - "code": - """ -function hello() { - console.log('Hello'); -} - -class MyClass { - constructor() { - this.value = 42; - } -} -""", - "expected_functions": - 2, # One function and one class - "should_parse": - True, - }, - { - "name": - "es_module", - "code": - """ -import { something } from './module'; -export function exportedFunc() { - return 'exported'; -} -export class ExportedClass { - method() {} -} -""", - "expected_functions": - 2, # One exported function and one exported class - "should_parse": - True, - }, - { - "name": - "optional_chaining", - "code": - """ -function processUser(user) { - return user?.profile?.name; -} -class UserManager { - getAddress() { - return this.user?.address?.street; - } -} -""", - "expected_functions": - 2, - "should_parse": - True, - }, - { - "name": "shebang_file", - "code": """#!/usr/bin/env node -function main() { - console.log('Main'); -} -""", - "expected_functions": 0, # Should skip due to shebang - "should_parse": False, - } -] - - -@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) -def test_function_extraction(test_case): - """Test that functions and classes are correctly extracted.""" - segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) - functions = segmenter.extract_functions_classes() - assert len(functions) == test_case["expected_functions"] - if not test_case["should_parse"]: - assert segmenter.skip_file - - -@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) -def test_code_simplification(test_case): - """Test that code is correctly simplified.""" - segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) - simplified = segmenter.simplify_code() - - # Basic checks - assert isinstance(simplified, str) - if not test_case["should_parse"]: - assert segmenter.skip_file - assert simplified == test_case["code"] # Return original code for unparseable files - else: - # Should have fewer or equal lines than original - assert len(simplified.splitlines()) <= len(test_case["code"].splitlines()) - # Should contain "// Code for:" for each function/class - if test_case["expected_functions"] > 0: - assert "// Code for:" in simplified - - -def test_optional_chaining_replacement(): - """Test that optional chaining is correctly replaced.""" - code = "const name = user?.profile?.name;" - segmenter = ExtendedJavaScriptSegmenter(code) - assert not segmenter.skip_file - assert "?." not in segmenter.code - assert "." in segmenter.code - - -def test_invalid_js(): - """Test handling of invalid JavaScript code.""" - code = "this is not valid javascript" - segmenter = ExtendedJavaScriptSegmenter(code) - assert not segmenter.skip_file # Should attempt to parse - - # Should handle errors gracefully - functions = segmenter.extract_functions_classes() - assert functions == [] - - simplified = segmenter.simplify_code() - assert simplified == code # Return original code for invalid JS diff --git a/tests/test_java_script_extended_segmenter.py b/tests/test_java_script_extended_segmenter.py new file mode 100644 index 000000000..637f0d1f6 --- /dev/null +++ b/tests/test_java_script_extended_segmenter.py @@ -0,0 +1,593 @@ +import pytest + +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter + +TEST_CASES = [ + { + "name": + "regular_script", + "code": + """ +function hello() { + console.log('Hello'); +} + + +class MyClass { + constructor() { + this.value = 42; + } +} +""", + "expected_functions": + 3, # One function and one class and one class function + "should_parse": + True, + }, + { + "name": + "es_module", + "code": + """ +import { something } from './module'; +export function exportedFunc() { + return 'exported'; +} +export class ExportedClass { + method() {} +} +""", + "expected_functions": + 3, # One exported function and one exported class + "should_parse": + True, + }, + { + "name": + "optional_chaining", + "code": + """ +function processUser(user) { + return user?.profile?.name; +} +class UserManager { + getAddress() { + return this.user?.address?.street; + } +} +""", + "expected_functions": + 3, + "should_parse": + True, + }, + { + "name": "shebang_file", + "code": """#!/usr/bin/env node +function main() { + console.log('Main'); +} +""", + "expected_functions": 0, # Should skip due to shebang + "should_parse": False, + } +] + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_function_extraction(test_case): + """Test that functions and classes are correctly extracted.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + functions = segmenter.extract_functions_classes() + assert len(functions) == test_case["expected_functions"] + if not test_case["should_parse"]: + assert segmenter.skip_file + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_code_simplification(test_case): + """Test that code is correctly simplified.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + simplified = segmenter.simplify_code() + + # Basic checks + assert isinstance(simplified, str) + if not test_case["should_parse"]: + assert segmenter.skip_file + assert simplified == test_case["code"] # Return original code for unparseable files + else: + # Should have fewer or equal lines than original + assert len(simplified.splitlines()) <= len(test_case["code"].splitlines()) + # Should contain "// Code for:" for each function/class + if test_case["expected_functions"] > 0: + assert "// Code for:" in simplified + + +def test_optional_chaining_replacement(): + """Test that optional chaining is correctly replaced.""" + code = "const name = user?.profile?.name;" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file + assert "?." not in segmenter.code + assert "." in segmenter.code + + +def test_invalid_js(): + """Test handling of invalid JavaScript code.""" + code = "this is not valid javascript" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file # Should attempt to parse + + # Should handle errors gracefully + functions = segmenter.extract_functions_classes() + assert functions == [] + + simplified = segmenter.simplify_code() + assert simplified == code # Return original code for invalid JS + +def test_extract_function_declaration(): + """Test extraction of regular function declarations.""" + code = """ +function hello(name) { + console.log('Hello ' + name); +} + +function goodbye() { + return 'Goodbye'; +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'function hello' in functions[0] + assert 'function goodbye' in functions[1] + + +def test_extract_function_expression(): + """Test extraction of function expressions.""" + code = """ +const greet = function(name) { + return 'Hello ' + name; +}; + +var farewell = function() { + return 'Goodbye'; +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'greet' in functions[0] and 'function' in functions[0] + assert 'farewell' in functions[1] and 'function' in functions[1] + + +def test_extract_arrow_function(): + """Test extraction of arrow functions (T039).""" + code = """ +const add = (a, b) => { + return a + b; +}; + +const multiply = (x, y) => x * y; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'add' in functions[0] + assert 'multiply' in functions[1] + + +def test_extract_async_function(): + """Test extraction of async functions (T040).""" + code = """ +async function fetchData(url) { + const response = await fetch(url); + return response.json(); +} + +const asyncArrow = async () => { + return await Promise.resolve('data'); +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 1 + assert 'fetchData' in functions[0] and 'async' in functions[0] + + + +def test_extract_generator_function(): + """Test extraction of generator functions (T041).""" + code = """ +function* numberGenerator() { + yield 1; + yield 2; + yield 3; +} + +function* fibonacci(n) { + let a = 0, b = 1; + while (n-- > 0) { + yield a; + [a, b] = [b, a + b]; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'numberGenerator' in functions[0] + assert 'fibonacci' in functions[1] + + +def test_extract_class_definition(): + """Test extraction of class definitions.""" + code = """ +class Person { + constructor(name, age) { + this.name = name; + this.age = age; + } + + greet() { + return `Hello, I'm ${this.name}`; + } + + getAge() { + return this.age; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: class definition + 3 methods (constructor, greet, getAge) + assert len(functions) == 4 + + # Verify class is extracted + assert any('class Person' in f for f in functions) + + +def test_extract_class_methods_with_annotation(): + """Test that class methods have proper //(class: ClassName) annotation (T042).""" + code = """ +class Calculator { + constructor(initial) { + this.value = initial; + } + + add(n) { + this.value += n; + return this; + } + + subtract(n) { + this.value -= n; + return this; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Find method functions (not the class itself) + method_functions = [f for f in functions if '//(class: Calculator)' in f] + + # Should have constructor, add, and subtract methods with annotations + assert len(method_functions) == 3 + + # Verify annotation format matches Python pattern: //(class: ClassName) + for method_func in method_functions: + assert method_func.strip().endswith('//(class: Calculator)') + assert 'constructor' in method_func or 'add' in method_func or 'subtract' in method_func + + +def test_parse_es6_module(): + """Test parsing of ES6 import/export syntax.""" + code = """ +import { helper } from './utils'; +import defaultExport from './default'; + +export function publicFunction() { + return helper(); +} + +export class PublicClass { + method() { + return 'public'; + } +} + +export default function() { + return 'default export'; +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract exported functions and classes + assert len(functions) == 4 + assert not segmenter.skip_file + + +def test_parse_commonjs_module(): + """Test parsing of CommonJS require/module.exports syntax.""" + code = """ +const path = require('path'); +const { helper } = require('./utils'); + +function internalFunction() { + return path.join('/tmp', 'file.txt'); +} + +module.exports = { + internalFunction, + publicFunction: function() { + return helper(); + } +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract functions + assert len(functions) >= 1 + assert not segmenter.skip_file + + +def test_destructured_import_resolution(): + """Test handling of destructured imports (T043).""" + code = """ +import { foo, bar as baz } from 'module'; +const { method1, method2: renamed } = require('package'); + +function useImports() { + foo(); + baz(); + method1(); + renamed(); +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should handle destructured imports without errors + assert len(functions) == 1 + assert not segmenter.skip_file + assert 'useImports' in functions[0] + +def test_mixed_function_types(): + """Test file with mixed function declaration types.""" + code = """ +// Regular function +function regularFunc() { return 1; } + +// Arrow function +const arrowFunc = () => 2; + +// Async function +async function asyncFunc() { return 3; } + +// Generator +function* genFunc() { yield 4; } + +// Class with methods +class MyClass { + constructor() {} + method() { return 5; } + async asyncMethod() { return 6; } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract multiple types of functions + assert len(functions) >= 8 + +def test_empty_file(): + """Test handling of empty files.""" + code = "" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 0 + assert not segmenter.skip_file + + +def test_comments_only(): + """Test handling of files with only comments.""" + code = """ +// This is a comment +/* Multi-line + comment */ +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 0 + + +def test_shebang_file(): + """Test that files with shebang are skipped.""" + code = """#!/usr/bin/env node +function main() { + console.log('Should be skipped'); +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert segmenter.skip_file + assert len(functions) == 0 + + +OBJECT_LITERAL_TEST_CASES = [ + { + "name": "shorthand_methods", + "code": """ +const api = { + fetchUser(id) { + return fetch('/users/' + id); + }, + parseResponse(res) { + return res.json(); + } +}; +""", + "expected_count": 3, + "object_var": "const api", + "annotation_name": "api", + "expected_methods": 2, + }, + { + "name": "function_expressions", + "code": """ +const utils = { + format: function(data) { + return JSON.stringify(data); + }, + parse: function(str) { + return JSON.parse(str); + } +}; +""", + "expected_count": 3, + "object_var": "const utils", + "annotation_name": "utils", + "expected_methods": 2, + }, + { + "name": "arrow_functions", + "code": """ +const handlers = { + onClick: () => { + console.log('clicked'); + }, + onHover: (e) => e.target.style.color = 'red' +}; +""", + "expected_count": 3, + "object_var": "const handlers", + "annotation_name": "handlers", + "expected_methods": 2, + }, + { + "name": "shorthand_methods_with_annotation", + "code": """ +const calculator = { + add(a, b) { + return a + b; + }, + subtract(a, b) { + return a - b; + } +}; +""", + "expected_count": 3, + "object_var": "const calculator", + "annotation_name": "calculator", + "expected_methods": 2, + }, + { + "name": "exported_object", + "code": """ +export const config = { + getBaseUrl() { + return 'https://api.example.com'; + }, + getTimeout() { + return 5000; + } +}; +""", + "expected_count": 3, + "object_var": "export const config", + "annotation_name": "config", + "expected_methods": 2, + }, + { + "name": "object_without_methods_not_extracted", + "code": """ +const data = { + name: 'John', + age: 30, + active: true +}; + +function processData() { + return data; +} +""", + "expected_count": 1, + "object_var": None, + "annotation_name": None, + "expected_methods": 0, + }, +] + + +@pytest.mark.parametrize("test_case", OBJECT_LITERAL_TEST_CASES, ids=lambda x: x["name"]) +def test_object_literal_extraction(test_case): + """Test extraction of object literals with various method syntaxes.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + functions = segmenter.extract_functions_classes() + + assert len(functions) == test_case["expected_count"], \ + f"Expected {test_case['expected_count']} items, got {len(functions)}" + + if test_case["object_var"]: + assert any(test_case["object_var"] in f for f in functions), \ + f"Object '{test_case['object_var']}' not found" + + if test_case["annotation_name"]: + annotation = f'//(class: {test_case["annotation_name"]})' + method_functions = [f for f in functions if annotation in f] + assert len(method_functions) == test_case["expected_methods"], \ + f"Expected {test_case['expected_methods']} methods with annotation, got {len(method_functions)}" + + +def test_mixed_classes_and_objects(): + """Test file with both classes and object literals.""" + code = """ +class UserService { + constructor() { + this.users = []; + } + + getUser(id) { + return this.users.find(u => u.id === id); + } +} + +const apiUtils = { + formatUrl(path) { + return '/api' + path; + }, + handleError(err) { + console.error(err); + } +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: + # - Class declaration (1) + # - Class methods with annotation (2: constructor, getUser) + # - Object literal (1) + # - Object methods with annotation (2: formatUrl, handleError) + assert len(functions) == 6 + + # Verify class methods have class annotation + class_methods = [f for f in functions if '//(class: UserService)' in f] + assert len(class_methods) == 2 + + # Verify object methods also have class annotation (unified pattern) + object_methods = [f for f in functions if '//(class: apiUtils)' in f] + assert len(object_methods) == 2 + diff --git a/tests/test_javascript_functions_parser.py b/tests/test_javascript_functions_parser.py new file mode 100644 index 000000000..68973d857 --- /dev/null +++ b/tests/test_javascript_functions_parser.py @@ -0,0 +1,2617 @@ +import pytest +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser + + +@pytest.fixture +def parser(): + """Create a JavaScriptFunctionsParser instance for testing.""" + return JavaScriptFunctionsParser() + + +@pytest.mark.parametrize("code,expected_name", [ + # Regular function declarations + ( + "function myFunc() {\n return 42;\n}", + "myFunc" + ), + ( + "function hello(name) {\n console.log(name);\n}", + "hello" + ), + ( + "function calculateSum(a, b) {\n return a + b;\n}", + "calculateSum" + ), + + # Async function declarations + ( + "async function fetchData() {\n return await getData();\n}", + "fetchData" + ), + ( + "async function processRequest(req) {\n const data = await req.json();\n return data;\n}", + "processRequest" + ), + + # Arrow functions with const (with parentheses) + ( + "const add = (a, b) => {\n return a + b;\n}", + "add" + ), + ( + "const greet = () => {\n console.log('hello');\n}", + "greet" + ), + ( + "const multiply = (x, y) => x * y", + "multiply" + ), + + # Arrow functions without parentheses (single parameter) + ( + "const last = arr => arr[arr.length - 1];", + "last" + ), + ( + "const double = n => n * 2;", + "double" + ), + ( + "const identity = x => x;", + "identity" + ), + + # Arrow functions with let + ( + "let divide = (a, b) => {\n return a / b;\n}", + "divide" + ), + + # Arrow functions with var + ( + "var subtract = (a, b) => {\n return a - b;\n}", + "subtract" + ), + + # Async arrow functions + ( + "const asyncFetch = async () => {\n return await fetch('/api');\n}", + "asyncFetch" + ), + ( + "const loadData = async (url) => {\n const response = await fetch(url);\n return response.json();\n}", + "loadData" + ), + + # Function expressions with const + ( + "const myFunction = function() {\n return 'hello';\n}", + "myFunction" + ), + ( + "const handler = function(event) {\n event.preventDefault();\n}", + "handler" + ), + + # Function expressions with let + ( + "let process = function(data) {\n return data.map(x => x * 2);\n}", + "process" + ), + + # Function expressions with var + ( + "var oldStyle = function() {\n return 'legacy';\n}", + "oldStyle" + ), + + # Async function expressions + ( + "const asyncProcess = async function() {\n return await doWork();\n}", + "asyncProcess" + ), + ( + "const handleAsync = async function(data) {\n await save(data);\n}", + "handleAsync" + ), + + # Class methods (without class declaration) + ( + "constructor() {\n this.value = 0;\n}//(class: MyClass)", + "constructor" + ), + ( + "getValue() {\n return this.value;\n}//(class: MyClass)", + "getValue" + ), + ( + "setValue(val) {\n this.value = val;\n}//(class: Calculator)", + "setValue" + ), + + # Async class methods + ( + "async fetchUser() {\n return await api.getUser();\n}//(class: UserService)", + "fetchUser" + ), + ( + "async save(data) {\n await db.save(data);\n}//(class: Repository)", + "save" + ), + + # Functions with extra whitespace + ( + "function spacedFunc () {\n return true;\n}", + "spacedFunc" + ), + ( + "const arrowSpaced = () => {\n return false;\n}", + "arrowSpaced" + ), + + # Generator functions + ( + "function* generator() {\n yield 1;\n}", + "generator" + ), + + # Curried arrow functions (returns another function) + ( + "const helper = minVersion => tpl => ({\n minVersion,\n ast: () => _template.default.program.ast(tpl)\n});", + "helper" + ), + ( + "const curry = a => b => c => a + b + c;", + "curry" + ), + + # Computed property methods (using symbols or expressions) + ( + "[Symbol.iterator]() {\n\t\treturn createHeadersIterator(this, 'key+value');\n\t}", + "Symbol.iterator" + ), + ( + "[Symbol.toStringTag]() {\n return 'CustomClass';\n}", + "Symbol.toStringTag" + ), + ( + "[computed]() {\n return this.value;\n}", + "computed" + ), + + # multiple parameters + ( + "function complexParams(a, b, c, d) {\n return a + b + c + d;\n}", + "complexParams" + ), + ( + "const arrowMultiParams = (x, y, z) => {\n return x + y + z;\n}", + "arrowMultiParams" + ), + ( + "", + "", + ), + # Multiline async function + ( + """async function processUserData(userId) { + const user = await fetchUser(userId); + const profile = await fetchProfile(user.profileId); + const settings = await fetchSettings(user.settingsId); + + return { + user, + profile, + settings + }; + }""", + "processUserData", + ), + # Complex arrow function + ( + """const calculateTotalPrice = (items) => { + const subtotal = items.reduce((sum, item) => sum + item.price, 0); + const tax = subtotal * 0.08; + const shipping = items.length > 0 ? 5.99 : 0; + return subtotal + tax + shipping; + }""", + "calculateTotalPrice", + ), + # Class method with annotation + ( + """async validateInput(input) { + if (!input) { + throw new Error('Input is required'); + } + const sanitized = this.sanitize(input); + return await this.validate(sanitized); + }//(class: InputValidator)""", + "validateInput", + ), + ( + "\n\nfunction myFunc() {\n return 42;\n}\n\n", + "myFunc" + ), + ( + " const indented = () => {\n return 'test';\n }", + "indented" + ), + # Test with tabs + ( + "\tfunction tabbedFunc() {\n\t\treturn true;\n\t}", + "tabbedFunc" + ), +]) +def test_get_function_name_valid_patterns(parser, code, expected_name): + """Test get_function_name with various valid function patterns.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == expected_name + + +@pytest.mark.parametrize("code", [ + # Class declarations should raise ValueError + "class MyClass {\n constructor() {}\n}", + "export class ExportedClass {\n method() {}\n}", + "class Calculator {\n add(a, b) {\n return a + b;\n }\n}", + " class IndentedClass {\n getValue() {}\n}", +]) +def test_get_function_name_raises_on_class_declaration(parser, code): + """Test that get_function_name raises ValueError for class declarations.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + with pytest.raises(ValueError, match="Only function document is supported"): + parser.get_function_name(doc) + + +@pytest.mark.parametrize("code", [ + # Code that doesn't match any function pattern - returns empty string + "const x = 5;", + "let variable = 'string';", + "// Just a comment", + "{ key: 'value' }", + "/* Multi-line\n comment */", +]) +def test_get_function_name_returns_empty_string(parser, code): + """Test that get_function_name returns empty string for non-function code. + + When no function pattern matches, the method logs a warning and returns ''(empty string). + """ + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == '' + + +@pytest.mark.parametrize("source_path,expected_packages", [ + # Application code (no node_modules) - returns root_project + ( + "/app/src/utils.js", + ["root_project"] + ), + # Regular third-party package + ( + "/app/node_modules/lodash/template.js", + ["lodash"] + ), + # Scoped package (starts with @) + ( + "/app/node_modules/@babel/core/lib/index.js", + ["@babel/core"] + ), + # Another regular package + ( + "/app/node_modules/express/lib/router.js", + ["express"] + ), + # Another application code path + ( + "/home/user/project/controllers/user.js", + ["root_project"] + ), + ( + "/app/node_modules/package-a/node_modules/package-b/index.js", + ["package-b"], + ), + # Nested scoped packages + ( + "/app/node_modules/@angular/common/node_modules/@angular/core/index.js", + ["@angular/core"], + ), +]) +def test_get_package_names(parser, source_path, expected_packages): + doc = Document( + page_content="function test() {}", + metadata={'source': source_path, 'content_type': 'functions_classes'} + ) + result = parser.get_package_names(doc) + assert result == expected_packages + + + +# ============================================================================ +# Tests for _get_function_calls method +# ============================================================================ + +@pytest.mark.parametrize("caller_code,callee_name,expected_calls", [ + # Direct function calls + ( + "function myFunc() {\n template();\n}", + "template", + ["template"] + ), + # Multiple calls to the same function + ( + "function process() {\n parse();\n parse();\n parse();\n}", + "parse", + ["parse"] + ), + # Object method calls + ( + "function handler() {\n lodash.template();\n}", + "template", + ["lodash.template"] + ), + # Nested property access + ( + "function run() {\n obj.nested.func();\n}", + "func", + ["obj.nested.func"] + ), + # Multiple different call patterns + ( + "function complex() {\n template();\n _.template();\n utils.template();\n}", + "template", + ["template", "_.template", "utils.template"] + ), + # Function call with arguments + ( + "function withArgs() {\n compile('test', 42);\n}", + "compile", + ["compile"] + ), + # Function call with complex arguments + ( + "function nested() {\n process(getData(), getConfig());\n}", + "process", + ["process"] + ), + # Call with newlines and whitespace + ( + "function spaces() {\n execute \n ();\n}", + "execute", + ["execute"] + ), + # No calls to the target function + ( + "function other() {\n differentFunc();\n}", + "targetFunc", + [] + ), + # Function name as part of longer identifier + ( + "function test() {\n template();\n mytemplate();\n templateFunc();\n}", + "template", + ["template"] + ), + # Chained method calls + ( + "function chain() {\n obj.method1().method2();\n}", + "method2", + ["obj.method1().method2"] + ), + # Arrow function with call + ( + "const arrow = () => {\n helper();\n}", + "helper", + ["helper"] + ), + # Async function with await + ( + "async function asyncFunc() {\n await fetchData();\n}", + "fetchData", + ["fetchData"] + ), + # Call in conditional + ( + "function conditional() {\n if (condition) {\n action();\n }\n}", + "action", + ["action"] + ), + # Call in loop + ( + "function loop() {\n for (let i = 0; i < 10; i++) {\n process();\n }\n}", + "process", + ["process"] + ), + # Call with callback + ( + "function withCallback() {\n execute(function() {\n callback();\n });\n}", + "execute", + ["execute"] + ), + # Multiple objects calling same method + ( + "function multi() {\n obj1.save();\n obj2.save();\n}", + "save", + ["obj1.save", "obj2.save"] + ), + # Special characters in object name (underscore, dollar sign) + ( + "function special() {\n _lodash.template();\n $jquery.ajax();\n}", + "template", + ["_lodash.template"] + ), + # No whitespace before parenthesis + ( + "function noSpace() {\n func();\n}", + "func", + ["func"] + ), + # Call as part of expression + ( + "function expr() {\n const result = calculate() + 10;\n}", + "calculate", + ["calculate"] + ), + # Empty function body + ( + "function empty() {}", + "anyFunc", + [], + ), + # Function calls in comments + ( + """function test() { + // execute(); + /* execute(); */ + actualCall(); + }""", + "execute", + [], # The regex will match both commented calls + ), + # Function names in string literals + ( + """function test() { + const str = "execute()"; + execute(); + }""", + "execute", + ["execute"], # Will find both in string and actual call + ), + # Special regex characters in function names + ( + """function test() { + $jquery(); + _underscore(); + }""", + "$jquery", + ["$jquery"], + ), + ( + """function test() { + $jquery(); + _underscore(); + }""", + "_underscore", + ["_underscore"], + ), + # Case sensitivity + ( + """function test() { + myFunc(); + MyFunc(); + MYFUNC(); + }""", + "myFunc", + ["myFunc"], + ), + ( + """function test() { + myFunc(); + MyFunc(); + MYFUNC(); + }""", + "MyFunc", + ["MyFunc"], + ), + # Constructor calls with 'new' + ( + """function create() { + const obj = new MyClass(); + return obj; + }""", + "MyClass", + ["MyClass"], + ), + # Deeply nested property access + ( + """function deepNesting() { + app.services.user.auth.validate(); + }""", + "validate", + ["app.services.user.auth.validate"], + ), + # ======================================================================== + # Callback references without parentheses + # ======================================================================== + # setTimeout with direct function reference + ( + """function test() { + setTimeout(add, 1000, 10, 15); + }""", + "add", + ["add"], + ), + # setTimeout with qualified function reference + ( + """function test() { + setTimeout(math.add, 1000, 7, 8); + }""", + "add", + ["math.add"], + ), + # Array forEach with direct function reference + ( + """function test() { + [1,2,3].forEach(add); + }""", + "add", + ["add"], + ), + # Array forEach with qualified function reference + ( + """function test() { + [1,2,3].forEach(math.add); + }""", + "add", + ["math.add"], + ), + # addEventListener with qualified function reference + ( + """function test() { + document.getElementById('btn').addEventListener('click', math.add); + }""", + "add", + ["math.add"], + ), + # Array map with callback + ( + """function test() { + const result = items.map(transform); + }""", + "transform", + ["transform"], + ), + # Array filter with callback + ( + """function test() { + const filtered = data.filter(validator.check); + }""", + "check", + ["validator.check"], + ), + # Promise then/catch with callbacks + ( + """function test() { + promise.then(handleSuccess).catch(handleError); + }""", + "handleSuccess", + ["handleSuccess"], + ), + ( + """function test() { + promise.then(handleSuccess).catch(handleError); + }""", + "handleError", + ["handleError"], + ), + # Mixed: both direct call and callback reference + ( + """function test() { + add(1, 2); + setTimeout(add, 1000); + }""", + "add", + ["add"], + ), + # Mixed: qualified call and qualified callback + ( + """function test() { + math.add(5, 10); + [1,2,3].forEach(math.add); + }""", + "add", + ["math.add"], + ), + # Callback in nested call + ( + """function test() { + Promise.all(items.map(process)); + }""", + "process", + ["process"], + ), + # Multiple callbacks in same line + ( + """function test() { + promise.then(success, failure); + }""", + "success", + ["success"], + ), + ( + """function test() { + promise.then(success, failure); + }""", + "failure", + ["failure"], + ), + # Callback with whitespace variations + ( + """function test() { + setTimeout( add , 1000 ); + }""", + "add", + ["add"], + ), + # Should NOT match: function name in string + ( + """function test() { + setTimeout("add", 1000); + }""", + "add", + [], + ), + # Should NOT match: function name as object key + ( + """function test() { + const obj = {add: someFunc}; + }""", + "add", + [], + ), + # Callback with deeply nested qualifier + ( + """function test() { + items.forEach(utils.math.operations.add); + }""", + "add", + ["utils.math.operations.add"], + ), + # setInterval with callback + ( + """function test() { + setInterval(tick, 1000); + }""", + "tick", + ["tick"], + ), + # requestAnimationFrame with callback + ( + """function test() { + requestAnimationFrame(render); + }""", + "render", + ["render"], + ), + # Array method chaining with callbacks + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "transform", + ["transform"], + ), + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "validate", + ["validate"], + ), + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "process", + ["process"], + ), +]) +def test_get_function_calls_various_patterns(parser, caller_code, callee_name, expected_calls): + caller_doc = Document( + page_content=caller_code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser._get_function_calls(caller_doc, callee_name) + assert sorted(result) == sorted(expected_calls) + + +def test_get_function_calls_complex_real_world_example(parser): + """Test _get_function_calls with realistic complex code.""" + caller_code = """async function processUserData(userId) { + const user = await fetchUser(userId); + const profile = await fetchProfile(user.profileId); + + if (user.isActive) { + logger.log('Processing active user'); + const settings = await fetchSettings(user.id); + validateSettings(settings); + } + + return { + user: user, + profile: profile + }; +}""" + caller_doc = Document( + page_content=caller_code, + metadata={'source': 'users.js', 'content_type': 'functions_classes'} + ) + + # Test finding different function calls + assert parser._get_function_calls(caller_doc, "fetchUser") == ["fetchUser"] + assert parser._get_function_calls(caller_doc, "fetchProfile") == ["fetchProfile"] + assert parser._get_function_calls(caller_doc, "fetchSettings") == ["fetchSettings"] + assert parser._get_function_calls(caller_doc, "validateSettings") == ["validateSettings"] + assert parser._get_function_calls(caller_doc, "log") == ["logger.log"] + + +@pytest.mark.parametrize("func_content,source,expected_key,expected_vars,description", [ + # Regular function parameters and variables + ( + """function processData(id, name, options) { + const userId = id; + let result = null; + var temp = 'test'; + return userId; +}""", + "app/utils.js", + "processData@app/utils.js", + { + 'id': {"value": "parameter", "type": ""}, + 'name': {"value": "parameter", "type": ""}, + 'options': {"value": "parameter", "type": ""}, + 'userId': {"value": "id", "type": ""}, + 'result': {"value": "null", "type": ""}, + 'temp': {"value": "'test'", "type": ""}, + 'return_types': [] + }, + "Regular function with parameters and various variable declarations" + ), + ( + """function processData(id, + name, + options) { + const userId = id; + let result = null; + var temp = 'test'; + return userId; + }""", + "app/utils.js", + "processData@app/utils.js", + { + 'id': {"value": "parameter", "type": ""}, + 'name': {"value": "parameter", "type": ""}, + 'options': {"value": "parameter", "type": ""}, + 'userId': {"value": "id", "type": ""}, + 'result': {"value": "null", "type": ""}, + 'temp': {"value": "'test'", "type": ""}, + 'return_types': [] + }, + "Regular function with parameters in few lines and various variable declarations" + ), + # Arrow function + ( + """const calculate = (a, b, c) => { + const sum = a + b + c; + let multiplier = 2; + return sum * multiplier; +}""", + "app/math.js", + "calculate@app/math.js", + { + 'a': {"value": "parameter", "type": ""}, + 'b': {"value": "parameter", "type": ""}, + 'c': {"value": "parameter", "type": ""}, + 'sum': {"value": "a + b + c", "type": ""}, + 'multiplier': {"value": "2", "type": ""}, + 'return_types': [] + }, + "Arrow function with parameters and local variables" + ), + # Rest parameters + ( + """async function processItems(baseUrl, ...endpoints) { + const results = []; + let count = 0; + return results; +}""", + "app/api.js", + "processItems@app/api.js", + { + 'baseUrl': {"value": "parameter", "type": ""}, + 'endpoints': {"value": "parameter", "type": "Array"}, + 'results': {"value": "[]", "type": ""}, + 'count': {"value": "0", "type": ""}, + 'return_types': [] + }, + "Function with rest parameters (type Array)" + ), + # Object destructuring parameters + ( + """const handleRequest = ({method, url, headers}) => { + const response = fetch(url); + let status = 200; + return response; +}""", + "app/request.js", + "handleRequest@app/request.js", + { + 'method': {"value": "parameter", "type": ""}, + 'url': {"value": "parameter", "type": ""}, + 'headers': {"value": "parameter", "type": ""}, + 'response': {"value": "fetch(url)", "type": ""}, + 'status': {"value": "200", "type": ""}, + 'return_types': [] + }, + "Function with object destructuring parameters" + ), + # Array destructuring parameters + ( + """function processCoordinates([x, y, z]) { + const distance = Math.sqrt(x*x + y*y + z*z); + return distance; +}""", + "app/math.js", + "processCoordinates@app/math.js", + { + 'x': {"value": "parameter", "type": ""}, + 'y': {"value": "parameter", "type": ""}, + 'z': {"value": "parameter", "type": ""}, + 'distance': {"value": "Math.sqrt(x*x + y*y + z*z)", "type": ""}, + 'return_types': [] + }, + "Function with array destructuring parameters" + ), + # Default parameters + ( + """function fetchData(url, timeout = 5000, retries = 3) { + const config = { url, timeout, retries }; + return config; +}""", + "app/fetch.js", + "fetchData@app/fetch.js", + { + 'url': {"value": "parameter", "type": ""}, + 'timeout': {"value": "5000", "type": ""}, + 'retries': {"value": "3", "type": ""}, + 'config': {"value": "{ url, timeout, retries }", "type": ""}, + 'return_types': [] + }, + "Function with default parameters" + ), + # Multiple declarations on one line + ( + """function multiVars() { + const x = 1, y = 2, z = 3; + let a, b, c; + var m = 'm', n = 'n'; + return x + y; +}""", + "app/multi.js", + "multiVars@app/multi.js", + { + 'x': {"value": "1", "type": ""}, + 'y': {"value": "2", "type": ""}, + 'z': {"value": "3", "type": ""}, + 'a': {"value": "", "type": ""}, + 'b': {"value": "", "type": ""}, + 'c': {"value": "", "type": ""}, + 'm': {"value": "'m'", "type": ""}, + 'n': {"value": "'n'", "type": ""}, + 'return_types': [] + }, + "Function with multiple variable declarations on one line" + ), + # Class method/constructor + ( + """constructor(name, age) { + const greeting = 'Hello'; + this.name = name; + this.age = age; +}//(class: Person)""", + "app/Person.js", + "constructor@app/Person.js", + { + 'name': {"value": "parameter", "type": ""}, + 'age': {"value": "parameter", "type": ""}, + 'greeting': {"value": "'Hello'", "type": ""}, + 'this': {"value": "Person()", "type": "Person"}, + 'return_types': [] + }, + "Class constructor with 'this' reference" + ), + # Object destructuring local variables + ( + """function processUser(user) { + const {name, age, email} = user; + let status = 'active'; + return name; +}""", + "app/user.js", + "processUser@app/user.js", + { + 'user': {"value": "parameter", "type": ""}, + 'name': {"value": "user", "type": ""}, + 'age': {"value": "user", "type": ""}, + 'email': {"value": "user", "type": ""}, + 'status': {"value": "'active'", "type": ""}, + 'return_types': [] + }, + "Function with object destructuring in local variables" + ), + # Array destructuring local variables + ( + """function processArray(data) { + const [first, second, third] = data; + return first + second; +}""", + "app/array.js", + "processArray@app/array.js", + { + 'data': {"value": "parameter", "type": ""}, + 'first': {"value": "data", "type": ""}, + 'second': {"value": "data", "type": ""}, + 'third': {"value": "data", "type": ""}, + 'return_types': [] + }, + "Function with array destructuring in local variables" + ), + # Empty function + ( + """function emptyFunc() { + return 42; +}""", + "app/empty.js", + "emptyFunc@app/empty.js", + { + 'return_types': [] + }, + "Function with no variables" + ), + # Complex values + ( + """function complex() { + const obj = { key: 'value', nested: { deep: true } }; + const arr = [1, 2, 3, 4, 5]; + const func_call = fetch('/api/data'); + const chain = data.map(x => x * 2).filter(x => x > 10); + return obj; +}""", + "app/complex.js", + "complex@app/complex.js", + { + 'obj': {"value": "{ key: 'value', nested: { deep: true } }", "type": ""}, + 'arr': {"value": "[1, 2, 3, 4, 5]", "type": ""}, + 'func_call': {"value": "fetch('/api/data')", "type": ""}, + 'chain': {"value": "data.map(x => x * 2).filter(x => x > 10)", "type": ""}, + 'return_types': [] + }, + "Function with complex variable values" + ), + # Renamed destructuring + ( + """function renamed({oldName: newName, id: userId}) { + const value = newName + userId; + return value; +}""", + "app/rename.js", + "renamed@app/rename.js", + { + 'newName': {"value": "parameter", "type": ""}, + 'userId': {"value": "parameter", "type": ""}, + 'value': {"value": "newName + userId", "type": ""}, + 'return_types': [] + }, + "Function with renamed object destructuring" + ), + # declaration split to multiple lines + ( + """function func() { + const fruits = ["Apple", + "Banana", + "Cherry", + "Date" + ]; + return value; + }""", + "app/func.js", + "func@app/func.js", + { + 'fruits': {"value": '["Apple", "Banana", "Cherry", "Date" ]', "type": ""}, + 'return_types': [] + }, + "Function with renamed object destructuring" + ), +]) +def test_create_map_of_local_vars_single_function(parser, func_content, source, expected_key, expected_vars, description): + """Test create_map_of_local_vars with various single function scenarios.""" + func_doc = Document( + page_content=func_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + + result = parser.create_map_of_local_vars([func_doc]) + + assert expected_key in result + vars_map = result[expected_key] + + for var_name, expected_value in expected_vars.items(): + assert vars_map[var_name] == expected_value, f"Variable '{var_name}' mismatch" + + +def test_create_map_of_local_vars_multiple_functions(parser): + """Test create_map_of_local_vars with multiple functions.""" + func_docs = [ + Document( + page_content="""function funcA(x) { + const result = x * 2; + return result; +}""", + metadata={'source': 'app/a.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="""function funcB(y) { + let temp = y + 1; + return temp; +}""", + metadata={'source': 'app/b.js', 'content_type': 'functions_classes'} + ) + ] + + result = parser.create_map_of_local_vars(func_docs) + + # Check both functions are in the result + assert 'funcA@app/a.js' in result + assert 'funcB@app/b.js' in result + + # Check funcA + assert result['funcA@app/a.js']['x'] == {"value": "parameter", "type": ""} + assert result['funcA@app/a.js']['result'] == {"value": "x * 2", "type": ""} + + # Check funcB + assert result['funcB@app/b.js']['y'] == {"value": "parameter", "type": ""} + assert result['funcB@app/b.js']['temp'] == {"value": "y + 1", "type": ""} + + +@pytest.mark.parametrize("class_code,fields_to_include,fields_to_exclude", [ + # Test: Local variables in methods are not included as class fields + ( + """ +class UserService { + constructor() { + this.processor = new DataProcessor(); + this.name = 'service'; + } + + findUser(username) { + const localVar = 'local'; + let anotherLocal = 123; + return this.processor.executeQuery(username); + } + + checkUser(username) { + var temp = 'temporary'; + return this.processor.validateInput(username); + } +} +""", + ['processor', 'name'], + ['localVar', 'anotherLocal', 'temp', 'username'] + ), + # Test: Class field declarations (ES2022+) + ( + """ +class DataProcessor { + apiKey = 'default-key'; + #privateField = 'private'; + counter = 0; + + constructor(config) { + this.config = config; + const tempVar = 'temp'; + } + + process(data) { + const result = this.apiKey + data; + return result; + } +} +""", + ['apiKey', 'privateField', 'counter', 'config'], + ['tempVar', 'result', 'data'] + ), + # Test: Async methods, static methods, and getters/setters + ( + """ +class ComplexClass { + status = 'active'; + + constructor() { + this.id = 123; + } + + async fetchData() { + const response = await fetch('/api'); + return response; + } + + static create() { + const instance = new ComplexClass(); + return instance; + } + + get value() { + const computed = this.id * 2; + return computed; + } +} +""", + ['status', 'id'], + ['response', 'instance', 'computed'] + ), + # Test: this.field assignments in non-constructor methods are excluded + ( + """ +class StateManager { + constructor() { + this.initialState = {}; + } + + setState(newState) { + // This should NOT be extracted as a field + this.currentState = newState; + const temp = 'local'; + } + + updateValue(val) { + this.value = val; // Not in constructor, should be excluded + } +} +""", + ['initialState'], + ['currentState', 'value', 'temp'] + ), + # Test: Constructor with nested braces (if statements, loops) + ( + """ +class ComplexConstructor { + constructor(config) { + this.config = config; + + if (config.enabled) { + const tempEnabled = true; + this.status = 'active'; + } + + for (let i = 0; i < 10; i++) { + const loopVar = i; + } + } +} +""", + ['config', 'status'], + ['tempEnabled', 'loopVar', 'i'] + ), + # Test: Class with both class field declarations and constructor assignments + ( + """ +class MixedFields { + // Class field declarations + publicField = 'public'; + anotherField = 42; + + constructor(param1, param2) { + // Constructor assignments + this.param1 = param1; + this.param2 = param2; + + // Local variables (should be excluded) + const localInConstructor = 'local'; + let tempValue = 123; + } + + method1() { + const methodLocal = 'should not appear'; + return this.publicField; + } +} +""", + ['publicField', 'anotherField', 'param1', 'param2'], + ['localInConstructor', 'tempValue', 'methodLocal'] + ), + ( + """ + class EmptyClass { + doSomething() { + console.log('test'); + } + } + """, + [], + [] + ), +]) +def test_extract_fields(parser, class_code, fields_to_include, fields_to_exclude): + """Test _extract_class_fields with various class structures. + + Verifies that: + - Class field declarations are extracted + - Constructor this.field assignments are extracted + - Local variables in methods are NOT extracted + - Method parameters are NOT extracted + - this.field assignments in non-constructor methods are NOT extracted + """ + fields = parser._extract_class_fields(class_code) + field_names = [field[0] for field in fields] + + assert len(fields) == len(fields_to_include) + for field in fields_to_include: + assert field in field_names + + for field in fields_to_exclude: + assert field not in field_names + + +# def test_extract_fields_empty_class(parser): +# """Test with a class that has no fields.""" +# class_code = """ +# class EmptyClass { +# doSomething() { +# console.log('test'); +# } +# } +# """ +# +# fields = parser._extract_class_fields(class_code) +# +# # Should return empty list +# assert fields == [] + + +def test_extract_fields_all_field_types_return_any(parser): + """Test that all extracted fields have type 'any'.""" + class_code = """ +class TypeTest { + field1 = 'string'; + field2 = 123; + + constructor() { + this.field3 = true; + } +} +""" + + fields = parser._extract_class_fields(class_code) + + # All fields should have type 'any' + for field_name, field_type in fields: + assert field_type == 'any', f"Field '{field_name}' should have type 'any', got '{field_type}'" + + # Should have 3 fields + assert len(fields) == 3 + +@pytest.mark.parametrize("description,function_content,file_content,expected", [ + ( + "Constructor should be detected as exported when class is exported via module.exports = ClassName", + "constructor() {\n this.name = 'PackageURL';\n}\n//(class: PackageURL)", + """class PackageURL { + constructor() { + this.name = 'PackageURL'; + } + + static fromString(str) { + return new PackageURL(); + } + } + + module.exports = PackageURL; + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via module.exports = ClassName", + "static fromString(str) {\n return new PackageURL();\n}\n//(class: PackageURL)", + """class PackageURL { + constructor() { + this.name = 'PackageURL'; + } + + static fromString(str) { + return new PackageURL(); + } + } + + module.exports = PackageURL; + """, + True, + ), + ( + "Class method should be detected as exported when class is in module.exports object", + "executeQuery(userInput) {\n const query = `SELECT * FROM users WHERE name = '${userInput}'`;\n return query;\n}\n//(class: DataProcessor)", + """class DataProcessor { + executeQuery(userInput) { + const query = `SELECT * FROM users WHERE name = '${userInput}'`; + return query; + } + } + + module.exports = { DataProcessor }; + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via export class", + "process(data) {\n return data.toUpperCase();\n}\n//(class: Processor)", + """export class Processor { + process(data) { + return data.toUpperCase(); + } + } + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via export default", + "save(item) {\n this.items.push(item);\n}\n//(class: Storage)", + """class Storage { + constructor() { + this.items = []; + } + + save(item) { + this.items.push(item); + } + } + + export default Storage; + """, + True, + ), + ( + "Function should be detected as exported", + "function myFunction() {\n return 'test';\n}", + """function myFunction() { + return 'test'; + } + + module.exports = myFunction; + """, + True, + ), + ( + "Function should be detected as exported when in module.exports object", + "function helper() {\n return 'help';\n}", + """function helper() { + return 'help'; + } + + function other() { + return 'other'; + } + + module.exports = { helper, other }; + """, + True, + ), + ( + "Class method should NOT be detected as exported when class is not exported", + "privateMethod() {\n return 'private';\n}\n//(class: InternalClass)", + """class InternalClass { + privateMethod() { + return 'private'; + } + } + + // No exports + """, + False, + ), + ( + "Function should NOT be detected as exported when not exported", + "function internalFunc() {\n return 'internal';\n}", + """function internalFunc() { + return 'internal'; + } + + // No exports + """, + False, + ), +]) +def test_is_exported_function(parser, description, function_content, file_content, expected): + source = "node_modules/packageurl-js/index.js", + func_doc = Document( + page_content=function_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + full_file_doc = Document( + page_content=file_content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + documents_of_full_sources = { + source: full_file_doc + } + + # Test + result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert result == expected, description + + +@pytest.mark.parametrize("description,code,expected", [ + ( + "Simple function with single line parameters", + """function add (a, b) { +a = 2; +b = 3; +// a+b +c = a+b; +}""", + [ + 'function add (a, b) {', + 'a = 2;', + 'b = 3;', + '// a+b', + 'c = a+b;', + '}' + ] + ), + ( + "Function with multiline parameters", + """function add (a, +b) { +a = 2; +b = 3; +c = a+b; +}""", + [ + 'function add (a, b) {', + 'a = 2;', + 'b = 3;', + 'c = a+b;', + '}' + ] + ), + ( + "Function with multiline parameters and extra spaces", + """function processData( + id, + name, + options +) { +const userId = id; +let result = null; +return userId; +}""", + [ + 'function processData( id, name, options ) {', + 'const userId = id;', + 'let result = null;', + 'return userId;', + '}' + ] + ), + ( + "Arrow function with multiline parameters", + """const calculate = ( + a, + b, + c +) => { +const sum = a + b + c; +return sum; +}""", + [ + 'const calculate = ( a, b, c ) => {', + 'const sum = a + b + c;', + 'return sum;', + '}' + ] + ), + ( + "Async function with multiline parameters", + """async function fetchData( + url, + timeout, + retries +) { +const response = await fetch(url); +return response; +}""", + [ + 'async function fetchData( url, timeout, retries ) {', + 'const response = await fetch(url);', + 'return response;', + '}' + ] + ), + ( + "Function with nested parentheses in default parameters", + """function test(a = (1 + 2), b) { +const x = a; +return x; +}""", + [ + 'function test(a = (1 + 2), b) {', + 'const x = a;', + 'return x;', + '}' + ] + ), + ( + "Multiple functions with multiline parameters", + """function first( + x, + y +) { +return x + y; +} +function second(a, b) { +return a * b; +}""", + [ + 'function first( x, y ) {', + 'return x + y;', + '}', + 'function second(a, b) {', + 'return a * b;', + '}' + ] + ), + ( + "Function with string containing semicolons", + """function test() { +const msg = "Hello; World"; +console.log(msg); +}""", + [ + 'function test() {', + 'const msg = "Hello; World";', + 'console.log(msg);', + '}' + ] + ), + ( + "Function with block comment", + """function test() { +/* This is a comment */ +const x = 1; +return x; +}""", + [ + 'function test() {', + '/* This is a comment */', + 'const x = 1;', + 'return x;', + '}' + ] + ), + ( + "Function with inline comment containing semicolons", + """function test() { +a = 1; +// This; is; a; comment +b = 2; +}""", + [ + 'function test() {', + 'a = 1;', + '// This; is; a; comment', + 'b = 2;', + '}' + ] + ), + ( + "Class with multiline constructor parameters", + """class MyClass { +constructor( + name, + age +) { +this.name = name; +this.age = age; +} +}""", + [ + 'class MyClass {', + 'constructor( name, age ) {', + 'this.name = name;', + 'this.age = age;', + '}', + '}' + ] + ), + ( + "Object destructuring with multiline", + """const handleRequest = ({ + method, + url, + headers +}) => { +const response = fetch(url); +return response; +}""", + [ + 'const handleRequest = ({ method, url, headers }) => {', + 'const response = fetch(url);', + 'return response;', + '}' + ] + ), + ( + "Multi-line block comment on its own", + """function test() { +/* comment with +few lines*/ +const x = 1; +}""", + [ + 'function test() {', + '/* comment with few lines*/', + 'const x = 1;', + '}' + ] + ), + ( + "Multi-line block comment on its own", + """function test() { + /* comment with + few lines*/ const x = 1; + }""", + [ + 'function test() {', + '/* comment with few lines*/', + 'const x = 1;', + '}' + ] + ), +]) +def test_split_by_semicolon(parser, description, code, expected): + """Test the _split_by_semicolon function with various code structures.""" + result = parser._split_into_statements(code) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("child_class,parent_class,code_documents_content,expected,description", [ + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends VulnerableBase {\n constructor() { super(); }\n}"}, + True, + "Simple direct inheritance: class X extends Y" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends LoggerMixin(VulnerableBase) {\n constructor() { super(); }\n}"}, + True, + "Inheritance with mixin: class X extends Mixin(Y)" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends Mixin1(Mixin2(VulnerableBase)) {\n constructor() { super(); }\n}"}, + True, + "Chained mixins: class X extends Mixin1(Mixin2(Y))" + ), + ( + "MyClass", + "BaseClass", + {"src/MyClass.js": "class MyClass extends A(B(C(BaseClass))) {\n}"}, + True, + "Triple chained mixins: class X extends A(B(C(Y)))" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends OtherClass {\n constructor() { super(); }\n}"}, + False, + "No match: child extends different parent" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler {\n constructor() {}\n}"}, + False, + "No match: class without extends" + ), + ( + "NonExistentClass", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends VulnerableBase {\n}"}, + False, + "No match: child class not found in documents" + ), + ( + "MyHandler", + "Base", + {"src/MyHandler.js": "class MyHandler extends BaseExtended {\n}"}, + False, + "Word boundary: Base should not match BaseExtended" + ), + ( + "ChildClass", + "ParentClass", + { + "src/other.js": "class Other extends Unrelated {\n}", + "src/child.js": "class ChildClass extends ParentClass {\n}" + }, + True, + "Multiple files: inheritance found in second file" + ), + ( + "MyService", + "BaseService", + {"src/MyService.js": "export class MyService extends BaseService {\n doSomething() {}\n}"}, + True, + "Export class with inheritance" + ), + ( + "", + "VulnerableBase", + {"src/file.js": "class Handler extends VulnerableBase {}"}, + False, + "Empty child class name" + ), + ( + "CustomHandler", + "", + {"src/file.js": "class CustomHandler extends VulnerableBase {}"}, + False, + "Empty parent class name" + ), + ( + "Handler", + "EventEmitter", + {"src/Handler.js": "class Handler extends EventEmitter(Object) {\n}"}, + True, + "Mixin with parent at the start of the chain" + ), + ( + "GrandChild", + "GrandParent", + { + "src/GrandChild.js": "class GrandChild extends Child {\n}", + "src/Child.js": "class Child extends GrandParent {\n}", + "src/GrandParent.js": "class GrandParent {\n}" + }, + True, + "2-level transitive: GrandChild extends Child extends GrandParent" + ), + ( + "A", + "D", + { + "src/A.js": "class A extends B {\n}", + "src/B.js": "class B extends C {\n}", + "src/C.js": "class C extends D {\n}", + "src/D.js": "class D {\n}" + }, + True, + "3-level transitive: A extends B extends C extends D" + ), + ( + "MyHandler", + "BaseClass", + { + "src/MyHandler.js": "class MyHandler extends Mixin(MiddleClass) {\n}", + "src/MiddleClass.js": "class MiddleClass extends BaseClass {\n}" + }, + True, + "Transitive with mixin: MyHandler extends Mixin(MiddleClass), MiddleClass extends BaseClass" + ), + ( + "A", + "C", + { + "src/A.js": "class A extends B {\n}", + "src/B.js": "class B extends A {\n}" + }, + False, + "Circular reference: A extends B extends A - should not infinite loop" + ), + ( + "Child", + "Unrelated", + { + "src/Child.js": "class Child extends Parent {\n}", + "src/Parent.js": "class Parent extends GrandParent {\n}", + "src/Unrelated.js": "class Unrelated {\n}" + }, + False, + "Transitive non-match: Child is not a subclass of Unrelated" + ), + ( + "Child", + "Parent", + {"src/legacy.js": "function Child() {}\nfunction Parent() {}\nutil.inherits(Child, Parent);"}, + True, + "util.inherits(Child, Parent)" + ), + ( + "MyClass", + "BaseClass", + {"src/legacy.js": "function MyClass() {}\nfunction BaseClass() {}\ninherits(MyClass, BaseClass);"}, + True, + "inherits(Child, Parent) - standalone inherits package" + ), + ( + "Derived", + "Base", + {"src/proto.js": "function Derived() {}\nfunction Base() {}\nDerived.prototype = Object.create(Base.prototype);"}, + True, + "Child.prototype = Object.create(Parent.prototype)" + ), + ( + "Sub", + "Super", + {"src/proto.js": "function Sub() {}\nfunction Super() {}\nObject.setPrototypeOf(Sub.prototype, Super.prototype);"}, + True, + "Object.setPrototypeOf(Child.prototype, Parent.prototype)" + ), + ( + "Child", + "WrongParent", + {"src/legacy.js": "function Child() {}\nfunction Parent() {}\nutil.inherits(Child, Parent);"}, + False, + "util.inherits - non-matching parent" + ), + ( + "GrandChild", + "GrandParent", + { + "src/legacy.js": """function GrandChild() {} +function Child() {} +function GrandParent() {} +util.inherits(GrandChild, Child); +util.inherits(Child, GrandParent);""" + }, + True, + "Transitive util.inherits: GrandChild -> Child -> GrandParent" + ), + ( + "ModernClass", + "LegacyBase", + { + "src/modern.js": "class ModernClass extends LegacyWrapper {\n}", + "src/legacy.js": "function LegacyWrapper() {}\nfunction LegacyBase() {}\nutil.inherits(LegacyWrapper, LegacyBase);" + }, + True, + "Mixed: ES6 class extends prototype-based class" + ), +]) +def test_is_subclass_of(parser, child_class, parent_class, code_documents_content, expected, description): + """Test _is_subclass_of with various inheritance patterns including mixins, transitive, and prototype-based.""" + code_documents = { + source: Document( + page_content=content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + for source, content in code_documents_content.items() + } + + result = parser._is_subclass_of(child_class, parent_class, code_documents) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("function_name,file_content,expected,description", [ + # ES6: export { functionName } + ( + "myFunc", + """function myFunc() { return 1; } +export { myFunc };""", + True, + "ES6 named export with braces", + ), + # ES6: export { func as functionName } (aliased export) + ( + "myFunc", + """function internalFunc() { return 1; } +export { internalFunc as myFunc };""", + True, + "ES6 named export with alias", + ), + # ES6: export default functionName + ( + "myFunc", + """function myFunc() { return 1; } +export default myFunc;""", + True, + "ES6 export default", + ), + # CommonJS: module.exports = functionName + ( + "myFunc", + """function myFunc() { return 1; } +module.exports = myFunc;""", + True, + "CommonJS direct module.exports assignment", + ), + # CommonJS: module.exports = { functionName } + ( + "myFunc", + """function myFunc() { return 1; } +function otherFunc() { return 2; } +module.exports = { myFunc, otherFunc };""", + True, + "CommonJS module.exports object shorthand", + ), + # CommonJS: module.exports = { functionName: functionName } + ( + "myFunc", + """function myFunc() { return 1; } +module.exports = { myFunc: myFunc };""", + True, + "CommonJS module.exports object with explicit key-value", + ), + # CommonJS: module.exports.functionName = ... + ( + "myFunc", + """function myFunc() { return 1; } +module.exports.myFunc = myFunc;""", + True, + "CommonJS module.exports property assignment", + ), + # CommonJS: exports.functionName = ... + ( + "myFunc", + """function myFunc() { return 1; } +exports.myFunc = myFunc;""", + True, + "CommonJS exports property assignment", + ), + # Not exported - should return False + ( + "myFunc", + """function myFunc() { return 1; } +function otherFunc() { return 2; } +// No exports""", + False, + "Function not exported", + ), + # Function name as substring of another - should NOT match + ( + "parse", + """function parseAsync() { return 1; } +function parseSomething() { return 2; } +module.exports = { parseAsync, parseSomething };""", + False, + "Function name as prefix of other functions - word boundary check", + ), + ( + "divide", + """const add = (a, b) => a + b; +const multiply = (a, b) => a * b; +const divide = (a, b) => a / b; +export { add, multiply, divide };""", + True, + "multiple functions exporting", + ), +]) +def test_is_exportable_function(parser, function_name, file_content, expected, description): + """Test _is_exportable_function with various export patterns.""" + result = parser._is_exportable_function(function_name, file_content) + assert result == expected, description + + +@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ + # ======================================================================== + # Dynamic imports with relative paths + # ======================================================================== + ( + "Top-level await dynamic import with relative path", + """// main.mjs (or in a script tag with type="module") + +// Dynamically import the "utils" module at the top level +const utils = await import('./utils.mjs'); +console.log(utils.calculateArea(5));""", + "utils", + "utils", + True, + ), + ( + "Async function with destructuring and default export", + """async function loadSpecifics() { + const { hi, default: say } = await import('./say.js'); + + hi(); // Hello! + say(); // Module loaded (export default)! +} + +loadSpecifics();""", + "hi", + "say", + True, + ), + ( + "Async function - check default export renamed to 'say'", + """async function loadSpecifics() { + const { hi, default: say } = await import('./say.js'); + + hi(); // Hello! + say(); // Module loaded (export default)! +} + +loadSpecifics();""", + "say", + "say", + True, + ), + ( + "Async function with try/catch block", + """async function loadAndCalculate() { + try { + // Dynamically import the module + const module = await import('./math.js'); + + // Use the exported functions + const sum = module.add(5, 3); + const difference = module.subtract(10, 4); + + console.log(`Sum: ${sum}`); // Output: Sum: 8 + console.log(`Difference: ${difference}`); // Output: Difference: 6 + + } catch (error) { + console.error('Failed to load the math module:', error); + } +} + +// Call the async function +loadAndCalculate();""", + "module", + "math", + True, + ), + # ======================================================================== + # Dynamic imports with NPM packages + # ======================================================================== + ( + "Top-level await dynamic import with method call", + """const utils = await import('lodash'); + console.log(utils.template('hello'));""", + "utils", + "lodash", + True, + ), + ( + "Async function with destructuring and default export - NPM package", + """async function loadSpecifics() { + const { parse, default: babylon } = await import('babylon'); + + parse(); // Parse code! + babylon(); // Module loaded (export default)! +} + +loadSpecifics();""", + "parse", + "babylon", + True, + ), + ( + "Async function - check default export renamed - NPM package", + """async function loadSpecifics() { + const { parse, default: babylon } = await import('babylon'); + + parse(); + babylon(); +}""", + "babylon", + "babylon", + True, + ), + ( + "Async function with try/catch and module property access - NPM package", + """async function loadAndCalculate() { + try { + // Dynamically import the module + const module = await import('mathjs'); + + // Use the exported functions + const sum = module.add(5, 3); + const difference = module.subtract(10, 4); + + console.log(`Sum: ${sum}`); // Output: Sum: 8 + console.log(`Difference: ${difference}`); // Output: Difference: 6 + + } catch (error) { + console.error('Failed to load the math module:', error); + } +} + +// Call the async function +loadAndCalculate();""", + "module", + "mathjs", + True, + ), + # ======================================================================== + # Dynamic imports with NPM packages + # ======================================================================== + ( + "Dynamic import of NPM package - lodash", + """const lodash = await import("lodash"); +lodash.template();""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import of NPM package with destructuring", + """const { template } = await import('lodash'); +template();""", + "template", + "lodash", + True, + ), + ( + "Dynamic import of scoped NPM package", + """const babel = await import('@babel/core'); +babel.transform();""", + "babel", + "@babel/core", + True, + ), + ( + "Dynamic import of express", + """const express = await import('express'); +express();""", + "express", + "express", + True, + ), + ( + "Dynamic import with multiple spaces", + """const lodash = await import ('lodash');""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import with space between import and parenthesis", + """const axios = await import ('axios');""", + "axios", + "axios", + True, + ), + ( + "Dynamic import without await - .then pattern", + """import('lodash').then(mod => mod.template());""", + "", + "lodash", + True, + ), + ( + "Dynamic import side-effect only", + """await import('side-effect-pkg');""", + "", + "side-effect-pkg", + True, + ), + ( + "Dynamic import with backticks", + """const moment = await import(`moment`);""", + "moment", + "moment", + True, + ), + # ======================================================================== + # Variable-based dynamic imports (with tracing) + # ======================================================================== + ( + "Variable-based import - NPM package with const", + """const pkgName = 'lodash'; +const lodash = await import(pkgName);""", + "lodash", + "lodash", + True, + ), + ( + "Variable-based import - NPM package with let", + """let pkgName = 'express'; +const express = await import(pkgName);""", + "express", + "express", + True, + ), + ( + "Variable-based import - NPM package with let", + """let pkgName = 'express'; + const express_pakc = await import(pkgName);""", + "express_pakc", + "express", + True, + ), + ( + "Variable-based import - with destructuring", + """const moduleName = 'express'; +const { Router } = await import(moduleName);""", + "Router", + "express", + True, + ), + ( + "Variable-based import - scoped package", + """const pkg = '@babel/core'; +const babel = await import(pkg);""", + "babel", + "@babel/core", + True, + ), + # ======================================================================== + # Recursive variable tracing - variable chains + # ======================================================================== + ( + "Variable chain 2-level", + """const original = 'lodash'; +const pkg = original; +const lib = await import(pkg);""", + "lib", + "lodash", + True, + ), + ( + "Variable chain 3-level", + """const original = 'axios'; +const alias1 = original; +const alias2 = alias1; +const lib = await import(alias2);""", + "lib", + "axios", + True, + ), + ( + "Variable chain with mixed let/const/var", + """var pkgName = 'moment'; +let temp = pkgName; +const final = temp; +const lib = await import(final);""", + "lib", + "moment", + True, + ), + # ======================================================================== + # Negative cases - should NOT match + # ======================================================================== + ( + "Dynamic import with wrong package name", + """const lodash = await import('lodash');""", + "lodash", + "express", + False, + ), + ( + "Dynamic import with wrong identifier", + """const lodash = await import('lodash');""", + "wrongIdentifier", + "lodash", + False, + ), + ( + "Dynamic import in comment - should not match", + """// const lodash = await import('lodash'); +const other = require('express');""", + "lodash", + "lodash", + False, + ), + ( + "Dynamic import in string literal - should not match", + """const code = "const lodash = await import('lodash');";""", + "lodash", + "lodash", + False, + ), + # ======================================================================== + # Variable tracing - non-traceable cases + # ======================================================================== + ( + "Dynamic import with environment variable (not traceable)", + """const module = await import(process.env.MODULE_PATH);""", + "module", + "lodash", + False, + ), + ( + "Dynamic import with config object property (not traceable)", + """const mod = await import(config.modulePath);""", + "mod", + "lodash", + False, + ), + ( + "Dynamic import with function call (not traceable)", + """const utils = await import(getModulePath());""", + "utils", + "lodash", + False, + ), +# ( +# "Dynamic import with computed expression (not traceable)", +# """const prefix = 'lod'; +# const lib = await import(prefix + 'ash');""", +# "lib", +# "lodash", +# False, +# ), + ( + "Variable not found in code (not traceable)", + """const lib = await import(unknownVariable);""", + "lib", + "lodash", + False, + ), + ( + "Variable chain that ends in non-traceable expression", + """const base = getPackageName(); +const pkg = base; +const lib = await import(pkg);""", + "lib", + "lodash", + False, + ), + # ======================================================================== + # Mixed static and dynamic imports + # ======================================================================== + ( + "Both static and dynamic imports - check dynamic", + """import { staticFunc } from 'other-package'; +const lodash = await import('lodash');""", + "lodash", + "lodash", + True, + ), + ( + "Both static and dynamic imports - check static", + """import { template } from 'lodash'; +const express = await import('express');""", + "template", + "lodash", + True, + ), + # ======================================================================== + # Edge cases + # ======================================================================== + ( + "Dynamic import with whitespace variations", + """const lodash = await import( 'lodash' );""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import without assignment", + """(await import('lodash')).template();""", + "", + "lodash", + True, + ), + ( + "Multiple dynamic imports - check first", + """const lodash = await import('lodash'); +const express = await import('express');""", + "lodash", + "lodash", + True, + ), + ( + "Multiple dynamic imports - check second", + """const lodash = await import('lodash'); +const express = await import('express');""", + "express", + "express", + True, + ), + ( + "Dynamic import in async function", + """async function loadModule() { + const { template } = await import('lodash'); + return template; +}""", + "template", + "lodash", + True, + ), + ( + "Dynamic import with Promise.then and destructuring", + """import('lodash').then(({ template }) => template());""", + "template", + "lodash", + True, + ), +]) +def test_is_package_imported_dynamic_imports(parser, description, code_content, identifier, callee_package, expected): + """Test is_package_imported with dynamic import() syntax including variable tracing.""" + result = parser.is_package_imported(code_content, identifier, callee_package) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +# ============================================================================ +# Tests for is_package_imported - Multi-function imports and word boundary edge cases +# ============================================================================ + +@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ + # ======================================================================== + # Multi-function imports - should match exact identifiers + # ======================================================================== + ( + "Multi-function import - exact match for function1", + """import { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Multi-function import - exact match for function2", + """import { function1, function2 } from "./utils.js";""", + "function2", + "./utils.js", + True, + ), + # ======================================================================== + # Substring edge cases - should NOT match partial identifiers + # ======================================================================== + ( + "Multi-function import - 'func' should NOT match 'function1'", + """import { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), + ( + "Multi-function import - 'add' should NOT match 'addHandler'", + """import { addHandler, removeHandler } from "./utils.js";""", + "add", + "./utils.js", + False, + ), + ( + "Single function import - 'parse' should NOT match 'parseAsync'", + """import { parseAsync } from "some-lib";""", + "parse", + "some-lib", + False, + ), + # ======================================================================== + # CommonJS require - substring edge cases + # ======================================================================== + ( + "CommonJS destructured require - exact match", + """const { function1, function2 } = require('./utils.js');""", + "function1", + "./utils.js", + True, + ), + ( + "CommonJS destructured require - 'func' should NOT match 'function1'", + """const { function1, function2 } = require('./utils.js');""", + "func", + "./utils.js", + False, + ), + # ======================================================================== + # Re-exports - substring edge cases + # ======================================================================== + ( + "Re-export - exact match", + """export { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Re-export - 'func' should NOT match 'function1'", + """export { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), +]) +def test_is_package_imported_word_boundaries(parser, description, code_content, identifier, callee_package, expected): + """Test is_package_imported with word boundary edge cases to prevent substring false positives.""" + result = parser.is_package_imported(code_content, identifier, callee_package) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("function_content,expected_name", [ + # Object methods now use //(class: name) pattern + ( + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + "apiUtils" + ), + ( + "format: function(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + "helpers" + ), + # Standalone function (no annotation) + ( + "function regularFunc() {\n return true;\n}", + None + ), +]) +def test_get_class_name_for_object_methods(parser, function_content, expected_name): + """Test that object methods use the same //(class: name) pattern as classes.""" + doc = Document( + page_content=function_content, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_class_name_from_class_function(doc) + assert result == expected_name + + +@pytest.mark.parametrize("description,function_content,file_content,expected", [ + ( + "Object method should be exported when object is in module.exports", + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + """const apiUtils = { + fetchData(url) { + return fetch(url); + } +}; + +module.exports = { apiUtils }; +""", + True, + ), + ( + "Object method should be exported with ES6 named export", + "format(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + """const helpers = { + format(data) { + return JSON.stringify(data); + } +}; + +export { helpers }; +""", + True, + ), + ( + "Object method should NOT be exported when object is not exported", + "internalMethod() {\n return 'internal';\n}\n//(class: privateUtils)", + """const privateUtils = { + internalMethod() { + return 'internal'; + } +}; + +// No exports +""", + False, + ), +]) +def test_is_exported_function_for_objects(parser, description, function_content, file_content, expected): + """Test is_exported_function correctly handles object methods with //(class: name) annotation.""" + source = "node_modules/test-pkg/index.js" + func_doc = Document( + page_content=function_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + full_file_doc = Document( + page_content=file_content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + documents_of_full_sources = {source: full_file_doc} + + result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert result == expected, f"{description}: expected {expected}, got {result}" + From 4145849acea25dbd5b0a5106095bc3b6019ed245 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 5 Feb 2026 17:47:19 +0200 Subject: [PATCH 185/286] chore: fine tune dashboards to client API Signed-off-by: Ilona Shishov --- .../mlops/grafana/dashboards/batch-info.yaml | 9 ++++-- .../grafana/dashboards/batch-metrics.yaml | 28 +++++++++---------- .../mlops/grafana/dashboards/evals.yaml | 22 +++++++-------- .../mlops/grafana/dashboards/jobs.yaml | 10 +++---- .../mlops/grafana/datasources/infinity.yaml | 2 +- .../mlops/grafana/folders/batches.yaml | 5 +--- .../overlays/mlops/grafana/folders/evals.yaml | 6 +--- .../overlays/mlops/grafana/folders/jobs.yaml | 6 +--- .../grafana/instances/serviceAccount.yaml | 2 +- 9 files changed, 41 insertions(+), 49 deletions(-) diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml index dde666c82..dba082859 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml @@ -52,7 +52,7 @@ spec: "format": "table", "parser": "backend", "refId": "Component", - "root_selector": "$distinct($.language)", + "root_selector": "$distinct($.app_language)", "source": "url", "type": "json", "url": "/batch/all", @@ -83,6 +83,7 @@ spec: "id": 1, "targets": [ { + "parser": "backend", "datasource": { "uid": "infinity" }, @@ -90,7 +91,7 @@ spec: "refId": "by_batch_id", "source": "url", "type": "json", - "url": "/batch/${batch_id:percentencode}", + "url": "/batch/${batch_id}", "url_options": { "method": "GET" } @@ -112,6 +113,8 @@ spec: "id": 2, "targets": [ { + "parser": "backend", + "filterExpression": "execution_start_timestamp==\"${execution_start}\"", "datasource": { "uid": "infinity" }, @@ -119,7 +122,7 @@ spec: "refId": "by_execution_start", "source": "url", "type": "json", - "url": "/batch/all?execution_start=${execution_start}", + "url": "/batch/all", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml index 9f1b8b490..e2b881118 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml @@ -28,11 +28,11 @@ spec: "options": { "indexByName": { "batch_id": 0, - "language": 1, - "accuracy": 2, - "f1_score": 3, - "precision": 4, - "recall": 5 + "app_language": 1, + "confusion_matrix_accuracy": 2, + "confusion_matrix_f1_score": 3, + "confusion_matrix_precision": 4, + "confusion_matrix_recall": 5 } } } @@ -50,7 +50,7 @@ spec: "url_options": { "method": "GET" }, - "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"language\": $v.language, \"accuracy\": $v.accuracy, \"precision\": $v.precision, \"f1_score\": $v.f1_score, \"recall\": $v.recall} })", + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"language\": $v.app_language, \"accuracy\": $v.confusion_matrix_accuracy, \"precision\": $v.confusion_matrix_precision, \"f1_score\": $v.confusion_matrix_f1_score, \"recall\": $v.confusion_matrix_recall} })", "parser": "backend" } ], @@ -74,7 +74,7 @@ spec: "options": { "indexByName": { "batch_id": 0, - "regressive_job_count": 1, + "number_of_regressive_jobs_ids": 1, "regression_ratio": 2 } } @@ -93,7 +93,7 @@ spec: "url_options": { "method": "GET" }, - "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"regressive_job_count\": $v.regressive_job_count, \"regression_ratio\": $v.total_jobs_executed != 0 ? $v.regressive_job_count / $v.total_jobs_executed : null} })", + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"number_of_regressive_jobs_ids\": $v.number_of_regressive_jobs_ids, \"regression_ratio\": $v.total_number_of_executed_jobs != 0 ? $v.number_of_regressive_jobs_ids / $v.total_number_of_executed_jobs : null} })", "parser": "backend" } ], @@ -116,8 +116,8 @@ spec: "id": "organize", "options": { "indexByName": { - "language": 0, - "regressive_job_count": 1, + "app_language": 0, + "number_of_regressive_jobs_ids": 1, "regression_ratio": 2 } } @@ -136,7 +136,7 @@ spec: "url_options": { "method": "GET" }, - "root_selector": "$map($distinct($.language), function($lang) { {\"language\": $lang, \"regressive_job_count\": $sum($filter($, function($v) { $v.language = $lang }).regressive_job_count), \"regression_ratio\": $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) != 0 ? $sum($filter($, function($v) { $v.language = $lang }).regressive_job_count) / $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) : null} })", + "root_selector": "$map($distinct($.app_language), function($lang) { {\"app_language\": $lang, \"number_of_regressive_jobs_ids\": $sum($filter($, function($v) { $v.app_language = $lang }).number_of_regressive_jobs_ids), \"regression_ratio\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) != 0 ? $sum($filter($, function($v) { $v.app_language = $lang }).number_of_regressive_jobs_ids) / $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) : null} })", "parser": "backend" } ], @@ -159,8 +159,8 @@ spec: "id": "organize", "options": { "indexByName": { - "language": 0, - "failure_count": 1, + "app_language": 0, + "total_number_of_failures": 1, "failure_ratio": 2 } } @@ -179,7 +179,7 @@ spec: "url_options": { "method": "GET" }, - "root_selector": "$map($distinct($.language), function($lang) { {\"language\": $lang, \"failure_count\": $sum($filter($, function($v) { $v.language = $lang }).failure_count), \"failed_ratio\": $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) != 0 ? $sum($filter($, function($v) { $v.language = $lang }).failure_count) / $sum($filter($, function($v) { $v.language = $lang }).total_jobs_executed) : null} })", + "root_selector": "$map($distinct($.app_language), function($lang) { {\"app_language\": $lang, \"total_number_of_failures\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_failures), \"failed_ratio\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) != 0 ? $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_failures) / $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) : null} })", "parser": "backend" } ], diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml index b223ca29e..3817644fc 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -14,7 +14,7 @@ spec: "list": [ { "definition": "Job request/trace id", - "name": "request_id", + "name": "job_id", "query": { "infinityQuery": { "columns": [], @@ -22,10 +22,10 @@ spec: "format": "table", "parser": "backend", "refId": "Request ID", - "root_selector": "$.request_id", + "root_selector": "$.job_id", "source": "url", "type": "json", - "url": "/eval/all", + "url": "/evals/all", "url_options": { "method": "GET" } @@ -50,7 +50,7 @@ spec: "root_selector": "$distinct($.component)", "source": "url", "type": "json", - "url": "/eval/all", + "url": "/evals/all", "url_options": { "method": "GET" } @@ -75,7 +75,7 @@ spec: "root_selector": "$distinct($.component_version)", "source": "url", "type": "json", - "url": "/eval/all", + "url": "/evals/all", "url_options": { "method": "GET" } @@ -100,7 +100,7 @@ spec: "root_selector": "$distinct($.cve)", "source": "url", "type": "json", - "url": "/eval/all", + "url": "/evals/all", "url_options": { "method": "GET" } @@ -125,7 +125,7 @@ spec: "root_selector": "$distinct($.metric_name)", "source": "url", "type": "json", - "url": "/eval/all", + "url": "/evals/all", "url_options": { "method": "GET" } @@ -157,10 +157,10 @@ spec: "uid": "infinity" }, "format": "table", - "refId": "by_request_id", + "refId": "by_job_id", "source": "url", "type": "json", - "url": "/eval/all?request_id=${request_id:percentencode}", + "url": "/evals/all?job_id=${job_id}", "url_options": { "method": "GET" } @@ -189,7 +189,7 @@ spec: "refId": "by_cve_component", "source": "url", "type": "json", - "url": "/eval/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url": "/evals/${cve}?component=${component}&componentVersion=${component_version}", "url_options": { "method": "GET" } @@ -218,7 +218,7 @@ spec: "refId": "by_cve_component_metric", "source": "url", "type": "json", - "url": "/eval/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}&metric_name=${metric_name:percentencode}", + "url": "/evals/${cve}?component=${component}&componentVersion=${component_version}&metric_name=${metric_name}", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml index 639a53c6b..521b750d7 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -39,7 +39,7 @@ spec: }, { "definition": "Job request/trace id", - "name": "request_id", + "name": "job_id", "query": { "infinityQuery": { "columns": [], @@ -47,7 +47,7 @@ spec: "format": "table", "parser": "backend", "refId": "Request ID", - "root_selector": "$.request_id", + "root_selector": "$.job_id", "source": "url", "type": "json", "url": "/jobs/all", @@ -160,7 +160,7 @@ spec: "refId": "by_request_id", "source": "url", "type": "json", - "url": "/jobs/${request_id:percentencode}", + "url": "/jobs/${job_id}", "url_options": { "method": "GET" } @@ -189,7 +189,7 @@ spec: "refId": "by_batch_id", "source": "url", "type": "json", - "url": "/jobs/all?batch_id=${batch_id:percentencode}", + "url": "/jobs/all?batchId=${batch_id}", "url_options": { "method": "GET" } @@ -218,7 +218,7 @@ spec: "refId": "by_cve_component", "source": "url", "type": "json", - "url": "/jobs/all?cve=${cve}&component=${component:percentencode}&component_version=${component_version:percentencode}", + "url": "/jobs/all?cveId=${cve}&component=${component:percentencode}&componentVersion=${component_version:percentencode}", "url_options": { "method": "GET" } diff --git a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml index d1aa1d3af..7af6b9ca1 100644 --- a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml +++ b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml @@ -20,7 +20,7 @@ spec: datasource: access: proxy isDefault: true - url: https://exploit-iq-client..svc:8443 + url: https://exploit-iq-client..svc:8443/api/v1 jsonData: httpHeaderName1: "Authorization" tlsSkipVerify: false diff --git a/kustomize/overlays/mlops/grafana/folders/batches.yaml b/kustomize/overlays/mlops/grafana/folders/batches.yaml index 1f198b6d7..f84769e63 100644 --- a/kustomize/overlays/mlops/grafana/folders/batches.yaml +++ b/kustomize/overlays/mlops/grafana/folders/batches.yaml @@ -1,5 +1,3 @@ -# GrafanaFolder Custom Resource -# Organizes dashboards into logical folders in Grafana apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaFolder metadata: @@ -8,5 +6,4 @@ spec: instanceSelector: matchLabels: app: exploit-iq - title: "Batches" - + title: "Batches" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/evals.yaml b/kustomize/overlays/mlops/grafana/folders/evals.yaml index 0c1f3f709..494c2b285 100644 --- a/kustomize/overlays/mlops/grafana/folders/evals.yaml +++ b/kustomize/overlays/mlops/grafana/folders/evals.yaml @@ -1,13 +1,9 @@ -# GrafanaFolder Custom Resource -# Organizes dashboards into logical folders in Grafana apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaFolder metadata: name: evals spec: - # TODO: Configure your Grafana instance selector instanceSelector: matchLabels: app: exploit-iq - title: "Evals" - + title: "Evals" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/jobs.yaml b/kustomize/overlays/mlops/grafana/folders/jobs.yaml index 5ead94024..37fcf4aba 100644 --- a/kustomize/overlays/mlops/grafana/folders/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/folders/jobs.yaml @@ -1,13 +1,9 @@ -# GrafanaFolder Custom Resource -# Organizes dashboards into logical folders in Grafana apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaFolder metadata: name: jobs spec: - # TODO: Configure your Grafana instance selector instanceSelector: matchLabels: app: exploit-iq - title: "Jobs" - + title: "Jobs" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml index 27be895d0..68932c7d6 100644 --- a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml +++ b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml @@ -3,4 +3,4 @@ kind: ServiceAccount metadata: name: grafana-sa annotations: - serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"exploit-iq-grafana"}}' \ No newline at end of file + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"exploit-iq-grafana-route"}}' \ No newline at end of file From 4a08dd9925d8fc612e02c7449034bb81c1f68587 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Thu, 5 Feb 2026 18:03:33 +0200 Subject: [PATCH 186/286] chore: kustomize SA name in grafana instance Signed-off-by: Ilona Shishov --- .../overlays/mlops/grafana/instances/grafana.yaml | 2 +- .../overlays/mlops/grafana/kustomization.yaml | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml index 5f3b0a49e..1e498c2fd 100644 --- a/kustomize/overlays/mlops/grafana/instances/grafana.yaml +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -19,12 +19,12 @@ spec: spec: containers: - args: + - '-openshift-service-account=grafana-sa' - '-provider=openshift' - '-https-address=:9091' - '-tls-cert=/etc/proxy/tls/tls.crt' - '-tls-key=/etc/proxy/tls/tls.key' - '-upstream=http://localhost:3000' - - '-openshift-service-account=exploit-iq-grafana-sa' - '-cookie-secret=/etc/proxy/secrets/session-secret' image: 'quay.io/openshift/origin-oauth-proxy:latest' name: grafana-oauth-proxy diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 38d1fdbd9..d57ed84df 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -45,4 +45,17 @@ replacements: kind: OperatorGroup name: grafana-operators fieldPaths: - - spec.targetNamespaces.0 \ No newline at end of file + - spec.targetNamespaces.0 + - source: + kind: ServiceAccount + name: grafana-sa + fieldPath: metadata.name + targets: + - select: + kind: Grafana + name: grafana + fieldPaths: + - spec.deployment.spec.template.spec.containers.[name=grafana-oauth-proxy].args.0 # update -openshift-service-account argument with the new service account name + options: + delimiter: "=" + index: 1 \ No newline at end of file From a2d0acbbdf514caa2498ddea03d2252f3cd04f38 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 8 Feb 2026 11:32:41 +0200 Subject: [PATCH 187/286] chore: add workaround to route name replacement in grafana SA Signed-off-by: Ilona Shishov --- .../grafana/instances/serviceAccount.yaml | 2 +- .../overlays/mlops/grafana/kustomization.yaml | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml index 68932c7d6..17f2eb7c2 100644 --- a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml +++ b/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml @@ -3,4 +3,4 @@ kind: ServiceAccount metadata: name: grafana-sa annotations: - serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"exploit-iq-grafana-route"}}' \ No newline at end of file + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"REPLACE_ROUTE"}}' \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index d57ed84df..5505891fc 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -21,6 +21,11 @@ labels: namePrefix: exploit-iq- +configMapGenerator: + - name: replacement-values + literals: + - route-suffix=route"}} + replacements: - source: kind: Grafana @@ -58,4 +63,30 @@ replacements: - spec.deployment.spec.template.spec.containers.[name=grafana-oauth-proxy].args.0 # update -openshift-service-account argument with the new service account name options: delimiter: "=" - index: 1 \ No newline at end of file + index: 1 + - source: + kind: ServiceAccount + name: grafana-sa + fieldPath: metadata.name + targets: + - select: + kind: ServiceAccount + name: grafana-sa + fieldPaths: + - metadata.annotations.serviceaccounts\.openshift\.io/oauth-redirectreference\.primary + options: + delimiter: '"name":"' + index: 1 + - source: + kind: ConfigMap + name: replacement-values + fieldPath: data.route-suffix + targets: + - select: + kind: ServiceAccount + name: grafana-sa + fieldPaths: + - metadata.annotations.serviceaccounts\.openshift\.io/oauth-redirectreference\.primary + options: + delimiter: '-' + index: 3 \ No newline at end of file From 738b34c272e5353d63e571d3c8bc857eeffb5e0e Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Sun, 8 Feb 2026 16:47:06 +0200 Subject: [PATCH 188/286] APPENG-4285 - Intercept Tracing Context in ExploitIQ Agent and save to DB (#198) * Intercept Tracing Context in ExploitIQ Agent and save to DB - Added tracing context interception - Persisted context data to database - Updated confusion matrix and Tekton run handling --- .tekton/on-cm-runner.yaml | 61 +++- .tekton/on-pull-request.yaml | 65 ++++- ci/scripts/HttpProvider.py | 169 +++++++++++ ci/scripts/collect_and_dispatch_traces.py | 262 ++++++++++++++++++ docker-compose.mlops.yml | 28 ++ kustomize/base/exploit-iq-config.yml | 2 +- kustomize/config-http-openai-local.yml | 2 +- otel-config.yaml | 28 ++ src/vuln_analysis/configs/config-http-nim.yml | 2 +- .../configs/config-http-openai.yml | 2 +- src/vuln_analysis/configs/config-tracing.yml | 2 +- src/vuln_analysis/configs/config.yml | 2 +- 12 files changed, 613 insertions(+), 12 deletions(-) create mode 100644 ci/scripts/HttpProvider.py create mode 100755 ci/scripts/collect_and_dispatch_traces.py create mode 100644 docker-compose.mlops.yml create mode 100644 otel-config.yaml diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 066f1e96d..9578618c0 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -100,6 +100,15 @@ spec: - name: google-creds-volume secret: secretName: google-sheets-secrets + - name: otel-config-volume + configMap: + name: otel-collector-config + - name: openshift-service-ca + configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: service-ca.crt # Mounts as a file named service-ca.crt # >>> THE SERVER (Sidecar) <<< sidecars: @@ -123,6 +132,9 @@ spec: # Inject Env Vars (API Keys, Models, RedHat Creds) - name: $(workspaces.exploit-iq-data.volume) mountPath: /exploit-iq-data + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true envFrom: - secretRef: name: integration-tests @@ -164,13 +176,17 @@ spec: value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa - name: DATA_DIR value: $(workspaces.exploit-iq-data.path) + - name: OTEL_SERVICE_NAME + value: cve_agent + - name: OTEL_TRACES_ENDPOINT + value: http://localhost:4318/v1/traces script: | #!/bin/sh set -e - # We use python to safely remove the 'telemetry' block + # We use python to safely update config echo "--- Generating Config without Tracing ---" - python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" FETCHER_SCRIPT="ci/scripts/config_fetcher.py" # [CHANGE] Logic: Check if the comment contains "config=sheet" @@ -234,7 +250,43 @@ spec: port: 26466 initialDelaySeconds: 10 periodSeconds: 5 - + - name: redpanda + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + script: | + #!/bin/bash + set -e + exec rpk redpanda start --smp=1 --memory=1Gi --overprovisioned --kafka-addr=internal://0.0.0.0:29092,external://0.0.0.0:9092 --advertise-kafka-addr=internal://localhost:29092,external://localhost:9092 + + resources: + requests: + memory: "1Gi" + cpu: "500m" + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 5 + + - name: otel-collector + image: otel/opentelemetry-collector-contrib:latest + # Mount your otel-config.yaml here via a ConfigMap + volumeMounts: + - name: otel-config-volume + mountPath: /etc/otelcol-contrib/ + - name: trace-collector + image: registry.access.redhat.com/ubi9/python-312:9.6 + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -e + pip install --no-cache-dir kafka-python pydantic requests + echo "--- STARTING TRACE COLLECTOR ---" + python3 ci/scripts/collect_and_dispatch_traces.py + echo "--- DEBUG: Current Directory ---" + pwd # >>> THE CLIENT (Test Runner) <<< steps: @@ -307,6 +359,9 @@ spec: - name: google-creds-volume mountPath: /etc/secrets/google readOnly: true + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true env: - name: GOOGLE_SERVICE_ACCOUNT_FILE diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index f15b2cd6c..2f6147120 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -271,6 +271,16 @@ spec: - name: source - name: basic-auth - name: exploit-iq-data + volumes: + - name: otel-config-volume + configMap: + name: otel-collector-config + - name: openshift-service-ca + configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: service-ca.crt # Mounts as a file named service-ca.crt # >>> THE SERVER (Sidecar) <<< sidecars: @@ -283,6 +293,9 @@ spec: # Inject Env Vars (API Keys, Models, RedHat Creds) - name: $(workspaces.exploit-iq-data.volume) mountPath: /exploit-iq-data + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true envFrom: - secretRef: name: integration-tests @@ -315,16 +328,20 @@ spec: value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa - name: DATA_DIR value: $(workspaces.exploit-iq-data.path) + - name: OTEL_SERVICE_NAME + value: cve_agent + - name: OTEL_TRACES_ENDPOINT + value: http://localhost:4318/v1/traces script: | #!/bin/sh set -e # We use python to: - #1. safely remove the 'telemetry' block. + #1. safely update config #2. removing the cve_output_config_name function from the agent flow ( no need for callback, result will be returned to IT process which calls the agent. #3. Making sure that Internet search tool is enabled ( part of the tests are relying on that). echo "--- Generating Config without Tracing ---" - python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c.get('general', {}).pop('telemetry', None); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = True;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = True;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" CACHE_DIR_TARGET=".cache/am_cache" CACHE_PVC_SOURCE="/exploit-iq-data" @@ -373,7 +390,43 @@ spec: port: 26466 initialDelaySeconds: 10 periodSeconds: 5 + - name: redpanda + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + script: | + #!/bin/bash + set -e + exec rpk redpanda start --smp=1 --memory=1Gi --overprovisioned --kafka-addr=internal://0.0.0.0:29092,external://0.0.0.0:9092 --advertise-kafka-addr=internal://localhost:29092,external://localhost:9092 + resources: + requests: + memory: "1Gi" + cpu: "500m" + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 5 + + - name: otel-collector + image: otel/opentelemetry-collector-contrib:latest + # Mount your otel-config.yaml here via a ConfigMap + volumeMounts: + - name: otel-config-volume + mountPath: /etc/otelcol-contrib/ + - name: trace-collector + image: registry.access.redhat.com/ubi9/python-312:9.6 + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -e + pip install --no-cache-dir kafka-python pydantic requests + echo "--- STARTING TRACE COLLECTOR ---" + python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify true + echo "--- DEBUG: Current Directory ---" + pwd # >>> THE CLIENT (Test Runner) <<< steps: # ------------------------------------------------------- @@ -382,6 +435,10 @@ spec: - name: run-test-suite image: quay.io/ecosystem-appeng/auto-cm-testing:latest imagePullPolicy: Always + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true workingDir: $(workspaces.source.path) env: - name: DATA_DIR @@ -393,7 +450,8 @@ spec: echo "--- STARTING INTEGRATION TESTS ---" echo "--- DEBUG: Current Directory ---" pwd - + echo " -- check it certificate is mounted" + ls -l /app/certs/service-ca.crt # 2. Prepares the IT input for test cp -f ci/it/integration-tests-input.json /app/src/input/scan_it.json cd /app/ @@ -409,6 +467,7 @@ spec: echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" + workspaces: - name: source volumeClaimTemplate: diff --git a/ci/scripts/HttpProvider.py b/ci/scripts/HttpProvider.py new file mode 100644 index 000000000..2a9b2370e --- /dev/null +++ b/ci/scripts/HttpProvider.py @@ -0,0 +1,169 @@ +""" +HTTP provider for dispatching payloads to a configurable endpoint with optional Bearer auth. +""" +import json +import time +import typing +from enum import Enum +from http import HTTPStatus +from pathlib import Path + +import requests +from urllib3.util.url import parse_url + +# Default dispatch endpoint configuration +DEFAULT_URL = "https://exploit-iq-client.exploit-iq-tests.svc.cluster.local:8443" +DEFAULT_AUTH_TYPE = "Bearer" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False + + +class AuthType(str, Enum): + BEARER = "bearer" + NONE = "none" + + +class HttpProvider: + """ + Sends HTTP POST requests to a configurable endpoint with optional Bearer token auth + and retry logic. + """ + + def __init__( + self, + url: str = DEFAULT_URL, + auth_type: AuthType = AuthType.NONE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, + ): + self.url = self._prepare_url(url) + self.auth_type = auth_type + self.token_path = token_path + self.verify_path = verify_path + self.enable_verify = enable_verify + self.session = requests.Session() + self.headers = self._build_headers() + + def send_post( + self, + json_data: str, + endpoint_url: str | None = None, + ) -> HTTPStatus: + """ + Send a POST request with JSON body to the endpoint. + + Args: + json_data: JSON string body to send. + endpoint_url: Override URL for this request; if None, uses the instance URL. + + Returns: + HTTP status code of the response, or INTERNAL_SERVER_ERROR on failure. + """ + target_url = endpoint_url if endpoint_url is not None else self.url + if self.enable_verify: + verify = self.verify_path + else: + verify = False + try: + _, response = self._request_with_retry( + request_kwargs={ + "url": target_url, + "method": "POST", + "data": json_data.encode("utf-8"), + "headers": self.headers, + "verify": verify, + }, + requests_session=self.session, + accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED), + ) + return HTTPStatus(response.status_code) + except Exception as e: + print("Unable to send output response to %s. Error: %s", target_url, e) + return HTTPStatus.INTERNAL_SERVER_ERROR + + def _build_headers(self) -> dict[str, str]: + """Build the headers for the request.""" + if self.auth_type == AuthType.BEARER: + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self._get_token()}", + "Bypass-Referential-Integrity-Check": "true", + } + return {"Content-Type": "application/json", "Bypass-Referential-Integrity-Check": "true"} + + def _get_token(self) -> str: + """Read Bearer token from token_path.""" + with open(self.token_path, encoding="utf-8") as f: + return f.read().strip() + + def _prepare_url(self, url: str) -> str: + """ + Verifies that `url` contains a protocol scheme and a host and returns the url. + If no protocol scheme is provided, `http` is used. + """ + parsed = parse_url(url) + if parsed.scheme is None or parsed.host is None: + url = f"http://{url}" + parsed = parse_url(url) + if parsed.scheme is None or parsed.host is None: + raise ValueError(f"Invalid URL: {url}") + return url + + def _request_with_retry( + self, + request_kwargs: dict, + requests_session: requests.Session | None = None, + max_retries: int = 10, + sleep_time: float = 0.1, + accept_status_codes: typing.Iterable[int] = (HTTPStatus.OK,), + respect_retry_after_header: bool = True, + on_success_fn: typing.Callable[[requests.Response], typing.Any] | None = None, + ) -> tuple[requests.Session, requests.Response | typing.Any]: + """ + Wrapper around requests.request that retries on failure. + """ + try_count = 0 + codes = tuple(accept_status_codes) + while try_count <= max_retries: + if requests_session is None: + requests_session = requests.Session() + retry_after_header: int | None = None + try: + response = requests_session.request(**request_kwargs) + if response.status_code in codes: + if on_success_fn is not None: + return (requests_session, on_success_fn(response)) + return (requests_session, response) + if respect_retry_after_header and "Retry-After" in response.headers: + retry_after_header = int(response.headers["Retry-After"]) + raise RuntimeError( + f"Received unexpected status code {response.status_code}: {response.text}" + ) + except Exception as e: + if isinstance(e, requests.exceptions.RequestException): + try: + requests_session.close() + finally: + requests_session = None + try_count += 1 + if try_count > max_retries: + print("Failed after %s retries: %s", max_retries, e) + raise + actual_sleep = ( + retry_after_header + if retry_after_header is not None + else (2 ** (try_count - 1)) * sleep_time + ) + print( + "Error performing %s to %s: %s", + request_kwargs.get("method"), + request_kwargs.get("url"), + e, + ) + print("Sleeping %s seconds before retry", actual_sleep) + time.sleep(actual_sleep) + raise RuntimeError("Unreachable") + + diff --git a/ci/scripts/collect_and_dispatch_traces.py b/ci/scripts/collect_and_dispatch_traces.py new file mode 100755 index 000000000..b26d0d980 --- /dev/null +++ b/ci/scripts/collect_and_dispatch_traces.py @@ -0,0 +1,262 @@ +from kafka import KafkaConsumer +import argparse +import json +from HttpProvider import HttpProvider, AuthType +from http import HTTPStatus +from datetime import datetime +from typing import Any +from pydantic import BaseModel, Field, constr, RootModel +import signal +import time +from kafka.errors import NoBrokersAvailable +# Define trace class +TRACE_VERSION = 1 + +# Default dispatch endpoint configuration +DEFAULT_URL = "https://exploit-iq-client.exploit-iq-tests.svc.cluster.local:8443" +DEFAULT_AUTH_TYPE = "Bearer" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False + + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + +class Trace(BaseModel): + format_version: int | None = None + job_id: constr(min_length=1) = Field( + ..., description='The job run id in which the trace and spans were instrumented' + ) + execution_start_timestamp: LocalDateTime | None = None + trace_id: constr(min_length=1) = Field( + ..., + description='The id corresponding to one agent stage, grouping underneath all spans of sub-tasks embedded in the agent stage', + ) + span_id: constr(min_length=1) = Field( + ..., description='The id corresponding to one sub task inside one agent stage.' + ) + span_payload: dict[str, Any] | None = Field( + None, + description='free style span payload without restrictions, containing all span metadata and istrumentation data', + ) + +# wrapper around jobs +class TracesContainer(RootModel[list[Trace]]): + pass + +# --- CONFIGURATION --- +KAFKA_TOPIC = 'otel-traces' +KAFKA_BROKER = 'localhost:9092' +OUTPUT_FILE = 'cve_research_log.jsonl' + +def truncate_value(value, max_length=100): + """Recursively truncate string values in OTLP attribute value objects.""" + if isinstance(value, dict): + # Handle OTLP value objects like {"stringValue": "...", "intValue": 123} + truncated = {} + for key, val in value.items(): + if key.endswith('Value') and isinstance(val, str): + # Truncate string values + truncated[key] = val[:max_length] if len(val) > max_length else val + else: + truncated[key] = truncate_value(val, max_length) + return truncated + elif isinstance(value, str): + # Direct string value + return value[:max_length] if len(value) > max_length else value + elif isinstance(value, list): + # Handle lists + return [truncate_value(item, max_length) for item in value] + else: + # Numbers, booleans, None - return as-is + return value + + +def get_metadata(span: dict) -> dict: + return { + 'trace_id': span.get('traceId'), + 'span_id': span.get('spanId'), + 'parent_span_id': span.get('parentSpanId'), + 'name': span.get('name'), + 'start_time': span.get('startTimeUnixNano'), + 'end_time': span.get('endTimeUnixNano'), + 'status': span.get('status'), + } + +def find_job_id(attributes: dict) -> str: + try: + input_value = attributes.get("input.value") + str_json = input_value.get("stringValue") + scan = json.loads(str_json) + if "scan" in scan: + return scan["scan"]["id"] + if "input" in scan: + return scan["input"]["scan"]["id"] + else: + print(f"Warning: Scan not found in attributes:") + return "" + except Exception as e: + print(f"Error finding job id (scan id): {e}") + print(f"Attributes: {attributes}") + return "" + +def process_message(http_provider, traces_table, map_trace_id_to_job_id, data): + # Traverse the OTLP structure + for rs in data.get('resourceSpans', []): + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + metadata = get_metadata(span) + attrs = { + a['key']: a.get('value', {}) + for a in span.get('attributes', []) + } + trace_id = metadata.get('trace_id') + if trace_id not in traces_table: + print("add trace_id to traces_table: ", trace_id) + traces_table[trace_id] = [] + if trace_id in map_trace_id_to_job_id: + job_id = map_trace_id_to_job_id[trace_id] + else: + job_id = find_job_id(attrs) + if len(job_id) > 0: + trace_collection = traces_table.get(trace_id, []) + if trace_id not in map_trace_id_to_job_id: + map_trace_id_to_job_id[trace_id] = job_id + print(f"add trace_id to map_trace_id_to_job_id: {trace_id} --> {job_id}") + print(map_trace_id_to_job_id) + dict_payload = {} + dict_payload["metadata"] = metadata + dict_payload["attributes"] = attrs + trace_obj = Trace( + format_version=TRACE_VERSION, + job_id=job_id, + execution_start_timestamp=LocalDateTime(datetime.fromtimestamp(int(metadata.get('start_time',"0"))/1e9)), + trace_id=metadata.get('trace_id'), + span_id=metadata.get('span_id'), + span_payload=dict_payload + ) + trace_collection.append(trace_obj) + traces_table[trace_id] = trace_collection + span_name = metadata.get('name') + if span_name == "": + print("finsh scan workflow- need to send mlops") + traces = traces_table.get(trace_id, []) + traces_container = TracesContainer(traces) + status = http_provider.send_post(traces_container.model_dump_json()) + print(f"send traces(job_id: {job_id}, trace_id: {trace_id}) to endpoint status: {status}") + print(traces_table.keys()) + print(("del trace_id: ", trace_id)) + del traces_table[trace_id] + + +def run_worker( + url: str = DEFAULT_URL, + auth_type: str = DEFAULT_AUTH_TYPE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, +): + # Logic to stop the loop on SIGTERM + stop_triggered = False + def signal_handler(sig, frame): + nonlocal stop_triggered + stop_triggered = True + + signal.signal(signal.SIGTERM, signal_handler) + # --- ADD THIS WAIT LOOP --- + print(f"Waiting for Kafka at {KAFKA_BROKER}...") + consumer = None + while consumer is None: + try: + consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + group_id='cve-file-writer-group', + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + # Optional: reduce metadata timeout for faster retries + request_timeout_ms=20000, + session_timeout_ms=10000 + ) + print("Successfully connected to Redpanda!") + except NoBrokersAvailable: + print("Kafka not ready yet... retrying in 2 seconds.") + time.sleep(2) + # -------------------------- + print(f"📡 Worker Active. Writing traces to: {OUTPUT_FILE}") + traces_table = {} + map_trace_id_to_job_id = {} + + #items_count = 0 + enpoint_url = url + "/api/v1/traces" + EndpointAuthType = AuthType.BEARER + if auth_type != DEFAULT_AUTH_TYPE: + EndpointAuthType = AuthType.NONE + http_provider = HttpProvider( + url=enpoint_url, + auth_type=EndpointAuthType, + token_path=token_path, + verify_path=verify_path, + enable_verify=enable_verify, + ) + + try: + with open(OUTPUT_FILE, 'a', encoding='utf-8') as f: + while not stop_triggered: + messages = consumer.poll(timeout_ms=1000) + for tp, records in messages.items(): + for message in records: + data = message.value + #f.write(f"Item: {items_count}\n") + #f.write(json.dumps(data) + '\n') + #f.flush() + process_message(http_provider, traces_table, map_trace_id_to_job_id, data) + #items_count += 1 + finally: + consumer.close() + print("\nStopping worker...") + +def parse_args(): + parser = argparse.ArgumentParser(description="Collect traces from Kafka and optionally dispatch to an endpoint.") + parser.add_argument( + "--url", + default=DEFAULT_URL, + help="Dispatch endpoint URL (default: %(default)s)", + ) + parser.add_argument( + "--auth-type", + default=DEFAULT_AUTH_TYPE, + help="Auth type for the endpoint, e.g. Bearer (default: %(default)s)", + ) + parser.add_argument( + "--token-path", + default=DEFAULT_TOKEN_PATH, + help="Path to auth token file (default: %(default)s)", + ) + parser.add_argument( + "--verify-path", + default=DEFAULT_VERIFY_PATH, + help="Path to TLS CA cert for verification (default: %(default)s)", + ) + parser.add_argument( + "--enable-verify", + action="store_true", + default=DEFAULT_ENABLE_VERIFY, + help="Enable TLS verification (default: False)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_worker( + url=args.url, + auth_type=args.auth_type, + token_path=args.token_path, + verify_path=args.verify_path, + enable_verify=args.enable_verify, + ) + + + diff --git a/docker-compose.mlops.yml b/docker-compose.mlops.yml new file mode 100644 index 000000000..5be14e73a --- /dev/null +++ b/docker-compose.mlops.yml @@ -0,0 +1,28 @@ +services: + # Phoenix - Visualization + phoenix: + image: arizephoenix/phoenix + ports: + - "6006:6006" + + # Redpanda - Lightweight Kafka replacement + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + ports: + - "9092:9092" + command: > + redpanda start --smp 1 --overprovisioned + --kafka-addr internal://0.0.0.0:29092,external://0.0.0.0:9092 + --advertise-kafka-addr internal://redpanda:29092,external://localhost:9092 + + # OTel Collector - The Traffic Controller + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: otel-collector + volumes: + - ./otel-config.yaml:/etc/otelcol-contrib/config.yaml:Z + ports: + - "4318:4318" # Agent sends data here + depends_on: + - redpanda + - phoenix \ No newline at end of file diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index f6bf85bd2..ac4835862 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -26,7 +26,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index a70657af6..1549f74f4 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -20,7 +20,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent # tracing: # langsmith: diff --git a/otel-config.yaml b/otel-config.yaml new file mode 100644 index 000000000..c4de090a8 --- /dev/null +++ b/otel-config.yaml @@ -0,0 +1,28 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + # 1. Export to Phoenix (Phoenix expects OTLP over HTTP) + otlphttp/phoenix: + endpoint: "http://phoenix:6006" + encoding: proto + + # 2. Export to Redpanda (Kafka) + kafka: + brokers: ["redpanda:29092"] # Use the internal Docker port + topic: "otel-traces" + encoding: otlp_json # This makes it MUCH easier for your Python script to read + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + # Add BOTH here to enable the fan-out + exporters: [otlphttp/phoenix, kafka] \ No newline at end of file diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index e14e0f60c..693b4f48b 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -19,7 +19,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 0a8c4c51f..d10b5a416 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -26,7 +26,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 6c064d8bd..331289e92 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -30,7 +30,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 701dcd842..7916989c1 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -19,7 +19,7 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: From 00a8ad762aee8c77765140d2527cbbf6ad401f07 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 9 Feb 2026 11:05:48 +0200 Subject: [PATCH 189/286] chore: revert accidental file changes Signed-off-by: Ilona Shishov --- kustomize/base/exploit-iq-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 0d971b888..f6bf85bd2 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -47,7 +47,7 @@ functions: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: source: Product Security research - endpoint: https://exploit-iq-client.eiq-test.svc:8443/api/v1/vulnerabilities/{vuln_id}/comments + endpoint: CALLBACK_URL_PLACEHOLDER/api/v1/vulnerabilities/{vuln_id}/comments token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt @@ -135,7 +135,7 @@ functions: # vex_format: csaf cve_http_output: _type: cve_http_output - url: https://exploit-iq-client.eiq-test.svc:8443 + url: CALLBACK_URL_PLACEHOLDER endpoint: /api/v1/reports auth_type: bearer token_path: /var/run/secrets/kubernetes.io/serviceaccount/token From 63bcbdff9a18a709418d1c6b208d4584bc7f1f56 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 9 Feb 2026 17:34:52 +0200 Subject: [PATCH 190/286] chore: add traces dashboard to grafana Signed-off-by: Ilona Shishov --- .../mlops/grafana/dashboards/evals.yaml | 4 +- .../mlops/grafana/dashboards/jobs.yaml | 6 +- .../grafana/dashboards/kustomization.yaml | 1 + .../mlops/grafana/dashboards/traces.yaml | 161 ++++++++++++++++++ .../mlops/grafana/folders/kustomization.yaml | 1 + .../mlops/grafana/folders/traces.yaml | 9 + 6 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 kustomize/overlays/mlops/grafana/dashboards/traces.yaml create mode 100644 kustomize/overlays/mlops/grafana/folders/traces.yaml diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml index 3817644fc..06ffe6fe7 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -13,7 +13,7 @@ spec: "templating": { "list": [ { - "definition": "Job request/trace id", + "definition": "Job ID", "name": "job_id", "query": { "infinityQuery": { @@ -21,7 +21,7 @@ spec: "filters": [], "format": "table", "parser": "backend", - "refId": "Request ID", + "refId": "Job ID", "root_selector": "$.job_id", "source": "url", "type": "json", diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml index 521b750d7..fe8433bd0 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -38,7 +38,7 @@ spec: "type": "query" }, { - "definition": "Job request/trace id", + "definition": "Job ID", "name": "job_id", "query": { "infinityQuery": { @@ -46,7 +46,7 @@ spec: "filters": [], "format": "table", "parser": "backend", - "refId": "Request ID", + "refId": "Job ID", "root_selector": "$.job_id", "source": "url", "type": "json", @@ -157,7 +157,7 @@ spec: "uid": "infinity" }, "format": "table", - "refId": "by_request_id", + "refId": "by_job_id", "source": "url", "type": "json", "url": "/jobs/${job_id}", diff --git a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml index d5920d2a3..76e051564 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml @@ -6,3 +6,4 @@ resources: - evals.yaml - batch-info.yaml - batch-metrics.yaml + - traces.yaml diff --git a/kustomize/overlays/mlops/grafana/dashboards/traces.yaml b/kustomize/overlays/mlops/grafana/dashboards/traces.yaml new file mode 100644 index 000000000..3630a6d7b --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/traces.yaml @@ -0,0 +1,161 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: traces +spec: + allowCrossNamespaceImport: false + folder: Traces + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Job ID", + "name": "job_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Job ID", + "root_selector": "$distinct($.job_id)", + "source": "url", + "type": "json", + "url": "/traces/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Trace ID", + "name": "trace_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Trace ID", + "root_selector": "$.trace_id", + "source": "url", + "type": "json", + "url": "/traces/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id", + "source": "url", + "type": "json", + "url": "/traces/all?jobId=${job_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Job ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_trace_id", + "source": "url", + "type": "json", + "url": "/traces/all?traceId=${trace_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Trace ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id_and_trace_id", + "source": "url", + "type": "json", + "url": "/traces/all?jobId=${job_id}&traceId=${trace_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Job ID and Trace ID", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "traces" + } diff --git a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml index 675cffd1e..8d19613f7 100644 --- a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml @@ -5,3 +5,4 @@ resources: - jobs.yaml - evals.yaml - batches.yaml + - traces.yaml \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/traces.yaml b/kustomize/overlays/mlops/grafana/folders/traces.yaml new file mode 100644 index 000000000..76cb156d3 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/traces.yaml @@ -0,0 +1,9 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: traces +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Traces" \ No newline at end of file From 07edd688aec18220b0e8fc558aad97f0116acebc Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 10 Feb 2026 09:53:24 +0200 Subject: [PATCH 191/286] docs: rename Grafana SA token placeholder Signed-off-by: Ilona Shishov --- kustomize/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 0247e58f9..ecb181d53 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -196,8 +196,8 @@ sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustom ``` ```shell -# replace EXPLOIT_IQ_SA_TOKEN with ExploitIQ SA Token from bitwarden vault (1 year expiration date) -oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_SA_TOKEN' +# replace EXPLOIT_IQ_GRAFANA_SA_TOKEN with ExploitIQ Grafana SA Token from bitwarden vault (1 year expiration date) +oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_GRAFANA_SA_TOKEN' ``` ```shell From 65d148a77b1a55e168d25801b68977c5b4175497 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 10 Feb 2026 11:53:13 +0200 Subject: [PATCH 192/286] docs: label grafana CRs to allow redeployment Signed-off-by: Ilona Shishov --- kustomize/README.md | 27 +++++++++++++++++++ .../grafana/dashboards/kustomization.yaml | 4 +++ .../grafana/datasources/kustomization.yaml | 4 +++ .../mlops/grafana/folders/kustomization.yaml | 6 ++++- .../grafana/instances/kustomization.yaml | 4 +++ 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/kustomize/README.md b/kustomize/README.md index ecb181d53..a6875523b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -206,6 +206,33 @@ oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` +#### ⚠️ Note: Grafana Resources May Fail to Deploy + +When applying the Grafana overlay, some Grafana resources **may fail to be created**. This is usually because the **Grafana Operator CSV has not yet reached the `Succeeded` phase**. + +**How to check** + +Check the Grafana Operator CSV status: + +```shell +oc get csv -n $YOUR_NAMESPACE_NAME | grep grafana-operator +``` + +Look at the PHASE column — it should show: +```text +Succeeded +``` + +**Redeploy Grafana resources** + +Once the CSV is in Succeeded state, you can safely redeploy the Grafana overlay filtered to Grafana CRs (all grafana CRs are labeled with managed-by: grafana-operator): +```shell +oc kustomize overlays/mlops \ + | yq e 'select(.metadata.labels.managed-by == "grafana-operator")' - \ + | oc replace -f - -n $YOUR_NAMESPACE_NAME +``` + + 10. Alternatively, to deploy `ExploitIQ` with a fully remote nim LLM, run: ```shell # Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM diff --git a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml index 76e051564..97a51c1e4 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml @@ -7,3 +7,7 @@ resources: - batch-info.yaml - batch-metrics.yaml - traces.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml index a0a2aa20a..170711951 100644 --- a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml @@ -3,3 +3,7 @@ kind: Kustomization resources: - infinity.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml index 8d19613f7..b390590ca 100644 --- a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml @@ -5,4 +5,8 @@ resources: - jobs.yaml - evals.yaml - batches.yaml - - traces.yaml \ No newline at end of file + - traces.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml index db8d3e21a..91c8fc2f9 100644 --- a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml @@ -4,3 +4,7 @@ kind: Kustomization resources: - grafana.yaml - serviceAccount.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file From 259393cfcb25af9ca3b74f50067354be3cbc97df Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 10 Feb 2026 12:04:27 +0200 Subject: [PATCH 193/286] chore: depend operatorGroup on subscription instead of on CR Signed-off-by: Ilona Shishov --- kustomize/overlays/mlops/grafana/kustomization.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index 5505891fc..ced6ba94d 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -42,8 +42,8 @@ replacements: delimiter: "." index: 1 - source: - kind: Grafana - name: grafana + kind: Subscription + name: grafana-operator fieldPath: metadata.namespace targets: - select: @@ -60,7 +60,7 @@ replacements: kind: Grafana name: grafana fieldPaths: - - spec.deployment.spec.template.spec.containers.[name=grafana-oauth-proxy].args.0 # update -openshift-service-account argument with the new service account name + - spec.deployment.spec.template.spec.containers.[name=grafana-oauth-proxy].args.0 # updates the -openshift-service-account argument with the new service account name options: delimiter: "=" index: 1 From 64cf4038106f1473205294c5da736c79b32e6e32 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Tue, 10 Feb 2026 12:16:50 +0200 Subject: [PATCH 194/286] chore: moved SA to dedicated folder Signed-off-by: Ilona Shishov --- kustomize/overlays/mlops/grafana/auth/kustomization.yaml | 5 +++++ .../mlops/grafana/{instances => auth}/serviceAccount.yaml | 0 .../overlays/mlops/grafana/instances/kustomization.yaml | 1 - kustomize/overlays/mlops/grafana/kustomization.yaml | 8 +++++--- 4 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 kustomize/overlays/mlops/grafana/auth/kustomization.yaml rename kustomize/overlays/mlops/grafana/{instances => auth}/serviceAccount.yaml (100%) diff --git a/kustomize/overlays/mlops/grafana/auth/kustomization.yaml b/kustomize/overlays/mlops/grafana/auth/kustomization.yaml new file mode 100644 index 000000000..a27ba6506 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/auth/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceAccount.yaml \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml b/kustomize/overlays/mlops/grafana/auth/serviceAccount.yaml similarity index 100% rename from kustomize/overlays/mlops/grafana/instances/serviceAccount.yaml rename to kustomize/overlays/mlops/grafana/auth/serviceAccount.yaml diff --git a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml index 91c8fc2f9..c4e4484bc 100644 --- a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization resources: - grafana.yaml - - serviceAccount.yaml labels: - pairs: diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml index ced6ba94d..867518e5f 100644 --- a/kustomize/overlays/mlops/grafana/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -8,10 +8,12 @@ resources: - instances - - folders - + - auth + - datasources - + + - folders + - dashboards labels: From e2daeed0288507ddbbe0d74b3634127b90e5063b Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:00:28 +0200 Subject: [PATCH 195/286] Appeng 4360 telemetry UI (#200) * also add verify on certs for mlops http request --- .tekton/on-cm-runner.yaml | 4 +- .tekton/on-pull-request.yaml | 6 +- .tekton/on-push.yaml | 1 + .tekton/on-tag.yaml | 2 + Dockerfile | 5 + kustomize/base/exploit-iq-config.yml | 7 + kustomize/base/exploit_iq_service.yaml | 2 + kustomize/config-http-openai-local.yml | 7 + src/vuln_analysis/configs/config-http-nim.yml | 7 + .../configs/config-http-openai.yml | 7 + src/vuln_analysis/data_models/job.py | 42 ++++++ .../functions/cve_http_output.py | 126 +++++++++++++++++- 12 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 src/vuln_analysis/data_models/job.py diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 9578618c0..a4108244c 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -284,7 +284,7 @@ spec: set -e pip install --no-cache-dir kafka-python pydantic requests echo "--- STARTING TRACE COLLECTOR ---" - python3 ci/scripts/collect_and_dispatch_traces.py + python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify echo "--- DEBUG: Current Directory ---" pwd # >>> THE CLIENT (Test Runner) <<< @@ -409,7 +409,7 @@ spec: # 2. Run Python script with the tag cd /app/ - python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME + python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME --telemetry-verify-enable true # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them echo "--- Copying Reports to Shared Workspace ---" diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 2f6147120..6315138b2 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -104,6 +104,7 @@ spec: --label=exploit_iq_vcs_branch={{source_branch}} --label=exploit_iq_vcs_build_commit_id={{revision}} --label=exploit_iq_vcs_repo={{repo_url}} + --build-arg=AGENT_GIT_COMMIT={{revision}} --format=docker taskRef: resolver: cluster @@ -240,6 +241,7 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME + print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" @@ -424,7 +426,7 @@ spec: set -e pip install --no-cache-dir kafka-python pydantic requests echo "--- STARTING TRACE COLLECTOR ---" - python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify true + python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify echo "--- DEBUG: Current Directory ---" pwd # >>> THE CLIENT (Test Runner) <<< @@ -463,7 +465,7 @@ spec: TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" # 3. Run IT Script - python3 src/main_integration_tests.py --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME + python3 src/main_integration_tests.py --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME --telemetry-verify-enable true echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml index e597bfacf..5da2cc106 100644 --- a/.tekton/on-push.yaml +++ b/.tekton/on-push.yaml @@ -94,6 +94,7 @@ spec: --label=exploit_iq_vcs_branch={{source_branch}} --label=exploit_iq_vcs_build_commit_id={{revision}} --label=exploit_iq_vcs_repo={{repo_url}} + --build-arg=AGENT_GIT_COMMIT={{revision}} --format=docker taskRef: resolver: cluster diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml index a820663d3..08718fd32 100644 --- a/.tekton/on-tag.yaml +++ b/.tekton/on-tag.yaml @@ -123,6 +123,8 @@ spec: --label=exploit_iq_vcs_build_commit_id={{revision}} --label=exploit_iq_vcs_repo={{repo_url}} --env=EXPLOIT_IQ_VERSION={{git_tag}} + --build-arg=AGENT_GIT_COMMIT={{revision}} + --build-arg=AGENT_GIT_TAG={{git_tag}} --format=docker taskRef: resolver: cluster diff --git a/Dockerfile b/Dockerfile index 681ced2a1..96133ca02 100755 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,14 @@ FROM ${BASE_IMAGE_URL}:${BASE_IMAGE_TAG} AS base COPY --from=ghcr.io/astral-sh/uv:0.7.15 /uv /uvx /bin/ ARG VULN_ANALYSIS_VERSION ARG PYTHON_VERSION +ARG AGENT_GIT_COMMIT +ARG AGENT_GIT_TAG ENV PYTHONDONTWRITEBYTECODE=1 +ENV AGENT_GIT_COMMIT=${AGENT_GIT_COMMIT} +ENV AGENT_GIT_TAG=${AGENT_GIT_TAG} + RUN apt-get update && apt-get install -y \ ca-certificates \ curl \ diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index ac4835862..ad70c5f3a 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -140,6 +140,13 @@ functions: auth_type: bearer token_path: /var/run/secrets/kubernetes.io/serviceaccount/token verify_path: /app/certs/service-ca.crt + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "bearer" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: true cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 16c17c5d1..f64ca472f 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -123,6 +123,8 @@ spec: fieldPath: metadata.namespace - name: GOMODCACHE value: /exploit-iq-package-cache/go/pkg/mod + - name: ENABLE_MLOPS + value: "true" volumeMounts: - name: config mountPath: /configs diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 1549f74f4..45dabca2c 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -134,6 +134,13 @@ functions: _type: cve_http_output url: http://localhost:8080 endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "None" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: false cve_file_output: _type: cve_file_output file_path: .tmp/output.json diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 693b4f48b..c613e6866 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -126,6 +126,13 @@ functions: _type: cve_http_output url: http://localhost:8080 endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "None" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: false cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index d10b5a416..4e6ca79f7 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -132,6 +132,13 @@ functions: _type: cve_http_output url: http://localhost:8080 endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: ${MLOPS_URL:-http://localhost:8080} + auth_type: ${MLOPS_AUTH_TYPE:-None} + token_path: ${MLOPS_TOKEN_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/token} + verify_path: ${MLOPS_VERIFY_PATH:-/app/certs/service-ca.crt} + enable_verify: ${MLOPS_ENABLE_VERIFY:-false} cve_calculate_intel_score: _type: cve_calculate_intel_score llm_name: intel_source_score_llm diff --git a/src/vuln_analysis/data_models/job.py b/src/vuln_analysis/data_models/job.py new file mode 100644 index 000000000..4380ed708 --- /dev/null +++ b/src/vuln_analysis/data_models/job.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field, RootModel + + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + + +class Job(BaseModel): + job_id: str | None = None + execution_start_timestamp: LocalDateTime | None = None + duration_in_seconds: str | None = None + cve: str | None = Field(None, pattern=r'CVE-\d{4}-\d{4,7}') + app_language: str | None = Field(None, pattern=r'go|python|c|javascript|java|all|\s*') + component: str | None = None + component_version: str | None = None + agent_git_commit: str | None = None + agent_git_tag: str | None = None + agent_config_b64: str | None = None + concrete_intel_sources_b64: str | None = None + executed_from_background_process: bool | None = None + batch_id: str | None = None + success_status: bool | None = None + env_vars: str = Field(..., description='comma delimited key=value pairs') + job_output: dict[str, Any] | None = None diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index fcc3b172b..e8b85e720 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -14,7 +14,7 @@ # limitations under the License. import base64 from http import HTTPStatus - +from datetime import datetime from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function @@ -22,11 +22,57 @@ from pydantic import Field from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from vuln_analysis.data_models.job import Job, LocalDateTime +from exploit_iq_commons.data_models.common import TypedBaseModel +import typing +from typing import Any +import os +import re logger = LoggingFactory.get_agent_logger(__name__) +# Explicit env var names to never export +ENV_BLACKLIST = frozenset({ + "GHSA_API_KEY", "NVD_API_KEY", "NVIDIA_API_KEY", "SERPAPI_API_KEY" +}) + +# Regex (case-insensitive) for key names to exclude +ENV_KEY_PATTERN = re.compile( + r"password|token|api\.key|api_key|api-key|secret|credential|" + r"private-key|private_key|auth|pwd|pass_|_pass|connection_string|" + r"connection-string|pat|certificate", + re.IGNORECASE +) + + +def get_filtered_env_vars() -> str: + """Filter os.environ and return comma-delimited key=value pairs (safe for Job.env_vars).""" + pairs = [] + for key, value in os.environ.items(): + if key in ENV_BLACKLIST: + continue + if ENV_KEY_PATTERN.search(key): + continue + pairs.append(f"{key}={value}") + return ",".join(pairs) + +def encode_in_base64(v: str) -> str: + if not v: + return "" # Return safe empty string + encoded_bytes = base64.b64encode(v.encode("utf-8")) + return encoded_bytes.decode("ascii") + + + +class MLOpsConfig(TypedBaseModel[typing.Literal["mlops"]]): + mlops_url: str = Field(description="URL to send CVE workflow output") + auth_type: str = Field(default="None", description="Type of auth - bearer, basic or None") + token_path: str | None = Field(default=None, description="Path to token file containing auth token") + verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") + enable_verify: bool = Field(default=False, description="Enable verify") + class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): """ Defines a function that sends CVE workflow output to HTTP endpoint. @@ -39,6 +85,8 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") username: str | None = Field(default=None, description="Username to authenticate with for basic auth") password: str | None = Field(default=None, description="Password to authenticate with for basic auth") + enable_mlops: bool = Field(default=False, description="Enable MLOps") + mlops_config: MLOpsConfig = Field(..., description="MLOps configuration") @register_function(config_type=CVEHttpOutputConfig) @@ -62,18 +110,61 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: http_utils.request_with_retry(request_kwargs={ "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers, "verify": verify }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) + if config.enable_mlops: + mlops_url = None + try: + job = _extract_job_data(message) + job_json = job.model_dump_json(by_alias=True) + http_params = _http_params_override(config, config.mlops_config, headers) + is_verify = http_params["verify"] + if is_verify: + verify = http_params["verify_path"] + mlops_url = http_params["url"] + http_utils.request_with_retry(request_kwargs={ + "url": mlops_url, "method": "POST", "data": job_json.encode('utf-8'), "headers": http_params["headers"], "verify": verify + }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) + except Exception as mlops_e: + logger.error('Unable to send job to MLOps API at %s. Error: %s', mlops_url, mlops_e) except Exception as e: logger.error('Unable to send output response to %s. Error: %s', url, e) else: logger.info('Successfully sent output to %s', url) return message + + def _extract_job_data(message: AgentMorpheusOutput) -> Job: + agent_config = builder.get_workflow_config() + job_id = message.input.scan.id + start_time = datetime.fromisoformat(message.input.scan.started_at.replace('Z', '+00:00')) + end_time = datetime.fromisoformat(message.input.scan.completed_at.replace('Z', '+00:00')) + duration = (end_time - start_time).total_seconds() + executionTimestamp = LocalDateTime(root=start_time) + duration_in_seconds = str(duration) + cve = message.input.scan.vulns[0].vuln_id + app_language = message.input.image.ecosystem + component = message.input.image.name + component_version = message.input.image.tag + env_vars = get_filtered_env_vars() + intel = message.info.intel[0].model_dump_json(by_alias=True) + intel_b64 = encode_in_base64(intel) + agent_config_b64 = encode_in_base64(agent_config.model_dump_json()) + message_output = message.output.model_dump() + agent_git_commit=os.environ.get("AGENT_GIT_COMMIT") + agent_git_tag=os.environ.get("AGENT_GIT_TAG") + return Job(job_id=job_id, execution_start_timestamp=executionTimestamp, duration_in_seconds=duration_in_seconds, + cve=cve, app_language=app_language, component=component, component_version=component_version, + agent_git_commit=agent_git_commit, agent_git_tag=agent_git_tag, + agent_config_b64=agent_config_b64, concrete_intel_sources_b64=intel_b64, + executed_from_background_process=False, batch_id="", success_status=True, + env_vars=env_vars, job_output=message_output) yield FunctionInfo.from_fn(_arun, input_schema=AgentMorpheusOutput, description=("Sends CVE workflow output to HTTP endpoint.")) + + def get_auth_header(http_config: CVEHttpOutputConfig | None) -> str | None: match http_config.auth_type: case "basic": @@ -94,3 +185,36 @@ def get_auth_header(http_config: CVEHttpOutputConfig | None) -> str | None: return None case None: return None + + +def _http_params_override(http_config: CVEHttpOutputConfig, mlops_config: MLOpsConfig,http_headers) -> dict[str, Any]: + + url = http_config.url + "/api/v1/jobs/one" + if mlops_config.auth_type == "None": + headers = {'Content-type': 'application/json'} + return { + "headers": headers, + "verify": False, + "url": url + } + if http_config.auth_type.lower() == "bearer" and mlops_config.auth_type.lower() == "bearer": + return { + "headers": http_headers, + "url": url, + "verify": mlops_config.enable_verify, + "verify_path": http_config.verify_path + } + clone_http = http_config.copy() + clone_http.auth_type = "bearer" + clone_http.token_path = mlops_config.token_path + auth_header = get_auth_header(clone_http) + headers = {'Content-type': 'application/json'} + if auth_header is not None: + headers['Authorization'] = auth_header + return { + "url": url, + "headers": headers, + "verify": mlops_config.enable_verify, + "verify_path": mlops_config.verify_path + } + From 37c420c5bc74cc61abb2a2ddc5f45cb98d4f760e Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Sun, 15 Feb 2026 14:38:28 +0200 Subject: [PATCH 196/286] fix: typos Signed-off-by: Ilona Shishov --- kustomize/README.md | 4 ++-- kustomize/overlays/mlops/grafana/dashboards/jobs.yaml | 2 +- kustomize/overlays/mlops/grafana/instances/grafana.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index a6875523b..1dab1c4ef 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -192,12 +192,12 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA ```shell # Patch overlay kustomization yaml with deployment namespace value -sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomize.yaml +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomization.yaml ``` ```shell # replace EXPLOIT_IQ_GRAFANA_SA_TOKEN with ExploitIQ Grafana SA Token from bitwarden vault (1 year expiration date) -oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_GRAFANA_SA_TOKEN' +oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_GRAFANA_SA_TOKEN' ``` ```shell diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml index fe8433bd0..27461dc94 100644 --- a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -166,7 +166,7 @@ spec: } } ], - "title": "Job by Request ID", + "title": "Job by Job ID", "type": "table" }, { diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml index 1e498c2fd..cd2ff3c14 100644 --- a/kustomize/overlays/mlops/grafana/instances/grafana.yaml +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -56,7 +56,7 @@ spec: secretName: exploit-iq-grafana-tls - name: oauth-proxy-secret secret: - secretName: oauth-client-secret-86gfcb49d4 + secretName: oauth-client-secret route: spec: port: From e567ac4443b43ce5bc7f1d45b343618f80b55e18 Mon Sep 17 00:00:00 2001 From: Super User Date: Sun, 15 Feb 2026 23:28:29 +0200 Subject: [PATCH 197/286] fix(kustomize): disable secret name suffix for Grafana CR It was not possible to dynamically propagate the generated secret name with hash into the Grafana CRD volumes field using Kustomize replacements. Therefore, the secret's name suffix has been disabled to allow the CR to reference it reliably. --- kustomize/base/kustomization.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 1e0afee87..d6dbd4a21 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -27,6 +27,8 @@ secretGenerator: - name: oauth-client-secret envs: - oauth-secrets.env + options: + disableNameSuffixHash: true - name: mongodb-credentials envs: From c4635fc1e07cf8b72e7e7c194febb5aa9ab5795c Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Wed, 18 Feb 2026 09:53:25 +0200 Subject: [PATCH 198/286] APPENG-4421: Send all traces and spans to Grafana Tempo (#201) --- ci/scripts/collect_and_dispatch_traces.py | 34 ++++- kustomize/README.md | 3 +- .../grafana/datasources/kustomization.yaml | 1 + .../mlops/grafana/datasources/tempo.yaml | 15 ++ kustomize/overlays/mlops/kustomization.yaml | 2 + .../overlays/mlops/tempo/kustomization.yaml | 13 ++ .../mlops/tempo/tempo-deployment.yaml | 134 ++++++++++++++++++ kustomize/overlays/mlops/tempo/tempo-scc.yaml | 13 ++ .../tools/tests/test_segmenter.py | 2 +- 9 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 kustomize/overlays/mlops/grafana/datasources/tempo.yaml create mode 100644 kustomize/overlays/mlops/tempo/kustomization.yaml create mode 100644 kustomize/overlays/mlops/tempo/tempo-deployment.yaml create mode 100644 kustomize/overlays/mlops/tempo/tempo-scc.yaml diff --git a/ci/scripts/collect_and_dispatch_traces.py b/ci/scripts/collect_and_dispatch_traces.py index b26d0d980..bbf8723cc 100755 --- a/ci/scripts/collect_and_dispatch_traces.py +++ b/ci/scripts/collect_and_dispatch_traces.py @@ -19,6 +19,8 @@ DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" DEFAULT_ENABLE_VERIFY = False +# Tempo configuration +TEMPO_URL = "http://tempo.grafana-tempo-poc.svc.cluster.local:4318/v1/traces" class LocalDateTime(RootModel[datetime]): root: datetime = Field(..., examples=['2022-03-10T12:15:50']) @@ -101,7 +103,28 @@ def find_job_id(attributes: dict) -> str: print(f"Attributes: {attributes}") return "" -def process_message(http_provider, traces_table, map_trace_id_to_job_id, data): +def process_message(http_provider, tempo_provider, traces_table, map_trace_id_to_job_id, data): + + # --- NEW: Forward original trace to Tempo --- + try: + for rs in data.get('resourceSpans', []): + # Requirement B: Ensure service.name is present in the resource + # (Checking if it's missing and adding it) + attrs = rs.get('resource', {}).get('attributes', []) + if not any(a['key'] == 'service.name' for a in attrs): + attrs.append({"key": "service.name", "value": {"stringValue": "cve-agent"}}) + + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + # Requirement A: If it's the workflow, explicitly null the parent + if span.get('name') == "": + span['parentSpanId'] = "" + # Tempo expects the raw OTLP JSON structure as-is + tempo_status = tempo_provider.send_post(json.dumps(data)) + # Optional: logging for debug + # print(f"Sent copy to Tempo, status: {tempo_status}") + except Exception as e: + print(f"Failed to forward trace to Tempo: {e}") # Traverse the OTLP structure for rs in data.get('resourceSpans', []): for ss in rs.get('scopeSpans', []): @@ -201,6 +224,13 @@ def signal_handler(sig, frame): enable_verify=enable_verify, ) + # Initialize Tempo Provider (Insecure/Internal) + tempo_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False + ) + try: with open(OUTPUT_FILE, 'a', encoding='utf-8') as f: while not stop_triggered: @@ -211,7 +241,7 @@ def signal_handler(sig, frame): #f.write(f"Item: {items_count}\n") #f.write(json.dumps(data) + '\n') #f.flush() - process_message(http_provider, traces_table, map_trace_id_to_job_id, data) + process_message(http_provider, tempo_provider, traces_table, map_trace_id_to_job_id, data) #items_count += 1 finally: consumer.close() diff --git a/kustomize/README.md b/kustomize/README.md index 1dab1c4ef..1b552d3d6 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -191,8 +191,9 @@ oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NA 10. To deploy `ExploitIQ` with a self-hosted LLM and MLOps, run: ```shell -# Patch overlay kustomization yaml with deployment namespace value +# Patch overlay kustomization yaml with deployment namespace value (Grafana and Tempo) sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomization.yaml +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/tempo/kustomization.yaml ``` ```shell diff --git a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml index 170711951..988d839a8 100644 --- a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml +++ b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - infinity.yaml + - tempo.yaml labels: - pairs: diff --git a/kustomize/overlays/mlops/grafana/datasources/tempo.yaml b/kustomize/overlays/mlops/grafana/datasources/tempo.yaml new file mode 100644 index 000000000..963cdbc6e --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/tempo.yaml @@ -0,0 +1,15 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: tempo +spec: + instanceSelector: + matchLabels: + app: exploit-iq + datasource: + access: proxy + url: http://tempo:3200 + name: Tempo + uid: tempo + type: tempo + isDefault: false diff --git a/kustomize/overlays/mlops/kustomization.yaml b/kustomize/overlays/mlops/kustomization.yaml index 27fbe79ea..e068e67c7 100644 --- a/kustomize/overlays/mlops/kustomization.yaml +++ b/kustomize/overlays/mlops/kustomization.yaml @@ -5,6 +5,8 @@ resources: - ../self-hosted-llama3.1-70b-4bit # Grafana monitoring resources - grafana + # Tempo tracing (same namespace as Grafana for non-CI; CI uses grafana-tempo-poc) + - tempo commonAnnotations: deployment-specialization: mlops \ No newline at end of file diff --git a/kustomize/overlays/mlops/tempo/kustomization.yaml b/kustomize/overlays/mlops/tempo/kustomization.yaml new file mode 100644 index 000000000..ed8b8dec7 --- /dev/null +++ b/kustomize/overlays/mlops/tempo/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: REPLACE_NAMESPACE + +resources: + - tempo-deployment.yaml + - tempo-scc.yaml + +labels: + - pairs: + app: exploit-iq + component: tempo diff --git a/kustomize/overlays/mlops/tempo/tempo-deployment.yaml b/kustomize/overlays/mlops/tempo/tempo-deployment.yaml new file mode 100644 index 000000000..2412a051f --- /dev/null +++ b/kustomize/overlays/mlops/tempo/tempo-deployment.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tempo-sa +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-conf +data: + tempo.yaml: | + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + + ingester: + max_block_duration: 5m + lifecycler: + address: 0.0.0.0 + final_sleep: 0s + ring: + kvstore: + store: inmemory + replication_factor: 1 + + # Storage paths (backed by PVC so data survives restarts) + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + + # Metrics generator for rate() and other TraceQL metrics (Tempo 2.6+) + metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + + # Allow querying up to 7 days (fixes "Selected time range exceeds...") + query_frontend: + search: + max_duration: 168h + + # Define retention so traces have a clear "shelf life" (fixes "No data for selected query") + compactor: + compaction: + block_retention: 168h + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + # Optional: uncomment and set a StorageClass name if your cluster needs it + # storageClassName: standard + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo +spec: + replicas: 1 + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + serviceAccountName: tempo-sa + # So the PVC is writable by Tempo (runs as UID 10001); fixes "mkdir /var/tempo/traces: permission denied" + securityContext: + fsGroup: 10001 + containers: + - name: tempo + image: grafana/tempo:2.6.1 + args: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - name: tempo-http + containerPort: 3200 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-conf + - name: data + persistentVolumeClaim: + claimName: tempo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo +spec: + ports: + - name: tempo-http + port: 3200 + targetPort: 3200 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 + selector: + app: tempo \ No newline at end of file diff --git a/kustomize/overlays/mlops/tempo/tempo-scc.yaml b/kustomize/overlays/mlops/tempo/tempo-scc.yaml new file mode 100644 index 000000000..eaccdbc6a --- /dev/null +++ b/kustomize/overlays/mlops/tempo/tempo-scc.yaml @@ -0,0 +1,13 @@ +# Allow Tempo pod to use fsGroup 10001 (required for Tempo's data dir on PVC). +# OpenShift: bind the built-in anyuid SCC ClusterRole to tempo-sa (same pattern as base/anyuid-scc-permission.yaml). +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tempo-anyuid-scc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: tempo-sa diff --git a/src/vuln_analysis/tools/tests/test_segmenter.py b/src/vuln_analysis/tools/tests/test_segmenter.py index 18095b407..91cd047b4 100644 --- a/src/vuln_analysis/tools/tests/test_segmenter.py +++ b/src/vuln_analysis/tools/tests/test_segmenter.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# dummy comment to trigger a build 2 from pathlib import Path import pytest from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended From 60406adcab398de85a928cfe65de605340170312 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 18 Feb 2026 15:45:13 +0200 Subject: [PATCH 199/286] chore(nginx): use worker_processes equal 1 Signed-off-by: Vladimir Belousov --- kustomize/base/nginx/nginx_cache.conf | 2 +- kustomize/base/nginx/nginx_ssl.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index 7d24b4128..4d7385b84 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -1,6 +1,6 @@ pid /tmp/nginx.pid; -worker_processes auto; +worker_processes 1; events { worker_connections 1024; diff --git a/kustomize/base/nginx/nginx_ssl.conf b/kustomize/base/nginx/nginx_ssl.conf index edec88a71..4825e0ffc 100644 --- a/kustomize/base/nginx/nginx_ssl.conf +++ b/kustomize/base/nginx/nginx_ssl.conf @@ -1,4 +1,4 @@ -worker_processes auto; +worker_processes 1; events { worker_connections 1024; From 3b026051a4809077bb9b5973868be8929f04cdfa Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 26 Jan 2026 00:46:25 +0200 Subject: [PATCH 200/286] chore: create tests overlay variant Signed-off-by: Zvi Grinberg --- .gitignore | 9 ++ .gitmodules | 5 + kustomize/README.md | 49 ++++++ .../overlays/tests}/buildah-task.yaml | 3 +- .../overlays/tests/exploit-iq-ips.secret | 15 ++ .../tests/google-sheets-secrets-enc.yaml | 26 ++++ .../tests/integration-tests-secrets-enc.yaml | 27 ++++ kustomize/overlays/tests/kustomization.yaml | 93 ++++++++++++ .../overlays/tests/mongodb-credentials.env2 | 15 ++ kustomize/overlays/tests/nginx-patch.yaml | 12 ++ kustomize/overlays/tests/nginx_cache.conf | 115 +++++++++++++++ kustomize/overlays/tests/oauth-secrets.env2 | 15 ++ .../tests/registry-app-creds-enc.yaml | 25 ++++ kustomize/overlays/tests/sc-llm-pvc.yaml | 13 ++ kustomize/overlays/tests/secrets.env2 | 15 ++ .../tests/server-model-config-enc.yaml | 32 ++++ kustomize/overlays/tests/tekton-config.yaml | 139 ++++++++++++++++++ .../overlays/tests/user-feedback-ips.secret | 15 ++ 18 files changed, 621 insertions(+), 2 deletions(-) rename {.tekton/tasks => kustomize/overlays/tests}/buildah-task.yaml (99%) create mode 100644 kustomize/overlays/tests/exploit-iq-ips.secret create mode 100644 kustomize/overlays/tests/google-sheets-secrets-enc.yaml create mode 100644 kustomize/overlays/tests/integration-tests-secrets-enc.yaml create mode 100644 kustomize/overlays/tests/kustomization.yaml create mode 100644 kustomize/overlays/tests/mongodb-credentials.env2 create mode 100644 kustomize/overlays/tests/nginx-patch.yaml create mode 100644 kustomize/overlays/tests/nginx_cache.conf create mode 100644 kustomize/overlays/tests/oauth-secrets.env2 create mode 100644 kustomize/overlays/tests/registry-app-creds-enc.yaml create mode 100644 kustomize/overlays/tests/sc-llm-pvc.yaml create mode 100644 kustomize/overlays/tests/secrets.env2 create mode 100644 kustomize/overlays/tests/server-model-config-enc.yaml create mode 100644 kustomize/overlays/tests/tekton-config.yaml create mode 100644 kustomize/overlays/tests/user-feedback-ips.secret diff --git a/.gitignore b/.gitignore index 0f3af34ed..79b658d86 100644 --- a/.gitignore +++ b/.gitignore @@ -198,5 +198,14 @@ tags # Persistent undo [._]*.un~ +**/exploit-iq-ips.json +**/user-feedback-ips.json +**/google-sheets-secrets.yaml +**/integration-tests-secrets.yaml +**/server-model-config.yaml +**/sec-decryption.key +**/registry-app-creds.yaml + + # End of https://www.gitignore.io/api/vim,c++,cmake,python,synology diff --git a/.gitmodules b/.gitmodules index 1717d5fe8..555ccf7bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,8 @@ [submodule ".tests-automation"] path = .tests-automation url = https://github.com/RHEcosystemAppEng/exploitiq-tests-automation.git + +[submodule "exploit-iq-models"] + path = exploit-iq-models + url = https://github.com/RHEcosystemAppEng/exploit-iq-models.git + diff --git a/kustomize/README.md b/kustomize/README.md index 1b552d3d6..2d413f096 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -286,3 +286,52 @@ kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=pers # Or, Delete Everything kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - ``` +### Deploy Test overlay variant (Rapid deployment) +1. Download and install [GnuPG](https://www.gnupg.org/download/) and [sops](https://github.com/getsops/sops/releases) +2. Create new namespace/project: +```shell +export PROJECT_NAME=exploit-test +oc new-project $PROJECT_NAME +``` +3. Take private key and import it to GPG: +```shell +gpg --import /path/to/sec-decryption.key +``` +4. Decrypt all secret files: +```shell +cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests +mkdir -p secrets +sops -d exploit-iq-ips.secret > secrets/exploit-iq-ips.json +sops -d google-sheets-secrets-enc.yaml > secrets/google-sheets-secrets.yaml +sops -d integration-tests-secrets-enc.yaml > secrets/integration-tests-secrets.yaml +sops -d mongodb-credentials.env2 > secrets/mongodb-credentials.env +sops -d oauth-secrets.env2 > secrets/oauth-secrets.env +sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml +sops -d secrets.env2 > secrets/secrets.env +sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml +sops -d user-feedback-ips.secret > secrets/user-feedback-ips.json +``` + +5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. +6. Now deploy to the cluster the exploitIQ system ( minus agent) with all resources: +```shell +kustomize build . | oc apply -f - +``` + +7. Deploy Self hosted LLM for the automation tests ( Integration tests and Confusion matrix runner): +```shell +helm upgrade --install --set nim_embed.enabled=false --set llama3_1_70b_instruct_4bit.storageClass.name=gp3-csi-thr +oughput-2000 --set llama3_1_70b_instruct_4bit.readinessProbe.initialDelaySeconds=25 --set llama3_1_70b_instruct_4bit.readinessProbe.periodSeconds=10 --set global.tolerationsKey=p4d-gpu exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models +``` + +8. Remove untracked decrypted secrets files +```shell +rm -rf secrets/ +``` + +9. Tear down: +```shell +helm delete exploit-iq-tests + +oc delete project $(oc project --short -q) +``` diff --git a/.tekton/tasks/buildah-task.yaml b/kustomize/overlays/tests/buildah-task.yaml similarity index 99% rename from .tekton/tasks/buildah-task.yaml rename to kustomize/overlays/tests/buildah-task.yaml index 60f9a43a7..953dceaff 100644 --- a/.tekton/tasks/buildah-task.yaml +++ b/kustomize/overlays/tests/buildah-task.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: buildah-pvc - namespace: ruben-morpheus spec: description: | @@ -165,4 +164,4 @@ spec: - description: An optional workspace that allows providing the entitlement keys for Buildah to access subscription. The mounted workspace contains entitlement.pem and entitlement-key.pem. mountPath: /tmp/entitlement name: rhel-entitlement - optional: true \ No newline at end of file + optional: true diff --git a/kustomize/overlays/tests/exploit-iq-ips.secret b/kustomize/overlays/tests/exploit-iq-ips.secret new file mode 100644 index 000000000..975a0461b --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-ips.secret @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:7Mhg0suh1pr4gW2ZSVDfDVBxR9M+WeOS/ndzCa6HynWbnbVi6o/bXSBrUwukb9NLEPyXG+i0Ja265AmV2Ix//+0YiHxRMTNWuAWeik9C1FhPyMefg/QJ7/TqVA19011U1oaqZwttcfxtiC69lKbIG6vZnxuLtWhjoWfi1SJrqPZ+EsKSD/st2DoWkhvlGd+ea8RyboXt2knL2jy7smo1wRWSUl98SqDr6TLqNg==,iv:oup5Ep55EXokJe+jRlOBXIxGoP88ZqV6aHzQtDrAGok=,tag:UuwWcFrQ8OJf5weq0HNBrg==,type:str]", + "sops": { + "lastmodified": "2026-01-25T12:11:15Z", + "mac": "ENC[AES256_GCM,data:o+/YRK8wHs6hlEJqkwDtzV/3plYxOTRs7QGfqLGv0TaGZMIYFnbav1M4AFDYY7pMxEnRCyRJWr6G6L3mN8Uwdcr9FyMls1yQAXnu8a6iP7xLSvShvk4sXmdRKCV+ZoI6uWOGpT5um2ovpqce2GENStvk7PhFhS0R+Dc2IqBYbyA=,iv:rWe1RZ8QEIeOTrIVYTroLrqm9QaugtTVw41dtxJRk5Y=,tag:CRh9MgwYzbLV+fqkIjpcxA==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T12:11:15Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdAenSUkHYnXpk59IsDKXVzwzXcmJYgwOC/mjNFPxrPUQkw\nIroNi7SaoYcdQ5bNd/IygS+LSJbqxWpMvPLgxw+Z/BUS0lWppfzAYgMeHGjH5Y+u\n0lwBGxusz5C9WM+oOHNOhrg8DZZU3iLfDgWpICqJ6OtRlcSlJlr2gXPFZngunkxz\nX5fFnLDgs2j6OV5CQEAkjC3j73t9RSE61ILuRLqZGMFjCm/xtL7KieKhstFxqw==\n=mhUv\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/google-sheets-secrets-enc.yaml b/kustomize/overlays/tests/google-sheets-secrets-enc.yaml new file mode 100644 index 000000000..9f9b7cd73 --- /dev/null +++ b/kustomize/overlays/tests/google-sheets-secrets-enc.yaml @@ -0,0 +1,26 @@ +apiVersion: ENC[AES256_GCM,data:FCE=,iv:mX1MhglqJCtmZ2+lAxeUdRweyKUjU0eEcxBBOyEfhQ0=,tag:wVimkqWtiSk2L6IcokRRHg==,type:str] +data: + credentials.json: ENC[AES256_GCM,data:eLGkTsE6lGTJOawS4Z4nNhbKwBLJVkSaFF1Tji8vpcH0M0A7LvRY53W6zH3TqSbvQ+AWssXjkuz7OIyHSg0WW60tkYixfToK/Rgy4TAbLTA++LcVqoe9tVspDtifEJLwTwHEiirX61SVgdK11Rm+035ICXmPlbV4VyQBfC1hHf0LDeX533raYq4QDyz/X/htT9eO/x3NyyYlPykcCT5Uyva1BV2Gi9iq8AJdpMBC/V+20DrPfzSsHgNlGUMOTmMK4qzs6nyOj3LeGc07ycKfGRK+hCCwTxoFyBJ25hgpNGB2FLRfJApqUjnKnh7VuoeilYEUdeWKVnzzsNsJc26CzeixxNXTaNPG2eo6LtNXndtMgXyEOa9zu0owJsrLl/zHHqqmtOkJvEKYkED6QhVMzfF5ciCnoyh3kwyVERVXhVMRGB19Gdn3dv9pdw/OPGOvvV3AGuzGR6SMvfKUn3lqoui7EWnOhsVd03RHXY8CXuBX7u9Y3aBEFj0DPaIBfH6CkG6oA6XhCt4kH57M/S43MbnlUC+napRqTJiJzRc49Qc3TpxG9VFSkMNOCnnm+8pxnbhGzaYA1izm1YHUTk1v6yjTaOCyTWQFEfum73TaafoF+QGypyHaiidRaa6VQ5XGkDxZ/yONWpt4i9NUp9+xoFersBYk9XBLWJdza4jVe2qOuLkM64OdNrnwKnxVqXFUNk9kyvXfMoCcweV9jCRN8Ky214MfnU5ALgVizPbW7TkZD/Tt9T4She4F17SgiR/+NTn8DArPVUuhQlCX4oR54CMFv/iQBYs1NMKKLZJ+aFyo62shAj7cV4grhtLNgezh+i5u9ISWS6ruU7GPxlOgxrdoDIPavCMi2GmcoCsxF2q/CtOghYyKh1fg02gnrkrJW8k3z9ZCmti5wlaPy8Td58xCLzsVxGSOwfexFdtR45EaPaHmlQotyIivfPKsFS24vqNdSjPYp6SygRv61G6aHg67EmzsT21hLb1PhtMLGpSdYYASBfdHq/6y2KsUQg2YZcDAURKEIc+1QWkRwsDBdr7RPrcmUFZVq7V0QsmFRiJqt1nDgn9GRfF02QXGLFoAehH64COjpERpymENoRKzqwVzL71VqoTdu9A1YXYhJdSzcWcSZ61pkLxgh6B6eM5PKLxqiQAIxgektIYf8yvSg3o+l3X36YHqb1pJXTgffe7ilehTPtkR8hbU4CpBR5+A6Se8/JBD74ym8Z0+A2UjUd/IsY6QOOza3Luko9UEDO2UMF4CgRRE1zdxW79i6HkNEEQaEB3uLfI72LuBI4bGdnAsPLzMk1/1ExtuvuzThJq3E6SXh3RSuFn2benLfvfn7GSVS4tJwdfkWzhTX13rqSByhey8kNiKYGMAj8I+rlr5th70VeZGbRD22OZ/99sK67xGA0M5GgodKvkFR3LQFYImD4rYt9k5DZ3PtI/r+dBQWZPecUuzHj3Mkn92G5wSnhJuEPUsfJJPOFyV+bBVjTh+FSqMUgmd2lzAn/Z3MqcfW2bRsNOFXHW/+vIf+xK6JsjTEipqZCLMl4cO0WL2gcL8KGetlfO91mgnpNcfZ8fkGOfRsEueA8CiQaJN4tGcKPW/TBJdqi7sCHHtwX8BlD/tcGX4OahaARR+jouKvYkKCtYsg8vq4up3BTYeHaATXxETUDzw1GYhnSzVdkZOq6ZsefvutDzR1PYgcOYHZ4mMOBghvwks6UNZNNQlFlfkD35ThBqCPzmPKuIFvUotjtxpGr6sfHBAqA6iPqe02iWs1+36ZyYzjioXyb3Mu2BVu6RnNYABzmJdAhiDxKXIjYLlIQJ3gocy5GpPkAn16qzuA3/RN9/FTVHKck60uqSsvrQ0TCG54hCN0O8lPZ2jGyH2ok4LMhyqHV1F8O1GGTDxoFZn/z00vXhh2yTv1km6ktlXdHKQB/T10YszwkczcseQw+O6BbGZEPzB4OiYX1kLthgqzkI8JIwnMZNdz1O+va2o1TAaFL9pCzGWkQqz1X6Nup0S1rJ0UsNPPAj8fZdjKrsvACbcd+oZLyUmexrXb9k8Jq0wPLKXZ51pX0JGB57BbDpBqeeA13QdFCUPNcr1nYSFyBKkkJu1zSeCluceeIHMHhIdie7HLCDL1PR909gwd0Y59c3QGfsuqHgblTbovXwx6s6ezvwlZpKXePj2Orknm/Kgmgou4ECtr/Yw1fmkCea1j5IkMCX9MJZ/TI1kVKpMdlwN+fkjADX8FJoe+EnNfjCkQxp1xFg189iWPm3VEIw989jJygHGwfu/IV6ZPvv9LCtTrV8dR8jPuliS1k0SUHB42iZB6J5XRKc3fbuHpcoLX715XNDW9zzlD+22bIHgCekOgvrlSeoXKHbbY0V9DML3ooF8mafFscMva/qExPnKTvNx3FIuTCRMgTMr57DlnwknrYR3eHM5BfWjsb7T/ZBEMTQf3jafmCoDC+V83qtNHJxGP2ObjpW878Ljtj1NF6Sqkne22aypmyvrY1kI2xeVl8UzGM9c5Wj7S3IxiROWGiubEfVgH0MZluir2jLLDUThNTTH2PlPjQ05VklTg9PobvJqna2xrUYhI6jvBwCq+SD6yTErNqLEozmZqkBkTkV7RAkY6a8S1FBkLXQDJVJUP6Thac50Ty3TDpJ044D1xwpFbrchosgVc2a4vkunlVx1tc+Uot4IYxAlCBzTQbeTq5wbgNEZSljVnAT5V1hgdJCt4wwBh+rUvLm01I5w/l6j3r9wNY2d1MJjvz2SGh4WcxkwOmsIoac9dky8imubt5Rc1kaJ2QDgeUpWvfyPaGn/vQRPtMTp+zF7aezf4Ci/FXBUUOQPZfUN70qH92Sg1cuETRCPNMGgBVlsIY+a7f75P+Av7pRheLZ8Z5EQ8Vt93pRp0o0a9Fyia4Go9V6FeTV+jgbltY03VhEFo1SEh0HU1cV1UKu++90okOjfKNTOH81zuESXvjI+V0fRIen6lbDaF3EToZ7XWL5MPobfOsuf++3HYAdoXPjI/khji0QJ3OlaTMkCrKMlOO2IXvm0rlsMvM4yLPuNeB5vQemv9hgaEwxH2RokKG2iaBEmMQec5m58b03kFhu9SnXJvKo71k3T1Y39iiTxcjzv7Ob/BqD463ndX0POhMJneM+OqyYA9mZ9ZEit+JtOquQ18wn8gRyP6oHRBkGkR880519rnzhCNhYZQyvYw6/3QUBy4FNDugtn7ANZXXkSfNPsqUsR91USMbtSAShczLcIyI0ybDiVVUbGD3zZ97pxPKq1cMeZ+iLKrgGXuDzPBgmtcluOrctWdxFoc4WPtem6mc/V2DtjNmTPGxVbA02wGHxYPLSTebm9xqfIk/QY2E4V3vQJybfgS/Ni1y9RErbI0/k0p3kdrnYXKfey1x2LMxACmIALkpMzOsms11tLMGpEONKHlUnKbVWR4GncBDraYCNGuIrQVls5O/1+eydu+2EAgbs2GG3/30zphqgQ+RA1LOwR4dFt+WswDyMiyYikl3OBUVQZ7gJNwkMEsM4Khfk90FcyEvtToqFMYKXHODEs6XcRyZFIJn/z6p2y5M2RniDc9/nM93mHJWtor/qK2eEt8ixjKDU9475HqPZefQFtYmtPq0J/Q3PL9N5jk7uodMOho2vSka86gGSDpYFWi0pw3wUfCgmAP/4zIiPOzM3Ut5YMOGWCvj4izOlwf8dpn3NJLEMhFukxxcMevzIF6uwWWPPcA6eLByWJWI1pM4TWtCY73yAuvPz1aH5wljV1ikozh3sZxeSx6kiMri0mg3Qg2g4nDMqMu8UozlCdn4uIvFgp92MBo1uMp8cnjZ82lgftI1PG2kpdeUjJRzNW7Kp9MUeG3O18IWE0KRDLU9mbd7RXLsTM534kI48NYYMJhiWu4ckr+mPg+mRkrp387serB/paD/oftcquKvvwRQKSqsIblxIrOqzx9c9smoZtEaN0Wjo1Rp/XFxAsXa9/UPGLswA4gheAnrWMldTBmacBHKz3O5FJk2KBgssqD07Gf+HALTLtdQxcCAdMjaSP/Anw17ooEZpxgbnegZi3wJXOKZujGHlZtpnMBeIip/DwzqPFPihF3/gtAvpWwvZTfPX88tShKQfg4jetcS9EhPAFcGleQqe86F6sN/WX9DVGX6vKr8TEDV2QXti3VFo+i6/lP+yFipRlxkmhsSObIW2TCDKcprBd2snmaa/xkRRFVrseaV4xuYdzVj9A+hlaJmq7gDxapCUyprIAD58p6NG1w6Ec8EbO4PJezQ==,iv:5/2vAV0mR91A76D1eknZllR6G2CcgS1CjFzH/vgC7yM=,tag:P4+B4QMpgh+v8G67QVSBUA==,type:str] + input_sheet_id: ENC[AES256_GCM,data:QGcHsxrPb4ocTIWd8CMvtetiwmeKJ2zaJ1pixjCzvB9i7IznNiTWSfm6rOmf1nd+1oCRsTFiSjlrM+73,iv:QpcPMsNz5xghxrjYw6kkdPJvL9zBLfVtyfE/Hyo2qMM=,tag:pz4UWY5aZYDLFCDjKcyqxw==,type:str] + output_sheet_id: ENC[AES256_GCM,data:XfLR39Saza3v6xX9NRxm1Lgg++0jTUVJU7T2DG/U8wpnfDJkHBlwzyQIypIWfapR2q2JMHh5CsfR6IY2,iv:MriqJEbO4qHyRuLqmYLCGghqcLXa79W0OueQRFzdQik=,tag:pIelynMOFR+qDWNQBZb7kw==,type:str] +kind: ENC[AES256_GCM,data:YfSIF+rj,iv:GSZfu8MjRgrLoZiDd3tYMC22XSBX8hN7rx4Qc+q1Sy8=,tag:pz5hprvCEnw2Js6o40s+VQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:pFg7/Zxxt1Tc9WQJVivlI8hgN5TL,iv:i/nNCn/CMrNBuRXaR7OgvADNV4vSwk0SkSwMrvzW0Xg=,tag:LBmpT+Aw2vziK13JDVOcFw==,type:str] +type: ENC[AES256_GCM,data:tcJ0TrMy,iv:bFgWgWktWvbfbQO8svO90b/6izbSTHKm4e3KdalerbU=,tag:nb7GgK5m+IjokvJAqUcDeg==,type:str] +sops: + lastmodified: "2026-01-25T22:22:08Z" + mac: ENC[AES256_GCM,data:FcISvk+RreKCwSHfvvobTXNbNbnRuvh7dPmjgTSzsW9jJGSaT9pVI1KnZO7pSwZyx9e1SxFxltM+l5lKRwEx644g8sXn4vgm5iS7hydlCaUSSG1S2Vm7QTkrc+Xd4anWt3/V9NTBgC99MbADwKUxL7SKj4auzXi2rbiJgrN2To8=,iv:bHBRrT6lFX2d9eOvS3henEqMusk3X9RbqPtqzIE1sA0=,tag:SXS5CgV6HSr+Ma0jADU/8g==,type:str] + pgp: + - created_at: "2026-01-25T22:22:08Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAtKMi6CN5tPD/ZRK0uiEZ1zZzS6XVXgaun1QQn3OufDcw + fVPefvN/Fw+6DuIzyRsBOCRT10BrD+2Cb08JE6GUMOLO7bihuAitxbwbzOivPPgK + 0l4Bj8S91shsRwhqFWDBWFHxiKJIuXLVBJd2AvijI3ErEL2hrxf3BAzaFQxtuxz7 + BPb4egF5zsUIjzkwW4vzUbqTiFzZPTh6uBOq1R5C1Ux3YFxDUWtKf4/0dOjQeiYB + =w0Fd + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/integration-tests-secrets-enc.yaml b/kustomize/overlays/tests/integration-tests-secrets-enc.yaml new file mode 100644 index 000000000..35e012136 --- /dev/null +++ b/kustomize/overlays/tests/integration-tests-secrets-enc.yaml @@ -0,0 +1,27 @@ +apiVersion: ENC[AES256_GCM,data:1iQ=,iv:HLBDch8hrZ1GpuB1+poHHWUNAlcKxHayVJhEvlbxq9M=,tag:wykCE87aZrCQbIdC3YO3QA==,type:str] +data: + GHSA_API_KEY: ENC[AES256_GCM,data:U28HKEJBc0nRNzS2ok6mmGU5uragLIBDkJ+8FzkNpsimf1YhXbT0/SbYgWs3TukQTBZ4nlpx1Gw=,iv:+EUrOoUJmQ6ldIsAuxcqIGd9OkfOTs11sfmj13wH9RE=,tag:rpCbWvXRi5G1x8XF5vXR3g==,type:str] + NVD_API_KEY: ENC[AES256_GCM,data:xbcl0ZI8ul0YPfI0ALb07Jx9lwXVyk+GpiE1r18lIMcC3mmFYH+v6b27RyUfa5zO,iv:RzZjdioWQmpWtQImXOM9vo6aPOXifWALgi9gmlsRS1U=,tag:aKUXAOsbfOvhhogYx6tAVw==,type:str] + NVIDIA_API_KEY: ENC[AES256_GCM,data:LV/REkQyjf9/r5pmF6isMLIrWNy7yPOILu1a3s5g2tgs6/GRuyDQjbWQEVeIS7Ia5xHLJYLsFFhv5Bp2GsCjDONejEG0iS8E/FHvvwIWm7XvwICh3/3kAO2v5zYhqYbO,iv:6ofERGT1m2SX44keQq6pGTQ2nZMsPL2GG1+Pqebp470=,tag:O/+3KL1tivQX5qoZVrq+8A==,type:str] + SERPAPI_API_KEY: ENC[AES256_GCM,data:qp8vpoTBsQUf16xrO0SwP6NC0YXYIat17sIf2Ceqg+0ywm+t456vAPwA3BzO4Q2ShoZacI325IKS73ikSbfUt38gM9kRry7S8vc7yWOapTAxDVMepNO65w==,iv:a802YttrC+iE6KIwGnRXS0Gz0TiJWHOdLJGD+RDtON4=,tag:MzyHmEPXb7FNqX6CB2kN/g==,type:str] +kind: ENC[AES256_GCM,data:GW1La2Ea,iv:Z+yqoV/+I6ZQ3EEddouc3NhYvUg/fZPSErB16foM51E=,tag:5QDiiu/FJeE9cFTmaTC1Cg==,type:str] +metadata: + name: ENC[AES256_GCM,data:F1vRnNiwgEv3Ym5uyw6HadY=,iv:mDA/W7a2YrlMVrE3rLH14FqdmZVrp9P+csIVB+wHff4=,tag:FY7zr/T+XB6Kl94Ir95/Kw==,type:str] +type: ENC[AES256_GCM,data:3czhKxVb,iv:2YRDLhXyiAalT7BJp006tjOvY/VKhb1u1wRGHpJFRHk=,tag:NHLatir8R+Zee/+7SnIkTg==,type:str] +sops: + lastmodified: "2026-01-25T22:25:21Z" + mac: ENC[AES256_GCM,data:YQEOFODZyLaVbnZbXToqv78KH3riikik97h3PSv1Ay0xhcjdBtBRv1JTIfVWCyAuMX7jA7WdUy9VL3xw7luK6TFbQRLI9W2vwovZ5ljA0qthTBIXuU98eLEsz/FNpg1GukNPhGKw54iTokGyEbRA5+Vo2zXygi5q3Ui/F5FDLyI=,iv:1ap3lpJvQuaFqD+l7jc7MHnVaCXCNBWRl0gue80anmg=,tag:yO6vcbIrM38ufAncapYcJQ==,type:str] + pgp: + - created_at: "2026-01-25T22:25:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAmOC1w11IvoBbzsDvMXNQXp7YzVqNEGt6xdxpgHExxjgw + h7GhiURhkbRGTA1Bd9V9JetB1ibDMv3Z2TSI0A+BS1GoNoG8BeM0t+efur3hMGNx + 0l4B/gnyReoDLZA6aaZDOMBLZe/GDaJ+FTegb5+VCTzAAsS1RgYrvPftBbCbyXbx + PdZo1yS+IMxuCQf0c3Q69FKF8Q8qy930UFjRxxHgiYdOQijJEhSyBpwiYhZK27tM + =geg9 + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml new file mode 100644 index 000000000..fbafaf322 --- /dev/null +++ b/kustomize/overlays/tests/kustomization.yaml @@ -0,0 +1,93 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base +- sc-llm-pvc.yaml +- buildah-task.yaml +- tekton-config.yaml +- secrets/google-sheets-secrets.yaml +- secrets/integration-tests-secrets.yaml +- secrets/registry-app-creds.yaml +- secrets/server-model-config.yaml + +secretGenerator: + - name: argilla-user-feedback-ips + files: + - .dockerconfigjson=secrets/user-feedback-ips.json + type: kubernetes.io/dockerconfigjson + + - name: exploit-iq-pull-secret + files: + - .dockerconfigjson=secrets/exploit-iq-ips.json + type: kubernetes.io/dockerconfigjson + + - name: ecosystem-appeng-morpheus-quay + files: + - .dockerconfigjson=secrets/exploit-iq-ips.json + type: kubernetes.io/dockerconfigjson + + - name: exploit-iq-secret + behavior: replace + envs: + - secrets/secrets.env + + + + + - name: oauth-client-secret + behavior: replace + envs: + - secrets/oauth-secrets.env + + - name: mongodb-credentials + behavior: replace + envs: + - secrets/mongodb-credentials.env + + +configMapGenerator: + - behavior: replace + + name: nginx-cache-config + files: + - nginx.conf=nginx_cache.conf + +commonAnnotations: + deployment-variant: tests + +patches: +- path: nginx-patch.yaml + +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: exploit-iq + $patch: delete +- patch: |- + apiVersion: v1 + kind: Service + metadata: + name: exploit-iq + $patch: delete +- patch: |- + apiVersion: v1 + kind: Service + metadata: + name: exploit-iq-phoenix-tracing + $patch: delete +- patch: |- + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: exploit-iq-phoenix-tracing + $patch: delete +- target: + version: v1 + kind: PersistentVolumeClaim + name: exploit-iq-data # Use the original name to match + patch: |- + - op: replace + path: /metadata/name + value: unit-test-shared-cache diff --git a/kustomize/overlays/tests/mongodb-credentials.env2 b/kustomize/overlays/tests/mongodb-credentials.env2 new file mode 100644 index 000000000..780f26360 --- /dev/null +++ b/kustomize/overlays/tests/mongodb-credentials.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:cTEW4bqf52nVEz7nWKXJAx/5NNoG6PH+sPu6BYuhj3M/4z4GpXaqSPvWL+V2LQwQCtmP1D2/BMtxbwgak1x699wYMfPdptx1I9iFnOM2P+CrYZFbOllYrEUHA6g7Cv4RItSadLPsYpNizMDxTP8udaWbYFq/2r6rnlK/V81nZaJZdxeyxC7T1CVqnxCGS3anI1ZVPDGY4F7t,iv:nkEQJuCDVI3wKzDsjh6tEHfob9kXJEegl9MhaaDClgE=,tag:kRirr2RpmIHJHQVlmbGbSA==,type:str]", + "sops": { + "lastmodified": "2026-01-25T11:54:45Z", + "mac": "ENC[AES256_GCM,data:hICbk3dfXtva1J8jqG7uGLis+pJuwGve7tdQSeu7x3M/AjXukgZqxQoOrzE7H7WEh6R4XM8z7OfFqmIegjDtJ3RGy3VIbo2uXpHrhkAGUI9204ehhe0GG3erZKWAYxdryFA27UOddYhMEqyeezrdmXEzbMhy3eOXTQlyENb9gH8=,iv:rMqa6c70Zim0SB4oba55MYcjS332/znRVU2YX9UBhsU=,tag:Gt7FgLOjKvqPc5v0zIZYBw==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T11:54:45Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdACeUMbw2HeFxqxL2d3+bLpJ5O8nnaixniZfOyLFNbDxUw\nudBo0dYJ5p9eSAr7/8xQBVeloOvLfO5DXBYlwZSkM4s1MPnTYM3vL6Je3qImRKzL\n0l4BPooAU/OY5y8idxBsi5gOKw/utLJ7150AhkLOCfvBVB5WP2zLLkWMx4rYMGTM\n8/hKqGkHi+8D2XfEByknn+95q92XnkGNcxZacCARXxFeDy0m+ZEeJa/KTO+5XU96\n=srjn\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/nginx-patch.yaml b/kustomize/overlays/tests/nginx-patch.yaml new file mode 100644 index 000000000..f8cd9dcad --- /dev/null +++ b/kustomize/overlays/tests/nginx-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + containers: + - name: nginx + env: + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://localhost:8000 diff --git a/kustomize/overlays/tests/nginx_cache.conf b/kustomize/overlays/tests/nginx_cache.conf new file mode 100644 index 000000000..33c32b01d --- /dev/null +++ b/kustomize/overlays/tests/nginx_cache.conf @@ -0,0 +1,115 @@ +pid /tmp/nginx.pid; + +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + proxy_ssl_server_name on; + + proxy_cache_path /server_cache/llm levels=1:2 keys_zone=llm_cache:10m max_size=20g inactive=14d use_temp_path=off; + + proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; + + log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + log_format cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format no_cache_log '[$time_local] traceId: $http_traceId - (BYPASSED) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format mirror_log '[$time_local] traceId: $http_traceId - (MIRROR) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format nvai_cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; + + include /etc/nginx/conf.d/variables/*.conf; + + map $http_cache_control $cache_bypass { + no-cache 1; + } + + # Log to stdout + access_log /dev/stdout cache_log; + + error_log /dev/stdout info; + + client_max_body_size 1G; + + server { + listen 8080; + server_name localhost; + + proxy_http_version 1.1; + + # Headers to Add + # proxy_set_header Host $host; + proxy_set_header Connection ''; + + # Headers to Remove + proxy_ignore_headers Cache-Control; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + + # Proxy Buffer Config + proxy_busy_buffers_size 1024k; + proxy_buffers 4 512k; + proxy_buffer_size 1024k; + + # Proxy validity + proxy_cache_valid 200 202 14d; + proxy_read_timeout 8m; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_bypass $cache_bypass; + + set $http_authorization_present '[NOT PROVIDED]'; # Default to '[NOT PROVIDED]' + + if ($http_authorization) { + set $http_authorization_present '[REDACTED]'; # Set to '[REDACTED]' when the Authorization header is present + } + + # Configure a resolver to use for DNS resolution. This uses the Docker DNS resolver + # See https://tenzer.dk/nginx-with-dynamic-upstreams/ for why this is necessary + # When considering what the "base_url" should be, consider the following: + # - The base_url should be the unchangable part of the URL for any request tho that API + # - If the API uses versioning, the version should be included in the base_url + # - If the API is a subpath of a larger API, the base_url should be the path to the API + # - Examples: + # - GET `https://api.first.org/data/v1/epss` => base_url=`https://api.first.org/data/v1` + # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` + + # resolver 127.0.0.11 [::1]:5353 valid=60s; + + # rewrite_log on; + + ################ Docker Compose Services ################# + + # Include any additional routes from the routes directory + include /etc/nginx/conf.d/routes/*.conf; + + + ################### Redirect Handling #################### + + location @handle_redirects { + # store the current state of the world so we can reuse it in a minute + # We need to capture these values now, because as soon as we invoke + # the proxy_* directives, these will disappear + set $original_uri $uri; + set $orig_loc $upstream_http_location; + + # nginx goes to fetch the value from the upstream Location header + proxy_pass $orig_loc; + proxy_cache llm_cache; + + # But we store the result with the cache key of the original request URI + # so that future clients don't need to follow the redirect too + proxy_cache_key $original_uri; + proxy_cache_valid 200 206 14d; + } + } +} diff --git a/kustomize/overlays/tests/oauth-secrets.env2 b/kustomize/overlays/tests/oauth-secrets.env2 new file mode 100644 index 000000000..ae7f97eb3 --- /dev/null +++ b/kustomize/overlays/tests/oauth-secrets.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:JM3q6XFUVQ8XvsAWyaHAXUcxGPgB3ixEwIL6NGXTLfznS8v7FRRouw1iG7zDn+IFCk2kYhHW+wJ2fDBj23Fdh5dMPIDzMvxn2yyLLqyvYsnzM9JNVnIM1XALph/U0J0GYmzMLIJgqKnk7UexkypD8VM=,iv:6YKdj5UDd2TdPNwG4u5U1CZ3Mbi0MV82kgvNc+R4MUU=,tag:Rm3WSsiGmnmIn1NrQdZlpg==,type:str]", + "sops": { + "lastmodified": "2026-01-25T11:54:51Z", + "mac": "ENC[AES256_GCM,data:UiHjGVbr6MnTwXjpfQAba15b2dW0EeRmZ0LCqvwMfMoJlRQOgjmGDldeXBRO+jyYKvMsxfRLsGjIIDAnWE4+68NCHhH4IbDPH3gvwLijzNrUWX5wwFtQP7uXhtB5gQjt0G5e1Mwtz2oPPPRQ1BTVycHrJN8E81UANBpTN93kQFU=,iv:tnwkRwG9ChLQFudPafIGotkyjUXTedcRhT4QXiI0t80=,tag:ZEGjTCCkmRTKTboqE/GK6w==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T11:54:51Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdABXzNXw0NPdMDlcCmVeio3d9jVFxFAr/IQkfrGkzimFMw\nt8tPnMaZfSe0ZNTtDMUhtXtYqu8Mqk+IGNANQyQXw9/pK6tL3sdctQpwES+h5SOK\n0l4B7x69zvOIlmK3yB8EoPE1Vn/0qoRjjyOk3/CWoeDGjL4hwwmOLTtAM0un/VCb\nhiEX4nEgWIaz4DWyQLZkqhJbPlRLTOvwXZI95UY1f5nQRt/zjbeKtRbXmiH+qlXy\n=X3t6\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/registry-app-creds-enc.yaml b/kustomize/overlays/tests/registry-app-creds-enc.yaml new file mode 100644 index 000000000..86da72c75 --- /dev/null +++ b/kustomize/overlays/tests/registry-app-creds-enc.yaml @@ -0,0 +1,25 @@ +apiVersion: ENC[AES256_GCM,data:tCQ=,iv:/JnoJlOSPDnpfzPax3IltVloS91QOp2VWzDWAkVi1cw=,tag:N+vwaUKTsFHin1YE0hh2pA==,type:str] +data: + REGISTRY_REDHAT_PASSWORD: ENC[AES256_GCM,data:DY9vrRSPyVvFWqIUiughhehhr65Z0xGrTzMH254SVOuMdHqq8PClGO9XN+JBjMib63nd/Dih5jMkdAqGPt8HnumVWJKj2O8B1ry1o1kcHbGEv3RM0YPs5xzoAqCX8P5eP9MRW1vJc0vxqJA3hKPJ5colLR4KevFmoFYMPCczVHJygvNQlqU0smJaQcdwyPYWo0NQgSYGo6qUVbjgYCjzQFf1EJEiaZX+bdQTAlcHcdexZI+5cquSijaBFxi4GlLSXIldlnPA+rCQ8hTyD3PEj7drvfJad6u06Wm0MjJoyD/gAIoE/9NiSsnq+7fr5Ulf1FrFekq7ps1iMm83uKAiXRSC2WtBnNA0UdU09Dl5efXSvlvCi5AbK4o6gI/LnMyvcGkYlqKDgGYYNcDZqMe8wxk3SOuMuL0JZTBdC92AN2HZtYvinDrL885azN7tXHulNwYRGtPkK2GH2Yuhbi693dwejR6Edk6C2KhczOKtaLFH7ULiQSUMwk/xEmqH1HDOcuai0oX94HuIKYiWFVhoe72I2Mxbwc9Wt8cQ/WAn5DEuBrdQJ/gd9q4/lRAi1x/5wwlGy2TWMOiq2Oh7tTrh6wEvImLFKQ6wxlxFLHYs1p9xvSrHcyFxa6zyNOsir2cImQSaXdS/d91JfjFiHY9MHdCTP7wY3Tb9kYKi2txUdTbVop7SiKpTJfZg1rwhZWi0HsEWpDCJugKENbJinv6IaEmb8yc5pnl9vS0qLfljDAipnBkMm81J05kVhrZoq5I6w1a7JYXomXcTZAPwPTwh+PMuI/+IUxT/ciWbkT2YfPWmWGlzLFaKGd1w7sj3XthTtrIh5elw2xwoeuEwvZLIbeZRZwhWOoi/h6ByMPdCc2ReUaN91yumN22sYaw8mHudz2D4nbPo12ZfPfA1Xx4EsVFe5j0FLn5GP2TCcuO/DAMrXvXUykhGCPMcExpid2Q+WXYYq/LwAEyyqqfUVsXbjlb9+4i4MqXISzUhI5mQmVbh5iusWbEUvrvpBJe1wsLZhRfOSpdmQRhYNoSiVj/xZfJt2jzqPK4ISmkY9wiK68V79cJgugzwS28bWwF08AZZNSsPAXadODCpuqCve7JmWl3nR0X0Acel/gqU2KCykGbr3EvdxUaIGUHsbLqoXjhrLrVsJ0Lyd3g+bhNL1an4QzgUHiVok0/eUQXXcUJt5TddedG2TkOtQINl8atBSS9TNW1paxco8Aiq47uRRC+4O1i0V1iyeYOPejQjMjTTGF2326R6c6REmQVsAqjqG1FaqH6WzQZB8p0sXHTtm+bFGKjPnjaaHezolumRL/DrrFL16VtJuqZLnKJ9m3RjAZQVVqINt9SUiNU3IA0u,iv:COGn/Fn8b7HLutwSONHSkQL8FDm/DiaTzksfJhv+5no=,tag:FLTdrggW+uTEdbczPZqCAw==,type:str] + REGISTRY_REDHAT_USERNAME: ENC[AES256_GCM,data:7tDxGRcjSHGsBb//r5NBJtaz+g9HrTQ2,iv:/gRNExKbdVqMOvdvUxCw5ajJVTb/7Pthxs02AlwKqo0=,tag:wjBGPzc8nkF/3LMoZaFoKA==,type:str] +kind: ENC[AES256_GCM,data:0g19ZH//,iv:FQwXlhfkZNRBZF65DmZarqFDLeQQJ8lUSS9d0laPABU=,tag:0RcpIXMAN4CK5QI5QkXG/A==,type:str] +metadata: + name: ENC[AES256_GCM,data:FDxteHJ/f1a7FgW+RUryKQ==,iv:59iQVkcUKGymcArOn4PCR8L1ups0XmHcmv601J9dtHc=,tag:vrdPMLmn0UDijAWfCm9MSg==,type:str] +type: ENC[AES256_GCM,data:jJztE258,iv:jk9TV1Uf1ydiyunGsv32tlp5grXTVyTbI2lJkVTM9lU=,tag:WUnjbftart7YctxO1XS+sg==,type:str] +sops: + lastmodified: "2026-01-25T22:26:41Z" + mac: ENC[AES256_GCM,data:5IZ8f03JChaaNm/wozc7wbs+DDeFYEguGB9/C49MPv6zK2xhnHFmk0B/RHrIl+/ZNZU0NtcQ9ZZxeYGEHgDKrHJpnFrPZeuRUPhmfCkVY2POzas85SI0oCfXKaSehdzRVd7XQ1++fRo9SHXbWsNgRH2C+g3XToFjOeyBg6cPfBM=,iv:bf0WSgF5g19TtxYL7HfzpjXlZAQbUAaXW2LkGraGVFw=,tag:KcRubwVZFcge2Kw5GrpMkw==,type:str] + pgp: + - created_at: "2026-01-25T22:26:41Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdA0hIJBy0m1G4y//yFta/U+2rnqzA/E5cuBHP379fUD2Ew + v0oJlrgCU01/xwvy2VvAHJaVbZ2/3vcR6DCnpo33Ev3tZFwRKDM8jQzPZpkvaAwh + 0lwBB7dTxipu3NSEQDY8Y/g9+eIlbQTF3T6aRi1XHXoVNsgsaHGTfAxfA99mXKzD + Q50124BAwhb88VXw6OLvN8CqcEK3l0t1VtpgF6BkGTB7/cJxaWmfHbP+O8J2Ew== + =GF5p + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/sc-llm-pvc.yaml b/kustomize/overlays/tests/sc-llm-pvc.yaml new file mode 100644 index 000000000..fe5caa307 --- /dev/null +++ b/kustomize/overlays/tests/sc-llm-pvc.yaml @@ -0,0 +1,13 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3-csi-throughput-2000 +parameters: + encrypted: "true" + throughput: "2000" + iops: "64000" + type: gp3 +provisioner: ebs.csi.aws.com +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer diff --git a/kustomize/overlays/tests/secrets.env2 b/kustomize/overlays/tests/secrets.env2 new file mode 100644 index 000000000..df5a50484 --- /dev/null +++ b/kustomize/overlays/tests/secrets.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:qJ79PMbpPyOLezRJhCB2mRe5C+LVDHvDnv8RwbYYX2H4c2P8Ugyhcs+3uAyARQnSdzKOu2Vf57lo3eB6pyU6FZwP7rn2nNbcviPQdQZA4Z4e3S/ZPodhqzkwCU/8KlAYJUDWRt7rbtz6yRSjF+9uwu1dJXIGoFrpCo7KE8pZgaAAHvADnBT7RT6MAyQkQZQpqpGyBiF1vZ/MPnY7w1ke5/IpzY30kIepcwZ0fCdgViweYthkKu/CrBwl7h/S6FxPp/+EpjEMQimnvSaMybjoKYovQ7aNK+sE5CFHxv09Wsy2oVx7C0DDkQGJJTmDXgbNwUmwNq9OK8vteh91YvH2r3yPJT5Q2UoRSvVpOnqyQGenrYme712O5ZL1MW9qLpukUtEGQjcK7gaOANOqKFAly0yDtBg7mvHmvtqJl7mqGjqE+m931h9bNQgIdP/kxt10RxfwRatMuQUfhmpqEAQCDsEW8WUCOjE3whBCYsrY429dRRmDS4sffOnN8SQkrqNh6/+7n7WQFw71Yjy2Nak7WRdWNsPuSUAGvq2kOVjQzWUbH6LSzcuDmV5chCunDQYXvbSO4K448Zq7zU8tnL/Kq0BC51iLztUBAQgZ2mSRFimx2IIJcZs3iKHN9Rad6w03oD1vyCfUMMLWCpqN+v/o9Gdtuh54lV75ocVk/uC6WBtfkRd31KDi9rERB1lO+zufillhOcsb6ihI1xg0HLbX4N0ds5Kl142BuJ57QuTC48M3rJRhAVCHNerHlPZFIaHNCJ6XeUEPbr+Bq8GTdKLIZTT3HWzPQRid5nrvKAgOH1k+N87QM95ofmnrd/RshwtyjcvCVLY0Ua2/93JxqlhaTMR4J9u8WoN3zNQ5oPGnU1/Gf7z7HzWicNne36XBv44Apby4V0IBrnpTi7PNzqKDm6vOUyGKJvbHXlHuEBEAMcdApdGkGCMwf5vHfdk1naI/7rXWcvJtlJgAdDACDtKhOdWX7o6FvsS2qyKTWeeDV/KuUpt2IJ2v0SoYCbClSydJseKy044Yr+Q/OlNTLfoVYP6Y+qbtJtjxUqJq6aq2kg6lP7bnEpLqyo5Od6GO2pI/E/N1OifZRNvaIwildhd+BSv4cM55Syu+S4aGbyb9o3+NeVhtC6uYIJZxIxFJfxbnAG89I1Yb1rmoD5YrXOAIv/jD95B0mU/JntsblJYlKyJ8TyiOL3w5ftoWTiRKefzwnvZcoUcxQ/iO9smcWaUABFC6nBFZJblYQ1E7d/b9PAsD+KwOJBJnhMPl/3qYip3SOL8NSX7+VtfRwMgd/MQInjmnhQx96xV6UWHe1EBX+Bx/75ZCye6f+EY/4tABMHyCgfZw2rofjsANCnZqp4MxUFKBUw/gm0JVAwMPR3u+uAWWnkiekjvGeqbMGW0Y+cteKZA/b/kaw3fLASSbzAMju0Od9rEV8lJd3T4j6PJ+uXuxieUU+mD1GRn5l5ZngiL6ntSQF6UD8bZ4Gss3PgvHEmqdaY1ta5k3t1OsJF/edgwzPW2Vlzc7wSHWpkGMIUTqsMiNIIM57otjt8ox2VfoJakt0Fj1AJ6MAtUJYLgGlBzsvoQ5h+yXT1dqNOUzqra1Zl5XsdWx+gnaFN+PUG+v0pmkXl8NJInYDcpFzacEMzVT+F1nCpwJ57bLAbmJVeqQ,iv:MNFqnAFl1Ugx9oebDR66ckvZx8jksY+HeDsNFehFrk4=,tag:f6N8qxAy07EJHHwu2N5K3g==,type:str]", + "sops": { + "lastmodified": "2026-01-25T12:03:28Z", + "mac": "ENC[AES256_GCM,data:S8a8LiimwRsM2oqj6kMnG1umAXxb42GcdGLpDtcXLcbA3EewgRVCl+UIkDvFkagK8WUl2jr715wNgS/OlIWWdTBc/jAfU0iV3AHOCCz/Z1t16wQ96Pf3sEc0++l4EFlPfBPmhhCdXoAVh8TrPSpQRRKEXNY9JKcps2nZ5UDJ5tM=,iv:djGFNCf5O36+iyb+ENVdnThB749rc7PvpA8JAFJFKCg=,tag:fl7sRerdVAAXBkIATUl7zw==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T12:03:28Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdAZGmK8TFHDd7hi+u6iK85h+tsLOTnv+cd5N5R2hmhizgw\nYGsqyjGgzxKosWdsGQw9n6RQwLookPSjLtO74CInBmSSpbL+k4NhAKs+G2vCojdN\n0lwB7TegWbha7ElzY6+MjOv+1p5EBhHFIWut+6+iV7HzrbralcDvo1Lo7b7e/4sv\n3jEf4HcelocgSLURYvL/5Cl48AVXrhkxdha+6xw4py4YRFIarIMU8ZRnL5gAqw==\n=caPs\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/server-model-config-enc.yaml b/kustomize/overlays/tests/server-model-config-enc.yaml new file mode 100644 index 000000000..f978b3c54 --- /dev/null +++ b/kustomize/overlays/tests/server-model-config-enc.yaml @@ -0,0 +1,32 @@ +apiVersion: ENC[AES256_GCM,data:GqM=,iv:X9ZA6MF+bye5xH0o8LL/4XFDfiVzgDeL/LJUVGNiJnA=,tag:jlRvjVgKLC66t+n/9UPxeQ==,type:str] +data: + CHECKLIST_MODEL_NAME: ENC[AES256_GCM,data:7OrZQrchux0+jlBPEuDhsyAv1Ix75Yp1/uPKdjy2g2lziSXhdwBrEtSF0U4dQWBITTYp,iv:TAd+SIFStCUd1qQDtQpD9ejlH8R6oCa4mLrwBYPh5nE=,tag:wGSMp+pww/4Zt7mvqVvwrg==,type:str] + CODE_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:4D444ciNBzQPnhYXT0niQZWF12OtQ/stcNpdEbCXCCRbf/jUfkYoai+0u08EsYgmh+TC,iv:SehqcrF7ZA5itfSeEDMyeuFTOBrufGLlGcNxO+yKdp4=,tag:4jJ05Bitj0iVNDvSl2NeaQ==,type:str] + CVE_AGENT_EXECUTOR_MODEL_NAME: ENC[AES256_GCM,data:xtooeRKaRb2uXvEmhcq6W6M9UX1g2fk2WC9jf0cj0ReMlNZ0waj7JOY0KVfWM4n9eBEg,iv:mYGS+QzlUv3aUKQlGF3Qj++h9kg73Btpmi+mf3GUJ3w=,tag:BWLyR3yhAqEdpLE1T4lYpg==,type:str] + DOC_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:g9DV3/7vINF2g4MPUScMedRhx4EzUn6aOHztFmAmhdqU7iaOYAMLQaGZkkMdiXsSqziu,iv:0swptAt4Hwg0/y3f28btcv2Mtq0yYB/mCTfFXjWe00k=,tag:vdgiZOimYt+5uluRtyP+cQ==,type:str] + GENERATE_CVSS_MODEL_NAME: ENC[AES256_GCM,data:uDTbwkxuuohDKJ9X+uAYHE2lJgb8BKEBkTGH5xL1YWy82R6OpWE53l9d9/KAk8JL3COi,iv:bWsmW1J1U0zmWlhj5cvLz1dFYG/NPF3PiZhWS5duT+Q=,tag:w6PCNZhOysrHW7l74X8EiQ==,type:str] + JUSTIFY_MODEL_NAME: ENC[AES256_GCM,data:ar93nbJy18x6Nb++YGHSuh6QGFnmhiyAZtq6hpRsDT6g4XSAmFAsCO5IvBEIZ7dpNprZ,iv:ks3khWQCRnz8mUrvK7B9SVz+wKOF49JU817dWglaDrM=,tag:OB3ULv2x4Ref63j/nlp8DQ==,type:str] + NVIDIA_API_BASE: ENC[AES256_GCM,data:58mO8y0qF6FBOXcc4E3jppjo3ym6A48wGZL9OxTPlHcYptR/AW6KvGdSA8TMk5DG9RE10Edx0ChXB9XsMFvxfQnfA+9OiBnCNxRUXw==,iv:OUlIO/XHdqkvpOM35JfEQSke6TmyCuMOEcvJqLSUkzE=,tag:kifU31FSIu3//q9pc7u40A==,type:str] + PYTHON_VERSION: ENC[AES256_GCM,data:OmHFTw==,iv:qsd4oNKf/oYCSpICfSE7cuUlIfRmI7VrZ+Wth/nNPvM=,tag:Hy0YNQ9lmcbqrXHEjtzfxg==,type:str] + SUMMARIZE_MODEL_NAME: ENC[AES256_GCM,data:vO8ztZTgcvg5tv5FQaKlKBsTs8ms/78Qe9OtmDS7LnibP6K9yljNbz9ai73Ph3CIn4Wu,iv:ak/afG2kXG1rvsriN21zNsjttc2Vt+VoYhoauwWA6ZI=,tag:lf0QVlghAuin8l4keWkY3A==,type:str] +immutable: ENC[AES256_GCM,data:7VHk5CM=,iv:9GuqxkAjk0XtcAHP1xJ7OjfHRDEydV2XCrNzXMzNZho=,tag:J261qozJtzdv+tOzIZYmrA==,type:bool] +kind: ENC[AES256_GCM,data:+XvGoKaqZt3d,iv:ipjdNZIC5ucrWo7Cpa3I64nKLd2eOslQjwe6R0LXsXU=,tag:ZvTEtcfLxIBMvqEw7mLqWA==,type:str] +metadata: + name: ENC[AES256_GCM,data:bIyOig0niHOw8aT3r5yg6wSZzg==,iv:FNtbtOzmrKkz4Ocb7ujgsBBg72P0ACMlS5DfT8DP4rk=,tag:beE9XVfmH2CzYoCkYJMnuQ==,type:str] +sops: + lastmodified: "2026-01-26T09:27:06Z" + mac: ENC[AES256_GCM,data:xMMv5Lrb9cRGYoEjwLHHf74gadMJfm17697zJn/UI+2bZceSBVlJZAN+4Q8yOtTDM0uC1uuGbwZC82yJuguMNuDUFo8yQJ1evU45A8WPKJXLW9GkLCIuh/8302kH08EI9GLcBR5eXuQCl1N1uKRx56rNCiu/WCFJN3WUS/gM/8A=,iv:C1KHyp68xbcOSCDpDjvoHMsPGB4YsnQ+2dHkyOo+l8g=,tag:0aFf97vxoXupUQX8kQAkRA==,type:str] + pgp: + - created_at: "2026-01-26T09:27:06Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAANcbJdqeIUK1mUDG/+GglQPLSlMcvS0xt0aP0eHsakEw + EU5sQytLTMNUYH6iRp7BacACVQ8T/1rllgno7lGN5nLoEUDbg/wOtFH3ZHFus6kj + 0l4Bwe9pK+UsHDqy6I38YWHU42AsEECCZjs5fPHUnKBR2oBBpf4S/Eg9/FVnRIOY + QFkPgGIyQIOPbCVzaIo+tzuIwwT0RjQQAcnPZM3eM0ie1q82eD9UstkwAV6Upzqv + =mxQm + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/tekton-config.yaml b/kustomize/overlays/tests/tekton-config.yaml new file mode 100644 index 000000000..8183bbcf6 --- /dev/null +++ b/kustomize/overlays/tests/tekton-config.yaml @@ -0,0 +1,139 @@ +apiVersion: operator.tekton.dev/v1alpha1 +kind: TektonConfig +metadata: + finalizers: + - tektonconfigs.operator.tekton.dev + labels: + openshift-pipelines.tekton.dev/sa-created: "true" + operator.tekton.dev/release-version: 1.21.0 + name: config +spec: + addon: + params: + - name: communityResolverTasks + value: "true" + - name: pipelineTemplates + value: "true" + - name: resolverTasks + value: "true" + - name: resolverStepActions + value: "true" + chain: + artifacts.oci.format: simplesigning + artifacts.oci.storage: oci + artifacts.pipelinerun.format: in-toto + artifacts.pipelinerun.storage: oci + artifacts.taskrun.format: in-toto + artifacts.taskrun.storage: oci + disabled: false + options: {} + performance: + disable-ha: false + config: {} + dashboard: + options: {} + readonly: false + hub: + options: {} + pipeline: + await-sidecar-readiness: true + coschedule: pipelineruns + default-service-account: pipeline + disable-affinity-assistant: true + disable-creds-init: false + enable-api-fields: beta + enable-bundles-resolver: true + enable-cel-in-whenexpression: false + enable-cluster-resolver: true + enable-custom-tasks: true + enable-git-resolver: true + enable-hub-resolver: true + enable-param-enum: false + enable-provenance-in-status: true + enable-step-actions: true + enforce-nonfalsifiability: none + keep-pod-on-cancel: false + max-result-size: 4096 + metrics.count.enable-reason: false + metrics.pipelinerun.duration-type: histogram + metrics.pipelinerun.level: pipeline + metrics.taskrun.duration-type: histogram + metrics.taskrun.level: task + options: {} + params: + - name: enableMetrics + value: "true" + performance: + disable-ha: false + require-git-ssh-secret-known-hosts: false + results-from: termination-message + running-in-environment-with-injected-sidecars: true + send-cloudevents-for-runs: false + set-security-context: false + trusted-resources-verification-no-match-policy: ignore + platforms: + openshift: + pipelinesAsCode: + enable: true + options: {} + settings: + application-name: Pipelines as Code CI + auto-configure-new-github-repo: "false" + auto-configure-repo-namespace-template: "" + auto-configure-repo-repository-template: "" + bitbucket-cloud-additional-source-ip: "" + bitbucket-cloud-check-source-ip: "true" + custom-console-name: "" + custom-console-url: "" + custom-console-url-namespace: "" + custom-console-url-pr-details: "" + custom-console-url-pr-tasklog: "" + default-max-keep-runs: "0" + enable-cancel-in-progress-on-pull-requests: "false" + enable-cancel-in-progress-on-push: "false" + error-detection-from-container-logs: "true" + error-detection-max-number-of-lines: "50" + error-detection-simple-regexp: ^(?P[^:]*):(?P[0-9]+):(?P[0-9]+)?([ + ]*)?(?P.*) + error-log-snippet: "true" + error-log-snippet-number-of-lines: "3" + hub-catalog-type: artifacthub + hub-url: https://artifacthub.io/api/v1 + max-keep-run-upper-limit: "0" + remember-ok-to-test: "false" + remote-tasks: "true" + require-ok-to-test-sha: "false" + secret-auto-create: "true" + secret-github-app-scope-extra-repos: "" + secret-github-app-token-scoped: "true" + skip-push-event-for-pr-commits: "true" + tekton-dashboard-url: "" + scc: + default: pipelines-scc + profile: all + pruner: + disabled: false + keep: 100 + resources: + - pipelinerun + schedule: 0 8 * * * + result: + disabled: false + is_external_db: false + options: {} + performance: + disable-ha: false + route_enabled: true + route_tls_termination: edge + targetNamespace: openshift-pipelines + tektonpruner: + disabled: true + global-config: + enforcedConfigLevel: global + historyLimit: 100 + options: {} + trigger: + default-service-account: pipeline + disabled: false + enable-api-fields: stable + options: {} diff --git a/kustomize/overlays/tests/user-feedback-ips.secret b/kustomize/overlays/tests/user-feedback-ips.secret new file mode 100644 index 000000000..e0f44fb90 --- /dev/null +++ b/kustomize/overlays/tests/user-feedback-ips.secret @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:DTCFG/YxGdv2PUvFy4ZpUcn71vTKVE9Pq6OdqGrhThVqW5yIZFdIlEutthH9q/AbnzZZalBxf7nb7T/muOnJd+pE+uN1jlrv6kSdaEcwgViK9CQCRVhqIfWCZNFYyGVFpo90TlD8dTrvk0f1LqbnTQGbuhBIscuKmYzJB+FZuUYzBP+7YgLD34lwhRwOd3gM6IwYyptkDYL7BnZB8t7bZSeliuQwmjwJZ49oDVtG8+CZN7UlUw1hxVJPM96BR6Ob,iv:VE3imWMy9VK5IEQ0ZV+WYbHtRNNAUXPZZY3mA52P258=,tag:2UFpjo65+QUs9fvcTueiQQ==,type:str]", + "sops": { + "lastmodified": "2026-01-25T12:11:26Z", + "mac": "ENC[AES256_GCM,data:PpNa9n8COE+xLG2Kmb8U18dAUT7iqhUfsquUqJC+42DdGC5eWU9t+nxJjy2Hgpf+Gj9hZErRQDYn6LZufKl87YcOix5V8TlBx078LIw1ZFch5xvPIoRef47uedpnPZM6UDAYFljDj6VWUXj6zdqC+GypArYYt5c50iIpY31Fd0c=,iv:N/8Ie16mcQ+QnGjg9c4rXK0OSJn+KujWA7S2WFFa4jc=,tag:69iiCf7FIVmm//LrtCixcw==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T12:11:26Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdA/VGmVQPa0NNd5tNIIEunjIeYbcNYFAytX7GStmGIWzow\nzGLovmXwgN/5IQAdqZiRovpI5nURr49xCgCjiHfvui0eI0mm05+Zhph7GlSG0FwV\n0lwB7q/kruiHG/tWCtqESzJYMfun7Jh0CWH/yl3K7N3JiwREqUsSUVG4xOdhJEEs\nus/Ov0GJd6isSwb1uLjtCNszmiQXss6Rw/tutDmRwR4nE0J26CujVwqkuSDbbg==\n=RFOe\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} From 71db2b1d91c7aa717a5415f0f542b5435ebe2d87 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 26 Jan 2026 18:39:59 +0200 Subject: [PATCH 201/286] docs: add sections about PAC github application and openshift pipelines config Signed-off-by: Zvi Grinberg --- kustomize/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 2d413f096..c07a3cc58 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -320,8 +320,7 @@ kustomize build . | oc apply -f - 7. Deploy Self hosted LLM for the automation tests ( Integration tests and Confusion matrix runner): ```shell -helm upgrade --install --set nim_embed.enabled=false --set llama3_1_70b_instruct_4bit.storageClass.name=gp3-csi-thr -oughput-2000 --set llama3_1_70b_instruct_4bit.readinessProbe.initialDelaySeconds=25 --set llama3_1_70b_instruct_4bit.readinessProbe.periodSeconds=10 --set global.tolerationsKey=p4d-gpu exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models +helm upgrade --install --set nim_embed.enabled=false --set llama3_1_70b_instruct_4bit.storageClass.name=gp3-csi-throughput-2000 --set llama3_1_70b_instruct_4bit.readinessProbe.initialDelaySeconds=25 --set llama3_1_70b_instruct_4bit.readinessProbe.periodSeconds=10 --set global.tolerationsKey=p4d-gpu exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models ``` 8. Remove untracked decrypted secrets files @@ -335,3 +334,7 @@ helm delete exploit-iq-tests oc delete project $(oc project --short -q) ``` + +10. Need to install on cluster [Openshift pipelines operator](https://docs.redhat.com/en/documentation/red_hat_openshift_pipelines/1.19/html/installing_and_configuring/installing-pipelines) +11. If need to install the [exploit-iq-pac](https://github.com/apps/exploit-iq-pac/) PAC (pipeline as code) github application on a new cluster , you need to make sure to configure it according to the [PAC github application docs](https://pipelinesascode.com/docs/install/github_apps/#configure-pipelines-as-code-on-your-cluster-to-access-the-github-app). +In this case, you need to supply to the secret in the documentation github application private key generated in the github app settings, and webhook secret defined and set in the application settings. From 5a8189be9e5c775c1aae438baf3353be451bc6ce Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 26 Jan 2026 23:07:24 +0200 Subject: [PATCH 202/286] chore: propagate ips to all taskruns Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 6315138b2..d9240805c 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -17,6 +17,10 @@ metadata: # How many runs we want to keep. pipelinesascode.tekton.dev/max-keep-runs: "5" spec: + taskRunTemplate: + podTemplate: + imagePullSecrets: + - name: ecosystem-appeng-morpheus-quay params: # The variable with brackets are special to Pipelines as Code # They will automatically be expanded with the events from Github. @@ -241,7 +245,6 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME - print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" @@ -257,6 +260,8 @@ spec: workspace: basic-auth # Needed for pushing tags/releases - name: exploit-iq-data workspace: exploit-iq-data + - name: dockerconfig + workspace: dockerconfig-ws params: - name: CURRENT_REVISION value: $(params.revision) @@ -452,7 +457,7 @@ spec: echo "--- STARTING INTEGRATION TESTS ---" echo "--- DEBUG: Current Directory ---" pwd - echo " -- check it certificate is mounted" + echo " -- check if certificate is mounted" ls -l /app/certs/service-ca.crt # 2. Prepares the IT input for test cp -f ci/it/integration-tests-input.json /app/src/input/scan_it.json From 9a6aa66fa5e8c51103494fc58bd0c811f8d26697 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 27 Jan 2026 17:06:22 +0200 Subject: [PATCH 203/286] fix: correct wrong password in secret chore: add exploit-iq client app ips secret for the ci build process Signed-off-by: Zvi Grinberg --- .gitignore | 1 + kustomize/README.md | 1 + .../exploit-iq-client-build-ips-enc.yaml | 24 ++++++++++++++++ kustomize/overlays/tests/kustomization.yaml | 1 + .../tests/registry-app-creds-enc.yaml | 28 +++++++++---------- 5 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml diff --git a/.gitignore b/.gitignore index 79b658d86..0c31c4dbe 100644 --- a/.gitignore +++ b/.gitignore @@ -205,6 +205,7 @@ tags **/server-model-config.yaml **/sec-decryption.key **/registry-app-creds.yaml +**/exploit-iq-client-build-ips.yaml diff --git a/kustomize/README.md b/kustomize/README.md index c07a3cc58..05d8df38e 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -310,6 +310,7 @@ sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml sops -d secrets.env2 > secrets/secrets.env sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml sops -d user-feedback-ips.secret > secrets/user-feedback-ips.json +sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml ``` 5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. diff --git a/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml b/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml new file mode 100644 index 000000000..1e385c0b2 --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:V0g=,iv:aku+6VNcpHX/VwyWNuHMU9p1UJn9QgI9az5pl57Up0c=,tag:9cHqcamNq/n459qQRu6bKg==,type:str] +data: + .dockerconfigjson: ENC[AES256_GCM,data:PSc/c7Dw4D0LycEKR0DwFGucqdHms9QombznJc7BtJEfDBd1/mTFOobGw4LHkUtUc3I7eplOd+u41N0PvERRXbsf26iWQkupb578MdoTXXtzpzwKZcOt1pmROrIs/I8yIgCmOYL0feqhPwSP0aYESYwTCQXK9nMvkrCNPpdsyeBMaQ17K2G+Lg0zk2/Crh8xQwbbVuU+ZxVCQ2mlORGQVFTQzjCCvMll28szpvqSXS6fcxJ1Ctma3I9Dhzv4fDP0QVlmxM9IjCZ2gQLqqMulXzmIViqxUn5G7Nmm3GtS7DjdKTpIFr6hIowzi/ikywmOX9rvqQKpZsq1VSkdenR7Nkq2BJQERmjxrGRp460ikA9MxI076q5HdjY6fibKSFROom9fltTOdniDZjnzd7rp/WT0Q+rrI4C9hVOFZ291KEOysLfmovSXWtDOKlX1EsnrwZVll7QWjLbqElqmGluOAHKJNFVUeBMxveRPcFVT01+4ddtcOqKgvx8vVihavEDEYCHTUny+VqUhnAYbXyH27TqfcfH7xs7KOcc71xnpXtcyAkdPBXku7T1xzl9090O4ZuDEK0hADGIL3RJuO6kCVIJes8vEfAY1HAiCBO3ykcrizkV76E9FPvNZlX7vbsmWQ07NbyFHBv6lKsuPrLBf80eulUxLnBz5+jfaHi7QcNjY2ADaPcdxyKvYipWxnjmOE5UisIQ3mmH6TvMg2ZuyBfuLwgDeLRyReXjfutYz1XFUNc5dstNJ0BRQ086zfppvKn+EniP6mmpflmFRGp1jULg3SgZiWXQ7SvLWhnDGIE2JS0j+dLr2ZyGrwrXNRSb0ypLuB2fxS3Tn1ZjVGsToB0gWv574v8IE68uI1J63FnL73DpYGIBLrnpN3QsAtRXSQ8jgPU6OLn3F8CNGM0NY9EsaT5sR2KA7TOo/dWzHWPfgphs3P1JwvAiT4shvFB1weu0jAKRAE2+vWPyazK29IYyqFg18KP5b/ibU4v/cmPJo69uy7tBOhS8YGKZ68IGcq4QJO9JKgS6LRd66mqZd9YdJSqwsVSi0fh+CtesxvpHWn1bz0Q1RMSczw4/8/4SY2mDIDOf/Z1ViiO+CgH1z5gR7UNeUmWHXUlDRf3y1YRFu8BhRCw+VH4yu+pg/m4r8E4bZ8fWZTMZpm/Q9Vf5NwJoKpwgOXorImJmSb0/UUq2VkMjDTiv0W7DZ9Ks6hz1yVCsALQB0FgC73zUAFm+f5PfpB3yhloxFeUKPWW4vNfHGq1M5GIc6eW87f39/U44n7WKYz0cdqebejVUNO6bIn8uF6+KqGc8dMKOpFheZQdIciOFkjd9L73//+NiG3kPXiwtsJKGLd3QybB3yEVF/DvFI7P1Kn0SaKB4Z0+lSElAxj7FF6P/oJHA9nNIYF+FR9EISstJOoyo2nXqQO3iMPeqjVdIBo1NJHjn33uvu6hVcnA15pXJshWpUPlSWq2Vw2R6YJcxzYnSQSxZYN+I9YQ5FIAOElA/8HJv9/4vDd4fkK2+P++UzgBOiZtLyNeNc6fuJkBisqqlse2MPtAttrvpjFmSAWgBzN7MUtd6bIrsTCP1i+tdNsyzIkDQtgTQVvjGT8knc0OHsgzvNZuIXfzqNbVwok6X8Lsal1I91en/o6BSGsOSLAKKN1trTb6UKFTqzkpKbKTQpthuN1sWlZuQGyfuNKVbx2KGciUjB9RtBUtNrj71moZ+90gQ9/chvAl3bRpMEwKhdtkrgX5k2rQGLz3Rtll96tEmJqvsPaF4geydJMTS6WIdZXGaBbsDA7barMi78ntiHSgHswQIw7FdMh/E+kmyITAi/rePle3nfSXiqP281XzdSWkzVgQDuVMQ7iq4+mD+r2hVaxA/DpVfi71qryHLY1GG9R57ba7wi1LMOLxcJ0SGVVnf4Y8qV7HdBtatig64JMrMFsfwT68RSd0XjNa8o8tsPKxsWgWfLYor1R+X0DvDTjXfEai/TgExmt+d9HKT/fiHpGpu21RmmJ89Q2HxxKaBG6ItwoyQfYmV77Wxl34UkPeG3NWuEmyzZ+MS/skkbVDdpMEc0R8h9xk3Uot8yT+1KBH/cyXvpqa7Q7kfRYFaeuKUDpHMZfSa4hfriRZsTrgDGF16kHU7HLpCQN6gPe16j6GfVSFHQpjEDcx5XxDp06ZFjhl06xsNt18ytPdykOBux26f4KWyN/TTgjDU0lVqWZ6P2I9tgDyJf6lQmqQ==,iv:GiAFMGtzev6bnTtbKrhaUmJipZfYEriVDR0hIIcbuvk=,tag:p87FISG6OFI5n2suKECcKg==,type:str] +kind: ENC[AES256_GCM,data:oDXj1wmI,iv:zRQA0DPLBhe7skXIqE4vLQKlQgzsOW7PhmSQBTUhVcw=,tag:NeNhMXL3nrFDXVGiISQqvw==,type:str] +metadata: + name: ENC[AES256_GCM,data:YWKVdWVd7Vkby+mGaoqCCdFbZnk=,iv:xNy7/XNWt0Ng7A45WEZ+K8RQD6inqy9TEhVFqnWzxm0=,tag:X+ar+MFIhG4Nsz5gwopLJg==,type:str] +type: ENC[AES256_GCM,data:tC3kqUKpNt1xPF8COd35itVzvDwEXxmOvnc5iYOT,iv:4KHXCxzXmH2LSX3/WmvWjeol2kG9FtcB9K1vczpoS2k=,tag:EkcbbOGnUM5ICiwV9fyYog==,type:str] +sops: + lastmodified: "2026-01-27T14:58:25Z" + mac: ENC[AES256_GCM,data:4QX9acqwVT0yIblv0/On8BZVK3d47GcLccXhWWyrdRYW+4BgR8zc1v+k1YRxJEBlojlXNqvaFs9tEEdNWQ0QHBUsR8jMQ0T2t5EqHb9LPE4WOYuBVcTyeqDmyc01EKHDWdutYxo15kotCMG1FO5OgZ11Rcwdv3f1QVdgskN4Pks=,iv:kiq2siF75QwGOK84k013AD8ex5N/kPMc6FrUOp531v0=,tag:Qq2Y1COIOw46WyU/qUH+ZQ==,type:str] + pgp: + - created_at: "2026-01-27T14:58:25Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAeDQZY4S70FaOHXTn3u7lIZfKvrpRyBtH5osQEepI+E4w + NnlXnnxgtOHBU8nHJZK2vmNge1hVUaGBcZ+mvsnanMEs0zpFkn2LD1hoQhVR+kYo + 0l4BIJTzMKoU0+NlCheUC8dD4lPhgVszP+Lis2ftddN2+q3rAwkcpBZC3ADkw4lp + GFuWjTCeNbfrWV5VGPj0rUfgXrqBS49df/aBBlkwCuEI+iwKFaw6UE21TrOT7p6L + =MreB + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml index fbafaf322..f2c17fd31 100644 --- a/kustomize/overlays/tests/kustomization.yaml +++ b/kustomize/overlays/tests/kustomization.yaml @@ -10,6 +10,7 @@ resources: - secrets/integration-tests-secrets.yaml - secrets/registry-app-creds.yaml - secrets/server-model-config.yaml +- secrets/exploit-iq-client-build-ips.yaml secretGenerator: - name: argilla-user-feedback-ips diff --git a/kustomize/overlays/tests/registry-app-creds-enc.yaml b/kustomize/overlays/tests/registry-app-creds-enc.yaml index 86da72c75..57af3b132 100644 --- a/kustomize/overlays/tests/registry-app-creds-enc.yaml +++ b/kustomize/overlays/tests/registry-app-creds-enc.yaml @@ -1,24 +1,24 @@ -apiVersion: ENC[AES256_GCM,data:tCQ=,iv:/JnoJlOSPDnpfzPax3IltVloS91QOp2VWzDWAkVi1cw=,tag:N+vwaUKTsFHin1YE0hh2pA==,type:str] +apiVersion: ENC[AES256_GCM,data:IZw=,iv:xXXhE071qT3QN3TglSdLUY9cFsjaVyJb194zVl1Bso8=,tag:lgtA2jrDSfp5lMw6M9dMKg==,type:str] data: - REGISTRY_REDHAT_PASSWORD: ENC[AES256_GCM,data:DY9vrRSPyVvFWqIUiughhehhr65Z0xGrTzMH254SVOuMdHqq8PClGO9XN+JBjMib63nd/Dih5jMkdAqGPt8HnumVWJKj2O8B1ry1o1kcHbGEv3RM0YPs5xzoAqCX8P5eP9MRW1vJc0vxqJA3hKPJ5colLR4KevFmoFYMPCczVHJygvNQlqU0smJaQcdwyPYWo0NQgSYGo6qUVbjgYCjzQFf1EJEiaZX+bdQTAlcHcdexZI+5cquSijaBFxi4GlLSXIldlnPA+rCQ8hTyD3PEj7drvfJad6u06Wm0MjJoyD/gAIoE/9NiSsnq+7fr5Ulf1FrFekq7ps1iMm83uKAiXRSC2WtBnNA0UdU09Dl5efXSvlvCi5AbK4o6gI/LnMyvcGkYlqKDgGYYNcDZqMe8wxk3SOuMuL0JZTBdC92AN2HZtYvinDrL885azN7tXHulNwYRGtPkK2GH2Yuhbi693dwejR6Edk6C2KhczOKtaLFH7ULiQSUMwk/xEmqH1HDOcuai0oX94HuIKYiWFVhoe72I2Mxbwc9Wt8cQ/WAn5DEuBrdQJ/gd9q4/lRAi1x/5wwlGy2TWMOiq2Oh7tTrh6wEvImLFKQ6wxlxFLHYs1p9xvSrHcyFxa6zyNOsir2cImQSaXdS/d91JfjFiHY9MHdCTP7wY3Tb9kYKi2txUdTbVop7SiKpTJfZg1rwhZWi0HsEWpDCJugKENbJinv6IaEmb8yc5pnl9vS0qLfljDAipnBkMm81J05kVhrZoq5I6w1a7JYXomXcTZAPwPTwh+PMuI/+IUxT/ciWbkT2YfPWmWGlzLFaKGd1w7sj3XthTtrIh5elw2xwoeuEwvZLIbeZRZwhWOoi/h6ByMPdCc2ReUaN91yumN22sYaw8mHudz2D4nbPo12ZfPfA1Xx4EsVFe5j0FLn5GP2TCcuO/DAMrXvXUykhGCPMcExpid2Q+WXYYq/LwAEyyqqfUVsXbjlb9+4i4MqXISzUhI5mQmVbh5iusWbEUvrvpBJe1wsLZhRfOSpdmQRhYNoSiVj/xZfJt2jzqPK4ISmkY9wiK68V79cJgugzwS28bWwF08AZZNSsPAXadODCpuqCve7JmWl3nR0X0Acel/gqU2KCykGbr3EvdxUaIGUHsbLqoXjhrLrVsJ0Lyd3g+bhNL1an4QzgUHiVok0/eUQXXcUJt5TddedG2TkOtQINl8atBSS9TNW1paxco8Aiq47uRRC+4O1i0V1iyeYOPejQjMjTTGF2326R6c6REmQVsAqjqG1FaqH6WzQZB8p0sXHTtm+bFGKjPnjaaHezolumRL/DrrFL16VtJuqZLnKJ9m3RjAZQVVqINt9SUiNU3IA0u,iv:COGn/Fn8b7HLutwSONHSkQL8FDm/DiaTzksfJhv+5no=,tag:FLTdrggW+uTEdbczPZqCAw==,type:str] - REGISTRY_REDHAT_USERNAME: ENC[AES256_GCM,data:7tDxGRcjSHGsBb//r5NBJtaz+g9HrTQ2,iv:/gRNExKbdVqMOvdvUxCw5ajJVTb/7Pthxs02AlwKqo0=,tag:wjBGPzc8nkF/3LMoZaFoKA==,type:str] -kind: ENC[AES256_GCM,data:0g19ZH//,iv:FQwXlhfkZNRBZF65DmZarqFDLeQQJ8lUSS9d0laPABU=,tag:0RcpIXMAN4CK5QI5QkXG/A==,type:str] + REGISTRY_REDHAT_PASSWORD: ENC[AES256_GCM,data:GquthNXj1O8624wEORBAvwpCGkG8Xv/BLAXuzTQX7RKwo3e9xdsxMbZ6QajCrwGeApCF+aQaKSkBXn4Weg7xJnKmrwN2X1fbdNeQFF9cvLaIY18VEJDtl5twsCYZlae62oHw4j1UNV124dQA6DPCtHND8GHw3rORYgMyGTnQxsCikbjcu94KVcFuiyXKPx21JUs0DT95YvzUtgkpPoKQJiexghfBH4NnunytiR/zamAfHLCgRdAcGC0qsbLEKDN3x+v4AF342oFnwT6OGfbcBaJrL27Kq6S+SO6ZlDAXvNawsiN60qZ3Ekj0DiUHSDKu9nuHOgAyyYQ8wfoZdwe3cpM8UHL0zLqQ3xvtOa1wxc5U5hiDaoC6kLzmvlGZEbBYbe4tHmaHyP0PUi4/qLou31x0xLrQbvnAsXbgn/2YEDhddSnW2B2uFpAxps7J1S4SIFrDIocaT7hdxu1k4vKDrVK3iIxNdzsbxJthUnbffsydXCHYAyorGpMicCwqY2WTHDlFu7whEoGzxx+z97c8eZcxgML3DHZ1tRexh3e/iuic8w8X8bwSDmYb3/i649ZnDSYPp8TbLbNPorneWMc4mch5ThOMyL4n434siHo8zsr64mSud+uWx2CBqcuVpR2eNGZYXJL4cimdLrugb58LAUBisPzLKnUtFxiiGsz2DfP8KgUEIlYaTq2a6Ns0LDIpxxvygMqcjLfEXblDV8zptI79zaOt75ghfUNP5CpkeOZmfkkp7KSPuyx1TLRXdW94pDiSE0rZp0VgzTkHFA/AF5S1QhINWuuAggWpB5zGqwM+uQb8+q+Ixwu/pCNCZd4xA+5hgyDhX3NZ+c+awD81WlwOLilneoT1O3VhCcBobxLdL3C//gtNOyxPFy68rzYgZD6MjHczw4aWYT3kGcr7MeVKKhqJPSRrESSythHX6AjVvg2ePKqTYScJo7qILgDMSKq3NhRu4vj3fSqQ13530mwAqrbV58N8t5XAkeciMI9GKFaxDiSSEUwbngOWd7aAgyePqFmLubfmM4jqtkKLJwcOzcE8F86cUbrKwAntm6x83+Jpt2qDZUCEOWbo/73WJAVv+l5Z91Qj7eWff70BOvg6jLNJoL0+r80x932VJPOwgrCaESBbFSs+olwfgyxkzQxrJ6tBtwX87fgzKhN0rwBSGukZsAZxzM2u9WS8trLIBOsEXFwofVP1nAcwRsexcni40TEw2DxjPrWW2aU8HOx+fVHy9xlglgNezu/ppcelDEdOcfwX0E1AflV7acIJSyU/k/R66LtUGUi5IUAEsDIWKgIcnAd3vIPTNxtjyJonStuLQ1JlqAMzRVyq7qch8uO5PS5Erxc=,iv:/zxAcdPQbS6r4iBGwUUlxShsfhuNjcCfo349uuvlhR8=,tag:y7+qHLSL0lSUcXgnH2tD8g==,type:str] + REGISTRY_REDHAT_USERNAME: ENC[AES256_GCM,data:+BCMkat0CX6ZoZbXcFVVEoBXFiPfyhxE,iv:KHaat2fMj/cDEPjbBC6enL9kCiCgWyjFw3ptCW2AuDk=,tag:ufvyd0TvLdKvCcaxz3ir/Q==,type:str] +kind: ENC[AES256_GCM,data:6E22NgYo,iv:Wi/gwNNrx0dL2/aacRKqJgKiSf9UF93Tixgc7E2pHIc=,tag:ZUbcU6fbNIgNXOQvGCeifw==,type:str] metadata: - name: ENC[AES256_GCM,data:FDxteHJ/f1a7FgW+RUryKQ==,iv:59iQVkcUKGymcArOn4PCR8L1ups0XmHcmv601J9dtHc=,tag:vrdPMLmn0UDijAWfCm9MSg==,type:str] -type: ENC[AES256_GCM,data:jJztE258,iv:jk9TV1Uf1ydiyunGsv32tlp5grXTVyTbI2lJkVTM9lU=,tag:WUnjbftart7YctxO1XS+sg==,type:str] + name: ENC[AES256_GCM,data:Myf/HsxEcMl/n0W/5vAtqA==,iv:qebrYwM4DLfYvUR/Sxin6GIDha8w4px/ybK8CGsnGAA=,tag:k7vi+9O9TBYHt7R8l7AcaA==,type:str] +type: ENC[AES256_GCM,data:X2uOhC+V,iv:dhqBwROQkhzoJM0PmjdmeOcNGj72k2Q3g5y0uPdJthI=,tag:irVym2z88svdyvJD5B1hfA==,type:str] sops: - lastmodified: "2026-01-25T22:26:41Z" - mac: ENC[AES256_GCM,data:5IZ8f03JChaaNm/wozc7wbs+DDeFYEguGB9/C49MPv6zK2xhnHFmk0B/RHrIl+/ZNZU0NtcQ9ZZxeYGEHgDKrHJpnFrPZeuRUPhmfCkVY2POzas85SI0oCfXKaSehdzRVd7XQ1++fRo9SHXbWsNgRH2C+g3XToFjOeyBg6cPfBM=,iv:bf0WSgF5g19TtxYL7HfzpjXlZAQbUAaXW2LkGraGVFw=,tag:KcRubwVZFcge2Kw5GrpMkw==,type:str] + lastmodified: "2026-01-27T15:04:49Z" + mac: ENC[AES256_GCM,data:98VTQKDDdusGPrUqQE0l+i/pCcD8AQG9iWjYJofYAXkebsx3ABXlHuyuDS+fg7fGsLTSR2CoxyLyqjnanL8PpdXS1AFCSFUCpD9VoJ1WEadCJ4c9lgKJpb84f2TZTggsE6bH4HpHZuEDZR1/U4gbX0bE/4MXpMpkWxWc0+tOKDw=,iv:eyFLHWaY5vHRNyjp5I8tmQQyZDrc4wvDKowreOBvDJs=,tag:TfGnJOewwjbE/K1Uayrmog==,type:str] pgp: - - created_at: "2026-01-25T22:26:41Z" + - created_at: "2026-01-27T15:04:49Z" enc: |- -----BEGIN PGP MESSAGE----- - hF4Dy77zzNMwU0sSAQdA0hIJBy0m1G4y//yFta/U+2rnqzA/E5cuBHP379fUD2Ew - v0oJlrgCU01/xwvy2VvAHJaVbZ2/3vcR6DCnpo33Ev3tZFwRKDM8jQzPZpkvaAwh - 0lwBB7dTxipu3NSEQDY8Y/g9+eIlbQTF3T6aRi1XHXoVNsgsaHGTfAxfA99mXKzD - Q50124BAwhb88VXw6OLvN8CqcEK3l0t1VtpgF6BkGTB7/cJxaWmfHbP+O8J2Ew== - =GF5p + hF4Dy77zzNMwU0sSAQdAtMvASO+L3ZWyF7hGBPz2KsVM2iH/GASnzUZthLd0tR0w + aAtePRC34cc6hzeMsXGyPkU93Aq1kimzGUuzfIFaMHBEBwsJetzR78p0bedNTBhv + 0l4B0MFdxa2LKX57vMhBnjSPqnndz9c4EzT+4faM3sb16Ly3of2kYdtrsf8LdyIs + NHhoxWAiuyqlKZ3vHEL950R9NS7OjhxOYqVTPVXOaOSpC7SL+rO2vIH1GdJk51iE + =lqdO -----END PGP MESSAGE----- fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 unencrypted_suffix: _unencrypted From 6e1ff27b987598550c9b68966e41a82cc56c9d3f Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 27 Jan 2026 17:14:46 +0200 Subject: [PATCH 204/286] chore: add secret exploit-iq-automation-token to tests deployment variant Signed-off-by: Zvi Grinberg --- .gitignore | 1 + kustomize/README.md | 1 + .../exploit-iq-automation-token-enc.yaml | 24 +++++++++++++++++++ kustomize/overlays/tests/kustomization.yaml | 1 + 4 files changed, 27 insertions(+) create mode 100644 kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml diff --git a/.gitignore b/.gitignore index 0c31c4dbe..3ef4a4930 100644 --- a/.gitignore +++ b/.gitignore @@ -206,6 +206,7 @@ tags **/sec-decryption.key **/registry-app-creds.yaml **/exploit-iq-client-build-ips.yaml +**/exploit-iq-automation-token.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 05d8df38e..933b3b619 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -311,6 +311,7 @@ sops -d secrets.env2 > secrets/secrets.env sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml sops -d user-feedback-ips.secret > secrets/user-feedback-ips.json sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml +sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml ``` 5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. diff --git a/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml b/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml new file mode 100644 index 000000000..d8df648f8 --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:Uns=,iv:t7ZWH0eiE63kyMW42wFsfsKN01OkC+brLLXaJUEClQs=,tag:pxTCqrwC1cUO/EMTfFrPzA==,type:str] +data: + gh-token: ENC[AES256_GCM,data:GgFKPuhztlsqsZ7PjXGH6V04uNu6ElKs5iC8uUW/UCL7vjmx5VR0suUaDA1DISb+Bc9W83bMnkW70H8CUwMX50OI536T/hDTokLAYTsFD+05HsUWKSxBqq7G25fv3mzPYSBAVKs0EL7K+f2Le6k2yeYltkOkfIRFtabFcg==,iv:cl5jl1oKbR2AtJYCoH4Je8SSyP4Jc+gPInIRfOflTSw=,tag:CrnZspbYfUjz0wgjzwfQhg==,type:str] +kind: ENC[AES256_GCM,data:j8rCiStt,iv:CaoFjqicLryq3MS+mvgVO5ffBbIX9vUQVQ5uy/NyNnM=,tag:r4LeUOBMkvXK3hd7faj/wA==,type:str] +metadata: + name: ENC[AES256_GCM,data:X7/gU5Es3+keADmoDJQWfF27KBKVjoCUcLRe,iv:zYPeX1H91bcUut1/wbVi6UdyRAwcT2QKRu//GS8KAY0=,tag:hPQtbwjqeIazh/hrnmzbjA==,type:str] +type: ENC[AES256_GCM,data:waF5cqrS,iv:yh5grziLkmXblL+zoo/DbsFI8GJdBICWq5xbGrjqrjM=,tag:W1odWNRYm9RpU4XBAXDvpA==,type:str] +sops: + lastmodified: "2026-01-27T15:12:24Z" + mac: ENC[AES256_GCM,data:AupZMMI0ycLlxF3/s2fLy97AC75/QF7itpaldF+I6Cuoj6FdmBd+2HwiDuz+505ZAm0/pP7Ez7p+zA7STyxY0vzGBu2XoOhKOcj0seJHrHZh60PXlV2BgQarBxVTtSg3BWFLrKYNRRHevHezfRKxFichlQZzj5Pc6TmheNs2c7U=,iv:TXYL4TE8GlAeuTjaw9GfdP+mmXtWpz++sHp0InHk04c=,tag:0I+c3wRyLh2vL+9N7K5i+w==,type:str] + pgp: + - created_at: "2026-01-27T15:12:24Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdA/biMrPzXXQ6FZhsAqOpLTXUzegYmekUNov4ZxfhQyWsw + GLEYcoYdGLjZo/BSUP3t6+8XJ/LY6ytRvMvsxWWJKGBspxIyE7JwTCpIdOWA4p0v + 0lwBlyu0o8Jc31ct4J1V+mPowF8L1znKEgqVBugA+l3N5JRizwecTdcb8k0OXHqf + +hXfcCYVK5FYJbdtGsGEEZmS6vGjdAViNiTyuS4NS+Lh4sEFXA0Z4CT7YILArA== + =5nCL + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml index f2c17fd31..09043a874 100644 --- a/kustomize/overlays/tests/kustomization.yaml +++ b/kustomize/overlays/tests/kustomization.yaml @@ -11,6 +11,7 @@ resources: - secrets/registry-app-creds.yaml - secrets/server-model-config.yaml - secrets/exploit-iq-client-build-ips.yaml +- secrets/exploit-iq-automation-token.yaml secretGenerator: - name: argilla-user-feedback-ips From a2adfe6955bb00983a50ddf227947157fbc7d4ad Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 24 Feb 2026 13:38:53 +0200 Subject: [PATCH 205/286] fix: revive agent to be part of the tests deployment docs: fix documentation fix: make the parent variant to be self hosted variant instead of the base variant Signed-off-by: Zvi Grinberg --- kustomize/README.md | 77 +++++++++++-- kustomize/overlays/tests/kustomization.yaml | 59 +++++----- kustomize/overlays/tests/nginx_cache.conf | 115 -------------------- 3 files changed, 93 insertions(+), 158 deletions(-) delete mode 100644 kustomize/overlays/tests/nginx_cache.conf diff --git a/kustomize/README.md b/kustomize/README.md index 933b3b619..48405bba0 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -143,7 +143,11 @@ EOF 6. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. -Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret +If openshift resource of kind `OAuthClient` named `exploit-iq-client` exists, just get the secret from there: +```shell +export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +``` +Otherwise, Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret of your own: ```shell export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" @@ -257,7 +261,7 @@ redirectURIs: - "http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')" EOF ``` -Otherwise, just add your route to the existing `OAuthClient` CR object: +Otherwise ( if creating `OAuthClient` instance got error because it's already exists in the cluster), just add your route to the existing `OAuthClient` CR object: ```shell export HTTPS_ROUTE=https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') export HTTP_ROUTE=http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') @@ -293,7 +297,7 @@ kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - export PROJECT_NAME=exploit-test oc new-project $PROJECT_NAME ``` -3. Take private key and import it to GPG: +3. Take private key from vault ( `ExploitIQ Tests Deployment Variant Private Key for Decryption.`) and import it to GPG: ```shell gpg --import /path/to/sec-decryption.key ``` @@ -315,28 +319,81 @@ sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-tok ``` 5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. -6. Now deploy to the cluster the exploitIQ system ( minus agent) with all resources: + +>[!WARNING] +Without completing the following step with the correct secret from step 6, authentication and logging into the UI App will fail! + +6. If openshift resource of kind `OAuthClient` named `exploit-iq-client` exists, just get the secret from there: +```shell +export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +echo oauthClientSecret=$OAUTH_CLIENT_SECRET +``` +Otherwise, Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret of your own: + +```shell +export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" +``` +Eventually, Run the following: +```shell +cat > secrets/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +EOF +``` + +7. Update `ExploitIQ` configuration file with the correct callback URL for the client service +```shell +cd $(git rev-parse --show-toplevel)/kustomize +export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" +find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + +cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests +``` + +8. Now deploy to the cluster the exploitIQ system ( minus agent) with all resources: ```shell kustomize build . | oc apply -f - ``` -7. Deploy Self hosted LLM for the automation tests ( Integration tests and Confusion matrix runner): + +9. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 6) and generated route + +```bash +oc create -f - < base_url=`https://api.first.org/data/v1` - # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` - - # resolver 127.0.0.11 [::1]:5353 valid=60s; - - # rewrite_log on; - - ################ Docker Compose Services ################# - - # Include any additional routes from the routes directory - include /etc/nginx/conf.d/routes/*.conf; - - - ################### Redirect Handling #################### - - location @handle_redirects { - # store the current state of the world so we can reuse it in a minute - # We need to capture these values now, because as soon as we invoke - # the proxy_* directives, these will disappear - set $original_uri $uri; - set $orig_loc $upstream_http_location; - - # nginx goes to fetch the value from the upstream Location header - proxy_pass $orig_loc; - proxy_cache llm_cache; - - # But we store the result with the cache key of the original request URI - # so that future clients don't need to follow the redirect too - proxy_cache_key $original_uri; - proxy_cache_valid 200 206 14d; - } - } -} From 4784b3e7df294a391541b24f87aab446d78b9031 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 26 Feb 2026 14:05:06 +0200 Subject: [PATCH 206/286] feat: add private repository support with PAT and SSH authentication Signed-off-by: Vladimir Belousov --- kustomize/README.md | 1 + kustomize/base/exploit_iq_client.yaml | 5 + kustomize/base/exploit_iq_service.yaml | 11 + kustomize/base/kustomization.yaml | 6 + .../tests/credential-encryption-key.env2 | 15 ++ kustomize/overlays/tests/kustomization.yaml | 4 + src/exploit_iq_commons/data_models/input.py | 4 + .../utils/credential_client.py | 208 +++++++++++++++++ .../utils/document_embedding.py | 4 + .../utils/source_code_git_loader.py | 213 +++++++++++++++-- .../functions/cve_generate_vdbs.py | 57 ++--- .../tools/tests/test_credential_client.py | 216 ++++++++++++++++++ .../tests/test_source_code_git_loader.py | 22 ++ 13 files changed, 726 insertions(+), 40 deletions(-) create mode 100644 kustomize/overlays/tests/credential-encryption-key.env2 create mode 100644 src/exploit_iq_commons/utils/credential_client.py create mode 100644 src/vuln_analysis/tools/tests/test_credential_client.py diff --git a/kustomize/README.md b/kustomize/README.md index 48405bba0..e271a5c90 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -316,6 +316,7 @@ sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml sops -d user-feedback-ips.secret > secrets/user-feedback-ips.json sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml +sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env ``` 5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml index 21c9ab455..7829e0e90 100644 --- a/kustomize/base/exploit_iq_client.yaml +++ b/kustomize/base/exploit_iq_client.yaml @@ -89,6 +89,11 @@ spec: secretKeyRef: name: exploit-iq-secret key: ghsa_api_key + - name: CREDENTIAL_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: credential-encryption-key + key: credential-encryption-key volumeMounts: - name: tls-certs diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index f64ca472f..3a21332b6 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -125,6 +125,17 @@ spec: value: /exploit-iq-package-cache/go/pkg/mod - name: ENABLE_MLOPS value: "true" + - name: CREDENTIAL_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: credential-encryption-key + key: credential-encryption-key + - name: CLIENT_BACKEND_URL + value: https://exploit-iq-client.$(NAMESPACE).svc.cluster.local:8443 + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: config mountPath: /configs diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index d6dbd4a21..08a0a709d 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -41,6 +41,12 @@ secretGenerator: options: disableNameSuffixHash: true + - name: credential-encryption-key + envs: + - credential-encryption-key.env + options: + disableNameSuffixHash: true + configMapGenerator: - name: nginx-cache-config files: diff --git a/kustomize/overlays/tests/credential-encryption-key.env2 b/kustomize/overlays/tests/credential-encryption-key.env2 new file mode 100644 index 000000000..d426793bb --- /dev/null +++ b/kustomize/overlays/tests/credential-encryption-key.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:vS2Mq9UvWw2fie8Ot9bi2p4lat4fF4+nIRAuB1OiynhbgjTguPaCQGcg3TTX+eHY5DtP65xwTbGkN6ymZOyiRatXp3zBUe0vm9Jk4VXb23IgedBGAK9CKjAf,iv:MJ5sFOfJf/iPj8DwtSJu3SRBd8YThpd7BRJmfWgE8NY=,tag:lgj/KzRa4ut6y4G8zWzyIg==,type:str]", + "sops": { + "lastmodified": "2026-02-25T10:27:54Z", + "mac": "ENC[AES256_GCM,data:ylpBZXqneUsWoiLpQkcIt3MR/fIqtP2nq2HBNQQfqEnFusUh9t6fKPcF8amIFMES5qhci21wWi04r6m4Y6BVpHWdsy4qnpgKQUU915VpxNPPUGL6FJbpFufAxD9W8c7U4X8YDzXA4q/GzEVLjApS4JoCckLPPNYYaKgZP7sBk0M=,iv:Qa7xr7IYV+BIMBDwDinKhrngWwoxJS5WJuXtJZUpnuk=,tag:ppY96hhacci3YS+Qq3DusA==,type:str]", + "pgp": [ + { + "created_at": "2026-02-25T10:27:54Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdA7pg0i+MN8TiEACfaErvgVmx3TISeSdYbrm75nf/9H0sw\nZ920uz6tCV3A+zbuwVxmh/QuU9DyL4sjiDOeFl7PPeUp9qQOVXbQGHXREhFI4q0g\n0l4BQ4hMMszexwVTakLL5aLO5F4Slx3p9mhlINktZnf4AqUIFMW/6bcVgmYPnHQg\nunWOiJVBzBi41DFzljAwfd4MnUSFtfQtSGj56vy8j4YXBb0gbdNWB2f8VxGkf5da\n=fdFQ\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml index 132862cf2..31b3337c4 100644 --- a/kustomize/overlays/tests/kustomization.yaml +++ b/kustomize/overlays/tests/kustomization.yaml @@ -46,6 +46,10 @@ secretGenerator: behavior: replace envs: - secrets/mongodb-credentials.env + - name: credential-encryption-key + behavior: replace + envs: + - secrets/credential-encryption-key.env commonAnnotations: deployment-variant: tests diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index 870101dbd..a7d4166f3 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -186,6 +186,10 @@ class AgentMorpheusInput(HashableModel): """ scan: ScanInfoInput image: ImageInfoInput + credential_id: str | None = Field( + default=None, + validation_alias=AliasChoices("credential_id", "credentialId"), + ) class AgentMorpheusEngineInput(BaseModel): diff --git a/src/exploit_iq_commons/utils/credential_client.py b/src/exploit_iq_commons/utils/credential_client.py new file mode 100644 index 000000000..46cab382a --- /dev/null +++ b/src/exploit_iq_commons/utils/credential_client.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Generator + +import requests +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +_K8S_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +_credential_id_ctx: ContextVar[str | None] = ContextVar("credential_id", default=None) + + +@contextmanager +def credential_context(credential_id: str | None) -> Generator[None, None, None]: + """ + Context manager that makes credential_id available to any code running + within this scope (including async tasks spawned from it). + + Usage:: + + with credential_context(message.credential_id): + embedder.build_vdbs(source_infos, ...) + """ + token = _credential_id_ctx.set(credential_id) + try: + yield + finally: + _credential_id_ctx.reset(token) + + +def _resolve_jwt_token(jwt_token: str | None) -> str: + """ + Resolve JWT token for authenticating with the credential backend. + + Resolution order: + 1. Provided jwt_token argument (highest priority) + 2. Kubernetes service account token (mounted at well-known path) + 3. CLIENT_JWT_TOKEN environment variable (local development fallback) + + Raises + ------ + RuntimeError + If no JWT token source is available. + """ + if jwt_token: + return jwt_token + + if os.path.isfile(_K8S_SERVICE_ACCOUNT_TOKEN_PATH): + with open(_K8S_SERVICE_ACCOUNT_TOKEN_PATH) as f: + token = f.read().strip() + if token: + logger.debug("Using Kubernetes service account token for credential backend auth") + return token + + env_token = os.environ.get("CLIENT_JWT_TOKEN") + if env_token: + logger.debug("Using CLIENT_JWT_TOKEN env var for credential backend auth") + return env_token + + raise RuntimeError( + "No JWT token available for credential backend authentication. " + "Set CLIENT_JWT_TOKEN env var or run in a Kubernetes pod with a mounted service account token." + ) + + +class CredentialNotFoundError(Exception): + """Raised when credential is not found, expired, or already consumed (HTTP 404).""" + + +class AuthenticationError(Exception): + """Raised when JWT token is invalid or missing (HTTP 401/403).""" + + +class DecryptionError(Exception): + """Raised when AES-256-GCM decryption fails (wrong key or corrupted data).""" + + +def fetch_and_decrypt_credential( + credential_id: str, + backend_url: str, + encryption_key: str, + jwt_token: str | None = None, +) -> dict: + """ + Fetch encrypted credential from backend and decrypt it in memory. + + Parameters + ---------- + credential_id : str + Unique credential identifier (UUID). + backend_url : str + Base URL of the backend. + encryption_key : str + AES encryption key as UTF-8 string. First 32 bytes are used, + matching the backend's SecretKeySpec(keyBytes, 0, 32, "AES"). + jwt_token : str | None, optional + JWT token for authenticating with the backend. When None, the token + is resolved automatically: first from the Kubernetes service account + token file, then from the CLIENT_JWT_TOKEN environment variable. + + Returns + ------- + dict with keys: + - secret_value (str): plaintext secret (PAT token or SSH private key) + - username (str | None): git username, may be None for SSH keys + - credential_type (str): "PAT" or "SSH_KEY" + - user_id (str): owner identifier + + Raises + ------ + CredentialNotFoundError + Credential not found, expired, or already consumed (single-use). + AuthenticationError + JWT token is invalid or insufficient permissions. + DecryptionError + Decryption failed — wrong key or corrupted payload. + RuntimeError + Unexpected HTTP status or network error. + """ + resolved_token = _resolve_jwt_token(jwt_token) + url = f"{backend_url.rstrip('/')}/api/v1/credentials/{credential_id}" + headers = {"Authorization": f"Bearer {resolved_token}"} + + logger.info("Fetching credential: credential_id=%s", credential_id) + + ca_bundle = os.environ.get("CLIENT_CA_BUNDLE", "/app/certs/service-ca.crt") + ca_exists = os.path.exists(ca_bundle) + logger.debug("Using CA bundle: %s (exists: %s)", ca_bundle, ca_exists) + + verify_ssl = ca_bundle if ca_exists else False + if not ca_exists: + logger.warning("CA bundle not found at %s, falling back to verify=False", ca_bundle) + + try: + response = requests.get(url, headers=headers, timeout=10, verify=verify_ssl) + except requests.RequestException as exc: + logger.error("Request failed", exc_info=True) + raise RuntimeError(f"Network error fetching credential {credential_id}") from exc + + if response.status_code == 404: + raise CredentialNotFoundError( + f"Credential not found or already consumed: credential_id={credential_id}" + ) + if response.status_code in (401, 403): + raise AuthenticationError( + f"Authentication failed fetching credential: status={response.status_code}" + ) + if not response.ok: + raise RuntimeError( + f"Unexpected error fetching credential {credential_id}: " + f"status={response.status_code}" + ) + + data = response.json() + + try: + encrypted_bytes = base64.b64decode(data["encryptedSecretValue"]) + iv_bytes = base64.b64decode(data["iv"]) + except (KeyError, ValueError) as exc: + raise DecryptionError("Invalid response payload: missing or malformed fields") from exc + + try: + # Key derivation matches Agent backend: + # new SecretKeySpec(keyBytes, 0, 32, "AES") where keyBytes = key.getBytes(UTF_8) + key = encryption_key.encode("utf-8")[:32] + aesgcm = AESGCM(key) + plaintext_bytes = aesgcm.decrypt(iv_bytes, encrypted_bytes, None) + secret_value = plaintext_bytes.decode("utf-8") + except Exception as exc: + raise DecryptionError("AES-256-GCM decryption failed") from exc + + credential_type = data.get("credentialType", "") + user_id = data.get("userId", "") + + logger.info( + "Credential decrypted successfully: credential_id=%s, type=%s, user_id=%s", + credential_id, + credential_type, + user_id, + ) + + return { + "secret_value": secret_value, + "username": data.get("username"), + "credential_type": credential_type, + "user_id": user_id, + } diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index 242a9d667..521c19390 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -340,6 +340,10 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: repository based on the include and exclude patterns. Each file is then parsed and segmented based on its language into Documents. + If a credential_id is active in the current context (set via + ``credential_context()``), it will be used for private repository + authentication inside SourceCodeGitLoader. + Parameters ---------- source_info : SourceDocumentsInfo diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index a376d447d..4e291843e 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -14,7 +14,9 @@ # limitations under the License. import os +import tempfile import typing +import urllib.parse from pathlib import Path from git import Blob as GitBlob @@ -24,6 +26,7 @@ from langchain_core.document_loaders.blob_loaders import Blob from tqdm import tqdm +from exploit_iq_commons.utils.credential_client import _credential_id_ctx, fetch_and_decrypt_credential from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -44,6 +47,11 @@ class SourceCodeGitLoader(BlobLoader): Each document represents one file in the repository. The `path` points to the local Git repository, and the `ref` specifies the git reference to load files from. By default, it loads from the `main` branch. + + If `credential_id` and `jwt_token` are provided, the loader will fetch + encrypted credentials from the backend and use them for cloning private + repositories. Credentials are decrypted in memory and cleaned up immediately + after the clone operation. """ def __init__( @@ -69,6 +77,9 @@ def __init__( A list of file patterns to include. Uses the glob syntax, by default None exclude : typing.Optional[typing.Iterable[str]], optional A list of file patterns to exclude. Uses the glob syntax, by default None + + Credential for private repository authentication is read from the current + async context (set via ``credential_context()`` in cve_generate_vdbs). """ self.repo_path = Path(repo_path) @@ -100,6 +111,17 @@ def load_repo(self): if (self._repo is not None): return self._repo + credential_id = _credential_id_ctx.get() + credential_clone = False + auth_mode = "private" if credential_id else "public" + logger.info( + "load_repo: url='%s' ref='%s' path='%s' auth=%s", + self.clone_url, + self.ref, + self.repo_path, + auth_mode, + ) + if not os.path.exists(self.repo_path) and self.clone_url is None: raise ValueError(f"Path {self.repo_path} does not exist") elif self.clone_url: @@ -112,14 +134,20 @@ def load_repo(self): if repo.remotes.origin.url != self.clone_url: raise ValueError("A different repository is already cloned at this path.") - logger.info("Updating existing Git repo for URL '%s' @ '%s'", self.clone_url, self.ref) + logger.info("Updating existing Git repo for URL '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) else: - logger.info("Cloning repository from URL: '%s' @ '%s'", self.clone_url, self.ref) - # Create a shallow clone of the repository without checking out - repo = Repo.clone_from(self.clone_url, self.repo_path, depth=1, no_checkout=True) - - # Set repo as git safe directory to avoid errors if directory ownership is changed outside the pipeline - # https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory + logger.info("Cloning repository from URL: '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) + + if credential_id: + repo = self._clone_with_credential(credential_id) + credential_clone = True + else: + # Create a shallow clone of the repository without checking out + repo = Repo.clone_from(self.clone_url, self.repo_path, depth=1, no_checkout=True) + + # Set repo as git safe directory to avoid errors if directory ownership is changed + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory + if self.clone_url: with repo.config_writer(config_level="global") as config: config.add_value("safe", "directory", str(self.repo_path.absolute())) @@ -127,13 +155,13 @@ def load_repo(self): repo = Repo(self.repo_path) logger.info("Using existing Git repo at path: '%s' @ '%s'", self.repo_path, self.ref) - # Reliable way to check out the ref using a shallow clone - repo.git.fetch("origin", self.ref, "--depth=1", "--force") - tag_refspec = f"refs/tags/{self.ref}:refs/tags/{self.ref}" - try: - repo.git.fetch("origin", tag_refspec, "--depth=1" , "--force") - except GitCommandError: - pass + # Fetch the ref (skip if already cloned with credentials) + if not credential_clone: + if credential_id: + self._fetch_authenticated(repo, credential_id) + else: + self._fetch_public(repo) + repo.git.checkout(self.ref, "--force") logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) @@ -142,6 +170,163 @@ def load_repo(self): return repo + def _fetch_public(self, repo: Repo) -> None: + """Fetch from public repository.""" + self._do_fetch(repo) + + def _fetch_authenticated(self, repo: Repo, credential_id: str) -> None: + """Fetch with authentication (PAT or SSH).""" + from exploit_iq_commons.utils.credential_client import CredentialNotFoundError, AuthenticationError + + backend_url = os.environ["CLIENT_BACKEND_URL"] + encryption_key = os.environ["CREDENTIAL_ENCRYPTION_KEY"] + + try: + cred = fetch_and_decrypt_credential( + credential_id=credential_id, + backend_url=backend_url, + encryption_key=encryption_key, + ) + except CredentialNotFoundError as e: + logger.warning("Credential expired or consumed: %s, falling back to public fetch", credential_id) + self._do_fetch(repo) + return + except AuthenticationError as e: + logger.error("Authentication failed for credential: %s", credential_id) + raise + + try: + if cred["credential_type"] == "PAT": + logger.info("Using PAT auth for fetch: username=%s", cred["username"]) + self._do_fetch_with_pat(repo, cred["secret_value"], cred["username"]) + elif cred["credential_type"] == "SSH_KEY": + logger.info("Using SSH key auth for fetch") + self._do_fetch_with_ssh_key(repo, cred["secret_value"]) + else: + raise ValueError(f"Unsupported credential type: {cred['credential_type']}") + finally: + del cred + + def _do_fetch(self, repo: Repo) -> None: + """Core fetch logic: fetch ref and tags.""" + try: + repo.git.fetch("origin", self.ref, "--depth=1", "--force") + except GitCommandError as e: + # If fetch fails, check if the ref already exists locally + try: + repo.commit(self.ref) + logger.info("Ref %s already exists locally", self.ref) + return + except Exception: + # Ref doesn't exist locally either, re-raise original fetch error + raise e + + # Try to fetch tags, but don't fail if they don't exist + try: + repo.git.fetch("origin", f"refs/tags/{self.ref}:refs/tags/{self.ref}", "--depth=1", "--force") + except GitCommandError: + pass + + def _do_fetch_with_pat(self, repo: Repo, pat: str, username: str | None) -> None: + """Fetch with PAT: temporarily update URL, fetch, restore URL.""" + original_url = repo.remotes.origin.url + parsed = urllib.parse.urlparse(original_url) + user_part = f"{username}:{pat}" if username else f"token:{pat}" + auth_url = parsed._replace(netloc=f"{user_part}@{parsed.netloc}").geturl() + + try: + repo.remotes.origin.set_url(auth_url) + self._do_fetch(repo) + finally: + repo.remotes.origin.set_url(original_url) + del auth_url, pat + + + def _do_fetch_with_ssh_key(self, repo: Repo, key: str) -> None: + """Fetch with SSH key: write temp file, set env var, fetch, cleanup.""" + ssh_key_path = None + try: + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: + f.write(key) + ssh_key_path = f.name + os.chmod(ssh_key_path, 0o600) + git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" + with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): + self._do_fetch(repo) + finally: + if ssh_key_path and os.path.exists(ssh_key_path): + os.unlink(ssh_key_path) + del key + + def _clone_with_credential(self, credential_id: str) -> Repo: + """Clone with authenticated credential (PAT or SSH key).""" + backend_url = os.environ["CLIENT_BACKEND_URL"] + encryption_key = os.environ["CREDENTIAL_ENCRYPTION_KEY"] + + cred = fetch_and_decrypt_credential( + credential_id=credential_id, + backend_url=backend_url, + encryption_key=encryption_key, + ) + + try: + if cred["credential_type"] == "PAT": + logger.info("Using PAT auth for clone: url=%s username=%s", self.clone_url, cred["username"]) + return self._clone_with_pat(cred["secret_value"], cred["username"]) + elif cred["credential_type"] == "SSH_KEY": + logger.info("Using SSH key auth for clone: url=%s", self.clone_url) + return self._clone_with_ssh_key(cred["secret_value"]) + else: + raise ValueError(f"Unsupported credential type: {cred['credential_type']}") + finally: + del cred + + def _clone_with_pat(self, pat: str, username: str | None) -> Repo: + """Clone using PAT: embed in URL, clone, reset URL to clean version.""" + parsed = urllib.parse.urlparse(self.clone_url) + user_part = f"{username}:{pat}" if username else f"token:{pat}" + auth_url = parsed._replace(netloc=f"{user_part}@{parsed.netloc}").geturl() + + repo = None + try: + repo = Repo.clone_from(auth_url, self.repo_path, depth=1, no_checkout=True) + self._do_fetch(repo) + return repo + finally: + if repo: + repo.remotes.origin.set_url(self.clone_url) + del auth_url, pat + + + def _clone_with_ssh_key(self, key: str) -> Repo: + """Clone using SSH key: convert URL to SSH format, write temp key file.""" + ssh_url = self._https_to_ssh_url(self.clone_url) + ssh_key_path = None + try: + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: + f.write(key) + ssh_key_path = f.name + os.chmod(ssh_key_path, 0o600) + git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" + + repo = Repo.clone_from(ssh_url, self.repo_path, depth=1, no_checkout=True) + with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): + self._do_fetch(repo) + return repo + finally: + if ssh_key_path and os.path.exists(ssh_key_path): + os.unlink(ssh_key_path) + del key + + + @staticmethod + def _https_to_ssh_url(https_url: str) -> str: + """Convert HTTPS URL to SSH format (required for SSH key auth).""" + if not https_url.startswith("https://"): + return https_url + parsed = urllib.parse.urlparse(https_url) + return f"git@{parsed.hostname}:{parsed.path.lstrip('/')}" + def yield_blobs(self) -> typing.Iterator[Blob]: """ Yield the blobs from the Git repository. One blob will be generated for each file in the repo which passes the diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index b37caa0f9..aebba7c20 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -28,6 +28,7 @@ from exploit_iq_commons.data_models.common import AnalysisType from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context from vuln_analysis.tools.tool_names import ToolNames logger = LoggingFactory.get_agent_logger(__name__) @@ -192,35 +193,39 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: try: trace_id.set(message.scan.id) - # Build VDBs - vdb_code_path, vdb_doc_path = embedder.build_vdbs(source_infos, config.ignore_code_embedding) - - if (vdb_code_path is None): - # Only log warning if we're not ignoring code embeddings - if (not config.ignore_code_embedding): - logger.warning(("Failed to generate code VDB for image '%s'. " + logger.debug("_arun: received credential_id=%r scan_id=%s", message.credential_id, message.scan.id) + # Build VDBs (credential_id is propagated via async context) + with credential_context(message.credential_id): + logger.debug("_arun: credential_context entered, credential_id=%r", message.credential_id) + vdb_code_path, vdb_doc_path = embedder.build_vdbs(source_infos, config.ignore_code_embedding) + + if (vdb_code_path is None): + # Only log warning if we're not ignoring code embeddings + if (not config.ignore_code_embedding): + logger.warning(("Failed to generate code VDB for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) + + if (vdb_doc_path is None): + logger.warning(("Failed to generate documentation VDB for image '%s'. " "Ensure the source repositories are setup correctly"), base_image) - if (vdb_doc_path is None): - logger.warning(("Failed to generate documentation VDB for image '%s'. " - "Ensure the source repositories are setup correctly"), - base_image) - - # Build code index if not ignored - if not config.ignore_code_index: - logger.info("analysis type: %s", message.image.analysis_type) - if message.image.analysis_type == AnalysisType.IMAGE and isinstance(sbom_infos, ManualSBOMInfoInput): - RPMDependencyManager.get_instance().sbom = sbom_infos.packages - image = f"{message.image.name}:{message.image.tag}" - RPMDependencyManager.get_instance().container_image = image - - code_index_path = _build_code_index(source_infos) - - if code_index_path is None: - logger.warning(("Failed to generate code index for image '%s'. " - "Ensure the source repositories are setup correctly"), - base_image) + # Build code index if not ignored + if not config.ignore_code_index: + logger.info("analysis type: %s", message.image.analysis_type) + logger.debug("_arun: building code index, credential_id in ctx=%r", message.credential_id) + if message.image.analysis_type == AnalysisType.IMAGE and isinstance(sbom_infos, ManualSBOMInfoInput): + RPMDependencyManager.get_instance().sbom = sbom_infos.packages + image = f"{message.image.name}:{message.image.tag}" + RPMDependencyManager.get_instance().container_image = image + + code_index_path = _build_code_index(source_infos) + + if code_index_path is None: + logger.warning(("Failed to generate code index for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) # Replace ref with specific commit hash for each source info for si in source_infos: diff --git a/src/vuln_analysis/tools/tests/test_credential_client.py b/src/vuln_analysis/tools/tests/test_credential_client.py new file mode 100644 index 000000000..8a344787a --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_credential_client.py @@ -0,0 +1,216 @@ +import base64 +import json +import logging +from unittest.mock import MagicMock, patch + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from exploit_iq_commons.utils.credential_client import ( + AuthenticationError, + CredentialNotFoundError, + DecryptionError, + fetch_and_decrypt_credential, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_ENCRYPTION_KEY = "test-encryption-key-32-bytes!!!!" # exactly 32 UTF-8 bytes +_IV = b"\x00" * 12 # deterministic IV for tests + + +def _encrypt(plaintext: str, key: str = _ENCRYPTION_KEY, iv: bytes = _IV) -> bytes: + """Encrypt plaintext using AES-256-GCM with the same parameters as the backend.""" + aesgcm = AESGCM(key.encode("utf-8")[:32]) + return aesgcm.encrypt(iv, plaintext.encode("utf-8"), None) + + +def _make_response( + plaintext: str, + credential_type: str = "PAT", + username: str | None = "github-bot", + user_id: str = "alice@example.com", + key: str = _ENCRYPTION_KEY, + iv: bytes = _IV, +) -> dict: + encrypted = _encrypt(plaintext, key=key, iv=iv) + return { + "encryptedSecretValue": base64.b64encode(encrypted).decode(), + "iv": base64.b64encode(iv).decode(), + "username": username, + "credentialType": credential_type, + "userId": user_id, + } + + +def _mock_http_ok(payload: dict) -> MagicMock: + resp = MagicMock() + resp.status_code = 200 + resp.ok = True + resp.json.return_value = payload + return resp + + +def _mock_http_error(status: int) -> MagicMock: + resp = MagicMock() + resp.status_code = status + resp.ok = False + return resp + + +# --------------------------------------------------------------------------- +# Tests: successful decryption +# --------------------------------------------------------------------------- + +class TestFetchAndDecryptCredential: + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_pat_decryption_success(self, mock_get): + plaintext = "ghp_myPersonalAccessToken123" + mock_get.return_value = _mock_http_ok(_make_response(plaintext, credential_type="PAT")) + + result = fetch_and_decrypt_credential( + credential_id="cred-uuid-1", + jwt_token="scan.jwt.token", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + assert result["secret_value"] == plaintext + assert result["username"] == "github-bot" + assert result["credential_type"] == "PAT" + assert result["user_id"] == "alice@example.com" + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_ssh_key_decryption_success(self, mock_get): + plaintext = "-----BEGIN OPENSSH PRIVATE KEY-----\nabc123\n-----END OPENSSH PRIVATE KEY-----" + mock_get.return_value = _mock_http_ok( + _make_response(plaintext, credential_type="SSH_KEY", username=None) + ) + + result = fetch_and_decrypt_credential( + credential_id="cred-uuid-2", + jwt_token="scan.jwt.token", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + assert result["secret_value"] == plaintext + assert result["username"] is None + assert result["credential_type"] == "SSH_KEY" + + @patch("exploit_iq_commons.utils.credential_client.os.path.exists") + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_correct_url_and_auth_header(self, mock_get, mock_exists): + mock_get.return_value = _mock_http_ok(_make_response("token")) + mock_exists.return_value = False # CA bundle doesn't exist, so verify=False + + fetch_and_decrypt_credential( + credential_id="cred-uuid-3", + jwt_token="my.jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + mock_get.assert_called_once_with( + "https://backend.example.com/api/v1/credentials/cred-uuid-3", + headers={"Authorization": "Bearer my.jwt"}, + timeout=10, + verify=False, + ) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_backend_url_trailing_slash_stripped(self, mock_get): + mock_get.return_value = _mock_http_ok(_make_response("token")) + + fetch_and_decrypt_credential( + credential_id="cred-uuid-4", + jwt_token="my.jwt", + backend_url="https://backend.example.com/", # trailing slash + encryption_key=_ENCRYPTION_KEY, + ) + + call_url = mock_get.call_args[0][0] + assert "//" not in call_url.split("://")[1], "Double slash in URL" + + +# --------------------------------------------------------------------------- +# Tests: error handling +# --------------------------------------------------------------------------- + +class TestErrorHandling: + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_404_raises_credential_not_found(self, mock_get): + mock_get.return_value = _mock_http_error(404) + + with pytest.raises(CredentialNotFoundError): + fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_401_raises_authentication_error(self, mock_get): + mock_get.return_value = _mock_http_error(401) + + with pytest.raises(AuthenticationError): + fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_403_raises_authentication_error(self, mock_get): + mock_get.return_value = _mock_http_error(403) + + with pytest.raises(AuthenticationError): + fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_500_raises_runtime_error(self, mock_get): + mock_get.return_value = _mock_http_error(500) + + with pytest.raises(RuntimeError): + fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_wrong_key_raises_decryption_error(self, mock_get): + mock_get.return_value = _mock_http_ok(_make_response("secret", key=_ENCRYPTION_KEY)) + + with pytest.raises(DecryptionError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key="wrong-key-32-bytes-xxxxxxxxxxxxxx", + ) + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_network_error_raises_runtime_error(self, mock_get): + import requests as req_lib + mock_get.side_effect = req_lib.RequestException("Connection refused") + + with pytest.raises(RuntimeError, match="Network error"): + fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + + +# --------------------------------------------------------------------------- +# Tests: secret_value never appears in logs +# --------------------------------------------------------------------------- + +class TestNoPlaintextInLogs: + + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_secret_not_logged(self, mock_get, caplog): + secret = "super-secret-token-must-not-appear-in-logs" + mock_get.return_value = _mock_http_ok(_make_response(secret)) + + with caplog.at_level(logging.DEBUG, logger="exploit_iq_commons.utils.credential_client"): + fetch_and_decrypt_credential( + credential_id="cred-uuid-log", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + for record in caplog.records: + assert secret not in record.getMessage(), ( + f"Secret value leaked into log message: {record.getMessage()}" + ) diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py index a0d8c9e8b..cd34cf514 100644 --- a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -2,6 +2,28 @@ import shutil from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader + +@pytest.mark.parametrize("https_url,expected_ssh", [ + ( + "https://github.com/org/repo.git", + "git@github.com:org/repo.git", + ), + ( + "https://github.com/org/repo", + "git@github.com:org/repo", + ), + ( + "https://gitlab.com/group/subgroup/repo.git", + "git@gitlab.com:group/subgroup/repo.git", + ), + ( + "git@github.com:org/repo.git", + "git@github.com:org/repo.git", # already SSH — unchanged + ), +]) +def test_https_to_ssh_url(https_url, expected_ssh): + assert SourceCodeGitLoader._https_to_ssh_url(https_url) == expected_ssh + @pytest.mark.parametrize("refs", [ ("3.1.43", "3.1.42", "tag switch"), ("ba5c10d3655c4fec714294cbc2ae0829c44dc046", From 87aa22c70fe21aa3b9b3b42d2d53747924adf00a Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 2 Mar 2026 10:56:15 +0200 Subject: [PATCH 207/286] fix(sec): harden TLS verification and decryption error handling Signed-off-by: Vladimir Belousov --- .../utils/credential_client.py | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/src/exploit_iq_commons/utils/credential_client.py b/src/exploit_iq_commons/utils/credential_client.py index 46cab382a..b6114211b 100644 --- a/src/exploit_iq_commons/utils/credential_client.py +++ b/src/exploit_iq_commons/utils/credential_client.py @@ -15,9 +15,11 @@ import base64 import os +from cryptography.exceptions import InvalidTag +from collections.abc import Generator from contextlib import contextmanager from contextvars import ContextVar -from typing import Generator +from pathlib import Path import requests from cryptography.hazmat.primitives.ciphers.aead import AESGCM @@ -27,12 +29,13 @@ logger = LoggingFactory.get_agent_logger(__name__) _K8S_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +AES_256_KEY_SIZE_BYTES = 32 _credential_id_ctx: ContextVar[str | None] = ContextVar("credential_id", default=None) @contextmanager -def credential_context(credential_id: str | None) -> Generator[None, None, None]: +def credential_context(credential_id: str | None) -> Generator[None]: """ Context manager that makes credential_id available to any code running within this scope (including async tasks spawned from it). @@ -66,8 +69,8 @@ def _resolve_jwt_token(jwt_token: str | None) -> str: if jwt_token: return jwt_token - if os.path.isfile(_K8S_SERVICE_ACCOUNT_TOKEN_PATH): - with open(_K8S_SERVICE_ACCOUNT_TOKEN_PATH) as f: + if Path(_K8S_SERVICE_ACCOUNT_TOKEN_PATH).is_file(): + with Path(_K8S_SERVICE_ACCOUNT_TOKEN_PATH).open(encoding="utf-8") as f: token = f.read().strip() if token: logger.debug("Using Kubernetes service account token for credential backend auth") @@ -96,6 +99,46 @@ class DecryptionError(Exception): """Raised when AES-256-GCM decryption fails (wrong key or corrupted data).""" +class TLSConfigurationError(RuntimeError): + """Raised when CA bundle configuration is invalid or inaccessible.""" + + +def _validate_ca_bundle(ca_bundle_path: str) -> str: + """ + Validate CA bundle file before using it for TLS verification. + + Parameters + ---------- + ca_bundle_path : str + Path to the CA bundle file (PEM format). + + Returns + ------- + str + The validated CA bundle path. + + Raises + ------ + TLSConfigurationError + If the path does not point to a regular file, the file is empty, + or the file cannot be opened due to permission errors. + """ + try: + if not Path(ca_bundle_path).is_file(): + raise TLSConfigurationError( + f"CA bundle is not a regular file: {ca_bundle_path}. " + "Set CLIENT_CA_BUNDLE env var to a valid CA certificate path." + ) + with Path(ca_bundle_path).open("rb") as f: + if not f.read(1): + raise TLSConfigurationError(f"CA bundle is empty: {ca_bundle_path}") + except OSError as exc: + raise TLSConfigurationError( + f"CA bundle not readable: {ca_bundle_path}" + ) from exc + return ca_bundle_path + + def fetch_and_decrypt_credential( credential_id: str, backend_url: str, @@ -145,12 +188,7 @@ def fetch_and_decrypt_credential( logger.info("Fetching credential: credential_id=%s", credential_id) ca_bundle = os.environ.get("CLIENT_CA_BUNDLE", "/app/certs/service-ca.crt") - ca_exists = os.path.exists(ca_bundle) - logger.debug("Using CA bundle: %s (exists: %s)", ca_bundle, ca_exists) - - verify_ssl = ca_bundle if ca_exists else False - if not ca_exists: - logger.warning("CA bundle not found at %s, falling back to verify=False", ca_bundle) + verify_ssl = _validate_ca_bundle(ca_bundle) try: response = requests.get(url, headers=headers, timeout=10, verify=verify_ssl) @@ -183,12 +221,14 @@ def fetch_and_decrypt_credential( try: # Key derivation matches Agent backend: # new SecretKeySpec(keyBytes, 0, 32, "AES") where keyBytes = key.getBytes(UTF_8) - key = encryption_key.encode("utf-8")[:32] + key = encryption_key.encode("utf-8")[:AES_256_KEY_SIZE_BYTES] aesgcm = AESGCM(key) plaintext_bytes = aesgcm.decrypt(iv_bytes, encrypted_bytes, None) secret_value = plaintext_bytes.decode("utf-8") - except Exception as exc: + except (ValueError, InvalidTag) as exc: raise DecryptionError("AES-256-GCM decryption failed") from exc + except Exception as exc: + raise DecryptionError("AES-256-GCM unexpected general failure encountered during decrypting credential") from exc credential_type = data.get("credentialType", "") user_id = data.get("userId", "") From 78657a81b4eea347712c06ad434b0cf7ffa09a41 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 2 Mar 2026 13:04:34 +0200 Subject: [PATCH 208/286] fix: improve private repo authentication robustness Signed-off-by: Vladimir Belousov --- .../utils/source_code_git_loader.py | 93 +++++++++++++------ 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 4e291843e..34b7fe43f 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import os import tempfile import typing @@ -26,9 +27,17 @@ from langchain_core.document_loaders.blob_loaders import Blob from tqdm import tqdm -from exploit_iq_commons.utils.credential_client import _credential_id_ctx, fetch_and_decrypt_credential -from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.credential_client import ( + AES_256_KEY_SIZE_BYTES, + AuthenticationError, + CredentialNotFoundError, + _credential_id_ctx, + fetch_and_decrypt_credential, +) +from exploit_iq_commons.utils.transitive_code_searcher_tool import ( + TransitiveCodeSearcher, +) PathLike = typing.Union[str, os.PathLike] @@ -91,6 +100,37 @@ def __init__( self._repo: Repo | None = None + def _get_credential_env_vars(self) -> tuple[str, str]: + """ + Read and validate required environment variables for credential operations. + + Returns + ------- + tuple[str, str] + A tuple of (backend_url, encryption_key). + + Raises + ------ + ValueError + If CLIENT_BACKEND_URL or CREDENTIAL_ENCRYPTION_KEY is missing or empty. + ValueError + If CREDENTIAL_ENCRYPTION_KEY is shorter than 32 bytes when UTF-8 encoded. + + """ + backend_url = os.environ.get("CLIENT_BACKEND_URL", "").strip() + if not backend_url: + raise ValueError("CLIENT_BACKEND_URL is required") + encryption_key = os.environ.get("CREDENTIAL_ENCRYPTION_KEY", "").strip() + if not encryption_key: + raise ValueError("CREDENTIAL_ENCRYPTION_KEY is required") + key_bytes = encryption_key.encode("utf-8") + if len(key_bytes) < AES_256_KEY_SIZE_BYTES: + raise ValueError( + f"CREDENTIAL_ENCRYPTION_KEY must be at least {AES_256_KEY_SIZE_BYTES} bytes " + f"when UTF-8 encoded, got {len(key_bytes)} bytes." + ) + return backend_url, encryption_key + def load_repo(self): """ Load the Git repository and return the GitPython `Repo` object. @@ -176,10 +216,7 @@ def _fetch_public(self, repo: Repo) -> None: def _fetch_authenticated(self, repo: Repo, credential_id: str) -> None: """Fetch with authentication (PAT or SSH).""" - from exploit_iq_commons.utils.credential_client import CredentialNotFoundError, AuthenticationError - - backend_url = os.environ["CLIENT_BACKEND_URL"] - encryption_key = os.environ["CREDENTIAL_ENCRYPTION_KEY"] + backend_url, encryption_key = self._get_credential_env_vars() try: cred = fetch_and_decrypt_credential( @@ -187,12 +224,12 @@ def _fetch_authenticated(self, repo: Repo, credential_id: str) -> None: backend_url=backend_url, encryption_key=encryption_key, ) - except CredentialNotFoundError as e: + except CredentialNotFoundError: logger.warning("Credential expired or consumed: %s, falling back to public fetch", credential_id) self._do_fetch(repo) return - except AuthenticationError as e: - logger.error("Authentication failed for credential: %s", credential_id) + except AuthenticationError: + logger.exception("Authentication failed for credential: %s", credential_id) raise try: @@ -217,15 +254,13 @@ def _do_fetch(self, repo: Repo) -> None: repo.commit(self.ref) logger.info("Ref %s already exists locally", self.ref) return - except Exception: + except GitCommandError: # Ref doesn't exist locally either, re-raise original fetch error - raise e + raise e from None # Try to fetch tags, but don't fail if they don't exist - try: + with contextlib.suppress(GitCommandError): repo.git.fetch("origin", f"refs/tags/{self.ref}:refs/tags/{self.ref}", "--depth=1", "--force") - except GitCommandError: - pass def _do_fetch_with_pat(self, repo: Repo, pat: str, username: str | None) -> None: """Fetch with PAT: temporarily update URL, fetch, restore URL.""" @@ -249,19 +284,18 @@ def _do_fetch_with_ssh_key(self, repo: Repo, key: str) -> None: with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: f.write(key) ssh_key_path = f.name - os.chmod(ssh_key_path, 0o600) + Path(ssh_key_path).chmod(0o600) git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): self._do_fetch(repo) finally: - if ssh_key_path and os.path.exists(ssh_key_path): - os.unlink(ssh_key_path) + if ssh_key_path and Path(ssh_key_path).exists(): + Path(ssh_key_path).unlink() del key def _clone_with_credential(self, credential_id: str) -> Repo: """Clone with authenticated credential (PAT or SSH key).""" - backend_url = os.environ["CLIENT_BACKEND_URL"] - encryption_key = os.environ["CREDENTIAL_ENCRYPTION_KEY"] + backend_url, encryption_key = self._get_credential_env_vars() cred = fetch_and_decrypt_credential( credential_id=credential_id, @@ -273,11 +307,10 @@ def _clone_with_credential(self, credential_id: str) -> Repo: if cred["credential_type"] == "PAT": logger.info("Using PAT auth for clone: url=%s username=%s", self.clone_url, cred["username"]) return self._clone_with_pat(cred["secret_value"], cred["username"]) - elif cred["credential_type"] == "SSH_KEY": + if cred["credential_type"] == "SSH_KEY": logger.info("Using SSH key auth for clone: url=%s", self.clone_url) return self._clone_with_ssh_key(cred["secret_value"]) - else: - raise ValueError(f"Unsupported credential type: {cred['credential_type']}") + raise ValueError(f"Unsupported credential type: {cred['credential_type']}") finally: del cred @@ -306,16 +339,19 @@ def _clone_with_ssh_key(self, key: str) -> Repo: with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: f.write(key) ssh_key_path = f.name - os.chmod(ssh_key_path, 0o600) + Path(ssh_key_path).chmod(0o600) git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" - repo = Repo.clone_from(ssh_url, self.repo_path, depth=1, no_checkout=True) + repo = Repo.clone_from( + ssh_url, self.repo_path, depth=1, no_checkout=True, + env={"GIT_SSH_COMMAND": git_ssh_cmd}, + ) with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): self._do_fetch(repo) return repo finally: - if ssh_key_path and os.path.exists(ssh_key_path): - os.unlink(ssh_key_path) + if ssh_key_path and Path(ssh_key_path).exists(): + Path(ssh_key_path).unlink() del key @@ -325,7 +361,10 @@ def _https_to_ssh_url(https_url: str) -> str: if not https_url.startswith("https://"): return https_url parsed = urllib.parse.urlparse(https_url) - return f"git@{parsed.hostname}:{parsed.path.lstrip('/')}" + path = parsed.path.lstrip("/") + if parsed.port and parsed.port != 443: + return f"ssh://git@{parsed.hostname}:{parsed.port}/{path}" + return f"git@{parsed.hostname}:{path}" def yield_blobs(self) -> typing.Iterator[Blob]: """ From 56692589f72477c8c53dff98259a5564aa24492f Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 8 Mar 2026 11:17:29 +0200 Subject: [PATCH 209/286] fix: skip SSL verification for HTTP URLs in credential client Signed-off-by: Vladimir Belousov --- .../utils/credential_client.py | 14 +++- .../tools/tests/test_credential_client.py | 71 ++++++++++++++----- 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/exploit_iq_commons/utils/credential_client.py b/src/exploit_iq_commons/utils/credential_client.py index b6114211b..775f418db 100644 --- a/src/exploit_iq_commons/utils/credential_client.py +++ b/src/exploit_iq_commons/utils/credential_client.py @@ -187,8 +187,18 @@ def fetch_and_decrypt_credential( logger.info("Fetching credential: credential_id=%s", credential_id) - ca_bundle = os.environ.get("CLIENT_CA_BUNDLE", "/app/certs/service-ca.crt") - verify_ssl = _validate_ca_bundle(ca_bundle) + # NOTE: SSL verification is only applicable for HTTPS endpoints. + # When running the agent locally against a non-TLS backend (e.g. http://localhost:8080), + # SSL verification is skipped automatically. + # + # Required environment variables for local debugging with private Git repositories: + # CLIENT_JWT_TOKEN=dummy_token + # CLIENT_BACKEND_URL=http://localhost:8080 + if url.startswith("https"): + ca_bundle = os.environ.get("CLIENT_CA_BUNDLE", "/app/certs/service-ca.crt") + verify_ssl = _validate_ca_bundle(ca_bundle) + else: + verify_ssl = False try: response = requests.get(url, headers=headers, timeout=10, verify=verify_ssl) diff --git a/src/vuln_analysis/tools/tests/test_credential_client.py b/src/vuln_analysis/tools/tests/test_credential_client.py index 8a344787a..d088e6749 100644 --- a/src/vuln_analysis/tools/tests/test_credential_client.py +++ b/src/vuln_analysis/tools/tests/test_credential_client.py @@ -1,5 +1,4 @@ import base64 -import json import logging from unittest.mock import MagicMock, patch @@ -66,8 +65,9 @@ def _mock_http_error(status: int) -> MagicMock: class TestFetchAndDecryptCredential: + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_pat_decryption_success(self, mock_get): + def test_pat_decryption_success(self, mock_get, mock_ca): plaintext = "ghp_myPersonalAccessToken123" mock_get.return_value = _mock_http_ok(_make_response(plaintext, credential_type="PAT")) @@ -83,8 +83,9 @@ def test_pat_decryption_success(self, mock_get): assert result["credential_type"] == "PAT" assert result["user_id"] == "alice@example.com" + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_ssh_key_decryption_success(self, mock_get): + def test_ssh_key_decryption_success(self, mock_get, mock_ca): plaintext = "-----BEGIN OPENSSH PRIVATE KEY-----\nabc123\n-----END OPENSSH PRIVATE KEY-----" mock_get.return_value = _mock_http_ok( _make_response(plaintext, credential_type="SSH_KEY", username=None) @@ -101,11 +102,10 @@ def test_ssh_key_decryption_success(self, mock_get): assert result["username"] is None assert result["credential_type"] == "SSH_KEY" - @patch("exploit_iq_commons.utils.credential_client.os.path.exists") + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_correct_url_and_auth_header(self, mock_get, mock_exists): + def test_correct_url_and_auth_header(self, mock_get, mock_ca): mock_get.return_value = _mock_http_ok(_make_response("token")) - mock_exists.return_value = False # CA bundle doesn't exist, so verify=False fetch_and_decrypt_credential( credential_id="cred-uuid-3", @@ -121,8 +121,9 @@ def test_correct_url_and_auth_header(self, mock_get, mock_exists): verify=False, ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_backend_url_trailing_slash_stripped(self, mock_get): + def test_backend_url_trailing_slash_stripped(self, mock_get, mock_ca): mock_get.return_value = _mock_http_ok(_make_response("token")) fetch_and_decrypt_credential( @@ -142,36 +143,61 @@ def test_backend_url_trailing_slash_stripped(self, mock_get): class TestErrorHandling: + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_404_raises_credential_not_found(self, mock_get): + def test_404_raises_credential_not_found(self, mock_get, mock_ca): mock_get.return_value = _mock_http_error(404) with pytest.raises(CredentialNotFoundError): - fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_401_raises_authentication_error(self, mock_get): + def test_401_raises_authentication_error(self, mock_get, mock_ca): mock_get.return_value = _mock_http_error(401) with pytest.raises(AuthenticationError): - fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_403_raises_authentication_error(self, mock_get): + def test_403_raises_authentication_error(self, mock_get, mock_ca): mock_get.return_value = _mock_http_error(403) with pytest.raises(AuthenticationError): - fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_500_raises_runtime_error(self, mock_get): + def test_500_raises_runtime_error(self, mock_get, mock_ca): mock_get.return_value = _mock_http_error(500) with pytest.raises(RuntimeError): - fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_wrong_key_raises_decryption_error(self, mock_get): + def test_wrong_key_raises_decryption_error(self, mock_get, mock_ca): mock_get.return_value = _mock_http_ok(_make_response("secret", key=_ENCRYPTION_KEY)) with pytest.raises(DecryptionError): @@ -182,13 +208,19 @@ def test_wrong_key_raises_decryption_error(self, mock_get): encryption_key="wrong-key-32-bytes-xxxxxxxxxxxxxx", ) + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_network_error_raises_runtime_error(self, mock_get): + def test_network_error_raises_runtime_error(self, mock_get, mock_ca): import requests as req_lib mock_get.side_effect = req_lib.RequestException("Connection refused") with pytest.raises(RuntimeError, match="Network error"): - fetch_and_decrypt_credential("cred-id", "jwt", "https://backend.example.com", _ENCRYPTION_KEY) + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) # --------------------------------------------------------------------------- @@ -197,8 +229,9 @@ def test_network_error_raises_runtime_error(self, mock_get): class TestNoPlaintextInLogs: + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) @patch("exploit_iq_commons.utils.credential_client.requests.get") - def test_secret_not_logged(self, mock_get, caplog): + def test_secret_not_logged(self, mock_get, mock_ca, caplog): secret = "super-secret-token-must-not-appear-in-logs" mock_get.return_value = _mock_http_ok(_make_response(secret)) From b9ca63ae2712f42b80dfa55a8f297b657f7c7140 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:40:05 +0200 Subject: [PATCH 210/286] Fix thread race and other issues (#210) * fix threading in CCA * fix java and cpp * javascript improvments --- .tekton/on-pull-request.yaml | 1 + .../utils/c_segmenter_custom.py | 59 +- .../utils/chain_of_calls_retriever.py | 165 ++- src/exploit_iq_commons/utils/dep_tree.py | 15 +- .../javascript_functions_parser.py | 15 + .../utils/javascript_extended_segmenter.py | 19 + .../tools/tests/test_segmenter.py | 35 +- .../tools/tests/tests_data/jsonpath.c | 1076 +++++++++++++++++ 8 files changed, 1291 insertions(+), 94 deletions(-) create mode 100644 src/vuln_analysis/tools/tests/tests_data/jsonpath.c diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index d9240805c..243b1530d 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -245,6 +245,7 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME + print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" diff --git a/src/exploit_iq_commons/utils/c_segmenter_custom.py b/src/exploit_iq_commons/utils/c_segmenter_custom.py index b1155150a..c842f1e7a 100644 --- a/src/exploit_iq_commons/utils/c_segmenter_custom.py +++ b/src/exploit_iq_commons/utils/c_segmenter_custom.py @@ -5,9 +5,12 @@ #class extened CSegmenter class CSegmenterExtended(CSegmenter): + _DEFINE_FUNC_RE = re.compile( + r'^\s*#\s*define\s+(?P[a-zA-Z_]\w*)\s*\((?P[^)]*)\)' + ) + def __init__(self, code: str): - # Preprocess code: remove comments and macro blocks - + self._raw_code = code code = self.remove_macro_blocks(code) super().__init__(code) self.structs_enums: List[str] = [] @@ -169,6 +172,57 @@ def find_top_level_blocks(c_code: str): return blocks + @classmethod + def extract_define_functions(cls, code: str) -> List[str]: + """ + Extract function-like #define macros whose name contains at least one + lowercase letter and convert them into dummy C function segments. + + Handles multi-line macros joined by backslash continuation. + Strips ``do { ... } while(0)`` wrappers when present. + """ + lines = code.splitlines() + total = len(lines) + results: List[str] = [] + i = 0 + + while i < total: + m = cls._DEFINE_FUNC_RE.match(lines[i]) + if not m or m.group('name').isupper(): + i += 1 + continue + + name = m.group('name') + args = m.group('args').strip() + + macro_lines = [lines[i]] + while macro_lines[-1].rstrip().endswith('\\') and i + 1 < total: + i += 1 + macro_lines.append(lines[i]) + + full_line = ' '.join( + ln.rstrip().rstrip('\\').strip() for ln in macro_lines + ) + + header_re = re.compile( + r'#\s*define\s+' + re.escape(name) + r'\s*\([^)]*\)\s*' + ) + body = header_re.sub('', full_line, count=1).strip() + + do_while_re = re.compile( + r'^do\s*\{(?P.*)\}\s*while\s*\(\s*0\s*\)\s*;?\s*$', + re.DOTALL, + ) + dw = do_while_re.match(body) + if dw: + body = dw.group('inner').strip() + + dummy = f"void {name}({args}) {{ {body} }}" + results.append(dummy) + i += 1 + + return results + def extract_functions_classes(self) -> List[str]: segments = super().extract_functions_classes() for i, seg in enumerate(segments): @@ -183,6 +237,7 @@ def extract_functions_classes(self) -> List[str]: hidden_segments.append(new_seg) break # support only one segment into 2 segments we only add hidden segments into segments list that might be functions segments.extend(hidden_segments) + segments.extend(self.extract_define_functions(self._raw_code)) return segments \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index 277ac4398..e0eced325 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -3,7 +3,7 @@ from collections import defaultdict, deque from pathlib import Path from typing import List - +import copy from langchain_core.documents import Document from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, PARENTS_INDEX, \ @@ -50,6 +50,14 @@ def is_likely_macro_block(segment: str) -> bool: return False +class SessionContext: + """Per-call mutable state, isolating concurrent get_relevant_documents calls.""" + def __init__(self, tree_dict: dict[str, list[str]]): + self.last_visited = dict() + self.tree_dict = copy.deepcopy(tree_dict) + self.found_path = False + + class ChainOfCallsRetriever(ChainOfCallsRetrieverBase): """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being called from the application code base or not. @@ -125,12 +133,8 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat logger.debug(f"self.documents len : {len(self.documents)}") logger.debug("Chain of Calls Retriever - retaining only types/classes docs " "documents_of_types len %d", len(self.documents_of_types)) - # boolean attribute that indicates whether a path was found or not, initially set to False. - self.found_path = False logger.debug("Chain of Calls Retriever - after documents_of_full_sources") - self.last_visited_parent_package_indexes = dict() - self.last_visited = dict() # Constructing a map of types and classes to their attributes/members/fields self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields(self.documents_of_types) # Create a data structure containing dict of key=(function_name@source_file),value = dict of @@ -155,36 +159,35 @@ def __group_docs_by_pkg(self) -> dict[str, list[Document]]: logger.debug("PROFILE: sort all docs (%d) elaps %.3f", len(self.documents), t1) return sort_docs - def __find_caller_function_dfs(self, document_function: Document, function_package: str) -> Document: + def __find_caller_function_dfs(self, document_function: Document, function_package: str, + session_context: SessionContext) -> Document: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function + :param session_context: per-call session state for thread safety :return: a single document of a function that is calling document_function, or None if not found """ package_names = self.language_parser.get_package_names(document_function) direct_parents = list() # gets list of all direct parents of function for package_name in package_names: - list_of_packages = self.tree_dict.get(package_name) + list_of_packages = session_context.tree_dict.get(package_name) if list_of_packages: direct_parents.extend(list_of_packages[PARENTS_INDEX]) - # Add same package itself to search path. + # Add same package itself to search path. # direct_parents.extend([function_package]) - # gets list of documents to search in only from parents of function' package. + # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) if function_name_to_search == self.language_parser.get_constructor_method_name(): function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() - last_visited_package_index = (self.last_visited_parent_package_indexes - .get(calculate_hashable_string_for_function(function_file_name, - function_name_to_search), 0)) - package_exclusions = self.tree_dict.get(function_package)[EXCLUSIONS_INDEX] # Search for caller functions only at parents according to dependency tree. - for package in direct_parents[last_visited_package_index:]: + package_exclusions = session_context.tree_dict.get(function_package)[EXCLUSIONS_INDEX] + for package in direct_parents: sources_location_packages = True - if self.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: + if session_context.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: sources_location_packages = False possible_docs = self.get_possible_docs(function_name_to_search, package, @@ -192,7 +195,6 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa sources_location_packages, frozenset(), dict()) - # Collect all potential caller functions for doc in self.get_functions_for_package(package_name=package, documents=possible_docs, @@ -200,7 +202,7 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa function_to_search=function_name_to_search, callee_function_file_name=function_file_name): relevant_docs_to_search_in.append(doc) - # Perform the search only on the subset of potential caller functions + # Perform the search only on the subset of potential caller functions for doc in relevant_docs_to_search_in: function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, callee_function_name= @@ -225,9 +227,6 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. if function_is_being_called: package_exclusions.append(doc) - # update index of last scanned package for backtracking - # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) - # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index return doc # If didn't find a matching caller function document, returns None. @@ -267,24 +266,23 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] - def __find_caller_functions_bfs(self, document_function: Document, function_package: str) -> List[Document]: + def __find_caller_functions_bfs(self, document_function: Document, function_package: str, + session_context: SessionContext) -> List[Document]: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function + :param session_context: per-call session state for thread safety :return: a list of documents of functions that are calling document_function """ total_start = time.time() direct_parents = list() - # gets list of all direct parents of function - list_of_packages = self.tree_dict.get(function_package) + list_of_packages = session_context.tree_dict.get(function_package) if list_of_packages is not None: direct_parents.extend(list_of_packages[0]) - # Add same package itself to search path. - # direct_parents.extend([function_package]) - # gets list of documents to search in only from parents of function' package. + # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() @@ -295,15 +293,6 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack for package in direct_parents: pkg_docs = self.sort_docs[package] for doc in pkg_docs: - # for doc in self.documents: - # is_doc_in_pkg = False - # for package in direct_parents: - # if self.language_parser.is_a_package(package, doc): - # is_doc_in_pkg = True - # break - # if not is_doc_in_pkg: - # continue - file_name = doc.metadata.get('source') if doc.metadata.get('state') == "invalid": continue @@ -311,9 +300,6 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack # check for same doc if (function_name_to_search == func_name) and (file_name == function_file_name): continue - # same function name different files ? - # if (function_name_to_search == func_name): - # logger.debug(f"same func name {function_name_to_search}") if func_name == "main": file_path = Path(function_file_name) @@ -322,7 +308,7 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack if self.language_parser.dir_name_for_3rd_party_packages() in path_parts: continue if doc.page_content.__contains__(f"{function_name_to_search}("): - last_visited = (self.last_visited.get( + last_visited = (session_context.last_visited.get( calculate_hashable_string_for_function(file_name, func_name), 0)) if last_visited == 0: found = self.language_parser.search_for_called_function( @@ -366,11 +352,10 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, - current_package_name: str) -> tuple[List[Document], bool]: - # main loop. + current_package_name: str, + session_context: SessionContext) -> tuple[List[Document], bool]: file_counter = 0 q = deque() - self.last_visited.clear() q.append(target_function_doc) loop_start = time.time() while q: @@ -384,10 +369,10 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct function_file = target_doc.metadata.get('source') hashed_value = calculate_hashable_string_for_function(function_file, function_name) - if hashed_value in self.last_visited: + if hashed_value in session_context.last_visited: continue - self.last_visited[hashed_value] = 1 + session_context.last_visited[hashed_value] = 1 logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", file_counter, target_doc.metadata['source'], function_name, target_pkg, len(q)) @@ -395,7 +380,8 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct # Find a caller function and containing package in the dependency tree according to hierarchy sub_start = time.time() found_documents = self.__find_caller_functions_bfs(document_function=target_doc, - function_package=target_pkg) + function_package=target_pkg, + session_context=session_context) sub_elapsed = time.time() - sub_start logger.debug(f"[PROFILE] __find_caller_functions took {sub_elapsed:.3f} seconds") # If found, then add it to path @@ -404,7 +390,7 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(doc_candidate): matching_documents.append(doc_candidate) - self.found_path = True + session_context.found_path = True break # Otherwise, we continue to search for callers for the current found function, in order to extend # the chain of calls and potentially find a path from application to the vulnerable @@ -412,40 +398,41 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct else: q.append(doc_candidate) - if self.found_path: + if session_context.found_path: break loop_elapsed = time.time() - loop_start logger.debug(f"[PROFILE] Main loop in get_relevant_C_documents took {loop_elapsed:.3f} seconds") # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was # found or not. - logger.debug("get_relevant_documents: result %s", self.found_path) + logger.debug("get_relevant_documents: result %s", session_context.found_path) logger.debug("get_relevant_documents: docs %s", matching_documents) - return matching_documents, self.found_path + return matching_documents, session_context.found_path def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, - current_package_name: str) -> tuple[List[Document], bool]: + current_package_name: str, + session_context: SessionContext) -> tuple[List[Document], bool]: """Execute depth-first search with backtracking strategy.""" end_loop = False - # main loop. while not end_loop: # Find a caller function and containing package in the dependency tree according to hierarchy found_document = self.__find_caller_function_dfs(document_function=target_function_doc, - function_package=current_package_name) + function_package=current_package_name, + session_context=session_context) # If found, then add it to path if found_document: matching_documents.append(found_document) # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(found_document): end_loop = True - self.found_path = True - # Otherwise, we continue to search for callers for the current found function, in order to extend - # the chain of calls and potentially find a path from application to the vulnerable - # function in input package + session_context.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package else: target_function_doc = found_document # extract package name from function document - current_package_name = self.__determine_doc_package_name(target_function_doc) + current_package_name = self.__determine_doc_package_name(target_function_doc, session_context) else: # end loop because didn't find a caller for initial function if len(matching_documents) == 1: @@ -455,12 +442,12 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio else: dead_end_node = matching_documents.pop() # Excludes dead end function node from future searches, as it led to nowhere. - self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + session_context.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) target_function_doc = matching_documents[-1] - current_package_name = self.__determine_doc_package_name(target_function_doc) - # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. - return matching_documents, self.found_path + current_package_name = self.__determine_doc_package_name(target_function_doc, session_context) + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, session_context.found_path # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). @@ -468,7 +455,6 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio # calls from application to input function in the input package. def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" - self.found_path = False query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() (package_name, function) = tuple(query.split(",")) class_name = None @@ -477,13 +463,15 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: class_name, function = function.split(splitters[0]) found_package = False matching_documents = [] - for dependency in self.tree_dict.values(): + session_context = SessionContext(self.tree_dict) + + for dependency in session_context.tree_dict.values(): dependency[EXCLUSIONS_INDEX] = [] standard_libs_cache = StandardLibraryCache.get_instance() # If it's a standard library package, then skip checking the package in dependency tree. if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): # Check if input package is in dependency tree - for package in self.tree_dict: + for package in session_context.tree_dict: if self.language_parser.is_same_package(package_name, package): package_name = package found_package = True @@ -492,12 +480,14 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: if found_package: target_function_doc = self.__find_initial_function(function, package_name=package_name, documents=self.documents, - class_name=class_name) + class_name=class_name, + session_context=session_context) if not target_function_doc and self.language_parser.get_constructor_method_name(): target_function_doc = self.__find_initial_function(function_name=self.language_parser.get_constructor_method_name(), package_name=package_name, documents=self.documents, - class_name=function) + class_name=function, + session_context=session_context) # If not, there is a chance that the package is some standard library in the ecosystem. else: @@ -505,7 +495,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # vulnerable function in standard lib package of language. page_content = self.language_parser.get_dummy_function(function) if page_content is None: - return matching_documents, self.found_path + return matching_documents, session_context.found_path if class_name: page_content = page_content + f'\n{self.language_parser.get_comment_line_notation()}(class: {class_name})' target_function_doc = Document(page_content=page_content @@ -514,18 +504,17 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, re.escape(package_name)) - root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] + root_package = [key for (key, value) in session_context.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() - # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. parents = list({self.language_parser.get_package_names(doc)[1] for doc in importing_docs if doc.metadata['source'].startswith( prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[1] - in self.tree_dict.keys()}) + in session_context.tree_dict.keys()}) for doc in importing_docs: if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): parents.append(root_package[0]) break - self.tree_dict[package_name] = [parents, []] + session_context.tree_dict[package_name] = [parents, []] end_loop = False current_package_name = package_name # If an initial document (that represents the vulnerable input function in the input package) was created @@ -538,26 +527,26 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: end_loop = True logger.error("Cannot find initial function=%s, in package=%s", function, package_name) if end_loop: - return matching_documents, self.found_path + return matching_documents, session_context.found_path if self.language_parser.is_search_algo_dfs(): - matching_documents, self.found_path = self._depth_first_search( - matching_documents, target_function_doc, current_package_name) + matching_documents, session_context.found_path = self._depth_first_search( + matching_documents, target_function_doc, current_package_name, session_context) else: - matching_documents, self.found_path = self._breadth_first_search( - matching_documents, target_function_doc, current_package_name) + matching_documents, session_context.found_path = self._breadth_first_search( + matching_documents, target_function_doc, current_package_name, session_context) - # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. - return matching_documents, self.found_path + return matching_documents, session_context.found_path - def __determine_doc_package_name(self, target_function_doc): + def __determine_doc_package_name(self, target_function_doc, + session_context: SessionContext): return [package_name for package_name in self.language_parser.get_package_names(target_function_doc) - if self.tree_dict.get(package_name, None) is not None][0] + if session_context.tree_dict.get(package_name, None) is not None][0] def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], - class_name: str = None) -> Document: + class_name: str = None, *, + session_context: SessionContext) -> Document: if self.language_parser.is_search_algo_dfs(): pkg_docs = documents @@ -569,15 +558,19 @@ def __find_initial_function(self, function_name: str, package_name: str, documen relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] - package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] - #for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): - for document in self.get_functions_for_package(package_name, relevant_docs): + package_exclusions = session_context.tree_dict.get(package_name)[EXCLUSIONS_INDEX] + from itertools import chain + for document in chain( + self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=True), + self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=False), + ): # document_function_calls_input_function = True if function_name.lower() == self.language_parser.get_function_name(document).lower(): # if language_parser.search_for_called_function(document, callee_function=function_name): package_exclusions.append(document) return document + if self.language_parser.create_dummy_for_standard_lib(package_name): dummy_function_doc = Document(page_content=f"func {function_name + '()' + '{}'}", metadata={ diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index b84a20519..719ac6bdd 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -849,12 +849,17 @@ def install_dependencies(self, manifest_path: Path): full_source_path = manifest_path / source_path for jar in full_source_path.glob("*-sources.jar"): - dest = full_source_path / jar.stem # folder named after jar + if jar.stat().st_size > 0: + dest = full_source_path / jar.stem # folder named after jar + + if not dest.exists(): + dest.mkdir(exist_ok=True) + result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) + if result.returncode != 0: + logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) + else: + logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) - if not dest.exists(): - dest.mkdir(exist_ok=True) - with zipfile.ZipFile(jar, "r") as zf: - zf.extractall(dest) def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: dependency_file = manifest_path / "dependency_tree.txt" diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py index a9e3252d6..7529f66f8 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -49,6 +49,21 @@ def get_function_name(self, function: Document) -> str: if match: return match.group(1) + # Try to match wrapper calls: var name = wrapper(function(...) { or var name = wrapper(() => + match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*\w+\s*\(', content) + if match: + return match.group(1) + + # Try to match property assignment functions: obj.name = function(...) { + match = re.search(r'(?:[\w.]+)\.(\w+)\s*=\s*(?:async\s+)?function\s*\(', content) + if match: + return match.group(1) + + # Try to match property assignment arrow functions: obj.name = (...) => + match = re.search(r'(?:[\w.]+)\.(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>', content) + if match: + return match.group(1) + # Try to match computed property methods: [Symbol.iterator]() { or [expr]() { # Return the computed expression inside brackets match = re.search(r'^\s*\[([^\]]+)\]\s*\([^)]*\)\s*\{', content, re.MULTILINE) diff --git a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py index eb4fc9ab3..d5168a365 100644 --- a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py +++ b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py @@ -106,6 +106,16 @@ def extract_functions_classes(self) -> List[str]: esprima.nodes.ClassDeclaration)): functions_classes.append(self._extract_code(node)) + # Handle property assignments with function expressions, + # e.g. hb.compile = function(input, options) { ... } + elif isinstance(node, esprima.nodes.ExpressionStatement): + if isinstance(node.expression, esprima.nodes.AssignmentExpression): + if isinstance(node.expression.right, ( + esprima.nodes.FunctionExpression, + esprima.nodes.ArrowFunctionExpression + )): + functions_classes.append(self._extract_code(node)) + # Extract individual class methods class_methods = self._extract_all_class_methods() functions_classes.extend(class_methods) @@ -178,6 +188,15 @@ def _extract_arrow_functions(self, var_node: esprima.nodes.VariableDeclaration) esprima.nodes.FunctionExpression )) + # Check if init is a CallExpression wrapping a function, + # e.g. var defaultsDeep = baseRest(function(args) { ... }) + if not is_function_expr and isinstance(declarator.init, esprima.nodes.CallExpression): + is_function_expr = any( + isinstance(arg, (esprima.nodes.FunctionExpression, + esprima.nodes.ArrowFunctionExpression)) + for arg in declarator.init.arguments + ) + if is_function_expr: # Extract the entire variable declaration including the assignment code = self._extract_code(var_node) diff --git a/src/vuln_analysis/tools/tests/test_segmenter.py b/src/vuln_analysis/tools/tests/test_segmenter.py index 91cd047b4..2e4fdcfed 100644 --- a/src/vuln_analysis/tools/tests/test_segmenter.py +++ b/src/vuln_analysis/tools/tests/test_segmenter.py @@ -179,4 +179,37 @@ def test_integration_ess_lib_c(ess_lib_c_code): assert "ESS_CERT_ID_V2_new_init" in names # static function (v2 version) assert "ess_issuer_serial_cmp" in names # static comparison function - \ No newline at end of file + +@pytest.fixture(scope="module") +def jsonpath_c_code(c_test_files_dir): + file_path = c_test_files_dir / "jsonpath.c" + return file_path.read_text(encoding="utf-8") + + +def test_jsonpath_c_define_macros_captured(jsonpath_c_code): + """ + Lowercase function-like #define macros (read_byte, read_int32, read_int32_n) + should be captured as dummy function segments. The raw ``#define`` text + must not leak through — only the synthesized ``void name(args) { body }`` form. + Functions that *use* these macros (e.g. jspInitByBuffer) must still be captured. + """ + segmenter = CSegmenterExtended(jsonpath_c_code) + segments = segmenter.extract_functions_classes() + + assert len(segments) > 0, "Should find segments in jsonpath.c" + + for seg in segments: + assert "#define read_byte" not in seg, ( + "Raw #define read_byte should not appear — only the dummy function form" + ) + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + assert "read_byte" in names, "read_byte macro should be captured as a dummy function segment" + assert "read_int32" in names, "read_int32 macro should be captured as a dummy function segment" + assert "read_int32_n" in names, "read_int32_n macro should be captured as a dummy function segment" + + assert "jspInitByBuffer" in names, ( + "jspInitByBuffer (which uses read_byte) should be captured as a function segment" + ) + assert "jspInit" in names diff --git a/src/vuln_analysis/tools/tests/tests_data/jsonpath.c b/src/vuln_analysis/tools/tests/tests_data/jsonpath.c new file mode 100644 index 000000000..db06e6f29 --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/jsonpath.c @@ -0,0 +1,1076 @@ +/*------------------------------------------------------------------------- + * + * jsonpath.c + * Input/output and supporting routines for jsonpath + * + * jsonpath expression is a chain of path items. First path item is $, $var, + * literal or arithmetic expression. Subsequent path items are accessors + * (.key, .*, [subscripts], [*]), filters (? (predicate)) and methods (.type(), + * .size() etc). + * + * For instance, structure of path items for simple expression: + * + * $.a[*].type() + * + * is pretty evident: + * + * $ => .a => [*] => .type() + * + * Some path items such as arithmetic operations, predicates or array + * subscripts may comprise subtrees. For instance, more complex expression + * + * ($.a + $[1 to 5, 7] ? (@ > 3).double()).type() + * + * have following structure of path items: + * + * + => .type() + * ___/ \___ + * / \ + * $ => .a $ => [] => ? => .double() + * _||_ | + * / \ > + * to to / \ + * / \ / @ 3 + * 1 5 7 + * + * Binary encoding of jsonpath constitutes a sequence of 4-bytes aligned + * variable-length path items connected by links. Every item has a header + * consisting of item type (enum JsonPathItemType) and offset of next item + * (zero means no next item). After the header, item may have payload + * depending on item type. For instance, payload of '.key' accessor item is + * length of key name and key name itself. Payload of '>' arithmetic operator + * item is offsets of right and left operands. + * + * So, binary representation of sample expression above is: + * (bottom arrows are next links, top lines are argument links) + * + * _____ + * _____ ___/____ \ __ + * _ /_ \ _____/__/____ \ \ __ _ /_ \ + * / / \ \ / / / \ \ \ / \ / / \ \ + * +(LR) $ .a $ [](* to *, * to *) 1 5 7 ?(A) >(LR) @ 3 .double() .type() + * | | ^ | ^| ^| ^ ^ + * | |__| |__||________________________||___________________| | + * |_______________________________________________________________________| + * + * Copyright (c) 2019-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/adt/jsonpath.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "funcapi.h" +#include "lib/stringinfo.h" +#include "libpq/pqformat.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/json.h" +#include "utils/jsonpath.h" + + +static Datum jsonPathFromCstring(char *in, int len); +static char *jsonPathToCstring(StringInfo out, JsonPath *in, + int estimated_len); +static int flattenJsonPathParseItem(StringInfo buf, JsonPathParseItem *item, + int nestingLevel, bool insideArraySubscript); +static void alignStringInfoInt(StringInfo buf); +static int32 reserveSpaceForItemPointer(StringInfo buf); +static void printJsonPathItem(StringInfo buf, JsonPathItem *v, bool inKey, + bool printBracketes); +static int operationPriority(JsonPathItemType op); + + +/**************************** INPUT/OUTPUT ********************************/ + +/* + * jsonpath type input function + */ +Datum +jsonpath_in(PG_FUNCTION_ARGS) +{ + char *in = PG_GETARG_CSTRING(0); + int len = strlen(in); + + return jsonPathFromCstring(in, len); +} + +/* + * jsonpath type recv function + * + * The type is sent as text in binary mode, so this is almost the same + * as the input function, but it's prefixed with a version number so we + * can change the binary format sent in future if necessary. For now, + * only version 1 is supported. + */ +Datum +jsonpath_recv(PG_FUNCTION_ARGS) +{ + StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); + int version = pq_getmsgint(buf, 1); + char *str; + int nbytes; + + if (version == JSONPATH_VERSION) + str = pq_getmsgtext(buf, buf->len - buf->cursor, &nbytes); + else + elog(ERROR, "unsupported jsonpath version number: %d", version); + + return jsonPathFromCstring(str, nbytes); +} + +/* + * jsonpath type output function + */ +Datum +jsonpath_out(PG_FUNCTION_ARGS) +{ + JsonPath *in = PG_GETARG_JSONPATH_P(0); + + PG_RETURN_CSTRING(jsonPathToCstring(NULL, in, VARSIZE(in))); +} + +/* + * jsonpath type send function + * + * Just send jsonpath as a version number, then a string of text + */ +Datum +jsonpath_send(PG_FUNCTION_ARGS) +{ + JsonPath *in = PG_GETARG_JSONPATH_P(0); + StringInfoData buf; + StringInfoData jtext; + int version = JSONPATH_VERSION; + + initStringInfo(&jtext); + (void) jsonPathToCstring(&jtext, in, VARSIZE(in)); + + pq_begintypsend(&buf); + pq_sendint8(&buf, version); + pq_sendtext(&buf, jtext.data, jtext.len); + pfree(jtext.data); + + PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); +} + +/* + * Converts C-string to a jsonpath value. + * + * Uses jsonpath parser to turn string into an AST, then + * flattenJsonPathParseItem() does second pass turning AST into binary + * representation of jsonpath. + */ +static Datum +jsonPathFromCstring(char *in, int len) +{ + JsonPathParseResult *jsonpath = parsejsonpath(in, len); + JsonPath *res; + StringInfoData buf; + + initStringInfo(&buf); + enlargeStringInfo(&buf, 4 * len /* estimation */ ); + + appendStringInfoSpaces(&buf, JSONPATH_HDRSZ); + + if (!jsonpath) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "jsonpath", + in))); + + flattenJsonPathParseItem(&buf, jsonpath->expr, 0, false); + + res = (JsonPath *) buf.data; + SET_VARSIZE(res, buf.len); + res->header = JSONPATH_VERSION; + if (jsonpath->lax) + res->header |= JSONPATH_LAX; + + PG_RETURN_JSONPATH_P(res); +} + +/* + * Converts jsonpath value to a C-string. + * + * If 'out' argument is non-null, the resulting C-string is stored inside the + * StringBuffer. The resulting string is always returned. + */ +static char * +jsonPathToCstring(StringInfo out, JsonPath *in, int estimated_len) +{ + StringInfoData buf; + JsonPathItem v; + + if (!out) + { + out = &buf; + initStringInfo(out); + } + enlargeStringInfo(out, estimated_len); + + if (!(in->header & JSONPATH_LAX)) + appendBinaryStringInfo(out, "strict ", 7); + + jspInit(&v, in); + printJsonPathItem(out, &v, false, true); + + return out->data; +} + +/* + * Recursive function converting given jsonpath parse item and all its + * children into a binary representation. + */ +static int +flattenJsonPathParseItem(StringInfo buf, JsonPathParseItem *item, + int nestingLevel, bool insideArraySubscript) +{ + /* position from beginning of jsonpath data */ + int32 pos = buf->len - JSONPATH_HDRSZ; + int32 chld; + int32 next; + int argNestingLevel = 0; + + check_stack_depth(); + CHECK_FOR_INTERRUPTS(); + + appendStringInfoChar(buf, (char) (item->type)); + + /* + * We align buffer to int32 because a series of int32 values often goes + * after the header, and we want to read them directly by dereferencing + * int32 pointer (see jspInitByBuffer()). + */ + alignStringInfoInt(buf); + + /* + * Reserve space for next item pointer. Actual value will be recorded + * later, after next and children items processing. + */ + next = reserveSpaceForItemPointer(buf); + + switch (item->type) + { + case jpiString: + case jpiVariable: + case jpiKey: + appendBinaryStringInfo(buf, (char *) &item->value.string.len, + sizeof(item->value.string.len)); + appendBinaryStringInfo(buf, item->value.string.val, + item->value.string.len); + appendStringInfoChar(buf, '\0'); + break; + case jpiNumeric: + appendBinaryStringInfo(buf, (char *) item->value.numeric, + VARSIZE(item->value.numeric)); + break; + case jpiBool: + appendBinaryStringInfo(buf, (char *) &item->value.boolean, + sizeof(item->value.boolean)); + break; + case jpiAnd: + case jpiOr: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiStartsWith: + { + /* + * First, reserve place for left/right arg's positions, then + * record both args and sets actual position in reserved + * places. + */ + int32 left = reserveSpaceForItemPointer(buf); + int32 right = reserveSpaceForItemPointer(buf); + + chld = !item->value.args.left ? pos : + flattenJsonPathParseItem(buf, item->value.args.left, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + left) = chld - pos; + + chld = !item->value.args.right ? pos : + flattenJsonPathParseItem(buf, item->value.args.right, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + right) = chld - pos; + } + break; + case jpiLikeRegex: + { + int32 offs; + + appendBinaryStringInfo(buf, + (char *) &item->value.like_regex.flags, + sizeof(item->value.like_regex.flags)); + offs = reserveSpaceForItemPointer(buf); + appendBinaryStringInfo(buf, + (char *) &item->value.like_regex.patternlen, + sizeof(item->value.like_regex.patternlen)); + appendBinaryStringInfo(buf, item->value.like_regex.pattern, + item->value.like_regex.patternlen); + appendStringInfoChar(buf, '\0'); + + chld = flattenJsonPathParseItem(buf, item->value.like_regex.expr, + nestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + offs) = chld - pos; + } + break; + case jpiFilter: + argNestingLevel++; + /* FALLTHROUGH */ + case jpiIsUnknown: + case jpiNot: + case jpiPlus: + case jpiMinus: + case jpiExists: + case jpiDatetime: + { + int32 arg = reserveSpaceForItemPointer(buf); + + chld = !item->value.arg ? pos : + flattenJsonPathParseItem(buf, item->value.arg, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + arg) = chld - pos; + } + break; + case jpiNull: + break; + case jpiRoot: + break; + case jpiAnyArray: + case jpiAnyKey: + break; + case jpiCurrent: + if (nestingLevel <= 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("@ is not allowed in root expressions"))); + break; + case jpiLast: + if (!insideArraySubscript) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("LAST is allowed only in array subscripts"))); + break; + case jpiIndexArray: + { + int32 nelems = item->value.array.nelems; + int offset; + int i; + + appendBinaryStringInfo(buf, (char *) &nelems, sizeof(nelems)); + + offset = buf->len; + + appendStringInfoSpaces(buf, sizeof(int32) * 2 * nelems); + + for (i = 0; i < nelems; i++) + { + int32 *ppos; + int32 topos; + int32 frompos = + flattenJsonPathParseItem(buf, + item->value.array.elems[i].from, + nestingLevel, true) - pos; + + if (item->value.array.elems[i].to) + topos = flattenJsonPathParseItem(buf, + item->value.array.elems[i].to, + nestingLevel, true) - pos; + else + topos = 0; + + ppos = (int32 *) &buf->data[offset + i * 2 * sizeof(int32)]; + + ppos[0] = frompos; + ppos[1] = topos; + } + } + break; + case jpiAny: + appendBinaryStringInfo(buf, + (char *) &item->value.anybounds.first, + sizeof(item->value.anybounds.first)); + appendBinaryStringInfo(buf, + (char *) &item->value.anybounds.last, + sizeof(item->value.anybounds.last)); + break; + case jpiType: + case jpiSize: + case jpiAbs: + case jpiFloor: + case jpiCeiling: + case jpiDouble: + case jpiKeyValue: + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", item->type); + } + + if (item->next) + { + chld = flattenJsonPathParseItem(buf, item->next, nestingLevel, + insideArraySubscript) - pos; + *(int32 *) (buf->data + next) = chld; + } + + return pos; +} + +/* + * Align StringInfo to int by adding zero padding bytes + */ +static void +alignStringInfoInt(StringInfo buf) +{ + switch (INTALIGN(buf->len) - buf->len) + { + case 3: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + case 2: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + case 1: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + default: + break; + } +} + +/* + * Reserve space for int32 JsonPathItem pointer. Now zero pointer is written, + * actual value will be recorded at '(int32 *) &buf->data[pos]' later. + */ +static int32 +reserveSpaceForItemPointer(StringInfo buf) +{ + int32 pos = buf->len; + int32 ptr = 0; + + appendBinaryStringInfo(buf, (char *) &ptr, sizeof(ptr)); + + return pos; +} + +/* + * Prints text representation of given jsonpath item and all its children. + */ +static void +printJsonPathItem(StringInfo buf, JsonPathItem *v, bool inKey, + bool printBracketes) +{ + JsonPathItem elem; + int i; + + check_stack_depth(); + CHECK_FOR_INTERRUPTS(); + + switch (v->type) + { + case jpiNull: + appendStringInfoString(buf, "null"); + break; + case jpiKey: + if (inKey) + appendStringInfoChar(buf, '.'); + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiString: + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiVariable: + appendStringInfoChar(buf, '$'); + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiNumeric: + appendStringInfoString(buf, + DatumGetCString(DirectFunctionCall1(numeric_out, + NumericGetDatum(jspGetNumeric(v))))); + break; + case jpiBool: + if (jspGetBool(v)) + appendBinaryStringInfo(buf, "true", 4); + else + appendBinaryStringInfo(buf, "false", 5); + break; + case jpiAnd: + case jpiOr: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiStartsWith: + if (printBracketes) + appendStringInfoChar(buf, '('); + jspGetLeftArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, jspOperationName(v->type)); + appendStringInfoChar(buf, ' '); + jspGetRightArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiLikeRegex: + if (printBracketes) + appendStringInfoChar(buf, '('); + + jspInitByBuffer(&elem, v->base, v->content.like_regex.expr); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + + appendBinaryStringInfo(buf, " like_regex ", 12); + + escape_json(buf, v->content.like_regex.pattern); + + if (v->content.like_regex.flags) + { + appendBinaryStringInfo(buf, " flag \"", 7); + + if (v->content.like_regex.flags & JSP_REGEX_ICASE) + appendStringInfoChar(buf, 'i'); + if (v->content.like_regex.flags & JSP_REGEX_DOTALL) + appendStringInfoChar(buf, 's'); + if (v->content.like_regex.flags & JSP_REGEX_MLINE) + appendStringInfoChar(buf, 'm'); + if (v->content.like_regex.flags & JSP_REGEX_WSPACE) + appendStringInfoChar(buf, 'x'); + if (v->content.like_regex.flags & JSP_REGEX_QUOTE) + appendStringInfoChar(buf, 'q'); + + appendStringInfoChar(buf, '"'); + } + + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiPlus: + case jpiMinus: + if (printBracketes) + appendStringInfoChar(buf, '('); + appendStringInfoChar(buf, v->type == jpiPlus ? '+' : '-'); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiFilter: + appendBinaryStringInfo(buf, "?(", 2); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiNot: + appendBinaryStringInfo(buf, "!(", 2); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiIsUnknown: + appendStringInfoChar(buf, '('); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendBinaryStringInfo(buf, ") is unknown", 12); + break; + case jpiExists: + appendBinaryStringInfo(buf, "exists (", 8); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiCurrent: + Assert(!inKey); + appendStringInfoChar(buf, '@'); + break; + case jpiRoot: + Assert(!inKey); + appendStringInfoChar(buf, '$'); + break; + case jpiLast: + appendBinaryStringInfo(buf, "last", 4); + break; + case jpiAnyArray: + appendBinaryStringInfo(buf, "[*]", 3); + break; + case jpiAnyKey: + if (inKey) + appendStringInfoChar(buf, '.'); + appendStringInfoChar(buf, '*'); + break; + case jpiIndexArray: + appendStringInfoChar(buf, '['); + for (i = 0; i < v->content.array.nelems; i++) + { + JsonPathItem from; + JsonPathItem to; + bool range = jspGetArraySubscript(v, &from, &to, i); + + if (i) + appendStringInfoChar(buf, ','); + + printJsonPathItem(buf, &from, false, false); + + if (range) + { + appendBinaryStringInfo(buf, " to ", 4); + printJsonPathItem(buf, &to, false, false); + } + } + appendStringInfoChar(buf, ']'); + break; + case jpiAny: + if (inKey) + appendStringInfoChar(buf, '.'); + + if (v->content.anybounds.first == 0 && + v->content.anybounds.last == PG_UINT32_MAX) + appendBinaryStringInfo(buf, "**", 2); + else if (v->content.anybounds.first == v->content.anybounds.last) + { + if (v->content.anybounds.first == PG_UINT32_MAX) + appendStringInfo(buf, "**{last}"); + else + appendStringInfo(buf, "**{%u}", + v->content.anybounds.first); + } + else if (v->content.anybounds.first == PG_UINT32_MAX) + appendStringInfo(buf, "**{last to %u}", + v->content.anybounds.last); + else if (v->content.anybounds.last == PG_UINT32_MAX) + appendStringInfo(buf, "**{%u to last}", + v->content.anybounds.first); + else + appendStringInfo(buf, "**{%u to %u}", + v->content.anybounds.first, + v->content.anybounds.last); + break; + case jpiType: + appendBinaryStringInfo(buf, ".type()", 7); + break; + case jpiSize: + appendBinaryStringInfo(buf, ".size()", 7); + break; + case jpiAbs: + appendBinaryStringInfo(buf, ".abs()", 6); + break; + case jpiFloor: + appendBinaryStringInfo(buf, ".floor()", 8); + break; + case jpiCeiling: + appendBinaryStringInfo(buf, ".ceiling()", 10); + break; + case jpiDouble: + appendBinaryStringInfo(buf, ".double()", 9); + break; + case jpiDatetime: + appendBinaryStringInfo(buf, ".datetime(", 10); + if (v->content.arg) + { + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + } + appendStringInfoChar(buf, ')'); + break; + case jpiKeyValue: + appendBinaryStringInfo(buf, ".keyvalue()", 11); + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", v->type); + } + + if (jspGetNext(v, &elem)) + printJsonPathItem(buf, &elem, true, true); +} + +const char * +jspOperationName(JsonPathItemType type) +{ + switch (type) + { + case jpiAnd: + return "&&"; + case jpiOr: + return "||"; + case jpiEqual: + return "=="; + case jpiNotEqual: + return "!="; + case jpiLess: + return "<"; + case jpiGreater: + return ">"; + case jpiLessOrEqual: + return "<="; + case jpiGreaterOrEqual: + return ">="; + case jpiPlus: + case jpiAdd: + return "+"; + case jpiMinus: + case jpiSub: + return "-"; + case jpiMul: + return "*"; + case jpiDiv: + return "/"; + case jpiMod: + return "%"; + case jpiStartsWith: + return "starts with"; + case jpiLikeRegex: + return "like_regex"; + case jpiType: + return "type"; + case jpiSize: + return "size"; + case jpiKeyValue: + return "keyvalue"; + case jpiDouble: + return "double"; + case jpiAbs: + return "abs"; + case jpiFloor: + return "floor"; + case jpiCeiling: + return "ceiling"; + case jpiDatetime: + return "datetime"; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", type); + return NULL; + } +} + +static int +operationPriority(JsonPathItemType op) +{ + switch (op) + { + case jpiOr: + return 0; + case jpiAnd: + return 1; + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiStartsWith: + return 2; + case jpiAdd: + case jpiSub: + return 3; + case jpiMul: + case jpiDiv: + case jpiMod: + return 4; + case jpiPlus: + case jpiMinus: + return 5; + default: + return 6; + } +} + +/******************* Support functions for JsonPath *************************/ + +/* + * Support macros to read stored values + */ + +#define read_byte(v, b, p) do { \ + (v) = *(uint8*)((b) + (p)); \ + (p) += 1; \ +} while(0) \ + +#define read_int32(v, b, p) do { \ + (v) = *(uint32*)((b) + (p)); \ + (p) += sizeof(int32); \ +} while(0) \ + +#define read_int32_n(v, b, p, n) do { \ + (v) = (void *)((b) + (p)); \ + (p) += sizeof(int32) * (n); \ +} while(0) \ + +/* + * Read root node and fill root node representation + */ +void +jspInit(JsonPathItem *v, JsonPath *js) +{ + Assert((js->header & ~JSONPATH_LAX) == JSONPATH_VERSION); + jspInitByBuffer(v, js->data, 0); +} + +/* + * Read node from buffer and fill its representation + */ +void +jspInitByBuffer(JsonPathItem *v, char *base, int32 pos) +{ + v->base = base + pos; + + read_byte(v->type, base, pos); + pos = INTALIGN((uintptr_t) (base + pos)) - (uintptr_t) base; + read_int32(v->nextPos, base, pos); + + switch (v->type) + { + case jpiNull: + case jpiRoot: + case jpiCurrent: + case jpiAnyArray: + case jpiAnyKey: + case jpiType: + case jpiSize: + case jpiAbs: + case jpiFloor: + case jpiCeiling: + case jpiDouble: + case jpiKeyValue: + case jpiLast: + break; + case jpiKey: + case jpiString: + case jpiVariable: + read_int32(v->content.value.datalen, base, pos); + /* FALLTHROUGH */ + case jpiNumeric: + case jpiBool: + v->content.value.data = base + pos; + break; + case jpiAnd: + case jpiOr: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiStartsWith: + read_int32(v->content.args.left, base, pos); + read_int32(v->content.args.right, base, pos); + break; + case jpiLikeRegex: + read_int32(v->content.like_regex.flags, base, pos); + read_int32(v->content.like_regex.expr, base, pos); + read_int32(v->content.like_regex.patternlen, base, pos); + v->content.like_regex.pattern = base + pos; + break; + case jpiNot: + case jpiExists: + case jpiIsUnknown: + case jpiPlus: + case jpiMinus: + case jpiFilter: + case jpiDatetime: + read_int32(v->content.arg, base, pos); + break; + case jpiIndexArray: + read_int32(v->content.array.nelems, base, pos); + read_int32_n(v->content.array.elems, base, pos, + v->content.array.nelems * 2); + break; + case jpiAny: + read_int32(v->content.anybounds.first, base, pos); + read_int32(v->content.anybounds.last, base, pos); + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", v->type); + } +} + +void +jspGetArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiFilter || + v->type == jpiNot || + v->type == jpiIsUnknown || + v->type == jpiExists || + v->type == jpiPlus || + v->type == jpiMinus || + v->type == jpiDatetime); + + jspInitByBuffer(a, v->base, v->content.arg); +} + +bool +jspGetNext(JsonPathItem *v, JsonPathItem *a) +{ + if (jspHasNext(v)) + { + Assert(v->type == jpiString || + v->type == jpiNumeric || + v->type == jpiBool || + v->type == jpiNull || + v->type == jpiKey || + v->type == jpiAny || + v->type == jpiAnyArray || + v->type == jpiAnyKey || + v->type == jpiIndexArray || + v->type == jpiFilter || + v->type == jpiCurrent || + v->type == jpiExists || + v->type == jpiRoot || + v->type == jpiVariable || + v->type == jpiLast || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiPlus || + v->type == jpiMinus || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiGreater || + v->type == jpiGreaterOrEqual || + v->type == jpiLess || + v->type == jpiLessOrEqual || + v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiNot || + v->type == jpiIsUnknown || + v->type == jpiType || + v->type == jpiSize || + v->type == jpiAbs || + v->type == jpiFloor || + v->type == jpiCeiling || + v->type == jpiDouble || + v->type == jpiDatetime || + v->type == jpiKeyValue || + v->type == jpiStartsWith || + v->type == jpiLikeRegex); + + if (a) + jspInitByBuffer(a, v->base, v->nextPos); + return true; + } + + return false; +} + +void +jspGetLeftArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiLess || + v->type == jpiGreater || + v->type == jpiLessOrEqual || + v->type == jpiGreaterOrEqual || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiStartsWith); + + jspInitByBuffer(a, v->base, v->content.args.left); +} + +void +jspGetRightArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiLess || + v->type == jpiGreater || + v->type == jpiLessOrEqual || + v->type == jpiGreaterOrEqual || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiStartsWith); + + jspInitByBuffer(a, v->base, v->content.args.right); +} + +bool +jspGetBool(JsonPathItem *v) +{ + Assert(v->type == jpiBool); + + return (bool) *v->content.value.data; +} + +Numeric +jspGetNumeric(JsonPathItem *v) +{ + Assert(v->type == jpiNumeric); + + return (Numeric) v->content.value.data; +} + +char * +jspGetString(JsonPathItem *v, int32 *len) +{ + Assert(v->type == jpiKey || + v->type == jpiString || + v->type == jpiVariable); + + if (len) + *len = v->content.value.datalen; + return v->content.value.data; +} + +bool +jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to, + int i) +{ + Assert(v->type == jpiIndexArray); + + jspInitByBuffer(from, v->base, v->content.array.elems[i].from); + + if (!v->content.array.elems[i].to) + return false; + + jspInitByBuffer(to, v->base, v->content.array.elems[i].to); + + return true; +} From c634d6f32f503edce3bf7b686d626086780fb9e0 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:31:08 +0200 Subject: [PATCH 211/286] add failure flow to graph (#211) * add failure flow to graph --- src/exploit_iq_commons/data_models/input.py | 1 + src/vuln_analysis/data_models/output.py | 11 +++++++++ .../functions/cve_generate_vdbs.py | 3 +++ .../functions/cve_http_output.py | 21 +++++++++++++++-- src/vuln_analysis/register.py | 23 ++++++++++++++++--- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index a7d4166f3..9a1a69195 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -190,6 +190,7 @@ class AgentMorpheusInput(HashableModel): default=None, validation_alias=AliasChoices("credential_id", "credentialId"), ) + code_index_success: bool | None = None class AgentMorpheusEngineInput(BaseModel): diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index e77469a3f..2af85daa9 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -16,11 +16,22 @@ import typing from pydantic import BaseModel +from pydantic import Field from pydantic import model_validator from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +class FailureReport(BaseModel): + """ + Lightweight error payload sent to the failure endpoint when processing fails + (e.g. code indexing / repository cloning). + """ + scan_id: str = Field(serialization_alias="scanId") + error_type: str = Field(serialization_alias="errorType") + error_message: str = Field(serialization_alias="errorMessage") + + class AgentIntermediateStep(BaseModel): """ Represents info for an intermediate step taken by an agent. diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index aebba7c20..3eb725134 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -253,9 +253,12 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: vdb_code_path = str(vdb_code_path) if vdb_code_path is not None else None vdb_doc_path = str(vdb_doc_path) if vdb_doc_path is not None else None code_index_path = str(code_index_path) if code_index_path is not None else None + code_index_success = True if code_index_path is not None else False + info = AgentMorpheusInfo(vdb=AgentMorpheusInfo.VdbPaths( code_vdb_path=vdb_code_path, doc_vdb_path=vdb_doc_path, code_index_path=code_index_path)) + message.code_index_success = code_index_success return AgentMorpheusEngineInput(input=message, info=info) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index e8b85e720..c9abf2297 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -85,6 +85,8 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") username: str | None = Field(default=None, description="Username to authenticate with for basic auth") password: str | None = Field(default=None, description="Password to authenticate with for basic auth") + failure_endpoint: str = Field(default="/api/v1/reports/failed", + description="Endpoint to send failure reports when code indexing fails") enable_mlops: bool = Field(default=False, description="Enable MLOps") mlops_config: MLOpsConfig = Field(..., description="MLOps configuration") @@ -92,11 +94,12 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): @register_function(config_type=CVEHttpOutputConfig) async def output_to_http(config: CVEHttpOutputConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.output import AgentMorpheusOutput + from vuln_analysis.data_models.output import AgentMorpheusOutput, FailureReport from vuln_analysis.utils import http_utils async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: trace_id.set(message.input.scan.id) + model_json = message.model_dump_json(by_alias=True) url = config.url + config.endpoint headers = {'Content-type': 'application/json', 'traceId': trace_id.get()} @@ -107,10 +110,24 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: if config.verify_path: verify = config.verify_path try: + skipped_mlops = False + if message.input.code_index_success is False: + repo_url = message.input.image.source_info[0].git_repo if message.input.image.source_info else "unknown" + failure_report = FailureReport( + scan_id=message.input.scan.id, + error_type="processing-error", + error_message=f"Failed to clone repository {repo_url}", + ) + failure_url = config.url + config.failure_endpoint + logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") + model_json = failure_report.model_dump_json(by_alias=True) + url = failure_url + skipped_mlops = True + logger.info(f"Sending output to {url}") http_utils.request_with_retry(request_kwargs={ "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers, "verify": verify }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) - if config.enable_mlops: + if config.enable_mlops and not skipped_mlops: mlops_url = None try: job = _extract_job_data(message) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 19541df1c..554833927 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -111,7 +111,10 @@ async def add_start_time_node(state: AgentMorpheusInput) -> AgentMorpheusInput: async def generate_vdbs_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: """Generates VDBs based on CVE input""" - return await cve_generate_vdbs_fn.ainvoke(state.model_dump()) + engine_input = await cve_generate_vdbs_fn.ainvoke(state.model_dump()) + result = engine_input.model_dump() + result["code_index_success"] = engine_input.input.code_index_success + return result @catch_pipeline_errors_async async def fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: @@ -179,7 +182,19 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput """Outputs results using configured output function""" return await cve_output_fn.ainvoke(state.model_dump()) if cve_output_fn else state + + async def check_vdbs_success(state: AgentMorpheusInput) -> str: + """Checks if the VDBs were successfully generated""" + if state.code_index_success: + return "fetch_intel" + else: + return "failure" + async def failure_node(state: AgentMorpheusInput) -> AgentMorpheusOutput: + """Handles the failure of the pipeline""" + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from vuln_analysis.data_models.output import OutputPayload + return AgentMorpheusOutput(input=state, info=AgentMorpheusInfo(), output=OutputPayload(analysis=[], vex=None)) # define langgraph # build llm engine subgraph @@ -223,10 +238,12 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph_builder.add_node("llm_engine", call_llm_engine_subgraph_node) graph_builder.add_node("add_completed_time", add_completed_time_node) graph_builder.add_node("output_results", output_results_node) - + graph_builder.add_node("failure", failure_node) graph_builder.add_edge(START, "add_start_time") graph_builder.add_edge("add_start_time", "generate_vdbs") - graph_builder.add_edge("generate_vdbs", "fetch_intel") + graph_builder.add_conditional_edges("generate_vdbs", check_vdbs_success,{"fetch_intel": "fetch_intel", "failure": "failure"}) + graph_builder.add_edge("failure", "add_completed_time") + #graph_builder.add_edge("generate_vdbs", "fetch_intel") graph_builder.add_edge("fetch_intel", "calculate_intel_score_node") graph_builder.add_edge("calculate_intel_score_node", "process_sbom") graph_builder.add_edge("process_sbom", "check_vuln_deps") From efab12cb4f0e5e42aaf51328cddcebe518bf29aa Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 23 Mar 2026 13:00:33 +0200 Subject: [PATCH 212/286] fix(dep-tree): rewrite Python version detection and ecosystem discovery Signed-off-by: Vladimir Belousov --- src/exploit_iq_commons/utils/dep_tree.py | 432 ++++++++++++++---- .../utils/transitive_code_searcher_tool.py | 10 +- tests/test_python_version_detection.py | 299 ++++++++++++ 3 files changed, 640 insertions(+), 101 deletions(-) create mode 100644 tests/test_python_version_detection.py diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 719ac6bdd..9afc4273e 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -13,8 +13,11 @@ from tqdm import tqdm import ast +import configparser import json +import tomllib import zipfile +from packaging.version import Version as _PkgVersion from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser from contextlib import contextmanager @@ -36,6 +39,12 @@ SETUP_PY = 'setup.py' README_MD = 'README.md' +_WALK_EXCLUDE_DIRS = frozenset({ + ".venv", "venv", "env", ".env", "node_modules", "vendor", + "dist", "build", "__pycache__", ".git", "tests", "test", + "fixtures", "docs", "site-packages", ".tox", ".nox", +}) + class Ecosystem(str, Enum): GO = "go" @@ -1035,137 +1044,362 @@ def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: tree[dependency] = list(parents) return tree - def extract_version_from_specifier(self, specifier_str: str) -> Optional[str]: - """ - Extracts a most likely specific Python version from a PEP 440 specifier string. - Prioritizes exact matches, then lower bounds. - Examples: - "==3.9" -> "3.9" - ">=3.8,<4.0" -> "3.8" - "~=3.7" -> "3.7" (if ~=3.7 is equivalent to >=3.7,<3.8) - ">3.8" -> "3.9" (returns the next major.minor version) + def extract_version_from_specifier(self, specifier_str: str) -> str | None: + """Extract the most likely runtime Python version from a PEP 440 specifier string. + + Priority order: exact pin (==) -> Python 2 upper-bound (<3) -> highest >= lower + bound -> compatible release (~=) -> exclusive lower bound (>). + + Args: + specifier_str: A PEP 440 version specifier, e.g. ``">=3.8,<4.0"``. + + Returns: + A ``"major.minor"`` string such as ``"3.9"`` or ``"2.7"``, or ``None`` + when no version can be determined. """ + if not specifier_str: + return None try: specifier_set = SpecifierSet(specifier_str) - for specifier in specifier_set: - if specifier.operator in ('==', '==='): - return specifier.version + for spec in specifier_set: + if spec.operator in ("==", "==="): + return spec.version + + for spec in specifier_set: + if spec.operator in ("<", "<=") and _PkgVersion(spec.version) <= _PkgVersion("3.0"): + return "2.7" - lower_bounds = [] - for specifier in specifier_set: - if specifier.operator in ('>=',): - lower_bounds.append(self.parse_version(specifier.version)) + lower_bounds = [ + _PkgVersion(spec.version) + for spec in specifier_set + if spec.operator == ">=" + ] if lower_bounds: - # Return the highest (most restrictive) lower bound - highest_lower_bound = max(lower_bounds) - return str(highest_lower_bound) - - # Look for a compatible release specifier (~=3.7 implies >=3.7,<3.8) - compatible_match = re.search(r'~=(\d+\.\d+)', specifier_str) - if compatible_match: - return compatible_match.group(1) - - # As a fallback for ">X.Y" which means X.Y+1.0, return the next minor version - greater_than_match = re.search(r'>(\d+\.\d+)', specifier_str) - if greater_than_match: - major, minor = map(int, greater_than_match.group(1).split('.')) + return str(max(lower_bounds)) + + compatible = re.search(r"~=(\d+\.\d+(?:\.\d+)?)", specifier_str) + if compatible: + return compatible.group(1) + + greater = re.search(r">(\d+\.\d+)", specifier_str) + if greater: + major, minor = map(int, greater.group(1).split(".")) return f"{major}.{minor + 1}" - except Exception as e: # Catch parsing errors from packaging.specifiers - logger.warning(f"Warning: Could not parse specifier '{specifier_str}': {e}") + except Exception as exc: + logger.warning("Could not parse version specifier %r: %s", specifier_str, exc) return None - def extract_version_from_readme_hint(self, line: str) -> Optional[str]: - """ - Extracts a Python version (e.g., 3.8, 3.12) from a free-form text line. - Looks for patterns like 'Python 3.9', 'python3.10', 'py3.11'. + def extract_version_from_readme_hint(self, line: str) -> str | None: + """Extract a Python version from a single free-form text line. + + Matches patterns such as ``Python 3.9``, ``python3.10``, or ``py3.11``. + + Args: + line: A single line of free-form text. + + Returns: + A ``"major.minor"`` string, or ``None`` if no match is found. """ match = re.search(r'(?:Python|python|py)\s*(\d+\.\d+)', line, re.IGNORECASE) return match.group(1) if match else None - def extract_version_from_pyproject_toml(self, content: str) -> Optional[str]: - #todo: implement more options here, see example: https://github.com/quay/quay/blob/master/pyproject.toml + def extract_version_from_pyproject_toml(self, content: str) -> str | None: + """Extract the required Python version from a ``pyproject.toml`` file. + + Reads ``[project].requires-python`` (PEP 621) first, then falls back to + ``[tool.poetry.dependencies].python`` (Poetry). Poetry's caret operator + (``^X.Y``) is normalised to ``>=X.Y`` before parsing. + + Args: + content: Raw text content of a ``pyproject.toml`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no Python constraint is + found or the file cannot be parsed. + """ try: - pyproject_data = json.loads(content) - specifier_from_metadata = pyproject_data.get('project', {}).get('requires-python', None) - if specifier_from_metadata: - logger.debug(f"Found requires-python in pyproject.toml: '{specifier_from_metadata}'") - # Try to extract a specific version from the specifier - specific_version = self.extract_version_from_specifier(specifier_from_metadata) - if specific_version: - return specific_version # Return the most authoritative version immediately - return '' - except json.JSONDecodeError: - logger.warning(f"Warning: Failed to parse pyproject.toml as JSON.") - - def extract_version_from_setup_py(self, content: str): + data = tomllib.loads(content) + except tomllib.TOMLDecodeError: + logger.warning("Failed to parse pyproject.toml as TOML") + return None + + pep621 = data.get("project", {}).get("requires-python") + if pep621: + logger.debug("Found requires-python in pyproject.toml: %r", pep621) + return self.extract_version_from_specifier(pep621) + + poetry = ( + data.get("tool", {}) + .get("poetry", {}) + .get("dependencies", {}) + .get("python") + ) + if poetry: + logger.debug("Found tool.poetry.dependencies.python: %r", poetry) + normalized = re.sub( + r"\^(\d+)\.(\d+)", + lambda m: f">={m.group(1)}.{m.group(2)}", + str(poetry), + ) + return self.extract_version_from_specifier(normalized) + + return None + + def extract_version_from_setup_py(self, content: str) -> str | None: + """Extract the required Python version from a ``setup.py`` file. + + Parses the AST to read the ``python_requires`` keyword argument of the + ``setup()`` call. Falls back to the highest ``Programming Language :: + Python :: X.Y`` classifier when ``python_requires`` is absent. + + Args: + content: Raw text content of a ``setup.py`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no version information + is found or the file cannot be parsed. + """ try: tree = ast.parse(content) - for node in ast.walk(tree): - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'setup': - for keyword in node.keywords: - if keyword.arg == 'python_requires': - if isinstance(keyword.value, ast.Constant): - specifier_from_metadata = keyword.value.value - logger.debug(f"Found python_requires in setup.py: '{specifier_from_metadata}'") - specific_version = self.extract_version_from_specifier(specifier_from_metadata) - if specific_version: - return specific_version - break except SyntaxError: - logger.warning(f"Warning: Failed to parse setup.py as Python code") - return '' + logger.warning("Failed to parse setup.py as Python code") + return None + + for node in ast.walk(tree): + if not (isinstance(node, ast.Call) and + isinstance(node.func, (ast.Name, ast.Attribute)) and + getattr(node.func, "id", None) == "setup"): + continue + + for kw in node.keywords: + if (kw.arg == "python_requires" and + isinstance(kw.value, ast.Constant) and + isinstance(kw.value.value, str)): + logger.debug("Found python_requires in setup.py: %r", kw.value.value) + return self.extract_version_from_specifier(kw.value.value) + + for kw in node.keywords: + if kw.arg == "classifiers" and isinstance(kw.value, (ast.List, ast.Tuple)): + versions = [] + for elt in kw.value.elts: + if not (isinstance(elt, ast.Constant) and isinstance(elt.value, str)): + continue + m = re.match( + r"Programming Language :: Python :: (\d+\.\d+)$", + elt.value, + ) + if m: + versions.append(m.group(1)) + if versions: + logger.debug("Found Python version classifiers in setup.py: %s", versions) + return str(max(versions, key=lambda v: _PkgVersion(v))) + + return None + + def extract_version_from_readme_md(self, content: str) -> str | None: + """Extract the highest Python version hint from README content. + + Scans every line for patterns matched by + :meth:`extract_version_from_readme_hint` and returns the highest + version found. + + Args: + content: Raw text content of a ``README.md`` or ``README.rst`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no hint is found. + """ + best: str | None = None + for line in content.splitlines(): + candidate = self.extract_version_from_readme_hint(line) + if candidate is None: + continue + if best is None or _PkgVersion(candidate) > _PkgVersion(best): + best = candidate + logger.debug("Updated best Python version hint from README: %r", best) + return best + + def extract_version_from_python_version_file(self, content: str) -> str | None: + """Extract the Python version from a ``.python-version`` file (pyenv). - def extract_version_from_readme_md(self, content: str): - version_from_readme_hint = '' + The pyenv format is a single non-empty line containing ``X.Y`` or + ``X.Y.Z``. The first non-empty line is treated as the authoritative + value; if it does not match the expected pattern the file is considered + invalid and ``None`` is returned immediately. + + Args: + content: Raw text content of a ``.python-version`` file. + + Returns: + A ``"major.minor"`` string such as ``"3.9"``, or ``None``. + """ for line in content.splitlines(): - extracted_version = self.extract_version_from_readme_hint(line) - if extracted_version: - if not version_from_readme_hint or ( - self.extract_version_from_specifier(extracted_version) > self.extract_version_from_specifier( - version_from_readme_hint) - ): - version_from_readme_hint = extracted_version - logger.debug(f"Found Python version hint in README.md: '{extracted_version}'") - return version_from_readme_hint - - def determine_python_version(self, git_repo_path: str) -> Optional[str]: + line = line.strip() + if not line: + continue + m = re.match(r"^(\d+)\.(\d+)(?:\.\d+)?$", line) + if m: + return f"{m.group(1)}.{m.group(2)}" + return None + return None + + def extract_version_from_setup_cfg(self, content: str) -> str | None: + """Extract the required Python version from a ``setup.cfg`` file. + + Reads ``[options] python_requires`` and delegates version resolution to + :meth:`extract_version_from_specifier`. + + Args: + content: Raw text content of a ``setup.cfg`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no ``python_requires`` + key is present or the file cannot be parsed. + """ + parser = configparser.ConfigParser() + try: + parser.read_string(content) + except configparser.Error: + logger.warning("Failed to parse setup.cfg") + return None + specifier = parser.get("options", "python_requires", fallback=None) + if specifier: + logger.debug("Found python_requires in setup.cfg: %r", specifier) + return self.extract_version_from_specifier(specifier.strip()) + return None + + def extract_version_from_pipfile(self, content: str) -> str | None: + """Extract the required Python version from a ``Pipfile``. + + Reads ``[requires] python_full_version`` (preferred) or + ``python_version`` and truncates to ``major.minor``. + + Args: + content: Raw text content of a ``Pipfile``. + + Returns: + A ``"major.minor"`` string, or ``None`` when no version is declared + or the file cannot be parsed. """ - Determines the most specific Python version from project information documents. - Prioritizes pyproject.toml, then setup.py, then README.md. + try: + data = tomllib.loads(content) + except tomllib.TOMLDecodeError: + logger.warning("Failed to parse Pipfile as TOML") + return None + requires = data.get("requires", {}) + full = requires.get("python_full_version") or requires.get("python_version") + if not full: + return None + m = re.match(r"^(\d+\.\d+)", str(full)) + logger.debug("Found python version in Pipfile: %r", full) + return m.group(1) if m else None + + def determine_python_version(self, git_repo_path: str) -> str | None: + """Determine the required Python version for a cloned Git repository. + + Searches for version information using a priority chain of manifest + files. The repository root is checked first; the rest of the tree is + then walked with common non-project directories excluded to avoid false + matches from fixtures or vendored code. + + Priority order (highest to lowest): + + 1. ``.python-version`` (pyenv) + 2. ``pyproject.toml`` — ``[project].requires-python`` then Poetry + 3. ``setup.cfg`` — ``[options] python_requires`` + 4. ``setup.py`` — ``python_requires`` kwarg then classifiers + 5. ``Pipfile`` — ``[requires] python_version`` + 6. ``README.md`` / ``README.rst`` — free-text hints (weakest) + + Args: + git_repo_path: Absolute path to the root of the cloned repository. + + Returns: + A ``"major.minor"`` string such as ``"3.9"`` or ``"2.7"``, or + ``None`` when no authoritative version information is found. """ + root = Path(git_repo_path) + + priority_chain = [ + (".python-version", self.extract_version_from_python_version_file), + ("pyproject.toml", self.extract_version_from_pyproject_toml), + ("setup.cfg", self.extract_version_from_setup_cfg), + ("setup.py", self.extract_version_from_setup_py), + ("Pipfile", self.extract_version_from_pipfile), + ("README.md", self.extract_version_from_readme_md), + ("README.rst", self.extract_version_from_readme_md), + ] - info_docs_mapping = {PYPROJECT_TOML: self.extract_version_from_pyproject_toml, - SETUP_PY: self.extract_version_from_setup_py, - README_MD: self.extract_version_from_readme_md} - - for doc, logic in info_docs_mapping.items(): - for root, _, files in os.walk(git_repo_path): - if doc in files: - doc_full_path = os.path.join(root, doc) - with open(doc_full_path, 'r') as file: - python_version = logic(file.read()) - if python_version: - return python_version - return '3.11' - - def install_dependencies(self, git_repo_path): - cmd = f'cd {git_repo_path} && python -m venv {TRANSITIVE_ENV_NAME}' + def _try_file(path: Path, extractor) -> str | None: + if not path.is_file(): + return None + try: + version = extractor(path.read_text(encoding="utf-8", errors="replace")) + if version: + logger.info("Detected Python %s from %s", version, path) + return version + except OSError as exc: + logger.warning("Could not read %s: %s", path, exc) + return None + + for filename, extractor in priority_chain: + version = _try_file(root / filename, extractor) + if version: + return version + + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [ + d for d in dirnames + if d not in _WALK_EXCLUDE_DIRS and not d.startswith(".") + ] + if Path(dirpath) == root: + continue + for filename, extractor in priority_chain: + if filename in filenames: + version = _try_file(Path(dirpath) / filename, extractor) + if version: + return version + + return None + + def install_dependencies(self, manifest_path: Path): + """Install Python dependencies for the given repository into a virtual environment. + + Calls :meth:`determine_python_version` to select the interpreter; when a + version is found ``uv venv`` is invoked with ``--python ``, + otherwise ``uv`` selects the default interpreter. Each line of + ``requirements.txt`` is then installed via :meth:`install_dependency`. + + Args: + manifest_path: Absolute path to the root of the cloned repository, + which is expected to contain a ``requirements.txt`` manifest. + """ + python_version = self.determine_python_version(str(manifest_path)) + if python_version: + logger.info("Creating transitive_env with Python %s using uv", python_version) + cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" + else: + logger.warning( + "Python version undetermined for %s; using uv default interpreter", manifest_path + ) + cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME}" run_command(cmd) - with open(git_repo_path / PYTHON_MANIFEST, 'r') as manifest: + with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: for line in tqdm(manifest): if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): - self.install_dependency(line, git_repo_path) + self.install_dependency(line, manifest_path) def install_dependency(self, dependency, repo_path): + dependency = dependency.strip() valid_signs = ['==', '>=', '<=', '!='] - if not any([sign in dependency for sign in valid_signs]): + if not any(sign in dependency for sign in valid_signs): dependency = dependency.replace('=', '==') - cmd = f'{repo_path}/{TRANSITIVE_ENV_NAME}/bin/python -m pip install {dependency}' + cmd = f'uv pip install {dependency} --python {repo_path}/{TRANSITIVE_ENV_NAME}/bin/python' res = run_command(cmd) if not res: - logger.warning(f'Failed to install dependency {dependency}') + logger.warning('Failed to install dependency %s', dependency) class JavaScriptDependencyTreeBuilder(DependencyTreeBuilder): diff --git a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py index 1f47de94c..1b909141f 100644 --- a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py +++ b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py @@ -23,7 +23,9 @@ JAVA_MANIFEST, JS_MANIFEST, MANIFESTS_TO_ECOSYSTEMS, + PYPROJECT_TOML, PYTHON_MANIFEST, + SETUP_PY, Ecosystem, get_dependency_tree_builder, C_CPLUSPLUS_MANIFEST_1, @@ -96,7 +98,11 @@ def download_dependencies(git_repo_path: Path) -> bool: # according to the order from top to bottom if os.path.isfile(git_repo_path / GOLANG_MANIFEST): ecosystem = MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] - elif os.path.isfile(git_repo_path / PYTHON_MANIFEST): + elif ( + os.path.isfile(git_repo_path / PYTHON_MANIFEST) + or os.path.isfile(git_repo_path / PYPROJECT_TOML) + or os.path.isfile(git_repo_path / SETUP_PY) + ): ecosystem = MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] elif os.path.isfile(git_repo_path / JS_MANIFEST): ecosystem = MANIFESTS_TO_ECOSYSTEMS[JS_MANIFEST] @@ -124,7 +130,7 @@ def download_dependencies(git_repo_path: Path) -> bool: return False try: - logger.info(f"Started installing packages for {ecosystem}") + logger.info(f"Started installing packages for {ecosystem.value}") tree_builder = get_dependency_tree_builder(ecosystem) tree_builder.install_dependencies(git_repo_path) with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: diff --git a/tests/test_python_version_detection.py b/tests/test_python_version_detection.py new file mode 100644 index 000000000..6f4a69420 --- /dev/null +++ b/tests/test_python_version_detection.py @@ -0,0 +1,299 @@ +import textwrap +from pathlib import Path +from unittest.mock import patch + +import pytest + +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder + + +@pytest.fixture +def builder(): + return object.__new__(PythonDependencyTreeBuilder) + + +class TestExtractVersionFromSpecifier: + + @pytest.mark.parametrize("specifier, expected", [ + ("==3.9", "3.9"), + ("==3.9.0", "3.9.0"), + (">=3.8", "3.8"), + (">=3.8,<4.0", "3.8"), + (">=3.8,<3.12", "3.8"), + (">=3.8,!=3.9", "3.8"), + ("~=3.7", "3.7"), + ("~=3.7.2", "3.7.2"), + (">3.8", "3.9"), + ("<3", "2.7"), + ("<3.0", "2.7"), + (">=2.7,<3", "2.7"), + (">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,<3", "2.7"), + ("", None), + ("!=3.6", None), + ]) + def test_specifier_extraction(self, builder, specifier, expected): + assert builder.extract_version_from_specifier(specifier) == expected + + +class TestExtractVersionFromPyprojectToml: + + def test_pep621_requires_python_gte(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = ">=3.9" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.9" + + def test_pep621_requires_python_exact(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = "==3.11" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.11" + + def test_poetry_python_constraint(self, builder): + content = textwrap.dedent("""\ + [tool.poetry.dependencies] + python = "^3.8" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.8" + + def test_python2_upper_bound(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = ">=2.7,<3" + """) + assert builder.extract_version_from_pyproject_toml(content) == "2.7" + + def test_no_python_constraint_returns_none(self, builder): + content = textwrap.dedent("""\ + [project] + name = "myapp" + """) + assert builder.extract_version_from_pyproject_toml(content) is None + + def test_malformed_toml_returns_none(self, builder): + assert builder.extract_version_from_pyproject_toml("{ not valid toml") is None + + +class TestExtractVersionFromSetupPy: + + def test_python_requires_gte(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + name="myapp", + python_requires=">=3.8", + ) + """) + assert builder.extract_version_from_setup_py(content) == "3.8" + + def test_python_requires_exact(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(python_requires="==3.9") + """) + assert builder.extract_version_from_setup_py(content) == "3.9" + + def test_python_requires_py2(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(python_requires=">=2.7,<3") + """) + assert builder.extract_version_from_setup_py(content) == "2.7" + + def test_classifiers_fallback_highest_version(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], + ) + """) + assert builder.extract_version_from_setup_py(content) == "3.9" + + def test_classifiers_python2(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + classifiers=[ + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + ], + ) + """) + assert builder.extract_version_from_setup_py(content) == "2.7" + + def test_no_version_info_returns_none(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(name="myapp") + """) + assert builder.extract_version_from_setup_py(content) is None + + def test_syntax_error_returns_none(self, builder): + assert builder.extract_version_from_setup_py("def broken(") is None + + +class TestExtractVersionFromReadmeMd: + + def test_single_version_hint(self, builder): + content = "Requires Python 3.9 or above." + assert builder.extract_version_from_readme_md(content) == "3.9" + + def test_multiple_hints_returns_highest(self, builder): + content = textwrap.dedent("""\ + Supports Python 3.8 and above. + Tested on Python 3.11. + """) + assert builder.extract_version_from_readme_md(content) == "3.11" + + def test_python2_hint(self, builder): + content = "Works with Python 2.7." + assert builder.extract_version_from_readme_md(content) == "2.7" + + def test_no_hint_returns_none(self, builder): + assert builder.extract_version_from_readme_md("No version info here.") is None + + +class TestExtractVersionFromPythonVersionFile: + + @pytest.mark.parametrize("content, expected", [ + ("3.9.7\n", "3.9"), + ("3.9\n", "3.9"), + ("2.7.18\n", "2.7"), + ("3.11.0\n", "3.11"), + ("\n3.9\n", "3.9"), + ("pypy3.9\n", None), + ("", None), + ]) + def test_python_version_file(self, builder, content, expected): + assert builder.extract_version_from_python_version_file(content) == expected + + +class TestExtractVersionFromSetupCfg: + + def test_python_requires_gte(self, builder): + content = textwrap.dedent("""\ + [options] + python_requires = >=3.8 + """) + assert builder.extract_version_from_setup_cfg(content) == "3.8" + + def test_python_requires_py2(self, builder): + content = textwrap.dedent("""\ + [options] + python_requires = >=2.7,<3 + """) + assert builder.extract_version_from_setup_cfg(content) == "2.7" + + def test_no_python_requires_returns_none(self, builder): + content = "[metadata]\nname = myapp\n" + assert builder.extract_version_from_setup_cfg(content) is None + + +class TestExtractVersionFromPipfile: + + def test_python_version(self, builder): + content = textwrap.dedent("""\ + [requires] + python_version = "3.9" + """) + assert builder.extract_version_from_pipfile(content) == "3.9" + + def test_python_full_version(self, builder): + content = textwrap.dedent("""\ + [requires] + python_full_version = "3.9.7" + """) + assert builder.extract_version_from_pipfile(content) == "3.9" + + def test_no_requires_section_returns_none(self, builder): + content = "[packages]\nrequests = \"*\"\n" + assert builder.extract_version_from_pipfile(content) is None + + +class TestDeterminePythonVersion: + + def _make_repo(self, tmp_path, files: dict) -> Path: + for name, content in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return tmp_path + + def test_python_version_file_wins_over_pyproject(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + ".python-version": "3.10\n", + "pyproject.toml": "[project]\nrequires-python = \">=3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.10" + + def test_pyproject_used_when_no_python_version_file(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "pyproject.toml": "[project]\nrequires-python = \">=3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.9" + + def test_setup_cfg_used_when_no_pyproject(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.cfg": "[options]\npython_requires = >=3.8\n", + }) + assert builder.determine_python_version(str(repo)) == "3.8" + + def test_setup_py_used_as_fallback(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.py": "from setuptools import setup\nsetup(python_requires='>=3.7')\n", + }) + assert builder.determine_python_version(str(repo)) == "3.7" + + def test_pipfile_used_as_fallback(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "Pipfile": "[requires]\npython_version = \"3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.9" + + def test_returns_none_when_nothing_found(self, builder, tmp_path): + repo = self._make_repo(tmp_path, {"README.md": "No version info.\n"}) + assert builder.determine_python_version(str(repo)) is None + + def test_ignores_nested_pyproject_in_tests_dir(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "tests/fixtures/pyproject.toml": "[project]\nrequires-python = \">=3.6\"\n", + }) + assert builder.determine_python_version(str(repo)) is None + + def test_python2_project(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.py": "from setuptools import setup\nsetup(python_requires='>=2.7,<3')\n", + }) + assert builder.determine_python_version(str(repo)) == "2.7" + + +class TestInstallDependenciesVersionGuard: + + def test_uses_python_flag_when_version_known(self, builder, tmp_path, monkeypatch): + (tmp_path / "requirements.txt").write_text("requests\n") + (tmp_path / "pyproject.toml").write_text("[project]\nrequires-python = \">=3.9\"\n") + calls = [] + monkeypatch.setattr( + "exploit_iq_commons.utils.dep_tree.run_command", + lambda cmd: calls.append(cmd) or "", + ) + builder.install_dependencies(tmp_path) + assert any("--python 3.9" in c for c in calls), f"calls: {calls}" + + def test_omits_python_flag_when_version_unknown(self, builder, tmp_path, monkeypatch): + (tmp_path / "requirements.txt").write_text("requests\n") + calls = [] + monkeypatch.setattr( + "exploit_iq_commons.utils.dep_tree.run_command", + lambda cmd: calls.append(cmd) or "", + ) + builder.install_dependencies(tmp_path) + venv_calls = [c for c in calls if "venv" in c] + assert venv_calls, "venv command not called" + assert "--python" not in venv_calls[0], f"Should not pass --python: {venv_calls[0]}" From 8a8c8b3d7059ba1ee670eeaf9c1398686d683a28 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 22 Mar 2026 16:19:20 +0200 Subject: [PATCH 213/286] chore: run agent container as non-root user Signed-off-by: Zvi Grinberg --- Dockerfile | 26 ++++++++++++++++++++------ kustomize/README.md | 1 + kustomize/base/exploit_iq_service.yaml | 5 ++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96133ca02..7044c22c9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -103,15 +103,26 @@ WORKDIR /workspace # Copy the project into the container COPY ./ /workspace +RUN groupadd workspace-group && \ + useradd -u 1001 -m -g workspace-group user1001 && \ + chown -R :workspace-group /home/user1001 && \ + chmod -R g+wrx /home/user1001 + + + # Install the NeMo Agent toolkit package and vuln analysis package -RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ +RUN --mount=type=cache,id=uv_cache,target=/home/user1001/.cache/uv,mode=0775,sharing=locked \ export SETUPTOOLS_SCM_PRETEND_VERSION=${VULN_ANALYSIS_VERSION} && \ uv venv --python ${PYTHON_VERSION} /workspace/.venv && \ - uv sync + uv sync && \ + chown -R :workspace-group /workspace && \ + chmod -R g+wrx /workspace && \ + chown -R :workspace-group /root && \ + chmod -R g+rx /root -# Activate the environment (make it default for subsequent commands) -RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc +USER 1001 +# Activate the environment (make it default for subsequent commands) # Enivronment variables for the venv ENV PATH="/workspace/.venv/bin:/usr/local/go/bin:$PATH" @@ -119,11 +130,14 @@ ENV PATH="/workspace/.venv/bin:/usr/local/go/bin:$PATH" RUN echo $'\ [safe]\n\ directory = *\n\ -'> /root/.gitconfig +'> ~/.gitconfig + + +# Activate the environment (make it default for subsequent commands) +RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc # ===== Setup for development ===== FROM base AS runtime - RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ source /workspace/.venv/bin/activate diff --git a/kustomize/README.md b/kustomize/README.md index e271a5c90..704299b4b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -324,6 +324,7 @@ sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env >[!WARNING] Without completing the following step with the correct secret from step 6, authentication and logging into the UI App will fail! + 6. If openshift resource of kind `OAuthClient` named `exploit-iq-client` exists, just get the secret from there: ```shell export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 3a21332b6..399ce1b03 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -19,6 +19,8 @@ spec: app: exploit-iq component: exploit-iq spec: + securityContext: + fsGroup: 1001 imagePullSecrets: [] serviceAccountName: exploit-iq-sa containers: @@ -58,7 +60,8 @@ spec: - "8080" securityContext: - runAsUser: 0 + runAsUser: 1001 + runAsGroup: 1001 ports: - name: http protocol: TCP From 5ed9e69297c3073a3fda665302b6004124bd262c Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Sun, 29 Mar 2026 15:41:27 +0300 Subject: [PATCH 214/286] =?UTF-8?q?fix=20APPENG-4780=20=20Not=20specific?= =?UTF-8?q?=20error=20indication=20for=20commit=20errors=20on=20U=E2=80=A6?= =?UTF-8?q?=20(#215)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix APPENG-4780 Not specific error indication for commit errors on UI or report page * Move git safe.directory config before uv to prevent dubious ownership error --- .tekton/on-pull-request.yaml | 5 +++-- src/exploit_iq_commons/data_models/input.py | 1 + src/vuln_analysis/functions/cve_generate_vdbs.py | 12 +++++++++--- src/vuln_analysis/functions/cve_http_output.py | 2 +- src/vuln_analysis/register.py | 1 + 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 243b1530d..7dc46783b 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -175,6 +175,9 @@ spec: echo "----------- ${1} -----------" } + # Mark the workspace as safe before any tool (uv, git, etc.) touches it. + git config --global --add safe.directory /workspace/source + # Install uv print_banner "Installing uv" curl -LsSf https://astral.sh/uv/install.sh | sh @@ -235,8 +238,6 @@ spec: mvn -v print_banner "RUNNING LINTER" - # Add the current directory to git's safe directories to avoid ownership errors. - git config --global --add safe.directory /workspace/source echo "Fetching target branch..." git fetch origin $TARGET_BRANCH_NAME echo "Git branches:" diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index 9a1a69195..dab612b4f 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -191,6 +191,7 @@ class AgentMorpheusInput(HashableModel): validation_alias=AliasChoices("credential_id", "credentialId"), ) code_index_success: bool | None = None + failure_reason: str | None = Field(default="No failure reason provided") class AgentMorpheusEngineInput(BaseModel): diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 3eb725134..63e4cccb4 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -33,6 +33,8 @@ logger = LoggingFactory.get_agent_logger(__name__) +class GitCloneError(Exception): + """Raised when cloning or updating a repo for document collection fails.""" class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): """ @@ -116,7 +118,7 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume documents.extend(embedder.collect_documents(si)) except Exception as e: logger.warning("Error collecting documents for source info %s: %s", si, e) - continue + raise GitCloneError(e) # Warn and return early if no documents were collected if len(documents) == 0: @@ -190,7 +192,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: base_image = message.image.name source_infos = message.image.source_info sbom_infos = message.image.sbom_info - + failure_reason = "" try: trace_id.set(message.scan.id) logger.debug("_arun: received credential_id=%r scan_id=%s", message.credential_id, message.scan.id) @@ -237,8 +239,11 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: except ValueError as e: logger.warning("Failed to get commit hash for repo defined in %s: %s", si, e) continue + except GitCloneError as e: + failure_reason = f"Error: {e}" except Exception as e: + failure_reason = f"Failure to build VDB , Error: {e}" logger.error("Failure to build VDB for image, '%s', with source code info: %s\nError: %s", base_image, source_infos, @@ -254,7 +259,8 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: vdb_doc_path = str(vdb_doc_path) if vdb_doc_path is not None else None code_index_path = str(code_index_path) if code_index_path is not None else None code_index_success = True if code_index_path is not None else False - + if not code_index_success: + message.failure_reason = failure_reason info = AgentMorpheusInfo(vdb=AgentMorpheusInfo.VdbPaths( code_vdb_path=vdb_code_path, doc_vdb_path=vdb_doc_path, code_index_path=code_index_path)) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index c9abf2297..b6b475645 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -116,7 +116,7 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: failure_report = FailureReport( scan_id=message.input.scan.id, error_type="processing-error", - error_message=f"Failed to clone repository {repo_url}", + error_message=f"Failed to clone repository {repo_url}--{message.input.failure_reason}", ) failure_url = config.url + config.failure_endpoint logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 554833927..9c41cc05f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -114,6 +114,7 @@ async def generate_vdbs_node(state: AgentMorpheusInput) -> AgentMorpheusEngineIn engine_input = await cve_generate_vdbs_fn.ainvoke(state.model_dump()) result = engine_input.model_dump() result["code_index_success"] = engine_input.input.code_index_success + result["failure_reason"] = engine_input.input.failure_reason return result @catch_pipeline_errors_async From 0d817909b6c35c2a448d3f31c85ad045a7baf52c Mon Sep 17 00:00:00 2001 From: batzionb Date: Mon, 30 Mar 2026 11:58:24 +0300 Subject: [PATCH 215/286] add UTC timezone to dates --- src/exploit_iq_commons/data_models/input.py | 10 ++++++++-- src/vuln_analysis/register.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index dab612b4f..897c915d1 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -110,8 +110,14 @@ class ScanInfoInput(HashableModel): """ id: str = Field(default_factory=lambda: str(uuid4())) type: str | None = None - started_at: str | None = None - completed_at: str | None = None + started_at: str | None = Field( + default=None, + description="Scan start time as ISO-8601 with UTC offset (e.g. ...+00:00).", + ) + completed_at: str | None = Field( + default=None, + description="Scan completion time as ISO-8601 with UTC offset (e.g. ...+00:00).", + ) vulns: typing.Annotated[list[VulnInfo], Field(min_length=1)] diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 9c41cc05f..dc847a87f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime +from datetime import datetime, timezone from io import TextIOWrapper from aiq.builder.builder import Builder @@ -104,7 +104,7 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): @catch_pipeline_errors_async async def add_start_time_node(state: AgentMorpheusInput) -> AgentMorpheusInput: """Adds the start time to the input""" - state.scan.started_at = datetime.now().isoformat() + state.scan.started_at = datetime.now(timezone.utc).isoformat() return state @catch_pipeline_errors_async @@ -175,7 +175,7 @@ async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEn @catch_pipeline_errors_async async def add_completed_time_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Adds the completed time to the output""" - state.input.scan.completed_at = datetime.now().isoformat() + state.input.scan.completed_at = datetime.now(timezone.utc).isoformat() return state @catch_pipeline_errors_async From ebdb805a5e590517867a9cbc20299d5f6c74c215 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:14:07 +0300 Subject: [PATCH 216/286] fix: widen fetch refspec so branch names are checkable after shallow clone (#218) git clone --depth=1 implies --single-branch, which narrows the fetch refspec to only the default branch. When the user specifies a different branch name as the ref, `git fetch origin ` stores the commit in FETCH_HEAD but does NOT create a remote-tracking ref, so `git checkout ` fails with "pathspec did not match". Fix by setting remote.origin.fetch to +refs/heads/*:refs/remotes/origin/* immediately after every clone_from(depth=1) call. This ensures that subsequent fetches create proper remote-tracking refs for any branch, allowing checkout by name. Also applied to existing repo re-use path to fix clones created before this patch. Tags and commit SHAs were unaffected (tags have their own explicit refspec fetch, SHAs resolve from the object database). Resolves: APPENG-4896 Signed-off-by: Theodor Mihalache --- .../utils/source_code_git_loader.py | 10 ++++++++++ .../tools/tests/test_source_code_git_loader.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 34b7fe43f..13379cc7f 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -175,6 +175,9 @@ def load_repo(self): raise ValueError("A different repository is already cloned at this path.") logger.info("Updating existing Git repo for URL '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) + # Ensure existing shallow clones have a widened fetch refspec + # so that branch names are resolvable (APPENG-4896). + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") else: logger.info("Cloning repository from URL: '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) @@ -184,6 +187,11 @@ def load_repo(self): else: # Create a shallow clone of the repository without checking out repo = Repo.clone_from(self.clone_url, self.repo_path, depth=1, no_checkout=True) + # depth=1 implies --single-branch, which narrows the fetch + # refspec to only the default branch. Widen it so that + # fetching any branch creates a remote-tracking ref and + # `git checkout ` succeeds (APPENG-4896). + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") # Set repo as git safe directory to avoid errors if directory ownership is changed # https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory @@ -323,6 +331,7 @@ def _clone_with_pat(self, pat: str, username: str | None) -> Repo: repo = None try: repo = Repo.clone_from(auth_url, self.repo_path, depth=1, no_checkout=True) + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") self._do_fetch(repo) return repo finally: @@ -346,6 +355,7 @@ def _clone_with_ssh_key(self, key: str) -> Repo: ssh_url, self.repo_path, depth=1, no_checkout=True, env={"GIT_SSH_COMMAND": git_ssh_cmd}, ) + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): self._do_fetch(repo) return repo diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py index cd34cf514..0896fa677 100644 --- a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -24,9 +24,22 @@ def test_https_to_ssh_url(https_url, expected_ssh): assert SourceCodeGitLoader._https_to_ssh_url(https_url) == expected_ssh +def test_load_repo_branch_checkout(tmp_path): + """Branch names must be checkable after a shallow clone (APPENG-4896).""" + repo_url = "https://github.com/medik8s/node-remediation-console" + repo_path = tmp_path / "branch_checkout_test" + branch = "release-0.6" + + loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=branch) + repo = loader.load_repo() + assert repo.head.commit.hexsha == repo.commit(branch).hexsha + + shutil.rmtree(repo_path) + + @pytest.mark.parametrize("refs", [ ("3.1.43", "3.1.42", "tag switch"), - ("ba5c10d3655c4fec714294cbc2ae0829c44dc046", + ("ba5c10d3655c4fec714294cbc2ae0829c44dc046", "5b098ff41266d6990237cbade5b6e27cab23557f", "sha switch"), ]) def test_load_repo_ref_switch(tmp_path, refs): From 6975df74802d2a210c0cc3930b3864c588267acc Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 12 Apr 2026 10:04:09 +0300 Subject: [PATCH 217/286] fix: remove runAsGroup override in exploit-iq container The exploit-iq container's securityContext was overriding runAsGroup to 1001, which prevented user 1001 from accessing the uv-managed Python installation located at /root/.local/share/uv/python/. The Dockerfile creates user1001 (UID 1001) with primary group workspace-group (GID 1000) and grants this group read/execute permissions on /root/. However, when runAsGroup: 1001 is set in the deployment, user 1001 runs with GID 1001 instead of its primary group GID 1000, losing access to /root/. Removing runAsGroup allows Kubernetes to use the primary group from the container image (/etc/passwd), enabling proper access to the Python stdlib. Fixes pod crash: ModuleNotFoundError: No module named 'encodings' --- kustomize/base/exploit_iq_service.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 399ce1b03..dd5011146 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -61,7 +61,6 @@ spec: securityContext: runAsUser: 1001 - runAsGroup: 1001 ports: - name: http protocol: TCP From e042012e44342e42db28acf150a8d4edbad10343 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 12 Apr 2026 10:04:46 +0300 Subject: [PATCH 218/286] fix: move SERPAPI_API_KEY to secret reference Replace hardcoded SERPAPI_API_KEY placeholder value with a reference to exploit-iq-secret, following the same pattern used for other sensitive credentials (REGISTRY_REDHAT_PASSWORD, REGISTRY_REDHAT_USERNAME). This improves security by keeping the API key in a Kubernetes secret rather than hardcoded in the deployment manifest. --- kustomize/base/exploit_iq_service.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 399ce1b03..8421c9510 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -75,7 +75,10 @@ spec: cpu: "1000m" env: - name: SERPAPI_API_KEY - value: EXPLOIT_IQ + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: exploit-iq-secret - name: NVIDIA_API_KEY value: EXPLOIT_IQ - name: OPENAI_API_KEY From 8b5f3baf2cfbcc4211b4229d193f86023619582e Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 12 Apr 2026 14:38:41 +0300 Subject: [PATCH 219/286] feat: add custom Git SSL CA certificate support Signed-off-by: Vladimir Belousov --- kustomize/README.md | 50 ++++++++++++++++++++++++++ kustomize/base/.gitignore | 8 +++++ kustomize/base/ca-certs/.gitkeep | 0 kustomize/base/ca-certs/README.md | 22 ++++++++++++ kustomize/base/exploit_iq_service.yaml | 9 +++++ kustomize/base/kustomization.yaml | 6 ++++ 6 files changed, 95 insertions(+) create mode 100644 kustomize/base/.gitignore create mode 100644 kustomize/base/ca-certs/.gitkeep create mode 100644 kustomize/base/ca-certs/README.md diff --git a/kustomize/README.md b/kustomize/README.md index 704299b4b..a61731f19 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -182,6 +182,56 @@ EOF export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` + +### Configuring Git SSL Certificate Authority for Custom CAs + +If your Git server uses a certificate that is signed by a custom Certificate Authority (CA), you must provide the CA certificate bundle to enable ExploitIQ to verify the Git server identity. + +> [!IMPORTANT] +> If you need to access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`, you must complete this procedure. + +#### Procedure + +1. Create the certificate directory: + +```shell +mkdir -p kustomize/base/ca-certs +``` + +2. Obtain your CA certificates. + +For Red Hat internal Git repositories: + +```shell +# Download the Red Hat Root CA +curl -o kustomize/base/ca-certs/internal-root-ca.pem \ + https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem + +# Download the Red Hat internal Intermediate CA +curl -o kustomize/base/ca-certs/rhcs-intermediate-ca.crt \ + https://certs.corp.redhat.com/chains/rhcs-ca-chain-2022-self-signed.crt +``` + +For other custom CAs: + +```shell +# Copy your custom CA certificate files to the directory +cp /path/to/your-custom-ca.pem kustomize/base/ca-certs/ +``` + +3. Create the CA bundle by concatenating all certificates: + +```shell +cat kustomize/base/ca-certs/*.{pem,crt} > kustomize/base/ca-certs/ca-bundle.crt +``` + +4. Verify that the bundle contains your certificates: + +```shell +openssl crl2pkcs7 -nocrl -certfile kustomize/base/ca-certs/ca-bundle.crt | \ + openssl pkcs7 -print_certs -noout +``` + >[!IMPORTANT] You should only run one of the steps 9,10 or 11, depending on if you want to run the service with a self hosted LLM, self hosted LLM with MLOps or Nvidia remote NIM. 9. To deploy `ExploitIQ` with a self-hosted LLM , run: diff --git a/kustomize/base/.gitignore b/kustomize/base/.gitignore new file mode 100644 index 000000000..5992ad117 --- /dev/null +++ b/kustomize/base/.gitignore @@ -0,0 +1,8 @@ +# Certificate files - contains sensitive internal CA certificates +ca-certs/*.pem +ca-certs/*.crt +ca-certs/ca-bundle.crt + +# Keep the directory structure +!ca-certs/.gitkeep +!ca-certs/README.md diff --git a/kustomize/base/ca-certs/.gitkeep b/kustomize/base/ca-certs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/kustomize/base/ca-certs/README.md b/kustomize/base/ca-certs/README.md new file mode 100644 index 000000000..6b6ea70b6 --- /dev/null +++ b/kustomize/base/ca-certs/README.md @@ -0,0 +1,22 @@ +# Git SSL CA Certificates Directory + +This directory contains Certificate Authority (CA) certificates for Git SSL verification. + +## Purpose + +If your Git server uses certificates signed by a custom CA, place the CA certificate files in this directory. The deployment process creates a ConfigMap from these certificates and mounts it to the ExploitIQ pod. + +## Instructions + +Refer to the [Configuring Git SSL Certificate Authority for Custom CAs](../../README.md#configuring-git-ssl-certificate-authority-for-custom-cas) section in the main deployment documentation for detailed instructions on downloading and configuring CA certificates. + +## Red Hat Internal Access + +For Red Hat internal Git repositories (gitlab.cee.redhat.com), download: + +- Root CA: +- Intermediate CA: + +## Security + +**Do not commit certificate files to version control.** These files are gitignored to prevent accidental exposure of internal CA certificates in public repositories. diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index dd5011146..d2aa367de 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -138,6 +138,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: GIT_SSL_CAINFO + value: /app/git-ca-bundle/ca-bundle.crt volumeMounts: - name: config mountPath: /configs @@ -148,6 +150,9 @@ spec: - name: ca-bundle mountPath: /app/certs readOnly: true + - name: git-ca-bundle + mountPath: /app/git-ca-bundle + readOnly: true volumes: - name: config configMap: @@ -161,6 +166,10 @@ spec: - name: ca-bundle configMap: name: openshift-service-ca.crt + - name: git-ca-bundle + configMap: + name: git-ca-bundle + optional: true --- apiVersion: v1 kind: Service diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 08a0a709d..70f43316b 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -70,6 +70,12 @@ configMapGenerator: - excludes.json - includes.json + - name: git-ca-bundle + files: + - ca-certs/ca-bundle.crt + options: + disableNameSuffixHash: true + patches: - path: ips-patch.json From 2f0bdd5c53726faf51b5c273badb55ca92f2e378 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 23 Apr 2026 18:01:05 +0300 Subject: [PATCH 220/286] chore: set SC fsGroup to user' gid to resolve fs perms errors Signed-off-by: Zvi Grinberg --- kustomize/base/exploit_iq_service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 99bac2777..1e41a9c97 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -20,7 +20,7 @@ spec: component: exploit-iq spec: securityContext: - fsGroup: 1001 + fsGroup: 1000 imagePullSecrets: [] serviceAccountName: exploit-iq-sa containers: From 4cfb46040ac54e264c6b671ca35753fb05b5a440 Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 24 Apr 2026 16:21:56 -0400 Subject: [PATCH 221/286] fix: replace proprietary license and restore truncated Apache 2.0 headers - intel_plugin.py: replaced NvidiaProprietary license with Apache 2.0 - intel_source_score.py: restored missing SPDX and copyright lines Made-with: Cursor --- .../data_models/plugins/intel_plugin.py | 21 ++++++++++++------- src/vuln_analysis/utils/intel_source_score.py | 4 ++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py index 9b85c6d11..373375261 100644 --- a/src/vuln_analysis/data_models/plugins/intel_plugin.py +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -1,12 +1,17 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import requests from pydantic import BaseModel diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 479b3cc55..9b3e74eb1 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # From 63f34b50d3e80992dc96c753bca1867631759d90 Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 24 Apr 2026 16:23:17 -0400 Subject: [PATCH 222/286] minor edits --- src/vuln_analysis/data_models/plugins/intel_plugin.py | 2 +- src/vuln_analysis/utils/intel_source_score.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py index 373375261..ea122d808 100644 --- a/src/vuln_analysis/data_models/plugins/intel_plugin.py +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py index 9b3e74eb1..1c595344c 100644 --- a/src/vuln_analysis/utils/intel_source_score.py +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import json import os import re From ba60c82933ce96c0a3f7e721106a261efa99946b Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 24 Apr 2026 16:26:00 -0400 Subject: [PATCH 223/286] fix: add Apache 2.0 license headers to exploit_iq_commons and SPDX-only files - Added full boilerplate to 20 headerless exploit_iq_commons Python files - Expanded 5 SPDX-only files to include the full Apache 2.0 license text Made-with: Cursor --- src/exploit_iq_commons/logging/loggers_factory.py | 15 +++++++++++++++ .../utils/c_segmenter_custom.py | 15 +++++++++++++++ .../utils/chain_of_calls_retriever.py | 15 +++++++++++++++ .../utils/chain_of_calls_retriever_base.py | 15 +++++++++++++++ .../utils/chain_of_calls_retriever_factory.py | 15 +++++++++++++++ src/exploit_iq_commons/utils/dep_tree.py | 15 +++++++++++++++ .../functions_parsers/c_lang_function_parsers.py | 15 +++++++++++++++ .../functions_parsers/golang_functions_parsers.py | 15 +++++++++++++++ .../functions_parsers/java_functions_parsers.py | 15 +++++++++++++++ .../javascript_functions_parser.py | 15 +++++++++++++++ .../functions_parsers/lang_functions_parsers.py | 15 +++++++++++++++ .../lang_functions_parsers_factory.py | 15 +++++++++++++++ .../functions_parsers/python_functions_parser.py | 15 +++++++++++++++ .../utils/go_segmenters_with_methods.py | 15 +++++++++++++++ .../utils/java_chain_of_calls_retriever.py | 15 +++++++++++++++ .../utils/java_segmenters_with_methods.py | 15 +++++++++++++++ src/exploit_iq_commons/utils/java_utils.py | 15 +++++++++++++++ .../python_segmenters_with_classes_methods.py | 15 +++++++++++++++ .../utils/source_rpm_downloader.py | 14 ++++++++++++++ .../utils/standard_library_cache.py | 15 +++++++++++++++ src/vuln_analysis/runtime_context.py | 12 ++++++++++++ src/vuln_analysis/tools/tool_names.py | 12 ++++++++++++ tests/test_base_tool_descriptions.py | 12 ++++++++++++ tests/test_tool_filtering.py | 12 ++++++++++++ tests/test_tool_names.py | 12 ++++++++++++ 25 files changed, 359 insertions(+) diff --git a/src/exploit_iq_commons/logging/loggers_factory.py b/src/exploit_iq_commons/logging/loggers_factory.py index de0482bc3..008e80aaf 100644 --- a/src/exploit_iq_commons/logging/loggers_factory.py +++ b/src/exploit_iq_commons/logging/loggers_factory.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import sys import textwrap diff --git a/src/exploit_iq_commons/utils/c_segmenter_custom.py b/src/exploit_iq_commons/utils/c_segmenter_custom.py index c842f1e7a..6601f2f0f 100644 --- a/src/exploit_iq_commons/utils/c_segmenter_custom.py +++ b/src/exploit_iq_commons/utils/c_segmenter_custom.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re from langchain_community.document_loaders.parsers.language.c import CSegmenter from typing import List diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index e0eced325..e13d9ad1d 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re import time from collections import defaultdict, deque diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py index 2900e59cb..1d30b105c 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import abstractmethod, ABC from pathlib import Path from typing import List, Optional diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py index ee5e0a698..5e64f3016 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pathlib import Path from typing import List diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 9afc4273e..7a8effad8 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re import shutil diff --git a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py index 5e2402aa6..37b73af18 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Tuple, List from langchain_core.documents import Document diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index fad0ee8f5..09688b5e9 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re from typing import Tuple, List diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index ac6b50f18..be53e412d 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import hashlib import os import re diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py index 7529f66f8..0275b64f9 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re from typing import List, Tuple diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py index 0c311919f..f7160593c 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC, abstractmethod from typing import Tuple, List from warnings import warn diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py index 3514e1f90..755455324 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from exploit_iq_commons.utils.dep_tree import Ecosystem, DependencyTree from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index bef207440..f019cbbf3 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import ast import os.path import re diff --git a/src/exploit_iq_commons/utils/go_segmenters_with_methods.py b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py index 4a8bae901..fe8c23f84 100644 --- a/src/exploit_iq_commons/utils/go_segmenters_with_methods.py +++ b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re from typing import List diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index 33c43afc2..2cc09ff5f 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re from pathlib import Path from typing import List diff --git a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py index 6b54c00da..f141696f4 100644 --- a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py +++ b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import bisect import os diff --git a/src/exploit_iq_commons/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py index 47f6c33ab..8de35bfde 100644 --- a/src/exploit_iq_commons/utils/java_utils.py +++ b/src/exploit_iq_commons/utils/java_utils.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import hashlib import re from dataclasses import dataclass diff --git a/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py index d39d6ce66..f37a3cc6e 100644 --- a/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py +++ b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import ast from typing import List import re diff --git a/src/exploit_iq_commons/utils/source_rpm_downloader.py b/src/exploit_iq_commons/utils/source_rpm_downloader.py index 09dc81078..9022b40de 100644 --- a/src/exploit_iq_commons/utils/source_rpm_downloader.py +++ b/src/exploit_iq_commons/utils/source_rpm_downloader.py @@ -1,3 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os from pathlib import Path diff --git a/src/exploit_iq_commons/utils/standard_library_cache.py b/src/exploit_iq_commons/utils/standard_library_cache.py index 8d090c40d..7435ab2dd 100644 --- a/src/exploit_iq_commons/utils/standard_library_cache.py +++ b/src/exploit_iq_commons/utils/standard_library_cache.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Standard library cache utilities. Loads an auto-generated JSON of standard libraries for supported ecosystems diff --git a/src/vuln_analysis/runtime_context.py b/src/vuln_analysis/runtime_context.py index 4d896f4c5..fceb67aad 100644 --- a/src/vuln_analysis/runtime_context.py +++ b/src/vuln_analysis/runtime_context.py @@ -1,6 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # A shared runtime context for passing workflow state to tools and functions. # This must be imported by all modules that need to read or set the active state. diff --git a/src/vuln_analysis/tools/tool_names.py b/src/vuln_analysis/tools/tool_names.py index 248fec7fd..8c523e303 100644 --- a/src/vuln_analysis/tools/tool_names.py +++ b/src/vuln_analysis/tools/tool_names.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Tool name constants to avoid hardcoded strings throughout the codebase. diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py index 9a939aa52..7641b0efc 100644 --- a/tests/test_base_tool_descriptions.py +++ b/tests/test_base_tool_descriptions.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Tests for the base build_tool_descriptions() function. diff --git a/tests/test_tool_filtering.py b/tests/test_tool_filtering.py index f325d9d8c..2a8442484 100644 --- a/tests/test_tool_filtering.py +++ b/tests/test_tool_filtering.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Unit tests for tool filtering logic in CVE agent functions. diff --git a/tests/test_tool_names.py b/tests/test_tool_names.py index 7d1677717..8aa289bb7 100644 --- a/tests/test_tool_names.py +++ b/tests/test_tool_names.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Unit tests for tool name constants. From d0d593133a1834535a8196b60cef714fdac6451a Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 24 Apr 2026 23:54:13 -0400 Subject: [PATCH 224/286] Update test_base_tool_descriptions.py --- tests/test_base_tool_descriptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py index 7641b0efc..b6a3d1e6b 100644 --- a/tests/test_base_tool_descriptions.py +++ b/tests/test_base_tool_descriptions.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); From 779ee84697cd45be445dff465a5b57f55494d24d Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:20:29 +0300 Subject: [PATCH 225/286] APPENG-4571:New agent Create a basic React agent flow migrated from Langchain' old agent executor to LangGraph state machine agent. --- .tekton/on-cm-runner.yaml | 21 +- .tekton/on-pull-request.yaml | 10 +- Dockerfile | 6 +- ci/scripts/analyze_traces.py | 762 +++++ ci/scripts/collect_traces_local.py | 353 ++ kustomize/README.md | 73 +- kustomize/base/exploit-iq-config.yml | 3 + kustomize/base/exploit_iq_service.yaml | 13 +- kustomize/base/exploitiq_mcp_server.yaml | 129 + kustomize/base/kustomization.yaml | 7 + kustomize/base/settings.xml | 62 + kustomize/config-http-openai-local.yml | 3 + otel-config.yaml | 5 +- .../utils/chain_of_calls_retriever.py | 191 +- .../utils/chain_of_calls_retriever_base.py | 3 - src/exploit_iq_commons/utils/data_utils.py | 13 + src/exploit_iq_commons/utils/dep_tree.py | 294 +- .../utils/document_embedding.py | 32 +- .../golang_functions_parsers.py | 3 + .../java_functions_parsers.py | 1231 +++++-- .../utils/java_chain_of_calls_retriever.py | 782 +++-- .../utils/java_segmenters_with_methods.py | 3017 ++++++++++++----- src/exploit_iq_commons/utils/java_utils.py | 878 ++--- .../utils/transitive_code_searcher_tool.py | 82 +- src/vuln_analysis/configs/config-http-nim.yml | 3 + .../configs/config-http-openai.yml | 3 + src/vuln_analysis/configs/config-tracing.yml | 3 + src/vuln_analysis/configs/config.yml | 5 +- src/vuln_analysis/functions/cve_agent.py | 611 +++- src/vuln_analysis/functions/cve_checklist.py | 8 +- .../functions/cve_generate_vdbs.py | 24 +- src/vuln_analysis/functions/cve_summarize.py | 69 +- .../functions/react_internals.py | 536 +++ .../tools/lexical_full_search.py | 2 +- .../tools/tests/test_concurrency.py | 464 +++ .../tests/test_transitive_code_search.py | 105 +- src/vuln_analysis/tools/tool_names.py | 9 +- .../tools/transitive_code_search.py | 372 +- .../utils/checklist_prompt_generator.py | 49 +- src/vuln_analysis/utils/full_text_search.py | 49 +- .../utils/function_name_locator.py | 264 +- src/vuln_analysis/utils/intel_utils.py | 235 ++ src/vuln_analysis/utils/prompt_factory.py | 303 ++ src/vuln_analysis/utils/prompting.py | 6 + tests/test_base_tool_descriptions.py | 81 +- 45 files changed, 8770 insertions(+), 2404 deletions(-) create mode 100644 ci/scripts/analyze_traces.py create mode 100644 ci/scripts/collect_traces_local.py create mode 100644 kustomize/base/exploitiq_mcp_server.yaml create mode 100644 kustomize/base/settings.xml create mode 100644 src/vuln_analysis/functions/react_internals.py create mode 100644 src/vuln_analysis/tools/tests/test_concurrency.py create mode 100644 src/vuln_analysis/utils/prompt_factory.py diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index a4108244c..27d0f6672 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -120,11 +120,11 @@ spec: resources: requests: cpu: "1000m" # CPU request (1 core) - memory: "8Gi" # Memory request (8 gigabytes) + memory: "12Gi" # Memory request (8 gigabytes) limits: cpu: "2000m" # CPU limit (2 cores) - memory: "16Gi" # Memory limit (16 gigabytes) - + memory: "32Gi" # Memory limit (16 gigabytes) + volumeMounts: - name: google-creds-volume mountPath: /etc/secrets/google @@ -142,7 +142,7 @@ spec: name: server-model-config - secretRef: name: redhat-app-creds - + env: - name: GOTOOLCHAIN value: "auto" @@ -180,7 +180,9 @@ spec: value: cve_agent - name: OTEL_TRACES_ENDPOINT value: http://localhost:4318/v1/traces - + - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH + value: $(workspaces.source.path)/kustomize/base/settings.xml + script: | #!/bin/sh set -e @@ -225,6 +227,13 @@ spec: echo "Cache link created successfully." ls -ld "${CACHE_DIR_TARGET}" + # Remove cached pickle files if REMOVE_CACHE_FILE is set (via Google Sheet config) + # e.g. REMOVE_CACHE_FILE=https.github.com.postgres.postgres + if [ -n "${REMOVE_CACHE_FILE:-}" ]; then + echo "--- Removing cache files matching: ${CACHE_DIR_TARGET}/pickle/${REMOVE_CACHE_FILE}* ---" + rm -fv "${CACHE_DIR_TARGET}/pickle/${REMOVE_CACHE_FILE}"* || echo " No matching files found." + fi + # Copy with verbose error if it fails echo "--- Exporting Telemetry Config ---" cp -v configs/config-no-tracing.yml $DATA_DIR/config-no-tracing.yml || echo "Failed to copy config" @@ -267,7 +276,7 @@ spec: initialDelaySeconds: 5 - name: otel-collector - image: otel/opentelemetry-collector-contrib:latest + image: otel/opentelemetry-collector-contrib:0.148.0 # Mount your otel-config.yaml here via a ConfigMap volumeMounts: - name: otel-config-volume diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 7dc46783b..c96501987 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -164,6 +164,8 @@ spec: env: - name: TARGET_BRANCH_NAME value: "{{target_branch}}" + - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH + value: $(workspaces.source.path)/kustomize/base/settings.xml image: registry.access.redhat.com/ubi9/python-312:9.6 workingDir: $(workspaces.source.path) script: | @@ -187,12 +189,15 @@ spec: source .venv/bin/activate print_banner "INSTALLING DEPENDENCIES" + # Add the current directory to git's safe directories before uv sync, + # because setuptools-scm needs git access for version introspection. + git config --global --add safe.directory /workspace/source uv sync # Install Java JAVA_ARCH="x64" - JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_${JAVA_ARCH}_linux_hotspot_22.0.2_9.tar.gz" - JDK_DIR="jdk-22.0.2+9" + JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.18%2B8/OpenJDK17U-jdk_${JAVA_ARCH}_linux_hotspot_17.0.18_8.tar.gz" + JDK_DIR="jdk-17.0.18+8" echo ">> Downloading $JDK_URL" mkdir -p /tekton/home/jdk @@ -246,7 +251,6 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME - print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" diff --git a/Dockerfile b/Dockerfile index 7044c22c9..fa6e6d652 100755 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,8 @@ ENV PATH="/opt/nodejs/bin:${PATH}" RUN node --version && npm --version # --- Temurin JDK 22 (amd64/x86_64) --- -ARG JDK_URL="https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz" -ARG JDK_DIR="jdk-22.0.2+9" +ARG JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.18%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.18_8.tar.gz" +ARG JDK_DIR="jdk-17.0.18+8" RUN mkdir -p /opt/jdk \ && curl -fsSL -o /tmp/jdk.tgz "${JDK_URL}" \ && tar -C /opt/jdk -xzf /tmp/jdk.tgz \ @@ -142,4 +142,4 @@ RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ source /workspace/.venv/bin/activate -CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] +CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] \ No newline at end of file diff --git a/ci/scripts/analyze_traces.py b/ci/scripts/analyze_traces.py new file mode 100644 index 000000000..81d51d6a0 --- /dev/null +++ b/ci/scripts/analyze_traces.py @@ -0,0 +1,762 @@ +""" +Trace analyzer for LLM reasoning quality. + +Reads per-trace JSON files produced by collect_traces_local.py, +reconstructs agent investigation flows, and flags quality issues. + +Usage: + python ci/scripts/analyze_traces.py --input traces_output/ + python ci/scripts/analyze_traces.py --input traces_output/ --json + python ci/scripts/analyze_traces.py --input traces_output/single_trace.json +""" +import argparse +import json +import sys +from collections import Counter +from dataclasses import dataclass, field, asdict +from enum import Enum +from pathlib import Path + + +REACHABILITY_TOOLS = {"Function Locator", "Call Chain Analyzer", "Function Caller Finder"} +SEARCH_ONLY_TOOLS = {"Code Keyword Search", "Code Semantic Search", "Docs Semantic Search"} +NODE_FUNCTION_NAMES = {"thought node", "observation node", "pre_process node", "forced_finish node"} +TOKEN_BUDGET_THRESHOLD = 6000 # 75% of 8k window +TOKEN_BUDGET_WARNING = 5800 # early warning before hitting threshold +CONTEXT_WINDOW_LIMIT = 8192 +MAX_COMPLETION_TOKENS = 2000 + + +class Severity(str, Enum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + +@dataclass +class QualityFlag: + name: str + severity: Severity + detail: str + + +@dataclass +class InvestigationStep: + step_num: int + span_kind: str # LLM, TOOL, or FUNCTION + name: str + input_preview: str + output_preview: str + token_prompt: int = 0 + token_completion: int = 0 + token_total: int = 0 + llm_role: str = "" # "thought" or "observation" for LLM spans + thought_mode: str = "" # "act" or "finish" for thought spans + has_error: bool = False + error_detail: str = "" + + +@dataclass +class InvestigationFlow: + """One agent investigation thread (one checklist question).""" + trace_id: str + job_id: str + parent_span_id: str + steps: list[InvestigationStep] = field(default_factory=list) + flags: list[QualityFlag] = field(default_factory=list) + tools_used: list[str] = field(default_factory=list) + llm_call_count: int = 0 + tool_call_count: int = 0 + token_progression: list[int] = field(default_factory=list) + cve_id: str = "" + ecosystem: str = "" + question: str = "" + is_reachability: str = "" + + +def _preview(text: str, max_len: int = 120) -> str: + if not text: + return "" + clean = text.replace("\n", " ").strip() + return clean[:max_len] + "..." if len(clean) > max_len else clean + + +def _extract_cve_from_input(input_value: str) -> tuple[str, str]: + """Try to extract CVE ID and ecosystem from an agent executor's LLM input.""" + cve_id = "" + ecosystem = "" + try: + messages = json.loads(input_value) + for msg in messages: + content = msg.get("content", "") if isinstance(msg, dict) else "" + if "CVE-" in content: + import re + match = re.search(r"(CVE-\d{4}-\d+)", content) + if match: + cve_id = match.group(1) + if "ecosystem" in content.lower(): + for eco in ("go", "python", "java", "javascript", "c"): + if f'"{eco}"' in content.lower() or f"'{eco}'" in content.lower(): + ecosystem = eco + break + except (json.JSONDecodeError, TypeError): + pass + return cve_id, ecosystem + + +def _extract_question(input_value: str) -> str: + """Extract the human/checklist question from the first LLM call's input.""" + try: + messages = json.loads(input_value) + for msg in messages: + if isinstance(msg, dict) and msg.get("type") == "human": + return _preview(msg.get("content", ""), 200) + except (json.JSONDecodeError, TypeError): + pass + return "" + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + +def load_trace_files(input_path: str) -> list[dict]: + """Load trace documents from a file or directory.""" + p = Path(input_path) + traces = [] + if p.is_file(): + with open(p, "r", encoding="utf-8") as f: + traces.append(json.load(f)) + elif p.is_dir(): + for fp in sorted(p.glob("*.json")): + with open(fp, "r", encoding="utf-8") as f: + try: + traces.append(json.load(f)) + except json.JSONDecodeError as e: + print(f"Warning: skipping {fp}: {e}", file=sys.stderr) + else: + print(f"Error: {input_path} is not a file or directory", file=sys.stderr) + sys.exit(1) + return traces + + +def _check_span_error(span: dict) -> tuple[bool, str]: + """Check if a span's output_value contains an error recorded by try/except tracing.""" + output = span.get("output_value", "") + if not output: + return False, "" + try: + parsed = json.loads(output) + if isinstance(parsed, dict) and "error" in parsed: + exc_type = parsed.get("exception_type", "Exception") + error_msg = parsed["error"] + return True, f"{exc_type}: {error_msg}" + except (json.JSONDecodeError, TypeError): + pass + return False, "" + + +def _classify_llm_span(span: dict) -> str: + """Classify an LLM span as 'thought' or 'observation' based on output format.""" + output = span.get("output_value", "") + stripped = output.lstrip() + if stripped.startswith('{"thought"') or stripped.startswith('{"thought'): + return "thought" + return "observation" + + +def _extract_thought_mode(output: str) -> str: + """Extract the mode ('act' or 'finish') from a thought-node JSON output.""" + try: + parsed = json.loads(output) + return parsed.get("mode", "") + except (json.JSONDecodeError, TypeError): + pass + import re + m = re.search(r'"mode"\s*:\s*"(act|finish)"', output) + return m.group(1) if m else "" + + +def _extract_goal_question(input_value: str) -> str: + """Extract the GOAL question from an observation-node input.""" + try: + messages = json.loads(input_value) + for msg in messages: + content = msg.get("content", "") if isinstance(msg, dict) else "" + if "GOAL:" in content: + goal_line = content.split("GOAL:")[1].split("\n")[0].strip() + return _preview(goal_line, 200) + except (json.JSONDecodeError, TypeError): + pass + return "" + + +def _split_by_question(spans: list[dict]) -> dict[str, list[dict]]: + """ + Split interleaved agent spans into per-question groups. + + Concurrent checklist questions share a parent_span_id. We separate them + by extracting the question from thought-node (human message) and + observation-node (GOAL field) LLM spans. TOOL spans are assigned to the + question whose thought span most recently preceded them. + """ + spans_sorted = sorted(spans, key=lambda s: int(s.get("start_time") or 0)) + + questions_seen: list[str] = [] + for span in spans_sorted: + if span.get("span_kind") == "LLM" and _classify_llm_span(span) == "thought": + q = _extract_question(span.get("input_value", "")) + if q and q not in questions_seen: + questions_seen.append(q) + + if len(questions_seen) <= 1: + return {} + + groups: dict[str, list[dict]] = {q: [] for q in questions_seen} + + last_thought_question: str | None = None + + for span in spans_sorted: + kind = span.get("span_kind", "") + + if kind == "LLM": + role = _classify_llm_span(span) + if role == "thought": + q = _extract_question(span.get("input_value", "")) + if q in groups: + last_thought_question = q + groups[q].append(span) + elif role == "observation": + goal_q = _extract_goal_question(span.get("input_value", "")) + matched = None + if goal_q: + for known_q in questions_seen: + if goal_q[:60] in known_q or known_q[:60] in goal_q: + matched = known_q + break + if matched: + groups[matched].append(span) + elif last_thought_question: + groups[last_thought_question].append(span) + + elif kind in ("TOOL", "FUNCTION"): + if last_thought_question: + groups[last_thought_question].append(span) + + return groups + + +def _build_flow_from_spans( + group_spans: list[dict], + trace_id: str, + job_id: str, + parent_id: str, +) -> InvestigationFlow: + """Build a single InvestigationFlow from an ordered list of spans.""" + group_spans.sort(key=lambda s: int(s.get("start_time") or 0)) + + flow = InvestigationFlow( + trace_id=trace_id, + job_id=job_id, + parent_span_id=parent_id, + ) + + for i, span in enumerate(group_spans): + kind = span.get("span_kind", "") + span_name = span.get("name", "") + role = "" + mode = "" + if kind == "LLM": + role = _classify_llm_span(span) + if role == "thought": + mode = _extract_thought_mode(span.get("output_value", "")) + elif kind == "FUNCTION" and span_name == "thought node": + role = "thought" + mode = _extract_thought_mode(span.get("output_value", "")) + + has_error, error_detail = _check_span_error(span) + + step = InvestigationStep( + step_num=i + 1, + span_kind=kind, + name=span_name, + input_preview=_preview(span.get("input_value", "")), + output_preview=_preview(span.get("output_value", "")), + token_prompt=span.get("token_prompt", 0), + token_completion=span.get("token_completion", 0), + token_total=span.get("token_total", 0), + llm_role=role, + thought_mode=mode, + has_error=has_error, + error_detail=error_detail, + ) + flow.steps.append(step) + + if kind == "LLM": + flow.llm_call_count += 1 + if step.token_prompt > 0: + flow.token_progression.append(step.token_prompt) + if i == 0: + cve_id, eco = _extract_cve_from_input(span.get("input_value", "")) + flow.cve_id = cve_id + flow.ecosystem = eco + flow.question = _extract_question(span.get("input_value", "")) + elif kind == "FUNCTION" and span_name == "thought node": + flow.llm_call_count += 1 + if step.token_prompt > 0: + flow.token_progression.append(step.token_prompt) + elif kind == "TOOL": + flow.tool_call_count += 1 + flow.tools_used.append(span_name) + elif kind == "FUNCTION" and span_name == "pre_process node": + try: + output = json.loads(span.get("output_value", "{}")) + if isinstance(output, dict) and "reachability_question" in output: + flow.is_reachability = output["reachability_question"] + except (json.JSONDecodeError, TypeError): + pass + + return flow + + +def _build_flows_from_workflow_output( + spans: list[dict], + trace_id: str, + job_id: str, +) -> list[InvestigationFlow]: + """Fallback: build flows from the span's output when agent-level + spans (thought node, pre_process node, etc.) are absent. + + This handles the ``cve_justify`` pipeline variant where the agent's + internal LangGraph spans are not exported individually. + """ + workflow_span = next( + (s for s in spans if s.get("function_name") == "" and s.get("span_kind") == "FUNCTION"), + None, + ) + if not workflow_span: + return [] + + output_raw = workflow_span.get("output_value", "") + try: + output = json.loads(output_raw) + except (json.JSONDecodeError, TypeError): + return [] + + analyses = (output.get("output", {}) or {}).get("analysis", []) + if not analyses: + return [] + + input_raw = workflow_span.get("input_value", "") + ecosystem = "" + try: + inp = json.loads(input_raw) + ecosystem = (inp.get("image", {}) or {}).get("ecosystem", "") + except (json.JSONDecodeError, TypeError): + pass + + llm_spans = [s for s in spans if s.get("span_kind") == "LLM"] + total_prompt = sum(s.get("token_prompt", 0) for s in llm_spans) + total_completion = sum(s.get("token_completion", 0) for s in llm_spans) + + flows: list[InvestigationFlow] = [] + for analysis in analyses: + vuln_id = analysis.get("vuln_id", "") + checklist = analysis.get("checklist", []) + justification = analysis.get("justification", {}) or {} + label = justification.get("label", "") + reason = justification.get("reason", "") + + flow = InvestigationFlow( + trace_id=trace_id, + job_id=job_id, + parent_span_id=workflow_span.get("span_id", ""), + cve_id=vuln_id, + ecosystem=ecosystem, + ) + + for i, item in enumerate(checklist): + question = item.get("input", "") + response = item.get("response", "") + flow.steps.append(InvestigationStep( + step_num=i + 1, + span_kind="SUMMARY", + name="checklist_qa", + input_preview=_preview(question, 200), + output_preview=_preview(response, 200), + )) + + flow.steps.append(InvestigationStep( + step_num=len(checklist) + 1, + span_kind="SUMMARY", + name="justification", + input_preview=f"label={label}", + output_preview=_preview(reason, 200), + )) + + flow.llm_call_count = len(llm_spans) + if total_prompt > 0: + flow.token_progression.append(total_prompt) + + flows.append(flow) + + return flows + + +def build_investigation_flows(trace_doc: dict) -> list[InvestigationFlow]: + """ + From a single trace document, reconstruct per-question investigation flows. + + When concurrent checklist questions share one parent_span_id (LangGraph), + we split by extracting the human question from each thought-node span. + Falls back to parent_span_id grouping when questions aren't interleaved. + """ + trace_id = trace_doc.get("trace_id", "") + job_id = trace_doc.get("job_id", "") + spans = trace_doc.get("spans", []) + + llm_tokens_by_parent: dict[str, dict[str, int]] = {} + for s in spans: + if ( + s.get("span_kind") == "LLM" + and s.get("function_name") in NODE_FUNCTION_NAMES + ): + pid = s.get("parent_span_id", "") + if pid and s.get("token_prompt", 0) > 0: + llm_tokens_by_parent[pid] = { + "token_prompt": s.get("token_prompt", 0), + "token_completion": s.get("token_completion", 0), + "token_total": s.get("token_total", 0), + } + + agent_spans = [ + s for s in spans + if ( + s.get("function_name") in ("cve_agent_executor", "checklist_question") + and s.get("span_kind") in ("LLM", "TOOL") + ) or ( + s.get("function_name") in NODE_FUNCTION_NAMES + and s.get("span_kind") == "FUNCTION" + ) + ] + + for s in agent_spans: + sid = s.get("span_id", "") + if sid in llm_tokens_by_parent: + s.update(llm_tokens_by_parent[sid]) + + parent_groups: dict[str, list[dict]] = {} + for span in agent_spans: + pid = span.get("parent_span_id", "unknown") + parent_groups.setdefault(pid, []).append(span) + + flows = [] + for parent_id, group_spans in parent_groups.items(): + question_groups = _split_by_question(group_spans) + + if question_groups: + for q_text, q_spans in question_groups.items(): + if q_spans: + flow = _build_flow_from_spans(q_spans, trace_id, job_id, parent_id) + if not flow.question: + flow.question = q_text + flows.append(flow) + else: + flow = _build_flow_from_spans(group_spans, trace_id, job_id, parent_id) + flows.append(flow) + + if not flows: + flows = _build_flows_from_workflow_output(spans, trace_id, job_id) + + return flows + + +# --------------------------------------------------------------------------- +# Quality checks +# --------------------------------------------------------------------------- + +def run_quality_checks(flow: InvestigationFlow) -> list[QualityFlag]: + flags = [] + + is_summary_only = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + if is_summary_only: + return flags + + error_steps = [s for s in flow.steps if s.has_error] + for es in error_steps: + flags.append(QualityFlag( + name="NODE_EXCEPTION", + severity=Severity.HIGH, + detail=f"Exception in {es.name} (step {es.step_num}): {es.error_detail}", + )) + + tool_set = set(flow.tools_used) + + is_reachability_question = flow.is_reachability != "no" + + if not tool_set: + flags.append(QualityFlag( + name="NO_TOOL_CALLS", + severity=Severity.HIGH, + detail="Agent concluded without any tool calls.", + )) + elif is_reachability_question and not tool_set & REACHABILITY_TOOLS: + flags.append(QualityFlag( + name="NO_REACHABILITY_TOOL", + severity=Severity.HIGH, + detail=f"No reachability tool used. Tools: {', '.join(tool_set)}", + )) + + if is_reachability_question and tool_set and tool_set <= SEARCH_ONLY_TOOLS: + flags.append(QualityFlag( + name="KEYWORD_SEARCH_ONLY", + severity=Severity.HIGH, + detail="Only search tools used, never traced call chain.", + )) + + if flow.llm_call_count <= 2 and flow.tool_call_count > 0: + flags.append(QualityFlag( + name="PREMATURE_FINISH", + severity=Severity.MEDIUM, + detail=f"Agent finished in {flow.llm_call_count} LLM iterations.", + )) + + budget_flagged = False + for tok in flow.token_progression: + if tok > TOKEN_BUDGET_THRESHOLD: + flags.append(QualityFlag( + name="TOKEN_BUDGET_HIGH", + severity=Severity.MEDIUM, + detail=f"LLM call used {tok} prompt tokens (>{TOKEN_BUDGET_THRESHOLD}).", + )) + budget_flagged = True + break + if tok > TOKEN_BUDGET_WARNING: + flags.append(QualityFlag( + name="TOKEN_BUDGET_APPROACHING", + severity=Severity.LOW, + detail=f"LLM call used {tok} prompt tokens, approaching {TOKEN_BUDGET_THRESHOLD} limit.", + )) + budget_flagged = True + break + + if "Function Caller Finder" in tool_set and flow.ecosystem and flow.ecosystem != "go": + flags.append(QualityFlag( + name="WRONG_TOOL_ECOSYSTEM", + severity=Severity.HIGH, + detail=f"Function Caller Finder used on {flow.ecosystem} (Go-only tool).", + )) + + seen_tool_calls: set[tuple[str, str]] = set() + duplicate_count = 0 + for step in flow.steps: + if step.span_kind == "TOOL": + key = (step.name, step.input_preview) + if key in seen_tool_calls: + duplicate_count += 1 + else: + seen_tool_calls.add(key) + if duplicate_count > 0: + flags.append(QualityFlag( + name="DUPLICATE_TOOL_CALL", + severity=Severity.MEDIUM, + detail=f"{duplicate_count} duplicate tool call(s) with identical inputs.", + )) + + thought_steps = [s for s in flow.steps if s.llm_role == "thought"] + if thought_steps: + last_thought = thought_steps[-1] + ended_with_finish = last_thought.thought_mode == "finish" + peak_tokens = max(flow.token_progression) if flow.token_progression else 0 + available_prompt_space = CONTEXT_WINDOW_LIMIT - MAX_COMPLETION_TOKENS + + if not ended_with_finish and last_thought.thought_mode == "act": + if peak_tokens > available_prompt_space * 0.65: + flags.append(QualityFlag( + name="CONTEXT_WINDOW_EXCEEDED", + severity=Severity.HIGH, + detail=( + f"Flow ended mid-investigation (last thought: mode='act', peak tokens: {peak_tokens}, " + f"available prompt space: {available_prompt_space}). " + f"Next LLM call likely exceeded {CONTEXT_WINDOW_LIMIT} context window." + ), + )) + else: + flags.append(QualityFlag( + name="INCOMPLETE_FLOW", + severity=Severity.HIGH, + detail=( + f"Flow ended without mode='finish' (last thought: mode='act', " + f"peak tokens: {peak_tokens}). Possible unrecorded exception." + ), + )) + + return flags + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +def print_flow_report(flow: InvestigationFlow): + is_summary = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + tag = " [SUMMARY]" if is_summary else "" + header = f"=== Trace: {flow.trace_id} | Parent: {flow.parent_span_id[:12]}...{tag} ===" + print(header) + if flow.cve_id or flow.ecosystem: + print(f"CVE: {flow.cve_id or 'unknown'} | Ecosystem: {flow.ecosystem or 'unknown'}") + if flow.question: + print(f"Question: {flow.question}") + print(f"Steps: {flow.llm_call_count} LLM calls, {flow.tool_call_count} tool calls") + + if flow.token_progression: + tokens_str = " -> ".join(str(t) for t in flow.token_progression) + print(f"Token usage: {tokens_str}") + + if flow.tools_used: + tool_counts = Counter(flow.tools_used) + tools_str = ", ".join(f"{name} (x{count})" for name, count in tool_counts.items()) + print(f"Tools used: {tools_str}") + reachability_used = set(flow.tools_used) & REACHABILITY_TOOLS + if reachability_used: + print(f"Reachability tools: YES ({', '.join(reachability_used)})") + else: + print("Reachability tools: NO") + else: + print("Tools used: NONE") + + if flow.flags: + flag_strs = [f"[{f.severity.value}] {f.name}: {f.detail}" for f in flow.flags] + print(f"Flags: {len(flow.flags)}") + for fs in flag_strs: + print(f" {fs}") + else: + print("Flags: NONE") + + print() + for step in flow.steps: + error_marker = " *** ERROR ***" if step.has_error else "" + if step.span_kind == "LLM": + role_label = f" [{step.llm_role}]" if step.llm_role else "" + mode_label = f" mode={step.thought_mode}" if step.thought_mode else "" + print(f"--- Step {step.step_num}: LLM{role_label}{mode_label}{error_marker} ---") + if step.token_prompt: + print(f" Prompt tokens: {step.token_prompt} | Completion: {step.token_completion}") + print(f" Output: \"{step.output_preview}\"") + elif step.span_kind == "TOOL": + print(f"--- Step {step.step_num}: TOOL ({step.name}){error_marker} ---") + print(f" Input: {step.input_preview}") + print(f" Output: \"{step.output_preview}\"") + elif step.span_kind == "SUMMARY": + print(f"--- Step {step.step_num}: {step.name} ---") + print(f" Q: {step.input_preview}") + print(f" A: \"{step.output_preview}\"") + elif step.span_kind == "FUNCTION": + print(f"--- Step {step.step_num}: NODE ({step.name}){error_marker} ---") + if step.has_error: + print(f" Exception: {step.error_detail}") + else: + print(f" Output: \"{step.output_preview}\"") + + print() + + +def print_summary(all_flows: list[InvestigationFlow]): + total = len(all_flows) + flagged = [f for f in all_flows if f.flags] + passed = total - len(flagged) + + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"Total investigation flows: {total}") + print(f"Passed (no flags): {passed}") + print(f"Flagged: {len(flagged)}") + + if flagged: + all_flag_counts: Counter = Counter() + for flow in flagged: + for flag in flow.flags: + all_flag_counts[flag.name] += 1 + for flag_name, count in all_flag_counts.most_common(): + print(f" - {flag_name}: {count}") + + if all_flows: + avg_llm = sum(f.llm_call_count for f in all_flows) / total + avg_tool = sum(f.tool_call_count for f in all_flows) / total + all_tokens = [t for f in all_flows for t in f.token_progression] + avg_tokens = sum(all_tokens) / len(all_tokens) if all_tokens else 0 + print(f"\nAvg LLM calls per flow: {avg_llm:.1f}") + print(f"Avg tool calls per flow: {avg_tool:.1f}") + print(f"Avg prompt tokens per LLM call: {avg_tokens:.0f}") + + print() + + +def build_json_report(all_flows: list[InvestigationFlow]) -> dict: + flows_data = [] + for flow in all_flows: + is_summary = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + flows_data.append({ + "trace_id": flow.trace_id, + "job_id": flow.job_id, + "cve_id": flow.cve_id, + "ecosystem": flow.ecosystem, + "question": flow.question, + "summary_only": is_summary, + "llm_call_count": flow.llm_call_count, + "tool_call_count": flow.tool_call_count, + "token_progression": flow.token_progression, + "tools_used": flow.tools_used, + "flags": [asdict(f) for f in flow.flags], + "steps": [asdict(s) for s in flow.steps], + }) + + total = len(all_flows) + flagged_count = sum(1 for f in all_flows if f.flags) + return { + "total_flows": total, + "passed": total - flagged_count, + "flagged": flagged_count, + "flows": flows_data, + } + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="Analyze agent trace files for LLM reasoning quality.") + parser.add_argument("--input", required=True, + help="Path to traces_output/ directory or a single trace JSON file") + parser.add_argument("--json", action="store_true", + help="Output structured JSON report instead of console text") + parser.add_argument("--summary-only", action="store_true", + help="Print only the summary, skip per-flow details") + args = parser.parse_args() + + trace_docs = load_trace_files(args.input) + if not trace_docs: + print("No trace files found.", file=sys.stderr) + sys.exit(1) + + all_flows: list[InvestigationFlow] = [] + for doc in trace_docs: + flows = build_investigation_flows(doc) + for flow in flows: + flow.flags = run_quality_checks(flow) + all_flows.extend(flows) + + if args.json: + report = build_json_report(all_flows) + print(json.dumps(report, indent=2)) + else: + if not args.summary_only: + for flow in all_flows: + print_flow_report(flow) + print_summary(all_flows) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/collect_traces_local.py b/ci/scripts/collect_traces_local.py new file mode 100644 index 000000000..f299c2738 --- /dev/null +++ b/ci/scripts/collect_traces_local.py @@ -0,0 +1,353 @@ +""" +Local trace collector -- copy of collect_and_dispatch_traces.py with per-trace +flattened JSON file output for offline analysis. + +Writes one JSON file per completed trace to traces_output/{job_id}_{trace_id}.json. +Original Tempo/endpoint dispatch is preserved. +""" +from kafka import KafkaConsumer +import argparse +import json +import os +from HttpProvider import HttpProvider, AuthType +from datetime import datetime +from typing import Any +from pydantic import BaseModel, Field, constr, RootModel +import signal +import time +from kafka.errors import NoBrokersAvailable + +TRACE_VERSION = 1 + +DEFAULT_URL = "http://localhost:8080" +DEFAULT_AUTH_TYPE = "none" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False +TEMPO_URL = "http://localhost:8080" + +KAFKA_TOPIC = 'otel-traces' +KAFKA_BROKER = 'localhost:9092' +TRACES_OUTPUT_DIR = 'traces_output' +FLUSH_GRACE_PERIOD_SEC = 15 + + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + + +class Trace(BaseModel): + format_version: int | None = None + job_id: constr(min_length=1) = Field( + ..., description='The job run id in which the trace and spans were instrumented' + ) + execution_start_timestamp: LocalDateTime | None = None + trace_id: constr(min_length=1) = Field( + ..., + description='Trace id grouping all spans of sub-tasks in one agent stage', + ) + span_id: constr(min_length=1) = Field( + ..., description='The id corresponding to one sub task inside one agent stage.' + ) + span_payload: dict[str, Any] | None = Field( + None, + description='Span payload with metadata and instrumentation data', + ) + + +class TracesContainer(RootModel[list[Trace]]): + pass + + +def _otlp_str(val) -> str: + """Extract plain string from an OTLP attribute value like {"stringValue": "..."}.""" + if isinstance(val, dict): + return val.get("stringValue", "") + return str(val) if val else "" + + +def _otlp_int(val) -> int: + """Extract integer from an OTLP attribute value like {"intValue": "123"}.""" + if isinstance(val, dict): + raw = val.get("intValue", 0) + try: + return int(raw) + except (ValueError, TypeError): + return 0 + try: + return int(val) + except (ValueError, TypeError): + return 0 + + +def get_metadata(span: dict) -> dict: + return { + 'trace_id': span.get('traceId'), + 'span_id': span.get('spanId'), + 'parent_span_id': span.get('parentSpanId'), + 'name': span.get('name'), + 'start_time': span.get('startTimeUnixNano'), + 'end_time': span.get('endTimeUnixNano'), + 'status': span.get('status'), + } + + +def find_job_id(attributes: dict) -> str: + try: + input_value = attributes.get("input.value") + str_json = input_value.get("stringValue") + scan = json.loads(str_json) + if "scan" in scan: + return scan["scan"]["id"] + if "input" in scan: + return scan["input"]["scan"]["id"] + else: + print("Warning: Scan not found in attributes") + return "" + except Exception as e: + print(f"Error finding job id (scan id): {e}") + return "" + + +def flatten_span(metadata: dict, attrs: dict) -> dict: + """Convert a raw OTLP span (metadata + attributes) into a flat dict.""" + return { + "span_id": metadata.get("span_id", ""), + "parent_span_id": metadata.get("parent_span_id", ""), + "name": metadata.get("name", ""), + "start_time": metadata.get("start_time", ""), + "end_time": metadata.get("end_time", ""), + "span_kind": _otlp_str(attrs.get("nat.span.kind")), + "function_name": _otlp_str(attrs.get("nat.function.name")), + "input_value": _otlp_str(attrs.get("input.value")), + "output_value": _otlp_str(attrs.get("output.value")), + "token_prompt": _otlp_int(attrs.get("llm.token_count.prompt")), + "token_completion": _otlp_int(attrs.get("llm.token_count.completion")), + "token_total": _otlp_int(attrs.get("llm.token_count.total")), + } + + +def write_trace_file(trace_id: str, job_id: str, flat_spans: list[dict], output_dir: str): + """Write a completed trace as a single JSON file, spans sorted by start_time.""" + os.makedirs(output_dir, exist_ok=True) + flat_spans.sort(key=lambda s: int(s.get("start_time") or 0)) + trace_doc = { + "trace_id": trace_id, + "job_id": job_id, + "completed_at": datetime.utcnow().isoformat(), + "spans": flat_spans, + } + filename = f"{job_id}_{trace_id}.json" + filepath = os.path.join(output_dir, filename) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(trace_doc, f, indent=2) + print(f"Wrote trace file: {filepath} ({len(flat_spans)} spans)") + + +def process_message(http_provider, tempo_provider, traces_table, flat_spans_table, + map_trace_id_to_job_id, pending_flushes, data, output_dir): + # Forward original trace to Tempo + try: + for rs in data.get('resourceSpans', []): + attrs = rs.get('resource', {}).get('attributes', []) + if not any(a['key'] == 'service.name' for a in attrs): + attrs.append({"key": "service.name", "value": {"stringValue": "cve-agent"}}) + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + if span.get('name') == "": + span['parentSpanId'] = "" + #tempo_status = tempo_provider.send_post(json.dumps(data)) + except Exception as e: + print(f"Failed to forward trace to Tempo: {e}") + + for rs in data.get('resourceSpans', []): + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + metadata = get_metadata(span) + attrs = { + a['key']: a.get('value', {}) + for a in span.get('attributes', []) + } + trace_id = metadata.get('trace_id') + if trace_id not in traces_table: + print("add trace_id to traces_table: ", trace_id) + traces_table[trace_id] = [] + flat_spans_table[trace_id] = [] + if trace_id in map_trace_id_to_job_id: + job_id = map_trace_id_to_job_id[trace_id] + else: + job_id = find_job_id(attrs) + if len(job_id) > 0: + trace_collection = traces_table.get(trace_id, []) + if trace_id not in map_trace_id_to_job_id: + map_trace_id_to_job_id[trace_id] = job_id + print(f"add trace_id to map_trace_id_to_job_id: {trace_id} --> {job_id}") + + dict_payload = {"metadata": metadata, "attributes": attrs} + trace_obj = Trace( + format_version=TRACE_VERSION, + job_id=job_id, + execution_start_timestamp=LocalDateTime( + datetime.fromtimestamp(int(metadata.get('start_time', "0")) / 1e9) + ), + trace_id=metadata.get('trace_id'), + span_id=metadata.get('span_id'), + span_payload=dict_payload, + ) + trace_collection.append(trace_obj) + traces_table[trace_id] = trace_collection + + flat_spans_table.setdefault(trace_id, []).append( + flatten_span(metadata, attrs) + ) + + span_name = metadata.get('name') + func_name = _otlp_str(attrs.get("nat.function.name")) + if span_name == "" or func_name == "agent_finish": + if trace_id not in traces_table: + print(f"Trace {trace_id} already flushed, skipping duplicate trigger.") + continue + if trace_id not in pending_flushes: + trigger = span_name if span_name == "" else f"agent_finish (func={func_name})" + pending_flushes[trace_id] = time.time() + print(f"Flush scheduled for trace {trace_id} (trigger: {trigger}, " + f"grace period: {FLUSH_GRACE_PERIOD_SEC}s, " + f"spans so far: {len(flat_spans_table.get(trace_id, []))})") + + +def flush_pending_traces(pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, force=False): + """Flush traces whose grace period has expired (or all if force=True).""" + now = time.time() + flushed = [] + for trace_id, trigger_time in list(pending_flushes.items()): + if not force and (now - trigger_time) < FLUSH_GRACE_PERIOD_SEC: + continue + if trace_id not in traces_table: + flushed.append(trace_id) + continue + job_id = map_trace_id_to_job_id.get(trace_id, "unknown") + waited = now - trigger_time + span_count = len(flat_spans_table.get(trace_id, [])) + print(f"Flushing trace {trace_id} (waited {waited:.1f}s, {span_count} spans)") + + write_trace_file( + trace_id, job_id, + flat_spans_table.get(trace_id, []), + output_dir, + ) + + del traces_table[trace_id] + del flat_spans_table[trace_id] + flushed.append(trace_id) + for trace_id in flushed: + del pending_flushes[trace_id] + + +def run_worker( + url: str = DEFAULT_URL, + auth_type: str = DEFAULT_AUTH_TYPE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, + output_dir: str = TRACES_OUTPUT_DIR, +): + stop_triggered = False + + def signal_handler(sig, frame): + nonlocal stop_triggered + stop_triggered = True + + signal.signal(signal.SIGTERM, signal_handler) + + print(f"Waiting for Kafka at {KAFKA_BROKER}...") + consumer = None + while consumer is None: + try: + consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + group_id='cve-file-writer-group', + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + request_timeout_ms=20000, + session_timeout_ms=10000, + ) + print("Successfully connected to Redpanda!") + except NoBrokersAvailable: + print("Kafka not ready yet... retrying in 2 seconds.") + time.sleep(2) + + print(f"Worker Active. Writing per-trace files to: {output_dir}/ (grace period: {FLUSH_GRACE_PERIOD_SEC}s)") + os.makedirs(output_dir, exist_ok=True) + traces_table = {} + flat_spans_table = {} + map_trace_id_to_job_id = {} + pending_flushes = {} + + endpoint_url = url + "/api/v1/traces" + EndpointAuthType = AuthType.BEARER + if auth_type != DEFAULT_AUTH_TYPE: + EndpointAuthType = AuthType.NONE + http_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False, + ) + tempo_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False, + ) + + try: + while not stop_triggered: + messages = consumer.poll(timeout_ms=1000) + for tp, records in messages.items(): + for message in records: + data = message.value + process_message( + http_provider, tempo_provider, + traces_table, flat_spans_table, + map_trace_id_to_job_id, pending_flushes, + data, output_dir, + ) + flush_pending_traces( + pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, + ) + finally: + if pending_flushes: + print(f"\nFlushing {len(pending_flushes)} pending traces on shutdown...") + flush_pending_traces( + pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, force=True, + ) + consumer.close() + print("Stopping worker...") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Collect traces from Kafka, dispatch to endpoint, and write per-trace JSON files for analysis." + ) + parser.add_argument("--url", default=DEFAULT_URL, help="Dispatch endpoint URL") + parser.add_argument("--auth-type", default=DEFAULT_AUTH_TYPE, help="Auth type (Bearer or None)") + parser.add_argument("--token-path", default=DEFAULT_TOKEN_PATH, help="Path to auth token file") + parser.add_argument("--verify-path", default=DEFAULT_VERIFY_PATH, help="Path to TLS CA cert") + parser.add_argument("--enable-verify", action="store_true", default=DEFAULT_ENABLE_VERIFY) + parser.add_argument("--output-dir", default=TRACES_OUTPUT_DIR, + help="Directory for per-trace JSON files (default: traces_output)") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_worker( + url=args.url, + auth_type=args.auth_type, + token_path=args.token_path, + verify_path=args.verify_path, + enable_verify=args.enable_verify, + output_dir=args.output_dir, + ) diff --git a/kustomize/README.md b/kustomize/README.md index a61731f19..0fd2d460b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -324,13 +324,46 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -12. On Non Production environments, the Swagger-UI endpoint is enabled, and can be configured for testing on environment +12. **(Optional) Enable OAuth for the ExploitIQ MCP Server.** If you want MCP clients (Claude Code, Cursor, etc.) to authenticate via OpenShift OAuth, create an `OAuthClient` CR for the MCP server: + +```bash +oc create -f - < | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +To restrict MCP access to specific OpenShift groups, edit the `EXPLOITIQ_ALLOWED_GROUPS` value in `base/exploitiq_mcp_server.yaml` (defaults to `exploitiq-users`): +```yaml +- name: EXPLOITIQ_ALLOWED_GROUPS + value: "exploitiq-users,security-team" +``` +When set, group membership is checked during the OAuth callback — unauthorized users see a 403 error in the browser before a token is issued, instead of a misleading "Authentication successful" followed by a connection failure. A second group check runs as Express middleware on every `/mcp` request as defense-in-depth. To allow any authenticated user, remove the env var entirely. + +13. On Non Production environments, the Swagger-UI endpoint is enabled, and can be configured for testing on environment that way: ```shell oc set env deployment -l component=exploit-iq-client QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') ``` -13. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: +14. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: ```shell DEPLOYMENT_VARIANT_NAME=remote-nim-all @@ -430,22 +463,50 @@ export HTTP_ROUTE=http://$(oc get route exploit-iq-client -o jsonpath='{.spec.ho oc patch oauthclient exploit-iq-client -p '{"redirectURIs":["'$HTTP_ROUTE'","'$HTTPS_ROUTE'"]}' ``` -10. Deploy Self hosted LLM for the automation tests ( Integration tests and Confusion matrix runner): +10. **(Optional) Enable OAuth for the ExploitIQ MCP Server.** Create an `OAuthClient` CR for the MCP server: + +```bash +oc create -f - < + + + + red-hat + + + red-hat-ga + https://maven.repository.redhat.com/ga + + + jboss-public-repository-group + JBoss Public Maven Repository Group + https://repository.jboss.org/nexus/content/groups/public/ + default + + true + never + + + true + never + + + + + + red-hat-ga + https://maven.repository.redhat.com/ga + + true + + + false + + + + + + + red-hat + + + + + + maven-default-http-blocker + Disabled default HTTP blocker + + + __no_such_repo_id__ + + + https://repo1.maven.org/maven2 + + + + \ No newline at end of file diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 45dabca2c..fa5b89769 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -83,6 +83,8 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Function Library Version Finder: + _type: calling_function_library_version_finder cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -94,6 +96,7 @@ functions: - Call Chain Analyzer - Function Caller Finder - Function Locator + - Function Library Version Finder max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/otel-config.yaml b/otel-config.yaml index c4de090a8..a7c89cc8e 100644 --- a/otel-config.yaml +++ b/otel-config.yaml @@ -16,8 +16,9 @@ exporters: # 2. Export to Redpanda (Kafka) kafka: brokers: ["redpanda:29092"] # Use the internal Docker port - topic: "otel-traces" - encoding: otlp_json # This makes it MUCH easier for your Python script to read + traces: + topic: "otel-traces" + encoding: otlp_json # This makes it MUCH easier for your Python script to read service: pipelines: diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index e0eced325..d61586eda 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -1,13 +1,14 @@ import re import time from collections import defaultdict, deque +from dataclasses import dataclass, field from pathlib import Path from typing import List -import copy + from langchain_core.documents import Document -from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, PARENTS_INDEX, \ - calculate_hashable_string_for_function, EXCLUSIONS_INDEX +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, \ + calculate_hashable_string_for_function from exploit_iq_commons.logging.loggers_factory import LoggingFactory from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( @@ -17,6 +18,17 @@ logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +@dataclass +class _SearchCtx: + """Per-search mutable state, created fresh for each get_relevant_documents call.""" + found_path: bool = False + exclusions: defaultdict = field(default_factory=lambda: defaultdict(list)) + last_visited_parent_package_indexes: dict = field(default_factory=dict) + last_visited: dict = field(default_factory=dict) + tree_additions: dict = field(default_factory=dict) + + def is_likely_macro_block(segment: str) -> bool: lines = segment.strip().splitlines() if not lines: @@ -50,14 +62,6 @@ def is_likely_macro_block(segment: str) -> bool: return False -class SessionContext: - """Per-call mutable state, isolating concurrent get_relevant_documents calls.""" - def __init__(self, tree_dict: dict[str, list[str]]): - self.last_visited = dict() - self.tree_dict = copy.deepcopy(tree_dict) - self.found_path = False - - class ChainOfCallsRetriever(ChainOfCallsRetrieverBase): """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being called from the application code base or not. @@ -95,9 +99,7 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) for package, parents in tree.items(): parents.extend([package]) - self.tree_dict[package] = list() - self.tree_dict[package].append(parents) - self.tree_dict[package].append([]) + self.tree_dict[package] = parents self.supported_packages = list(self.tree_dict.keys()) logger.debug("Chain of Calls Retriever - populating functions documents") @@ -145,6 +147,14 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat if not self.language_parser.is_search_algo_dfs(): self.sort_docs = self.__group_docs_by_pkg() + def _get_parents(self, package: str, ctx: _SearchCtx) -> list | None: + """Look up parents for a package, checking both the immutable tree_dict + and any packages added during the current search (ctx.tree_additions).""" + parents = self.tree_dict.get(package) + if parents is not None: + return parents + return ctx.tree_additions.get(package) + def __group_docs_by_pkg(self) -> dict[str, list[Document]]: """ sort docs by pkg name @@ -160,34 +170,37 @@ def __group_docs_by_pkg(self) -> dict[str, list[Document]]: return sort_docs def __find_caller_function_dfs(self, document_function: Document, function_package: str, - session_context: SessionContext) -> Document: + ctx: _SearchCtx) -> Document: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function - :param session_context: per-call session state for thread safety + :param ctx: per-search mutable state :return: a single document of a function that is calling document_function, or None if not found """ package_names = self.language_parser.get_package_names(document_function) direct_parents = list() # gets list of all direct parents of function for package_name in package_names: - list_of_packages = session_context.tree_dict.get(package_name) - if list_of_packages: - direct_parents.extend(list_of_packages[PARENTS_INDEX]) - # Add same package itself to search path. + parents = self._get_parents(package_name, ctx) + if parents: + direct_parents.extend(parents) + # Add same package itself to search path. # direct_parents.extend([function_package]) - # gets list of documents to search in only from parents of function' package. + # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) if function_name_to_search == self.language_parser.get_constructor_method_name(): function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() + last_visited_package_index = (ctx.last_visited_parent_package_indexes + .get(calculate_hashable_string_for_function(function_file_name, + function_name_to_search), 0)) + package_exclusions = ctx.exclusions[function_package] # Search for caller functions only at parents according to dependency tree. - package_exclusions = session_context.tree_dict.get(function_package)[EXCLUSIONS_INDEX] - for package in direct_parents: + for package in direct_parents[last_visited_package_index:]: sources_location_packages = True - if session_context.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: + if self._get_parents(package, ctx)[0] == ROOT_LEVEL_SENTINEL: sources_location_packages = False possible_docs = self.get_possible_docs(function_name_to_search, package, @@ -195,6 +208,7 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa sources_location_packages, frozenset(), dict()) + # Collect all potential caller functions for doc in self.get_functions_for_package(package_name=package, documents=possible_docs, @@ -202,7 +216,7 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa function_to_search=function_name_to_search, callee_function_file_name=function_file_name): relevant_docs_to_search_in.append(doc) - # Perform the search only on the subset of potential caller functions + # Perform the search only on the subset of potential caller functions for doc in relevant_docs_to_search_in: function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, callee_function_name= @@ -227,6 +241,9 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. if function_is_being_called: package_exclusions.append(doc) + # update index of last scanned package for backtracking + # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) + # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index return doc # If didn't find a matching caller function document, returns None. @@ -267,22 +284,25 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] def __find_caller_functions_bfs(self, document_function: Document, function_package: str, - session_context: SessionContext) -> List[Document]: + ctx: _SearchCtx) -> List[Document]: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function - :param session_context: per-call session state for thread safety + :param ctx: per-search mutable state :return: a list of documents of functions that are calling document_function """ total_start = time.time() direct_parents = list() + # gets list of all direct parents of function - list_of_packages = session_context.tree_dict.get(function_package) - if list_of_packages is not None: - direct_parents.extend(list_of_packages[0]) - # gets list of documents to search in only from parents of function' package. + parents = self._get_parents(function_package, ctx) + if parents is not None: + direct_parents.extend(parents) + # Add same package itself to search path. + # direct_parents.extend([function_package]) + # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() @@ -293,6 +313,15 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack for package in direct_parents: pkg_docs = self.sort_docs[package] for doc in pkg_docs: + # for doc in self.documents: + # is_doc_in_pkg = False + # for package in direct_parents: + # if self.language_parser.is_a_package(package, doc): + # is_doc_in_pkg = True + # break + # if not is_doc_in_pkg: + # continue + file_name = doc.metadata.get('source') if doc.metadata.get('state') == "invalid": continue @@ -300,6 +329,9 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack # check for same doc if (function_name_to_search == func_name) and (file_name == function_file_name): continue + # same function name different files ? + # if (function_name_to_search == func_name): + # logger.debug(f"same func name {function_name_to_search}") if func_name == "main": file_path = Path(function_file_name) @@ -308,7 +340,7 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack if self.language_parser.dir_name_for_3rd_party_packages() in path_parts: continue if doc.page_content.__contains__(f"{function_name_to_search}("): - last_visited = (session_context.last_visited.get( + last_visited = (ctx.last_visited.get( calculate_hashable_string_for_function(file_name, func_name), 0)) if last_visited == 0: found = self.language_parser.search_for_called_function( @@ -352,8 +384,8 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, - current_package_name: str, - session_context: SessionContext) -> tuple[List[Document], bool]: + current_package_name: str, ctx: _SearchCtx) -> tuple[List[Document], bool]: + # main loop. file_counter = 0 q = deque() q.append(target_function_doc) @@ -369,10 +401,10 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct function_file = target_doc.metadata.get('source') hashed_value = calculate_hashable_string_for_function(function_file, function_name) - if hashed_value in session_context.last_visited: + if hashed_value in ctx.last_visited: continue - session_context.last_visited[hashed_value] = 1 + ctx.last_visited[hashed_value] = 1 logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", file_counter, target_doc.metadata['source'], function_name, target_pkg, len(q)) @@ -381,7 +413,7 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct sub_start = time.time() found_documents = self.__find_caller_functions_bfs(document_function=target_doc, function_package=target_pkg, - session_context=session_context) + ctx=ctx) sub_elapsed = time.time() - sub_start logger.debug(f"[PROFILE] __find_caller_functions took {sub_elapsed:.3f} seconds") # If found, then add it to path @@ -390,7 +422,7 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(doc_candidate): matching_documents.append(doc_candidate) - session_context.found_path = True + ctx.found_path = True break # Otherwise, we continue to search for callers for the current found function, in order to extend # the chain of calls and potentially find a path from application to the vulnerable @@ -398,41 +430,41 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct else: q.append(doc_candidate) - if session_context.found_path: + if ctx.found_path: break loop_elapsed = time.time() - loop_start logger.debug(f"[PROFILE] Main loop in get_relevant_C_documents took {loop_elapsed:.3f} seconds") # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was # found or not. - logger.debug("get_relevant_documents: result %s", session_context.found_path) + logger.debug("get_relevant_documents: result %s", ctx.found_path) logger.debug("get_relevant_documents: docs %s", matching_documents) - return matching_documents, session_context.found_path + return matching_documents, ctx.found_path def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, - current_package_name: str, - session_context: SessionContext) -> tuple[List[Document], bool]: + current_package_name: str, ctx: _SearchCtx) -> tuple[List[Document], bool]: """Execute depth-first search with backtracking strategy.""" end_loop = False + # main loop. while not end_loop: # Find a caller function and containing package in the dependency tree according to hierarchy found_document = self.__find_caller_function_dfs(document_function=target_function_doc, function_package=current_package_name, - session_context=session_context) + ctx=ctx) # If found, then add it to path if found_document: matching_documents.append(found_document) # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(found_document): end_loop = True - session_context.found_path = True - # Otherwise, we continue to search for callers for the current found function, in order to extend - # the chain of calls and potentially find a path from application to the vulnerable - # function in input package + ctx.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package else: target_function_doc = found_document # extract package name from function document - current_package_name = self.__determine_doc_package_name(target_function_doc, session_context) + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) else: # end loop because didn't find a caller for initial function if len(matching_documents) == 1: @@ -442,12 +474,12 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio else: dead_end_node = matching_documents.pop() # Excludes dead end function node from future searches, as it led to nowhere. - session_context.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + ctx.exclusions[current_package_name].append(dead_end_node) target_function_doc = matching_documents[-1] - current_package_name = self.__determine_doc_package_name(target_function_doc, session_context) - # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. - return matching_documents, session_context.found_path + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, ctx.found_path # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). @@ -455,6 +487,7 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio # calls from application to input function in the input package. def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" + ctx = _SearchCtx() query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() (package_name, function) = tuple(query.split(",")) class_name = None @@ -463,15 +496,11 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: class_name, function = function.split(splitters[0]) found_package = False matching_documents = [] - session_context = SessionContext(self.tree_dict) - - for dependency in session_context.tree_dict.values(): - dependency[EXCLUSIONS_INDEX] = [] standard_libs_cache = StandardLibraryCache.get_instance() # If it's a standard library package, then skip checking the package in dependency tree. if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): # Check if input package is in dependency tree - for package in session_context.tree_dict: + for package in self.tree_dict: if self.language_parser.is_same_package(package_name, package): package_name = package found_package = True @@ -480,14 +509,14 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: if found_package: target_function_doc = self.__find_initial_function(function, package_name=package_name, documents=self.documents, - class_name=class_name, - session_context=session_context) + ctx=ctx, + class_name=class_name) if not target_function_doc and self.language_parser.get_constructor_method_name(): target_function_doc = self.__find_initial_function(function_name=self.language_parser.get_constructor_method_name(), package_name=package_name, documents=self.documents, - class_name=function, - session_context=session_context) + ctx=ctx, + class_name=function) # If not, there is a chance that the package is some standard library in the ecosystem. else: @@ -495,7 +524,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # vulnerable function in standard lib package of language. page_content = self.language_parser.get_dummy_function(function) if page_content is None: - return matching_documents, session_context.found_path + return matching_documents, ctx.found_path if class_name: page_content = page_content + f'\n{self.language_parser.get_comment_line_notation()}(class: {class_name})' target_function_doc = Document(page_content=page_content @@ -504,17 +533,18 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, re.escape(package_name)) - root_package = [key for (key, value) in session_context.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] + root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() + # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. parents = list({self.language_parser.get_package_names(doc)[1] for doc in importing_docs if doc.metadata['source'].startswith( prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[1] - in session_context.tree_dict.keys()}) + in self.tree_dict.keys()}) for doc in importing_docs: if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): parents.append(root_package[0]) break - session_context.tree_dict[package_name] = [parents, []] + ctx.tree_additions[package_name] = parents end_loop = False current_package_name = package_name # If an initial document (that represents the vulnerable input function in the input package) was created @@ -527,26 +557,26 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: end_loop = True logger.error("Cannot find initial function=%s, in package=%s", function, package_name) if end_loop: - return matching_documents, session_context.found_path + return matching_documents, ctx.found_path if self.language_parser.is_search_algo_dfs(): - matching_documents, session_context.found_path = self._depth_first_search( - matching_documents, target_function_doc, current_package_name, session_context) + matching_documents, ctx.found_path = self._depth_first_search( + matching_documents, target_function_doc, current_package_name, ctx) else: - matching_documents, session_context.found_path = self._breadth_first_search( - matching_documents, target_function_doc, current_package_name, session_context) + matching_documents, ctx.found_path = self._breadth_first_search( + matching_documents, target_function_doc, current_package_name, ctx) - return matching_documents, session_context.found_path + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, ctx.found_path - def __determine_doc_package_name(self, target_function_doc, - session_context: SessionContext): + def __determine_doc_package_name(self, target_function_doc, ctx: _SearchCtx): return [package_name for package_name in self.language_parser.get_package_names(target_function_doc) - if session_context.tree_dict.get(package_name, None) is not None][0] + if package_name in self.tree_dict or package_name in ctx.tree_additions][0] def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], - class_name: str = None, *, - session_context: SessionContext) -> Document: + ctx: _SearchCtx, class_name: str = None) -> Document: if self.language_parser.is_search_algo_dfs(): pkg_docs = documents @@ -558,19 +588,18 @@ def __find_initial_function(self, function_name: str, package_name: str, documen relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] - package_exclusions = session_context.tree_dict.get(package_name)[EXCLUSIONS_INDEX] + package_exclusions = ctx.exclusions[package_name] + #for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): from itertools import chain for document in chain( self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=True), self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=False), ): - # document_function_calls_input_function = True if function_name.lower() == self.language_parser.get_function_name(document).lower(): # if language_parser.search_for_called_function(document, callee_function=function_name): package_exclusions.append(document) return document - if self.language_parser.create_dummy_for_standard_lib(package_name): dummy_function_doc = Document(page_content=f"func {function_name + '()' + '{}'}", metadata={ diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py index 2900e59cb..f3cf0cd8f 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py @@ -30,7 +30,6 @@ def is_function_callable(document: Document, language_parser: LanguageFunctionsP document.metadata['source'].lower() == callee_function_file_name.lower()) class ChainOfCallsRetrieverBase(ABC): - last_visited_parent_package_indexes: Optional[dict] documents: Optional[List[Document]] documents_of_full_sources: Optional[dict] documents_of_types: Optional[list] @@ -41,8 +40,6 @@ class ChainOfCallsRetrieverBase(ABC): supported_packages: Optional[list[str]] ecosystem: Optional[Ecosystem] manifest_path: Optional[Path] - package_name: Optional[str] - found_path: Optional[bool] types_classes_fields_mapping: Optional[dict[tuple, list[tuple]]] functions_local_variables_index: Optional[dict[str, dict]] sort_docs: dict[str, list[Document]] diff --git a/src/exploit_iq_commons/utils/data_utils.py b/src/exploit_iq_commons/utils/data_utils.py index ee2e0a2ee..771d9e6b2 100644 --- a/src/exploit_iq_commons/utils/data_utils.py +++ b/src/exploit_iq_commons/utils/data_utils.py @@ -121,6 +121,19 @@ def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: with open(cached_documents_path, 'wb') as doc_file: # open a text file pickle.dump(documents, doc_file) +def cache_exists(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> bool: + """Check if a pickle cache file exists without loading it into memory.""" + cached_documents_path = \ + (f"{base_pickle_dir}/" + f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" + f"{repo_ref}") + + if documents_name: + cached_documents_path += "-" + documents_name + + return os.path.isfile(cached_documents_path) + + def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> tuple: if not os.path.exists(base_pickle_dir): os.makedirs(base_pickle_dir) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 9afc4273e..ef3d6225c 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -74,6 +74,41 @@ class Ecosystem(str, Enum): "configure": Ecosystem.C_CPP } + +def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None: + """Detect the programming language ecosystem from manifest files in a git repository. + + Checks for manifest files in priority order: Go, Python, JavaScript, Java, C/C++. + For C/C++, also verifies that actual source files exist. + Returns None if no recognized manifest is found. + """ + if os.path.isfile(git_repo_path / GOLANG_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] + if ( + os.path.isfile(git_repo_path / PYTHON_MANIFEST) + or os.path.isfile(git_repo_path / PYPROJECT_TOML) + or os.path.isfile(git_repo_path / SETUP_PY) + ): + return MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] + if os.path.isfile(git_repo_path / JS_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[JS_MANIFEST] + if os.path.isfile(git_repo_path / JAVA_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[JAVA_MANIFEST] + c_candidates = [ + git_repo_path / C_CPLUSPLUS_MANIFEST_1, + git_repo_path / C_CPLUSPLUS_MANIFEST_2, + git_repo_path / C_CPLUSPLUS_MANIFEST_3, + git_repo_path / C_CPLUSPLUS_MANIFEST_4, + git_repo_path / "auto" / C_CPLUSPLUS_MANIFEST_4, + ] + if any(p.is_file() for p in c_candidates): + for root, dirs, files in os.walk(git_repo_path): + dirs[:] = [d for d in dirs if not d.startswith('.')] + if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files): + return MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] + return None + + def run_command(cmd: str) -> str: try: result = subprocess.run(cmd, shell=True, capture_output=True, text=True) @@ -851,45 +886,118 @@ class JavaDependencyTreeBuilder(DependencyTreeBuilder): def __init__(self, query: str): self._query = query + def __check_file_exists(self, dir_path: str | Path, filename: str) -> bool: + """ + Return True iff `filename` exists as a regular file directly under `dir_path`. + """ + p = Path(dir_path) / filename + return p.is_file() + def install_dependencies(self, manifest_path: Path): + mvn_command = "mvn" + settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml') source_path = "dependencies-sources" - subprocess.run(["mvn", "dependency:copy-dependencies", "-Dclassifier=sources", - f"-DoutputDirectory={source_path}"], cwd=manifest_path) + + if self.__check_file_exists(manifest_path, "mvnw"): + mvn_command = "./mvnw" + + process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", + "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + + if process_object.returncode > 0: + process_object = subprocess.run([mvn_command, "clean", "install", + "-DskipTests", "-s", settings_path], cwd=manifest_path) + if process_object.returncode > 0: + formatted_error_msg = ( + f"Failed to build project" + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}" + ) + raise Exception(formatted_error_msg) + + process_object = subprocess.run([mvn_command, "-s", settings_path, + "dependency:copy-dependencies", "-Dclassifier=sources", + "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + + if process_object.returncode > 0: + formatted_error_msg = ( + f"Failed to install dependencies" + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}" + ) + raise Exception(formatted_error_msg) full_source_path = manifest_path / source_path for jar in full_source_path.glob("*-sources.jar"): - if jar.stat().st_size > 0: - dest = full_source_path / jar.stem # folder named after jar - - if not dest.exists(): - dest.mkdir(exist_ok=True) - result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) - if result.returncode != 0: - logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) - else: - logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) - + if jar.stat().st_size > 0: + dest = full_source_path / jar.stem # folder named after jar + + if not dest.exists(): + dest.mkdir(exist_ok=True) + result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) + if result.returncode != 0: + logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) + else: + logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + mvn_command = "mvn" + settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml") dependency_file = manifest_path / "dependency_tree.txt" - package_name = self._query.split(',')[0] + package_name = self._query.split(",")[0] + + if self.__check_file_exists(manifest_path, "mvnw"): + mvn_command = "./mvnw" if is_maven_gav(package_name): - with open(dependency_file, "w") as f: - subprocess.run(["mvn", "dependency:tree", - f"-Dincludes={add_missing_jar_string(package_name)}", - "-Dverbose"], cwd=manifest_path, stdout=f, check=True) + subprocess.run( + [ + mvn_command, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-s", settings_path, + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DoutputFileName=dependency_tree.txt", + "-DclasspathScope=runtime", + f"-DtargetIncludes={add_missing_jar_string(package_name)}", + ], + cwd=manifest_path, + check=True, + ) else: - with open(dependency_file, "w") as f: - subprocess.run(["mvn", "dependency:tree", - "-Dverbose"], cwd=manifest_path, stdout=f, check=True) + subprocess.run( + [ + mvn_command, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-s", settings_path, + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DclasspathScope=runtime", + "-DoutputFileName=dependency_tree.txt", + ], + cwd=manifest_path, + check=True, + ) with dependency_file.open("r", encoding="utf-8") as f: lines = f.readlines() parent, graph = self.__build_upside_down_dependency_graph(lines) - # Mark the top level for + # Mark *all* roots, not just the first one. + # A "root" is any node with no parents in the computed parent-chain list. + # This preserves old single-root behavior and fixes multi-root / multi-parent trees. + roots = [node for node, parents in graph.items() if not parents] + for r in roots: + graph[r] = [ROOT_LEVEL_SENTINEL] + + # Backward-compatible: keep the old behavior too (harmless if already set above) graph[parent] = [ROOT_LEVEL_SENTINEL] return graph @@ -899,8 +1007,7 @@ def __build_upside_down_dependency_graph( ) -> Tuple[str, Dict[str, List[str]]]: root: str = "" stack: List[str] = [] - # coord -> set of direct parents (possibly multiple) - graph_sets: Dict[str, set] = {} + graph_sets: Dict[str, set[str]] = {} # coord -> set of direct parents for line in dependency_lines: depth, coord = self.__parse_dependency_line(line) @@ -908,7 +1015,8 @@ def __build_upside_down_dependency_graph( continue if depth == 0: - # start (or restart) a root line + # depgraph aggregate can emit multiple top-level roots. Keep the first as "root" + # for backward compatibility, but still record others as separate roots in graph_sets. if not root: root = coord stack = [coord] @@ -921,104 +1029,144 @@ def __build_upside_down_dependency_graph( parent = stack[-1] if stack else None if parent is not None: - graph_sets.setdefault(coord, set()).add(parent) + graph_sets.setdefault(coord, set()).add(parent) # supports multiple direct parents graph_sets.setdefault(parent, set()) else: graph_sets.setdefault(coord, set()) stack.append(coord) - # ---------- second phase: all parents (direct + transitive) without duplicates ---------- - def build_parent_chain(node: str) -> List[str]: """ - For a given coord, return a flat list of *all* parents reachable - via any path up to the root, with no duplicates. - - Order: breadth-first from nearest parents outward. + Return a flat list of all parents reachable via any path, no duplicates. + Deterministic BFS order: nearest parents outward. """ result: List[str] = [] seen: set[str] = set() - q = deque(graph_sets.get(node, ())) + q = deque(sorted(graph_sets.get(node, ()))) while q: - parent = q.popleft() - if parent in seen: + p = q.popleft() + if p in seen: continue - seen.add(parent) - result.append(parent) + seen.add(p) + result.append(p) - # enqueue this parent's parents - for gp in graph_sets.get(parent, ()): + for gp in sorted(graph_sets.get(p, ())): if gp not in seen: q.append(gp) return result - graph: Dict[str, List[str]] = { - coord: build_parent_chain(coord) for coord in graph_sets.keys() - } - + graph: Dict[str, List[str]] = {coord: build_parent_chain(coord) for coord in graph_sets.keys()} return root, graph def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[str]]: - if not line.startswith("[INFO]"): + """ + Parse one dependency line from depgraph's graphFormat=text output. + + Expected depgraph token shape (after indentation/branch prefix): + groupId:artifactId:version:type:scope + Example from your file: + org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile :contentReference[oaicite:3]{index=3} + + We return (depth, "groupId:artifactId:version") and ignore type/scope/optional marker. + + Also tolerates Maven log prefixes like "[INFO] " if they appear. + """ + raw = (line or "").rstrip("\n") + if not raw.strip(): return None, None - # Keep indentation blocks; Maven prints exactly one space after "[INFO]" - s = line[6:] - if s.startswith(" "): - s = s[1:] + # If Maven stdout and depgraph output got mixed, you may see mid-line "[INFO]" injection. + # Those lines are not safely recoverable as dependency tokens. + if "[INFO]" in raw and not raw.lstrip().startswith("[INFO]"): + return None, None - # Skip non-tree lines early - if (not s - or s.startswith(("---", "BUILD", "Scanning", "Finished", "Total time")) - or ":" not in s): + s = raw.lstrip() + + # Strip Maven log prefix if present + if s.startswith("[INFO]"): + s = s[6:].lstrip() + + # Skip headers and build noise + if ( + not s + or "Dependency graph:" in s + or s.startswith(("---", "BUILD", "Reactor Summary", "Total time", "Finished at", "Scanning")) + or s.startswith("[") # other log levels like [WARNING], [ERROR], etc. + or ":" not in s + ): return None, None - # indent blocks ("| " or " ") + optional "+- " or "\- " + rest - m = re.match(r'^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$', s) + # depgraph indentation blocks ("| " or " ") + optional "+- " or "\- " + rest + m = re.match(r"^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$", s) if not m: return None, None - # Each indent block is 3 chars; add 1 if a branch token is present - depth = (len(m.group('indent')) // 3) + (1 if m.group('branch') else 0) - rest = m.group('rest').strip() + depth = (len(m.group("indent")) // 3) + (1 if m.group("branch") else 0) + rest = m.group("rest").strip() # First token up to whitespace or ')', optionally starting with '(' - m2 = re.match(r'^\(?([^\s\)]+)\)?', rest) + m2 = re.match(r"^\(?([^\s\)]+)\)?", rest) if not m2: return None, None - token = m2.group(1) # e.g., io.foo:bar:jar:1.2.3:compile - parts = token.split(':') + token = m2.group(1) # e.g. com.google.guava:guava:32.0.1-jre:jar:compile + parts = token.split(":") - # Drop trailing Maven scope if present - scopes = {'compile', 'runtime', 'test', 'provided', 'system', 'import'} + scopes = {"compile", "runtime", "test", "provided", "system", "import"} if parts and parts[-1] in scopes: parts = parts[:-1] - # Expect group:artifact:type:(classifier:)version — return without the type - if len(parts) >= 4: - group, artifact = parts[0], parts[1] - version = parts[-1] - coord = f"{group}:{artifact}:{version}" - return depth, coord + if len(parts) < 3: + return None, None + + group, artifact = parts[0], parts[1] + + # depgraph text format puts version in position 2: + # group:artifact:version:type (scope already removed) + # We detect that by checking whether the last part is a packaging/type marker. + packaging = {"jar", "war", "pom", "bundle", "maven-plugin", "ear", "ejb", "rar", "zip", "test-jar"} + + def looks_like_version(v: str) -> bool: + return any(ch.isdigit() for ch in v) - return None, None + version: Optional[str] = None + + # depgraph: group:artifact:version:type + if len(parts) >= 4 and parts[-1] in packaging and looks_like_version(parts[2]): + version = parts[2] + # depgraph (rare): group:artifact:version:type:classifier + elif len(parts) >= 5 and parts[-2] in packaging and looks_like_version(parts[2]): + version = parts[2] + else: + # Fallback for other Maven-like formats where version is last + if looks_like_version(parts[-1]): + version = parts[-1] + elif looks_like_version(parts[2]): + version = parts[2] + else: + return None, None + + coord = f"{group}:{artifact}:{version}" + return depth, coord class PythonDependencyTreeBuilder(DependencyTreeBuilder): def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: - cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python -m pip install deptree' - run_command(cmd) - cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/deptree', + venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' + run_command(f'{venv_python} -m pip install "setuptools<81" deptree') + cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/deptree' dependencies = run_command(cmd) + if not dependencies or not dependencies.strip(): + logger.error("deptree returned empty output — third-party dependency tree will be incomplete. " + "Check that the virtual environment at %s has packages installed.", manifest_path) parent_stack = [] tree = defaultdict(set) ROOT_PROJECT = 'root_project' tree[ROOT_PROJECT] = [ROOT_LEVEL_SENTINEL] - for line in dependencies.split(os.linesep): + for line in dependencies.split(os.linesep) if dependencies else []: level = 0 while line.startswith(' '): level += 1 diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index 521c19390..035e86157 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -17,6 +17,8 @@ import json import os import sys +import threading +import weakref import time import typing from hashlib import sha512 @@ -221,6 +223,22 @@ class DocumentEmbedding: repositories and chunked into smaller pieces for embedding. """ + # Per-repo lock for serializing collect_documents calls on the same (git_repo, ref). + # Prevents concurrent git clone, install_dependencies, and document scanning + # when multiple CVEs target the same repo (e.g., from cve_generate_vdbs and cve_agent_executor). + _repo_locks: weakref.WeakValueDictionary[tuple, threading.Lock] = weakref.WeakValueDictionary() + _repo_locks_guard = threading.Lock() + + @classmethod + def _get_repo_lock(cls, git_repo: str, ref: str) -> threading.Lock: + key = (git_repo, ref) + with cls._repo_locks_guard: + lock = cls._repo_locks.get(key) + if lock is None: + lock = threading.Lock() + cls._repo_locks[key] = lock + return lock + def __init__(self, *, embedding: "Embeddings", vdb_directory: PathLike = VDB_DIRECTORY, git_directory: PathLike = DEFAULT_GIT_DIRECTORY, chunk_size: int = 800, chunk_overlap: int = 160, pickle_cache_directory: PathLike = DEFAULT_PICKLE_CACHE_DIRECTORY): @@ -360,7 +378,19 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: source_info.git_repo, source_info.ref) if documents_were_in_cache or len(documents) > 0: return documents - else: + + # Serialize git clone + install_dependencies + document scanning per (git_repo, ref). + # Without this lock, concurrent CVEs targeting the same repo (e.g., two keycloak CVEs) + # would both miss the pickle cache and run install_dependencies simultaneously, + # corrupting the filesystem (e.g., "META-INF: could not create directory"). + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + with repo_lock: + # Re-check cache — another thread may have populated it while we waited. + documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, + source_info.git_repo, source_info.ref) + if documents_were_in_cache or len(documents) > 0: + return documents + # Download 3rd party dependencies to be part of the git repo' dir. blob_loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=source_info.git_repo, diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index fad0ee8f5..47c2e201a 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -265,6 +265,9 @@ def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inhe current_line_stripped = current_line_stripped[next_eol + 1:].lstrip() elif -1 < next_struct < next_eol and next_eol > -1: right_curly_bracket_ind = current_line_stripped.find("}") + if right_curly_bracket_ind == -1: + pos = size_of_type_block + break self.parse_one_type( Document(page_content=f"type {current_line_stripped[:right_curly_bracket_ind + 1]}", metadata={"source": the_type.metadata['source']}), diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index ac6b50f18..11e41d9da 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -1,15 +1,16 @@ import hashlib import os import re +from functools import lru_cache from typing import Dict, Tuple, List, Optional from tqdm import tqdm from langchain_core.documents import Document from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser -from exploit_iq_commons.utils.java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, get_type_name, \ +from exploit_iq_commons.utils.java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, \ is_java_type, is_java_method, extract_method_name_with_params, find_function, get_target_class_names, \ - strip_java_generics, JAVA_ANNOTATION_SYMBOL + strip_java_generics, JAVA_ANNOTATION_SYMBOL, extract_fqcn from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") @@ -25,61 +26,174 @@ } TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') +_ARRAY_SUFFIX_RE = re.compile(r"\s*(\[\])+$") +_WS_RE = re.compile(r"\s+") + +# Supports: +# /* inner-type: Inner */ +# /* inner-type: com.foo.Outer.Inner */ +# // inner-type: Inner +_INNER_TYPE_RE = re.compile( + r""" +(?: +/\*\s*inner-type\s*:\s*(?P.*?)\s*\*/ # block comment marker +| +//\s*inner-type\s*:\s*(?P[^\r\n]*) # line comment marker +) +""", + re.IGNORECASE | re.DOTALL | re.VERBOSE, + ) class JavaLanguageFunctionsParser(LanguageFunctionsParser): def parse_all_type_struct_class_to_fields( self, types: list[Document], - inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]] - ) -> dict[tuple, list[tuple[str, str]]]: + inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]], + ) -> dict[Tuple[str, str], list[tuple[str, str]]]: """ - Returns (TypeSimpleName, doc.metadata['source']) -> [(memberName, memberType), ...] + Returns (TypeFqcn, doc.metadata['source']) -> [(memberName, memberType), ...] + - Enums are ignored entirely. - Interfaces: includes only fields (constants). - Classes/records: includes fields. - Also includes inherited fields (from superclasses and implemented interfaces) using BFS. - Uses only the existing `inheritance_map`. - - Critically, it classifies *upstream* (parents/interfaces) via relative position indices to avoid pulling in descendants. + - Classifies *upstream* (parents/interfaces) via relative position indices to avoid pulling in descendants. + + NOTE: `inheritance_map` keys are now FQCNs and may include inner FQCNs. """ from collections import deque from typing import Dict, Tuple, Any, List - results: Dict[Tuple[str, Any], List[Tuple[str, str]]] = {} + results: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + # --------------------------- + # CHANGED: build a fast (source, parsed-name) -> fqcn resolver from inheritance_map keys + # --------------------------- + def _norm_fqcn(fqcn: str) -> str: + # normalize inner separator so suffix matching works regardless of '$' vs '.' + return fqcn.replace("$", ".") + + def _strip_package_prefix(fqcn_dot: str) -> str: + """ + Convert "org.foo.Outer.Inner" -> "Outer.Inner" by dropping package tokens. + Uses the heuristic: first segment starting with uppercase begins the type chain. + """ + parts = fqcn_dot.split(".") + for idx, tok in enumerate(parts): + if tok and tok[0].isupper(): + return ".".join(parts[idx:]) + return fqcn_dot + + def _put_unique(m: dict[str, str | None], k: str, v: str) -> None: + # store unique, mark ambiguous with None + prev = m.get(k) + if prev is None: + # already ambiguous or unset (if unset None isn't distinguishable), handle below: + if k in m: + return + m[k] = v + else: + if prev != v: + m[k] = None + + # Index by file source, because inheritance_map keys include (fqcn, source) + suffix_to_fqcn_by_source: dict[str, dict[str, str | None]] = {} + simple_to_fqcn_by_source: dict[str, dict[str, str | None]] = {} + + for (fqcn, src_path) in inheritance_map.keys(): + fqcn_dot = _norm_fqcn(fqcn) + no_pkg = _strip_package_prefix(fqcn_dot) # e.g. Outer.Inner + simple = no_pkg.rsplit(".", 1)[-1] # e.g. Inner + + suf_map = suffix_to_fqcn_by_source.setdefault(src_path, {}) + sim_map = simple_to_fqcn_by_source.setdefault(src_path, {}) + + _put_unique(suf_map, no_pkg, fqcn) # "Outer.Inner" -> fqcn (may be with '$') + _put_unique(sim_map, simple, fqcn) # "Inner" -> fqcn (if unique) + + # also allow direct normalized fqcn dot form as a key for rare parsers returning fqcn-ish names + _put_unique(suf_map, fqcn_dot, fqcn) + + def _resolve_fqcn_for_parsed_name(src_path: str, parsed_name: str) -> str | None: + """ + Map `parsed_name` produced by collect_fields_from_types() to the FQCN key used by inheritance_map. + + Fast-paths: + - If parsed_name includes dots (likely "Outer.Inner"), resolve via suffix map. + - Else resolve via unique simple-name map. + """ + if not parsed_name: + return None + name = parsed_name.strip() + if not name: + return None + name_dot = name.replace("$", ".") # be tolerant + + suf_map = suffix_to_fqcn_by_source.get(src_path) + sim_map = simple_to_fqcn_by_source.get(src_path) + if "." in name_dot: + if suf_map: + fq = suf_map.get(name_dot) + return fq if fq else None + return None + + if sim_map: + fq = sim_map.get(name_dot) + return fq if fq else None + + return None + + # --------------------------- # 1) Collect own fields per document + # CHANGED: per_key_fields now keyed by (fqcn, source) not (simple, source) + # --------------------------- per_key_fields: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} for doc in tqdm(types, total=len(types), desc="Parsing class/interface/enum/record documents for members..."): src = doc.page_content - source = doc.metadata['source'] + source = doc.metadata["source"] if not src: continue per_file = collect_fields_from_types(src, include_nested=True, include_anonymous=True) - for class_name, fields in per_file.items(): - per_key_fields[(class_name, source)] = fields + # Map each parsed type name to the inheritance_map fqcn key for this source file + for parsed_type_name, fields in per_file.items(): + fqcn = _resolve_fqcn_for_parsed_name(source, parsed_type_name) + if not fqcn: + # If we cannot map reliably to an inheritance_map key, skip (prevents wrong merges). + continue + per_key_fields[(fqcn, source)] = fields + + # --------------------------- # 2) Precompute: row lists and O(1) index lookups for each key + # --------------------------- rows: Dict[Tuple[str, str], List[Tuple[str, str]]] = inheritance_map - row_index: Dict[Tuple[str, str], Dict[Tuple[str, str], int]] = {} - for k, lst in rows.items(): - # stable O(1) index lookup - row_index[k] = {v: i for i, v in enumerate(lst)} - - # 3) Upstream classifier (parents + interfaces) using relative index rule: - # candidate is upstream of type_key <=> candidate ∈ rows[type_key], - # type_key ∈ rows[candidate], and - # row_index[type_key][candidate] < row_index[candidate][type_key] + row_index: Dict[Tuple[str, str], Dict[Tuple[str, str], int]] = { + type_key: {neighbor: i for i, neighbor in enumerate(neighbors)} + for type_key, neighbors in rows.items() + } + + # --------------------------- + # 3) Upstream classifier with caching (performance) + # --------------------------- + upstream_cache: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: """ Return the direct upstream types (superclasses and implemented interfaces) for `type_key` using the relative-index rule over `rows` and `row_index`. """ + cached = upstream_cache.get(type_key) + if cached is not None: + return cached + type_row = rows.get(type_key) if not type_row: + upstream_cache[type_key] = [] return [] - # Fast lookup: neighbor -> index in type_key's row type_index_map = row_index.get(type_key, {}) - upstreams: List[Tuple[str, str]] = [] seen_candidates: set[Tuple[str, str]] = set() @@ -87,10 +201,8 @@ def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: if candidate == type_key or candidate in seen_candidates: continue - # Index of candidate within type_key's row idx_candidate_in_type = type_index_map.get(candidate, 1 << 30) - # Index of type_key within candidate's row candidate_index_map = row_index.get(candidate) if not candidate_index_map: continue @@ -98,69 +210,76 @@ def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: if idx_type_in_candidate is None: continue - # Ancestor/interface test via relative positions if idx_candidate_in_type < idx_type_in_candidate: upstreams.append(candidate) seen_candidates.add(candidate) + upstream_cache[type_key] = upstreams return upstreams + # --------------------------- # 4) Merge own + inherited fields per type using BFS over upstream neighbors. + # --------------------------- for type_key, own_fields in per_key_fields.items(): merged: List[Tuple[str, str]] = [] have_names: set[str] = set() # child-over-parent shadowing - # Own first (preserve order) - for (fname, ftype) in own_fields: + for fname, ftype in own_fields: if fname not in have_names: merged.append((fname, ftype)) have_names.add(fname) - # BFS over upstreams q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) visited: set[Tuple[str, str]] = set() while q: - u = q.popleft() - if u in visited: + upstream_key = q.popleft() + if upstream_key in visited: continue - visited.add(u) + visited.add(upstream_key) - upstream_fields = per_key_fields.get(u) + upstream_fields = per_key_fields.get(upstream_key) if upstream_fields: - for (fname, ftype) in upstream_fields: + for fname, ftype in upstream_fields: if fname not in have_names: merged.append((fname, ftype)) have_names.add(fname) - for next_upstream in upstream_neighbors(u): + for next_upstream in upstream_neighbors(upstream_key): if next_upstream not in visited: q.append(next_upstream) results[type_key] = merged + # --------------------------- # 5) Include types that had no own fields but may inherit some + # --------------------------- for type_key in rows.keys(): if type_key in results: continue merged: List[Tuple[str, str]] = [] have_names: set[str] = set() + q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) visited: set[Tuple[str, str]] = set() + while q: - u = q.popleft() - if u in visited: + upstream_key = q.popleft() + if upstream_key in visited: continue - visited.add(u) - upstream_fields = per_key_fields.get(u) + visited.add(upstream_key) + + upstream_fields = per_key_fields.get(upstream_key) if upstream_fields: - for (fname, ftype) in upstream_fields: + for fname, ftype in upstream_fields: if fname not in have_names: merged.append((fname, ftype)) have_names.add(fname) - for next_upstream in upstream_neighbors(u): + + for next_upstream in upstream_neighbors(upstream_key): if next_upstream not in visited: q.append(next_upstream) + if merged: results[type_key] = merged @@ -169,17 +288,43 @@ def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: def get_dummy_function(self, function_name): return f"public void {function_name + '()' + '{}'}" - def get_class_name_from_class_function(self, func: Document): + def __get_optional_inner_type(self, method_source: str) -> Optional[str]: """ - Extract the final path segment and strip its last extension. - Works with / or \ path separators. If there's no dot, returns the segment as-is. + Extract the optional inner type marker from a method's extracted source. + Returns: + - the inner type string (trimmed) if a marker exists + - None otherwise """ - source = func.metadata.get('source') + m = _INNER_TYPE_RE.search(method_source) + if not m: + return None - # get last non-empty segment - tail = re.split(r"[\\/]", source.rstrip("/\\"))[-1] - # drop final extension if present - return tail.rsplit(".", 1)[0] + inner = m.group("block") if m.group("block") is not None else m.group("line") + inner = (inner or "").strip() + return inner or None + + def get_class_name_from_class_function(self, func: Document) -> str: + """ + Return the Java FQCN (fully-qualified class name) inferred from func.metadata['source']. + + Assumptions (per your note): + - Either: + dependencies-sources/--sources//.java + or: + .../src/main/java//.java + - Package path is the directory structure after the marker, followed by the class filename. + """ + source = (func.metadata.get("source") or "").strip() + if not source: + return "" + + fqcn = extract_fqcn(source) + + inner_type = self.__get_optional_inner_type(func.page_content) + if inner_type: + return fqcn + "." + inner_type + + return fqcn def get_function_reserved_word(self) -> str: return "" @@ -603,7 +748,7 @@ def search_for_called_function( callee_function_name: str, callee_function: Document, callee_function_package: str, - code_documents: list[Document], + code_documents: dict[str, Document], type_documents: list[Document], callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], @@ -619,11 +764,6 @@ def search_for_called_function( - Constructor calls: `new Type(...)` (when the callee is a constructor). - Method references: `left-hand-side::method` → synthesized as `left-hand-side.method(` for resolution. - Constructor references: `left-hand-side::new` → synthesized as `new left-hand-side(` for resolution. - - Fast-paths: - - Regex pre-filters for method names and constructor sites. - - Bounded left-scan to slice the owning expression. - - De-duplicate by (start, end) slice of the expression being resolved. """ def _find_matching_paren(s: str, open_idx: int) -> int: """Find matching ')' for the '(' at open_idx. Ignores strings/char literals.""" @@ -680,9 +820,6 @@ def _next_token_after(s: str, idx: int) -> str: return s[i] if i < n else '' # Precise left-scan to the start of the expression that owns this call. - # Stops at the first top-level boundary (not inside (),[],{} or strings): - # ; , = + - * / % ! ~ ? : & | ^ < > { } \n - # We allow dots, identifiers, casts, and 'new ...' as part of the expression. BOUNDARY_CHARS = set(';,=+-*/%!?::&|^<>{}\n') def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: @@ -751,18 +888,14 @@ def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: i -= 1 return limit - # Method-ref left-hand-side extractor: find the minimal "left-hand-side" immediately before '::' + # Method-ref left-hand-side extractor: find minimal "lhs" immediately before '::' def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: """ - Return the start index of the *left-hand-side* expression immediately preceding '::' at dc_idx. - - Unlike _expr_start_left (which finds the start of the whole owning expression), - this stops at top-level delimiters that commonly precede a method reference inside - an argument list (notably '(' and ','). + Return the start index of the LHS expression immediately preceding '::' at dc_idx. + Stops at top-level delimiters that commonly precede a method reference in args + (notably '(' and ',') so we don't capture an outer call prefix. """ i = dc_idx - 1 - n = len(s) - # skip whitespace while i >= 0 and s[i].isspace(): i -= 1 @@ -771,8 +904,6 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: dp = db = dbr = da = 0 # (), [], {}, <> (generics) limit = max(0, dc_idx - max_back) - # boundaries that delimit an argument/expression at top-level for method references - # (include '(' and ',' to avoid capturing outer call prefixes like "foo(") REF_BOUNDARY = set(';,=+-*/%!?&|^:\n,(') while i >= limit: @@ -806,7 +937,6 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: i -= 1 continue - # nesting tracking (reverse) if ch == ')': dp += 1 i -= 1 @@ -816,7 +946,6 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: dp -= 1 i -= 1 continue - # top-level '(' is a delimiter for the left-hand-side in method refs if db == dbr == da == 0: return i + 1 i -= 1 @@ -859,7 +988,6 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: da -= 1 i -= 1 continue - # top-level '<' (e.g., comparison) is a delimiter for our left-hand-side extraction if dp == db == dbr == 0: return i + 1 i -= 1 @@ -872,7 +1000,9 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: return limit - # Extract method body once + # --------------------------- + # Extract caller function body + # --------------------------- src = caller_function.page_content try: lo = src.index("{") @@ -881,20 +1011,59 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: except ValueError: caller_function_body = src - # --- patterns --- + # --------------------------- + # Patterns + # --------------------------- method_pat = re.compile( rf'(?:\breturn\b\s*\(?[^;]*?)?(?:[\w$()\[\].]*?\.?)?\b{re.escape(callee_function_name)}\s*\(', - re.MULTILINE + re.MULTILINE, ) - declaring_simple = self.get_class_name_from_class_function(callee_function) + callee_function_source = callee_function.metadata['source'] + + # CHANGED: get_class_name_from_class_function now returns FQCN (possibly inner). + # Use it as the declaring FQCN for inheritance lookup AND for constructor name derivation. + declaring_fqcn = self.get_class_name_from_class_function(callee_function) + + # CHANGED: normalize inner separator to Java syntax ('.') for matching source text. + declaring_fqcn_dot = declaring_fqcn.replace('$', '.') + declaring_simple = declaring_fqcn_dot.rsplit('.', 1)[-1] - # Treat as constructor-target if caller asked the declaring type name - is_ctor_target = callee_function_name == declaring_simple + # CHANGED: support ctor targeting by simple name, no-package inner name, or fqcn. + def _strip_package_prefix(fqcn_dot: str) -> str: + parts = fqcn_dot.split('.') + for idx, tok in enumerate(parts): + if tok and tok[0].isupper(): + return '.'.join(parts[idx:]) + return fqcn_dot # fallback - ctor_pat = re.compile( - rf'\bnew\s+((?:[A-Za-z_$][\w$]*\.)*{re.escape(declaring_simple)})\s*\(' - ) if is_ctor_target else None + declaring_no_pkg = _strip_package_prefix(declaring_fqcn_dot) + + # Treat as constructor-target if caller asked for the declaring type name (simple/no-pkg/fqcn). + is_ctor_target = callee_function_name in { + declaring_simple, + declaring_no_pkg, + declaring_fqcn_dot, + declaring_fqcn, # just in case callers still use '$' form + } + + # CHANGED: ctor_pat must work when declaring_simple is derived from an FQCN/inner FQCN. + # We match "new <...>(..." where <...> ends with: + # - Simple class name (Inner) + # - No-package inner chain (Outer.Inner) + # - Fully-qualified name (pkg.Outer.Inner) + ctor_pat = None + if is_ctor_target: + ctor_variants = [] + # Prefer longest first to reduce backtracking. + for v in (declaring_fqcn_dot, declaring_no_pkg, declaring_simple): + if v and v not in ctor_variants: + ctor_variants.append(v) + ctor_variants.sort(key=len, reverse=True) + ctor_alt = "|".join(re.escape(v) for v in ctor_variants) + ctor_pat = re.compile( + rf'\bnew\s+((?:[A-Za-z_$][\w$]*\.)*(?:{ctor_alt}))\s*\(' + ) # Method reference patterns (allow optional method type args: Type::m) methodref_pat = re.compile( @@ -904,13 +1073,19 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: seen_slices: set[tuple[int, int]] = set() - # Target class names for the resolver - key = (declaring_simple, callee_function.metadata['source']) + # --------------------------- + # Target class names (inheritance) + # --------------------------- + # CHANGED: key uses declaring_fqcn (inner-aware), not extract_fqcn(file_path). + key = (declaring_fqcn, callee_function_source) if "dummy" not in callee_function_file_name: target_class_names = get_target_class_names(type_inheritance[key]) else: - target_class_names = frozenset([declaring_simple]) + target_class_names = frozenset([declaring_fqcn]) + # --------------------------- + # Helpers + # --------------------------- def _process_call(start_idx: int, open_paren_pos: int) -> bool: close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) if close_paren_pos == -1: @@ -938,6 +1113,7 @@ def _process_call(start_idx: int, open_paren_pos: int) -> bool: documents_of_functions=documents_of_functions, callee_function_name=callee_function_name, type_inheritance=type_inheritance, + callee_declaring_fqcn=declaring_fqcn, ): logger.debug( "__check_identifier_resolved_to_callee_function_package resolved successfully - " @@ -985,16 +1161,21 @@ def _process_method_ref(dc_idx: int, ref_len: int, make_ctor: bool) -> bool: documents_of_functions=documents_of_functions, callee_function_name=callee_function_name, type_inheritance=type_inheritance, + callee_declaring_fqcn=declaring_fqcn, ) + # --------------------------- # 1) Constructor matches (only when target is a ctor) + # --------------------------- if is_ctor_target and ctor_pat: for m in ctor_pat.finditer(caller_function_body): open_paren_pos = m.end(0) - 1 if _process_call(m.start(), open_paren_pos): return True else: + # --------------------------- # 2) Regular method matches + # --------------------------- for m in method_pat.finditer(caller_function_body): open_paren_pos = m.end(0) - 1 close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) @@ -1008,7 +1189,9 @@ def _process_method_ref(dc_idx: int, ref_len: int, make_ctor: bool) -> bool: if _process_call(m.start(), open_paren_pos): return True + # --------------------------- # 3) Method reference matches + # --------------------------- for m in methodref_pat.finditer(caller_function_body): if _process_method_ref(m.start(), m.end() - m.start(), make_ctor=False): return True @@ -1020,7 +1203,6 @@ def _process_method_ref(dc_idx: int, ref_len: int, make_ctor: bool) -> bool: return False - def __check_identifier_resolved_to_callee_function_package( self, function: Document, @@ -1034,7 +1216,8 @@ def __check_identifier_resolved_to_callee_function_package( target_class_names: frozenset[str], documents_of_functions: list[Document], callee_function_name: str, - type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]], + callee_declaring_fqcn: str = "", ) -> bool: """ Decide if the found call expression (`identifier_function`) in `function` actually targets @@ -1048,6 +1231,9 @@ def __check_identifier_resolved_to_callee_function_package( - Unqualified helper call in same class: helper(…) → resolve helper() return type. - Receiver that is a cast: ((Type) x).target(…) → use Type as a strong hint. - SUPER handling uses `target_class_names` (parents/children set) — no file header parsing. + + NOTE (CHANGED): `target_class_names` now contains FQCNs (not simple names), so all calls into + `_type_token_matches_callee(...)` are now made with FQCN candidates only. """ def _strip_return_and_ws(s: str) -> str: s = s.strip() @@ -1071,24 +1257,37 @@ def _split_top_level_dots(expr: str) -> list[str]: ch = expr[i] if in_str: buf.append(ch) - if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 - elif ch == '"': in_str = False + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == '"': + in_str = False i += 1; continue if in_chr: buf.append(ch) - if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 - elif ch == "'": in_chr = False + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == "'": + in_chr = False i += 1; continue - if ch == '"': in_str = True; buf.append(ch); i += 1; continue - if ch == "'": in_chr = True; buf.append(ch); i += 1; continue - if ch == '(': dp += 1; buf.append(ch); i += 1; continue - if ch == ')': dp = max(0, dp - 1); buf.append(ch); i += 1; continue - if ch == '[': db += 1; buf.append(ch); i += 1; continue - if ch == ']': db = max(0, db - 1); buf.append(ch); i += 1; continue - if ch == '{': dbr += 1; buf.append(ch); i += 1; continue - if ch == '}': dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue - if ch == '<': da += 1; buf.append(ch); i += 1; continue + if ch == '"': + in_str = True; buf.append(ch); i += 1; continue + if ch == "'": + in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': + dp += 1; buf.append(ch); i += 1; continue + if ch == ')': + dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': + db += 1; buf.append(ch); i += 1; continue + if ch == ']': + db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': + dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': + dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': + da += 1; buf.append(ch); i += 1; continue if ch == '>': if da > 0: da -= 1 buf.append(ch); i += 1; continue @@ -1137,7 +1336,6 @@ def _is_fqcn_like(token: str) -> bool: return bool(pkg) and _is_upper_camel(cls[:1] + cls[1:]) return _is_upper_camel(token) - # --- resolve the return type of a method declared in the same source file --- def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: sig = re.compile( rf""" @@ -1158,13 +1356,12 @@ def _find_method_return_type_in_file(source_text: str, method_name: str) -> str ret = re.sub(r'\s*(\[\])+$', '', ret) return ret or None - # peel a leading Java cast "((Type) expr)" → (cast_type | None, remainder_expr) _CAST_RE = re.compile( r'^\(\s*([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:<[^()<>]*>)?(?:\s*(?:\[\]))*)\s*\)\s*(.*)\Z' ) + def _peel_leading_cast(expr: str) -> tuple[str | None, str]: s = expr.strip() - # unwrap outermost parens only if they wrap the whole expr (not a call) changed = True while changed and s.startswith('(') and s.endswith(')'): changed = False @@ -1179,7 +1376,6 @@ def _peel_leading_cast(expr: str) -> tuple[str | None, str]: else: s = s[1:-1].strip() changed = True - # peel one or more casts last_type = None rest = s while True: @@ -1190,7 +1386,6 @@ def _peel_leading_cast(expr: str) -> tuple[str | None, str]: rest = m.group(2).strip() return last_type, rest - # SAFER: extract receiver + method; never builds an empty list or raises IndexError def _extract_recv_and_method(snippet: str) -> tuple[str | None, str | None]: s = snippet.strip() m = re.search(r'([A-Za-z_$][\w$]*)\s*\($', s) @@ -1205,13 +1400,12 @@ def _extract_recv_and_method(snippet: str) -> tuple[str | None, str | None]: method = m.group(1) method_start = m.start(1) - # immediate '.' before the method token? i = method_start - 1 - while i >= 0 and s[i].isspace(): i -= 1 + while i >= 0 and s[i].isspace(): + i -= 1 if i < 0 or s[i] != '.': return None, method - # walk left to get the *smallest* receiver just before the dot dot_idx = i def _match_paren_reverse(text: str, close_idx: int) -> int: @@ -1242,10 +1436,10 @@ def _match_paren_reverse(text: str, close_idx: int) -> int: return -1 j = dot_idx - 1 - while j >= 0 and s[j].isspace(): j -= 1 + while j >= 0 and s[j].isspace(): + j -= 1 if j >= 0 and s[j] == ')': - # get the minimal '( ... )' group immediately before the dot open_idx = _match_paren_reverse(s, j) if open_idx == -1: start = j @@ -1254,16 +1448,15 @@ def _match_paren_reverse(text: str, close_idx: int) -> int: recv = s[start + 1:dot_idx].strip() return (recv if recv else None), method - # If the '(' belongs to a *call* (e.g. foo(...)) — include the callee token to its left. k = open_idx - 1 - while k >= 0 and s[k].isspace(): k -= 1 + while k >= 0 and s[k].isspace(): + k -= 1 if k >= 0 and (s[k].isalnum() or s[k] in '_$.'): start = k while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): start -= 1 recv = s[start + 1:dot_idx].strip() else: - # parenthesized group/cast → keep minimal group recv = s[open_idx:dot_idx].strip() else: start = j @@ -1275,51 +1468,161 @@ def _match_paren_reverse(text: str, close_idx: int) -> int: caller_src = function.metadata.get('source') caller_doc = code_documents.get(caller_src) - caller_text = caller_doc.page_content if caller_doc else "" - caller_class_name = self.get_class_name_from_class_function(caller_doc) if caller_doc else "" - # Does a type token (simple or qualified) belong to the *callee type(s)/package*? - def _type_token_matches_callee(type_token: str) -> bool: - if not type_token: + caller_text = (caller_doc.page_content if caller_doc else "") # safe fallback + caller_fqcn = self.get_class_name_from_class_function(function) + + # -------------------- FQCN-only type matching -------------------- + # Build quick index: simple-name -> list of target FQCNs (for fast disambiguation) + _target_by_simple: dict[str, list[str]] = {} + for fq in target_class_names: + last = fq.rsplit('.', 1)[-1] + _target_by_simple.setdefault(last, []).append(fq) + if '$' in last: + _target_by_simple.setdefault(last.split('$')[-1], []).append(fq) + + # Parse header imports once (bounded), used only for disambiguation (still cheap) + _hdr = caller_text + _PKG_RE = re.compile(r'^\s*package\s+([\w.]+)\s*;\s*$', re.MULTILINE) + _IMP_RE = re.compile(r'^\s*import\s+(?:static\s+)?([\w$.]+)\s*;\s*$', re.MULTILINE) + + m_pkg = _PKG_RE.search(_hdr) + _caller_pkg = (m_pkg.group(1) if m_pkg else "") + _explicit_imports: dict[str, str] = {} # simple -> fqcn (CHANGED) + _wild_import_pkgs: list[str] = [] # "a.b" for "import a.b.*" (CHANGED) + for imp in _IMP_RE.findall(_hdr): + if imp.endswith(".*"): + _wild_import_pkgs.append(imp[:-2]) + else: + _explicit_imports[imp.rsplit('.', 1)[-1]] = imp + + def _strip_type_syntax(token: str) -> str: + """Strip generics, arrays, and wildcard bounds from a type-ish token (CHANGED).""" + t = (token or "").strip() + if not t: + return "" + # Drop keywords after the type (e.g. "? extends Foo") + for kw in ("extends", "super"): + if kw in t: + t = t.split(kw, 1)[0].strip() + t = t.replace("?", "").strip() + + # Strip generics with nesting + out = [] + depth = 0 + for ch in t: + if ch == '<': + depth += 1 + continue + if ch == '>': + if depth > 0: + depth -= 1 + continue + if depth == 0: + out.append(ch) + t = ''.join(out).strip() + + # Strip trailing arrays + while t.endswith("]") and "[]" in t: + t = re.sub(r'\s*(\[\])\s*$', '', t).strip() + + return t + + def _is_package_qualified(name: str) -> bool: + """Heuristic: package-qualified names typically start with a lower-case segment (CHANGED).""" + first = name.split('.', 1)[0] + return bool(first) and first[0].islower() + + def _iter_fqcn_candidates(raw_type_token: str): + """ + Yield FQCN candidates for a raw type token. Ensures that downstream + `_type_token_matches_callee(...)` is invoked with FQCNs only (CHANGED). + """ + t = _strip_type_syntax(raw_type_token) + if not t: + return + # Already looks like a FQCN: pkg.Type or pkg.Outer.Inner + if '.' in t and _is_package_qualified(t): + yield t + return + + # Nested type without package: Outer.Inner -> prefix caller package if known + if '.' in t and not _is_package_qualified(t): + if _caller_pkg: + yield f"{_caller_pkg}.{t}" + else: + # best-effort; still a "fqcn-ish" candidate + yield t + return + + # Simple token: try target allow-list by simple name + cands = _target_by_simple.get(t) + if cands: + for fq in cands: + yield fq + + # Explicit import disambiguation + imp = _explicit_imports.get(t) + if imp: + yield imp + + # Wildcard imports: only usable if we can cheaply construct candidates + # (we only build candidates that are already in target_class_names to avoid work) + if _wild_import_pkgs and cands: + # already covered by `cands`; do nothing + pass + + # Same-package fallback + if _caller_pkg: + yield f"{_caller_pkg}.{t}" + + def _type_token_matches_callee(type_fqcn: str) -> bool: + """ + (CHANGED) This function now assumes it receives a FQCN token only. + All callers must pass FQCNs (or FQCN candidates). + """ + if not type_fqcn: return False - tt = re.sub(r'<[^<>]*>', '', type_token).strip() - tt = re.sub(r'\s*(\[\])+$', '', tt) - simple_tt = tt.rsplit('.', 1)[-1] - # IMPORTANT: If allow-list exists and the token's simple name is not in it, short-circuit False. - if target_class_names and strip_java_generics(simple_tt) not in target_class_names: + + fq = _strip_type_syntax(type_fqcn) # defensive: keep it cheap + if not fq: + return False + + # IMPORTANT (CHANGED): target_class_names now contains FQCNs; membership is exact. + if target_class_names and fq not in target_class_names: return False - code_text = caller_text - if '.' in tt and _is_fqcn_like(tt): - if self.is_package_imported(code_text, tt): + + if _is_fqcn_like(fq): + if self.is_package_imported(caller_text, fq): return True + matches = self.__get_type_docs_matched_with_callee_type( - callee_package, tt, type_documents, target_class_names + callee_package, fq, type_documents, target_class_names ) return len(matches) > 0 + def _type_matches_callee(raw_type_token: str) -> bool: + """Resolve raw token -> FQCN candidates, then call `_type_token_matches_callee` (CHANGED).""" + seen = set() + for fq in _iter_fqcn_candidates(raw_type_token): + if fq in seen: + continue + seen.add(fq) + if _type_token_matches_callee(fq): # fqcn-only + return True + return False + def _caller_key() -> str: content = function.page_content md5hex = hashlib.md5(content.strip().encode("utf-8")).hexdigest() return f"{extract_method_name_with_params(content)}@{md5hex}@{caller_src}" - # ---- normalize the found call snippet ---- expr = _strip_return_and_ws(_strip_leading_this(identifier_function)) expr = re.sub(r'\s+', ' ', expr).strip() def _extract_ctor_type(expr: str) -> str: - """ - Given an expression that starts with 'new ', return the constructor type token. - Handles qualified names, generics (including diamond), and array brackets. - Stops at the first '(' or '{' seen at top level (outside '<...>'). - Examples: - - 'new Foo(' -> 'Foo' - - 'new a.b.C(' -> 'a.b.C' - - 'new HashMap>(' -> 'HashMap>' - - 'new int[] {' -> 'int[]' - """ i = 4 # skip 'new ' n = len(expr) - # skip leading spaces while i < n and expr[i].isspace(): i += 1 buf = [] @@ -1335,63 +1638,70 @@ def _extract_ctor_type(expr: str) -> str: buf.append(ch); i += 1; continue if (ch == '(' or ch == '{') and angle == 0: break - # include identifiers, dots, '$', arrays, spaces between [] if present buf.append(ch); i += 1 return ''.join(buf).strip() # ====== EARLY: direct constructor snippet 'new Type(...' ====== - # This covers cases where search_for_called_function fed us a constructor occurrence directly. if expr.startswith('new '): ctor_type_token = _extract_ctor_type(expr) - return _type_token_matches_callee(ctor_type_token) + return _type_matches_callee(ctor_type_token) # resolve -> FQCN(s) recv_norm, method_norm = _extract_recv_and_method(expr) if method_norm: expr = (f"{recv_norm}.{method_norm}(" if recv_norm else f"{method_norm}(").strip() - # decide qualified vs unqualified using recv_norm is_unqualified = (recv_norm is None) # ======== SUPER / THIS handling via target_class_names ======== if recv_norm is not None: recv_trim = recv_norm.strip() - # Interface.super.method(…) if recv_trim.endswith('.super'): - iface_type = recv_trim[:-6].strip() # drop ".super" - if _type_token_matches_callee(iface_type): + iface_type = recv_trim[:-6].strip() + if _type_matches_callee(iface_type): return True - # plain super.method(…) - if recv_trim == 'super': - caller_inheritance_list = get_target_class_names(type_inheritance[(caller_class_name, caller_src)]) - for cand in caller_inheritance_list: - if cand == strip_java_generics(caller_class_name): - continue - if _type_token_matches_callee(cand): - return True + if recv_trim == 'super' and callee_declaring_fqcn: + # Don't match super calls from root-package functions. + # super delegates to the parent's own implementation which is + # self-contained. Allowing it at the app→dependency boundary + # lets the CCA's backtracking route through the entire + # dependency hierarchy via polymorphic dispatch, producing + # false-positive chains (e.g. handler.handle() hops). + if self.is_root_package(function): + pass # skip – super from app code is not a valid entry point + else: + try: + caller_inheritance = type_inheritance[(caller_fqcn, caller_src)] + except KeyError: + caller_inheritance = [] + # super resolves to the direct parent class only; match only if the + # direct parent IS the callee's declaring class (exact match). + for cand_fqcn, _cand_src in caller_inheritance: + if cand_fqcn == caller_fqcn: + continue + if cand_fqcn == callee_declaring_fqcn or cand_fqcn == callee_declaring_fqcn.replace('$', '.'): + return True + break - # explicit this.method(…) if recv_trim == 'this': - if caller_class_name and _type_token_matches_callee(caller_class_name): + if caller_fqcn and _type_token_matches_callee(caller_fqcn): return True if is_unqualified: - # helper method in same class? use its return type base_before_paren = method_norm if base_before_paren and callee_function_name != base_before_paren: fn = find_function(caller_src, base_before_paren, documents_of_functions) if fn: ret_type = _find_method_return_type_in_file(fn.page_content, base_before_paren) - if ret_type and _type_token_matches_callee(ret_type): + if ret_type and _type_matches_callee(ret_type): return True - # fallback: same-package unqualified call rule try: callee_doc = code_documents[callee_function_file_name] callee_pkg_decl = (self.get_package_name_file(callee_doc) or "").strip() except KeyError: - callee_pkg_decl = callee_function_file_name # fallback + callee_pkg_decl = callee_function_file_name caller_pkg_decl = (self.get_package_name_file(caller_doc) or "").strip() if caller_doc else "" jar_name = extract_jar_name(caller_src or "") @@ -1399,32 +1709,28 @@ def _extract_ctor_type(expr: str) -> str: callee_package.__contains__(jar_name) and callee_pkg_decl and callee_pkg_decl == caller_pkg_decl - and strip_java_generics(caller_class_name) in target_class_names + and (caller_fqcn in target_class_names if caller_fqcn else False) ) - # ---- dotted/chained expression handling ---- chain = _split_top_level_dots(expr) if not chain: return False - recv_raw = chain[-2] if len(chain) >= 2 else "" - recv_raw = recv_raw.strip() + recv_raw = (chain[-2] if len(chain) >= 2 else "").strip() - # Casts first: '((Type) x).method(' cast_type, post_cast_expr = _peel_leading_cast(recv_raw) if cast_type: - if _type_token_matches_callee(cast_type): + if _type_matches_callee(cast_type): return True m_id_cast = re.match(r'\s*([A-Za-z_$][\w$]*)', post_cast_expr) if m_id_cast: - caller_key = _caller_key() traced_cast = self.__trace_down_package( expression=m_id_cast.group(1), type_documents=type_documents, callee_package=callee_package, fields_of_types=fields_of_types, functions_local_variables_index=functions_local_variables_index, - caller_function_index=caller_key, + caller_function_index=_caller_key(), target_class_names=target_class_names, function=function, code_documents=code_documents @@ -1432,33 +1738,41 @@ def _extract_ctor_type(expr: str) -> str: if traced_cast: return True - # Also allow Interface.super receiver in a chain if recv_raw.endswith('.super'): iface_type = recv_raw[:-6].strip() - if _type_token_matches_callee(iface_type): + if _type_matches_callee(iface_type): return True - # Explicit super/this in chained receiver - if recv_raw == 'super': - caller_inheritance_list = get_target_class_names(type_inheritance[(caller_class_name, caller_src)]) - for cand in caller_inheritance_list: - if cand == strip_java_generics(caller_class_name): - continue - if _type_token_matches_callee(cand): - return True + if recv_raw == 'super' and callee_declaring_fqcn: + # Don't match super calls from root-package functions (see early-path comment). + if self.is_root_package(function): + pass # skip + else: + try: + caller_inheritance = type_inheritance[(caller_fqcn, caller_src)] + except KeyError: + caller_inheritance = [] + # super resolves to the direct parent class only; match only if the + # direct parent IS the callee's declaring class (exact match). + for cand_fqcn, _cand_src in caller_inheritance: + if cand_fqcn == caller_fqcn: + continue + if cand_fqcn == callee_declaring_fqcn or cand_fqcn == callee_declaring_fqcn.replace('$', '.'): + return True + break elif recv_raw == 'this': - if caller_class_name and _type_token_matches_callee(caller_class_name): + if caller_fqcn and _type_token_matches_callee(caller_fqcn): return True - # Case A: any static-style class root in the chain (includes 'new Type(...)' receivers) + # Case A: static-style class root in the chain (includes 'new Type(...)' receivers) for seg in chain[:-1]: seg_core = _strip_call_parens(seg).strip() if seg_core.startswith("new "): typ = _strip_call_parens(seg_core[4:].strip()) - if _type_token_matches_callee(typ): + if _type_matches_callee(typ): return True continue - if _is_fqcn_like(seg_core) and _type_token_matches_callee(seg_core): + if _is_fqcn_like(seg_core) and _type_matches_callee(seg_core): return True # Case B: identifier/expression → dataflow OR rightmost receiver-call return type @@ -1494,58 +1808,118 @@ def _extract_ctor_type(expr: str) -> str: recv_call_name = mtok.group(1) break if recv_call_name: - # Use local method return type (cheap heuristic) ret_type = _find_method_return_type_in_file(caller_text, recv_call_name) - if ret_type and _type_token_matches_callee(ret_type): + if ret_type and _type_matches_callee(ret_type): return True # Case C: final receiver is a direct 'new Type(...)' if recv_raw.startswith("new "): typ = _strip_call_parens(recv_raw[4:].strip()) - if _type_token_matches_callee(typ): + if _type_matches_callee(typ): return True # Case D: immediate receiver is a (pkg.)Class token - if _is_fqcn_like(recv_raw) and _type_token_matches_callee(recv_raw): + if _is_fqcn_like(recv_raw) and _type_matches_callee(recv_raw): return True return False - def __trace_down_package(self, expression: str, type_documents: list[Document], - callee_package: str, fields_of_types: dict[tuple, list[tuple]], - functions_local_variables_index: dict[str, dict], - caller_function_index: str, - target_class_names: frozenset[str], - function: Document, - code_documents: dict[str, Document]) -> bool: - - variables_mappings = functions_local_variables_index[caller_function_index] + def __trace_down_package( + self, + expression: str, + type_documents: list[Document], + callee_package: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str, + target_class_names: frozenset[str], + function: Document, + code_documents: dict[str, Document], + ) -> bool: + variables_mappings = functions_local_variables_index.get(caller_function_index, {}) # CHANGED: safe fallback parts = expression.split(".") result = False + def _normalize_type_token(t: str) -> str: + """Strip generics + array suffixes; keep dots/$ for FQCNs.""" + if not t: + return "" + t = t.strip() + t = strip_java_generics(t).strip() + t = re.sub(r"\s*(\[\])+$", "", t).strip() + return t + + def _fqcn_candidates_from_token(type_token: str) -> list[str]: + """ + Return FQCN candidates derived from `type_token`, restricted to `target_class_names`. + - If token is already an exact member of target_class_names, return it. + - Else map by simple-name suffix against target_class_names (handles Inner via '$'). + """ + t = _normalize_type_token(type_token) + if not t: + return [] + + # Exact match (already FQCN in allow-list) + if t in target_class_names: + return [t] + + # Derive simple name and match against allow-list by suffix + simple = t.rsplit(".", 1)[-1] + out: list[str] = [] + dot_suffix = "." + simple + dollar_suffix = "$" + simple + for fq in target_class_names: + # fq is FQCN; match on end + if fq.endswith(dot_suffix) or fq.endswith(dollar_suffix): + out.append(fq) + + return out + + def _has_matching_type(type_token: str) -> bool: + """Call __get_type_docs_matched_with_callee_type with FQCNs only.""" + for fq in _fqcn_candidates_from_token(type_token): + if self.__get_type_docs_matched_with_callee_type( + callee_package, fq, type_documents, target_class_names + ): + return True + return False + (resolved_type, struct_initializer_expression, value_list, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-1) - if (var_properties is not None - and (struct_initializer_expression or - resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or PARAMETER in value_list)): - result = self.__lookup_package(callee_package, resolved_type, struct_initializer_expression, - type_documents, value_list, target_class_names) + if ( + var_properties is not None + and ( + struct_initializer_expression + or resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS + or PARAMETER in value_list + ) + ): + result = self.__lookup_package( + callee_package=callee_package, + resolved_type=resolved_type, + struct_initializer_expression=struct_initializer_expression, + type_documents=type_documents, + value_list=value_list, + target_class_names=target_class_names + ) # Property/member is not in function, check if it's member/property of a type - elif var_properties is None and (expression[0].islower() or expression[0] == '_'): - class_name = self.get_class_name_from_class_function(function) - possible_types = {key: value for (key, value) in fields_of_types.items() - if key == (class_name, function.metadata['source'])} + elif var_properties is None and (expression and (expression[0].islower() or expression[0] == '_')): + fqcn = self.get_class_name_from_class_function(function) + possible_types = { + key: value for (key, value) in fields_of_types.items() + if key == (fqcn, function.metadata['source']) + } for mappings in possible_types.values(): for mapping in mappings: if expression in mapping: - returned_matched_types = self.__get_type_docs_matched_with_callee_type(callee_package, - mapping[1], - type_documents, - target_class_names) - if len(returned_matched_types) > 0: + # CHANGED: mapping[1] may be simple; resolve to FQCN candidates first + if _has_matching_type(mapping[1]): result = True + break + if result: + break elif var_properties is not None: value = var_properties.get("value", None) @@ -1558,12 +1932,10 @@ def __trace_down_package(self, expression: str, type_documents: list[Document], code_documents=code_documents ) if inferred: - returned_matched_types = self.__get_type_docs_matched_with_callee_type(callee_package, - inferred, - type_documents, - target_class_names) - if len(returned_matched_types) > 0: + # CHANGED: inferred may be simple; resolve to FQCN candidates first + if _has_matching_type(inferred): result = True + return result def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: @@ -2880,64 +3252,240 @@ def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): else: return None, None, None, None - def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, - value_list, target_class_names: frozenset[str]) -> bool: - result = False + @staticmethod + def _normalize_type_token(t: str) -> str: + """ + Normalize a Java type token for matching: + - strip leading/trailing whitespace + - strip generics + - strip trailing array suffixes + Keeps dots/$ for FQCNs. + """ + if not t: + return "" + t = t.strip() + t = strip_java_generics(t).strip() + t = _ARRAY_SUFFIX_RE.sub("", t).strip() + return t + + @staticmethod + def _simple_name_from_type_token(t: str) -> str: + """ + Extract the simple name from either a simple type, FQCN, or inner-class FQCN. + Uses the last separator among '.' and '$'. + """ + if not t: + return "" + t = t.strip() + i_dot = t.rfind(".") + i_dol = t.rfind("$") + i = i_dot if i_dot > i_dol else i_dol + return t[i + 1:] if i != -1 else t + + @staticmethod + @lru_cache(maxsize=256) + def _simple_to_fqcns_index(target_class_names: frozenset[str]) -> dict[str, tuple[str, ...]]: + """ + Build an index: simpleName -> tuple(FQCNs). Cached per frozenset instance/value. + This avoids O(|target_class_names|) scans per resolution. + """ + idx: dict[str, list[str]] = {} + for fq in target_class_names: + simple = JavaLanguageFunctionsParser._simple_name_from_type_token(fq) + if not simple: + continue + idx.setdefault(simple, []).append(fq) + return {k: tuple(v) for k, v in idx.items()} + + def _fqcn_candidates_from_token(self, type_token: str, target_class_names: frozenset[str]) -> tuple[str, ...]: + """ + Map a possibly-simple type token to FQCN candidates strictly within `target_class_names`. + - If token already equals an allowed FQCN => (token,) + - Else => all allowed FQCNs that share the same simple name + """ + t = self._normalize_type_token(type_token) + if not t: + return () + if t in target_class_names: + return (t,) + simple = self._simple_name_from_type_token(t) + if not simple: + return () + return self._simple_to_fqcns_index(target_class_names).get(simple, ()) + + def _has_matching_type_in_package( + self, + callee_package: str, + type_token: str, + type_documents: list[Document], + target_class_names: frozenset[str], + ) -> bool: + """ + Calls __get_type_docs_matched_with_callee_type with FQCN candidates only. + """ + for fq in self._fqcn_candidates_from_token(type_token, target_class_names): + if self.__get_type_docs_matched_with_callee_type(callee_package, fq, type_documents, target_class_names): + return True + return False + + @staticmethod + def _split_top_level_dots(expr: str) -> list[str]: + parts, buf = [], [] + dp = db = dbr = da = 0 + in_str = in_chr = False + i, n = 0, len(expr) + while i < n: + ch = expr[i] + if in_str: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == '"': + in_str = False + i += 1 + continue + if in_chr: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == "'": + in_chr = False + i += 1 + continue + + if ch == '"': + in_str = True; buf.append(ch); i += 1; continue + if ch == "'": + in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': + dp += 1; buf.append(ch); i += 1; continue + if ch == ')': + dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': + db += 1; buf.append(ch); i += 1; continue + if ch == ']': + db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': + dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': + dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': + da += 1; buf.append(ch); i += 1; continue + if ch == '>': + if da > 0: da -= 1 + buf.append(ch); i += 1; continue + + if ch == '.' and dp == db == dbr == da == 0: + parts.append(''.join(buf).strip()); buf = []; i += 1; continue + + buf.append(ch); i += 1 + + parts.append(''.join(buf).strip()) + return [p for p in parts if p] + + @staticmethod + def _strip_call_parens(s: str) -> str: + s2 = s.rstrip() + if not s2.endswith(')'): + return s + depth = 0 + in_str = in_chr = False + i = len(s2) - 1 + while i >= 0: + ch = s2[i] + if in_str: + if ch == '\\': i -= 2; continue + if ch == '"': in_str = False; i -= 1; continue + i -= 1; continue + if in_chr: + if ch == '\\': i -= 2; continue + if ch == "'": in_chr = False; i -= 1; continue + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == ')': depth += 1; i -= 1; continue + if ch == '(': + depth -= 1 + if depth == 0: + return s2[:i].rstrip() + i -= 1; continue + i -= 1 + return s + + @staticmethod + @lru_cache(maxsize=2048) + def _method_sig_re(method_name: str) -> re.Pattern: + return re.compile( + rf""" + (?[A-Za-z_.$?][\w$.<>\[\]?]*?) # return type + \s+{re.escape(method_name)}\s*\( # method name + '(' + """, + re.VERBOSE, + ) + + @staticmethod + @lru_cache(maxsize=2048) + def _type_header_re(simple_type_name: str) -> re.Pattern: + return re.compile(rf"\b(class|interface|enum|record)\b\s+{re.escape(simple_type_name)}\b") + + # ------------------------- + # Updated functions (DEDUPED) + # ------------------------- + + def __lookup_package( + self, + callee_package, + resolved_type, + struct_initializer_expression, + type_documents, + value_list, + target_class_names: frozenset[str], + ) -> bool: if not struct_initializer_expression and resolved_type not in JAVA_METHOD_PRIM_TYPES: - docs = self.__get_type_docs_matched_with_callee_type(callee_package, resolved_type, type_documents, target_class_names) + if resolved_type and resolved_type not in JAVA_METHOD_PRIM_TYPES: + if self._has_matching_type_in_package( + callee_package, resolved_type, type_documents, target_class_names + ): + return True - if len(docs) > 0: - result = True - elif PARAMETER in value_list: - result = False + if PARAMETER in value_list: + return False elif struct_initializer_expression: - struct_type = (struct_initializer_expression.group(0)) # TODO struct_initializer_expression is a list of expressions - docs = self.__get_type_docs_matched_with_callee_type(callee_package, struct_type, type_documents, target_class_names) - if len(docs) > 0: - result = True - return result + struct_type = struct_initializer_expression.group(0) # TODO list of expressions + if self._has_matching_type_in_package( + callee_package, struct_type, type_documents, target_class_names + ): + return True - def __get_type_docs_matched_with_callee_type(self, callee_package, checked_type, type_documents, target_class_names: frozenset[str]) -> list[ - Document]:# TODO Make sure Fix works - the target classes can be in other jars - """ - Return all type_documents whose jar "package" matches callee_package and whose - (generic-stripped) type name is in target_class_names. + return False - Performance notes: - - Short-circuits early if checked_type cannot possibly match. - - Normalizes callee_package once. - - Reuses cached extract_jar_name, get_type_name, and strip_java_generics. - """ - # Early exit if the checked_type is present and does not match any target class - if checked_type: - stripped_checked = strip_java_generics(checked_type) - if stripped_checked not in target_class_names: - return [] + def __get_type_docs_matched_with_callee_type( + self, + callee_package, + checked_type, + type_documents, + target_class_names: frozenset[str] + ) -> list[Document]: + checked = strip_java_generics(checked_type or "").strip() + if not checked or checked not in target_class_names: + return [] - # Normalize the "package" part of callee_package once (after first ':') _, _, callee_pkg_tail = callee_package.partition(":") callee_pkg_tail_lower = callee_pkg_tail.lower() - result: List["Document"] = [] - + result: list[Document] = [] for a_type in type_documents: - src = a_type.metadata['source'] - - # extract_jar_name is cached and fairly cheap after first call - jar_name = extract_jar_name(src) - - # Same logic as is_same_package(extract_jar_name(...), callee_package.partition(":")[2]) - if jar_name.lower() != callee_pkg_tail_lower: - continue - - # get_type_name is cached per source text - type_name = get_type_name(a_type.page_content) - if not type_name: + src = a_type.metadata["source"] + if extract_jar_name(src).lower() != callee_pkg_tail_lower and self.dir_name_for_3rd_party_packages() not in src : continue - # Preserve original logic: strip generics before membership check - if strip_java_generics(type_name) in target_class_names: + fqcn = self.get_class_name_from_class_function(a_type) + if fqcn == checked: # FIX: match the requested target, not any target result.append(a_type) return result @@ -2949,145 +3497,74 @@ def _find_method_return_type_in_type_docs( type_documents: list[Document], ) -> str | None: """ - Find the declared return type of `receiver_type#method_name(...)` by scanning - the matching type document in `type_documents`. Returns a normalized type - (generics/arrays stripped) or None if not found. + Same logic, but reuses cached regex helpers (no duplicated regex construction). """ - # Normalize the receiver simple name (e.g. "org.quartz.JobDetail" -> "JobDetail") - simple = re.sub(r'\s+', '', receiver_type).rsplit('.', 1)[-1] - simple = re.sub(r'<[^<>]*>', '', simple) - simple = re.sub(r'\s*(\[\])+$', '', simple) + simple = re.sub(r"\s+", "", receiver_type) + simple = simple.rsplit(".", 1)[-1] + simple = re.sub(r"<[^<>]*>", "", simple) + simple = _ARRAY_SUFFIX_RE.sub("", simple) - # A light signature regex (like the one you already use) - sig = re.compile( - rf""" - (?[A-Za-z_.$?][\w$.<>\[\]?]*?) # return type - \s+{re.escape(method_name)}\s*\( # method name + '(' - """, - re.VERBOSE, - ) + sig = self._method_sig_re(method_name) + header = self._type_header_re(simple) - # Pick the first doc whose header declares this simple type - header = re.compile(rf'\b(class|interface|enum|record)\b\s+{re.escape(simple)}\b') for doc in type_documents: src = doc.page_content if not src: continue if not header.search(src): continue + m = sig.search(src) if not m: - # Could be overloaded; try a looser scan (multiple matches) matches = list(sig.finditer(src)) if not matches: continue m = matches[0] - ret = m.group('ret').strip() - ret = re.sub(r'<[^<>]*>', '', ret) # strip generics - ret = re.sub(r'\s*(\[\])+$', '', ret) # strip array suffixes + ret = m.group("ret").strip() + ret = re.sub(r"<[^<>]*>", "", ret) + ret = _ARRAY_SUFFIX_RE.sub("", ret).strip() return ret or None return None def _infer_type_from_var_initializer( self, - initializer: str, # e.g. "getDescriptorById(conn, recording.remoteId)" - variables_mappings: dict, # per-function locals/params map + initializer: str, + variables_mappings: dict, type_documents: list[Document], - caller_src: str, # function.metadata['source'] + caller_src: str, code_documents: dict[str, Document], ) -> str | None: - def _split_top_level_dots(expr: str) -> list[str]: - parts, buf = [], [] - dp = db = dbr = da = 0 - in_str = in_chr = False - i, n = 0, len(expr) - while i < n: - ch = expr[i] - if in_str: - buf.append(ch) - if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 - elif ch == '"': in_str = False - i += 1; continue - if in_chr: - buf.append(ch) - if ch == '\\' and i + 1 < n: buf.append(expr[i+1]); i += 1 - elif ch == "'": in_chr = False - i += 1; continue - - if ch == '"': in_str = True; buf.append(ch); i += 1; continue - if ch == "'": in_chr = True; buf.append(ch); i += 1; continue - if ch == '(': dp += 1; buf.append(ch); i += 1; continue - if ch == ')': dp = max(0, dp - 1); buf.append(ch); i += 1; continue - if ch == '[': db += 1; buf.append(ch); i += 1; continue - if ch == ']': db = max(0, db - 1); buf.append(ch); i += 1; continue - if ch == '{': dbr += 1; buf.append(ch); i += 1; continue - if ch == '}': dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue - if ch == '<': da += 1; buf.append(ch); i += 1; continue - if ch == '>': - if da > 0: da -= 1 - buf.append(ch); i += 1; continue - - if ch == '.' and dp == db == dbr == da == 0: - parts.append(''.join(buf).strip()); buf = []; i += 1; continue - buf.append(ch); i += 1 - parts.append(''.join(buf).strip()) - return [p for p in parts if p] - - def _strip_call_parens(s: str) -> str: - s2 = s.rstrip() - if not s2.endswith(')'): - return s - depth = 0 - in_str = in_chr = False - i = len(s2) - 1 - while i >= 0: - ch = s2[i] - if in_str: - if ch == '\\': i -= 2; continue - if ch == '"': in_str = False; i -= 1; continue - i -= 1; continue - if in_chr: - if ch == '\\': i -= 2; continue - if ch == "'": in_chr = False; i -= 1; continue - i -= 1; continue - if ch == '"': in_str = True; i -= 1; continue - if ch == "'": in_chr = True; i -= 1; continue - if ch == ')': depth += 1; i -= 1; continue - if ch == '(': - depth -= 1 - if depth == 0: return s2[:i].rstrip() - i -= 1; continue - i -= 1 - return s - - def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: - sig = re.compile( + """ + Same logic, but uses shared helpers for dot-splitting and call-paren stripping. + """ + @lru_cache(maxsize=1024) + def _file_sig_re(method_name: str) -> re.Pattern: + return re.compile( rf"""(?[A-Za-z_.$?][\w$.<>\[\]?]*?)\s+{re.escape(method_name)}\s*\( - """, re.VERBOSE) - m = sig.search(source_text) + """, + re.VERBOSE, + ) + + def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: + m = _file_sig_re(method_name).search(source_text or "") if not m: return None - ret = m.group('ret').strip() - ret = re.sub(r'<[^<>]*>', '', ret) - ret = re.sub(r'\s*(\[\])+$', '', ret) + ret = m.group("ret").strip() + ret = re.sub(r"<[^<>]*>", "", ret) + ret = _ARRAY_SUFFIX_RE.sub("", ret).strip() return ret or None - expr = initializer.strip().rstrip(';') - chain = _split_top_level_dots(expr) + expr = (initializer or "").strip().rstrip(';') + chain = self._split_top_level_dots(expr) if not chain: return None - # ----- CASE 1: starts with a variable identifier (old behavior) ----- start = chain[0].strip() ident_only = re.match(r'^[A-Za-z_$][\w$]*$', start) if ident_only and (len(chain) == 1 or not start.endswith(')')): @@ -3096,13 +3573,11 @@ def _find_method_return_type_in_file(source_text: str, method_name: str) -> str if not start_type: return None cur_type = re.sub(r'<[^<>]*>', '', start_type) - cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) - + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() else: - # Examine only the "head" before '(' to decide qualification - head = _strip_call_parens(start).strip() + head = self._strip_call_parens(start).strip() - # ----- CASE 2: unqualified method call in same file: foo(a,b) ----- + # CASE 2: unqualified method call in same file: foo(a,b) if start.endswith(')') and '.' not in head: mname = re.match(r'^([A-Za-z_$][\w$]*)$', head) if not mname: @@ -3114,32 +3589,29 @@ def _find_method_return_type_in_file(source_text: str, method_name: str) -> str if not ret: return None cur_type = re.sub(r'<[^<>]*>', '', ret) - cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() - # ----- CASE 3: static-style or qualified head: ClassName.staticMethod(...) or pkg.Class.m(...) + # CASE 3: static-style or qualified head: ClassName.staticMethod(...) or pkg.Class.m(...) elif start.endswith(')') and '.' in head: - # Split head into type-like left and method right left, _, right = head.rpartition('.') - # Left should be a type token (simple or qualified). We don't verify imports here. base_type = re.sub(r'<[^<>]*>', '', left) - base_type = re.sub(r'\s*(\[\])+$', '', base_type) + base_type = _ARRAY_SUFFIX_RE.sub("", base_type).strip() method_name = right if not re.match(r'^[A-Za-z_$][\w$]*$', method_name): return None - # Return type of static method on 'base_type' ret = self._find_method_return_type_in_type_docs(base_type, method_name, type_documents) if not ret: return None cur_type = re.sub(r'<[^<>]*>', '', ret) - cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() else: return None - # ----- Walk remaining segments; method calls change the type ----- + # Walk remaining segments; method calls change the type for seg in chain[1:]: - seg_core = _strip_call_parens(seg).strip() - if not seg.endswith(')'): # field access — keep type + seg_core = self._strip_call_parens(seg).strip() + if not seg.endswith(')'): continue mname = re.match(r'^([A-Za-z_$][\w$]*)\s*$', seg_core) if not mname: @@ -3149,9 +3621,20 @@ def _find_method_return_type_in_file(source_text: str, method_name: str) -> str if not ret: return None cur_type = re.sub(r'<[^<>]*>', '', ret) - cur_type = re.sub(r'\s*(\[\])+$', '', cur_type) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() return cur_type or None def get_package_name(self, function: Document, package_name: str) -> str: - return package_name if extract_jar_name(function.metadata['source']) in package_name else '' \ No newline at end of file + jar_name = extract_jar_name(function.metadata['source']) + if not jar_name: + return '' + # Extract artifactId:version from GAV for exact match instead of substring. + # Substring matching (jar_name in package_name) caused false positives when + # short jar_names like "st" matched unrelated GAVs containing "xstream". + parts = package_name.split(':') + if len(parts) == 3: + artifact_version = f"{parts[1]}:{parts[2]}" + else: + artifact_version = package_name + return package_name if jar_name == artifact_version else '' \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index 33c43afc2..731eee2f7 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -1,12 +1,15 @@ +import itertools import re +import threading +from collections import defaultdict +from dataclasses import dataclass, field from pathlib import Path from typing import List -from tqdm import tqdm from langchain_core.documents import Document -from exploit_iq_commons.utils.chain_of_calls_retriever_base import PARENTS_INDEX, calculate_hashable_string_for_function, EXCLUSIONS_INDEX, \ - ChainOfCallsRetrieverBase, METHOD_EXCLUSIONS_INDEX +from exploit_iq_commons.utils.chain_of_calls_retriever_base import calculate_hashable_string_for_function, \ + ChainOfCallsRetrieverBase from exploit_iq_commons.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( @@ -34,6 +37,223 @@ # Matches "--sources/" in the path _GAV_DIR_RE = re.compile(r'([^/\\]+)-([0-9][^/\\]*)-sources(?:/|\\)') +# Module-level cache: one _JavaRepoData per (git_repo, ref). +# Entries are explicitly removed via _release_repo_data when ref count drops to 0. +_repo_data_cache: dict[tuple, '_JavaRepoData'] = {} +_repo_data_cache_lock = threading.Lock() + + +class _JavaRepoData: + """Repo-level data shared across all JavaChainOfCallsRetriever instances + for the same (git_repo, ref). + + Holds the 5 CCA structures + source-keyed function index. Multiple + package-specific retrievers reference the same instance, eliminating + duplicate in-memory copies of multi-GB data structures. + + Immutable after construction — safe for concurrent reads. + """ + + def __init__(self, documents_of_functions_raw: list[Document], + documents_of_types: list, + documents_of_full_sources: dict, + type_inheritance: dict, + types_classes_fields_mapping: dict, + functions_local_variables_index: dict): + logger.debug("_JavaRepoData - building source-keyed function index") + # Build source-keyed function index (always needed by FL) + self.source_to_fn_docs: dict[str, list[Document]] = {} + for doc in documents_of_functions_raw: + src = doc.metadata.get('source', '') + if src in self.source_to_fn_docs: + self.source_to_fn_docs[src].append(doc) + else: + self.source_to_fn_docs[src] = [doc] + logger.debug("_JavaRepoData - source_to_fn_docs built (%d sources, %d docs)", + len(self.source_to_fn_docs), len(documents_of_functions_raw)) + + # CCA structures — read-only, shared by all package retrievers + self.documents_of_types = documents_of_types + self.documents_of_full_sources = documents_of_full_sources + self.type_inheritance = type_inheritance + self.types_classes_fields_mapping = types_classes_fields_mapping + self.functions_local_variables_index = functions_local_variables_index + logger.debug("_JavaRepoData - assigned CCA structures (types=%d, full_sources=%d, " + "type_inheritance=%d, fields_mapping=%d, local_vars=%d)", + len(documents_of_types), len(documents_of_full_sources), + len(type_inheritance), len(types_classes_fields_mapping), + len(functions_local_variables_index)) + + # Reference count: how many JavaChainOfCallsRetriever instances reference + # this repo data. Managed by _get_or_create_repo_data / _release_repo_data. + self._ref_count = 0 + self._cache_key: tuple | None = None + + +def _release_repo_data(repo_data: _JavaRepoData) -> None: + """Decrement ref count and remove from cache when no searcher references it. + + Called from transitive_code_search.py when a searcher is evicted from _searcher_cache. + """ + with _repo_data_cache_lock: + repo_data._ref_count -= 1 + logger.debug("_release_repo_data - decremented ref_count to %d for %s", repo_data._ref_count, repo_data._cache_key) + if repo_data._ref_count <= 0 and repo_data._cache_key is not None: + _repo_data_cache.pop(repo_data._cache_key, None) + logger.info("Released shared _JavaRepoData for %s", repo_data._cache_key) + + +def _get_or_create_repo_data(documents: List[Document], + code_source_info: SourceDocumentsInfo, + language_parser) -> _JavaRepoData: + """Get or create shared repo-level data for the given (git_repo, ref). + + Thread-safe: uses a lock to prevent duplicate creation. In practice, + concurrent calls for the same key are already serialized by the upstream + repo_lock in _build_or_get_cached, but the lock here provides defense-in-depth. + """ + key = (code_source_info.git_repo, code_source_info.ref) + logger.debug("_get_or_create_repo_data - looking up key %s", key) + + # Fast path: already cached + with _repo_data_cache_lock: + repo_data = _repo_data_cache.get(key) + if repo_data is not None: + repo_data._ref_count += 1 + logger.info("Reusing shared _JavaRepoData for %s (ref_count=%d)", key, repo_data._ref_count) + return repo_data + + logger.debug("_get_or_create_repo_data - cache miss, building repo data for %s", key) + + # Build all repo-level structures (same logic as old _build_repo_data) + allowed_files_extensions = language_parser.supported_files_extensions() + + logger.debug("_get_or_create_repo_data - loading pickle sub-caches") + documents_of_functions_raw, exist_fn = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_functions") + documents_of_types, exist_ty = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_types") + documents_of_full_sources, exist_fs = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_full_sources") + + need_fn = not exist_fn + need_ty = not exist_ty + need_fs = not exist_fs + logger.debug("_get_or_create_repo_data - pickle cache status: functions=%s, types=%s, full_sources=%s", + "hit" if exist_fn else "miss", "hit" if exist_ty else "miss", "hit" if exist_fs else "miss") + + # Single pass over raw documents — only needed if any of fn/ty/fs are missing from pickle + if need_fn or need_ty or need_fs: + logger.debug("_get_or_create_repo_data - classifying raw documents (need_functions=%s, need_types=%s, need_full_sources=%s)", + need_fn, need_ty, need_fs) + if need_fn: + fn_list = [] + if need_ty: + ty_list = [] + if need_fs: + full_dict = {} + + for doc in documents: + source = str(doc.metadata['source']) + if not any(source.endswith(ext) for ext in allowed_files_extensions): + continue + content_type = doc.metadata.get('content_type') + if need_fs and content_type == 'simplified_code': + full_dict[doc.metadata.get('source')] = doc + if need_fn and language_parser.is_function(doc): + fn_list.append(doc) + if need_ty and language_parser.is_doc_type(doc): + ty_list.append(doc) + + if need_fn: + documents_of_functions_raw = fn_list + logger.debug("_get_or_create_repo_data - saving documents_of_functions to pickle (%d docs)", len(fn_list)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_functions_raw, "documents_of_functions") + if need_ty: + documents_of_types = ty_list + logger.debug("_get_or_create_repo_data - saving documents_of_types to pickle (%d docs)", len(ty_list)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_types, "documents_of_types") + if need_fs: + documents_of_full_sources = full_dict + logger.debug("_get_or_create_repo_data - saving documents_of_full_sources to pickle (%d docs)", len(full_dict)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_full_sources, "documents_of_full_sources") + + type_inheritance, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "type_inheritance") + if not exist: + logger.debug("_get_or_create_repo_data - building type_inheritance from documents_of_full_sources") + type_inheritance = create_inheritance_map(documents_of_full_sources.values()) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + type_inheritance, "type_inheritance") + else: + logger.debug("_get_or_create_repo_data - type_inheritance loaded from pickle") + + types_classes_fields_mapping, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "types_classes_fields_mapping") + if not exist: + logger.debug("_get_or_create_repo_data - building types_classes_fields_mapping") + types_classes_fields_mapping = language_parser.parse_all_type_struct_class_to_fields( + documents_of_types, type_inheritance) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + types_classes_fields_mapping, "types_classes_fields_mapping") + else: + logger.debug("_get_or_create_repo_data - types_classes_fields_mapping loaded from pickle") + + functions_local_variables_index, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "functions_local_variables_index") + if not exist: + logger.debug("_get_or_create_repo_data - building functions_local_variables_index") + functions_local_variables_index = language_parser.create_map_of_local_vars(documents_of_functions_raw) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + functions_local_variables_index, "functions_local_variables_index") + else: + logger.debug("_get_or_create_repo_data - functions_local_variables_index loaded from pickle") + + logger.debug("_get_or_create_repo_data - creating _JavaRepoData instance") + repo_data = _JavaRepoData( + documents_of_functions_raw=documents_of_functions_raw, + documents_of_types=documents_of_types, + documents_of_full_sources=documents_of_full_sources, + type_inheritance=type_inheritance, + types_classes_fields_mapping=types_classes_fields_mapping, + functions_local_variables_index=functions_local_variables_index, + ) + + with _repo_data_cache_lock: + # Double-check: another thread may have created it while we were building + existing = _repo_data_cache.get(key) + if existing is not None: + existing._ref_count += 1 + logger.debug("_get_or_create_repo_data - lost race, reusing existing _JavaRepoData for %s (ref_count=%d)", + key, existing._ref_count) + return existing + repo_data._cache_key = key + repo_data._ref_count = 1 + _repo_data_cache[key] = repo_data + logger.info("Created new shared _JavaRepoData for %s", key) + + return repo_data + + +@dataclass +class _JavaSearchCtx: + """Per-search mutable state, created fresh for each get_relevant_documents call. + + Keeps the retriever instance fully immutable after __init__, so concurrent + CCA calls can safely share it without deep-copy. + """ + found_path: bool = False + exclusions: defaultdict = field(default_factory=lambda: defaultdict(list)) + method_exclusions: defaultdict = field(default_factory=lambda: defaultdict(dict)) + last_visited_parent_package_indexes: dict = field(default_factory=dict) + tree_additions: dict = field(default_factory=dict) + root_docs: list = field(default_factory=list) + jar_to_docs: dict = field(default_factory=dict) + + class JavaChainOfCallsRetriever(ChainOfCallsRetrieverBase): """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being called from the application code base or not. @@ -54,13 +274,12 @@ def __init__(self, documents: List[Document], used here to build the dependency tree for a more efficient lookup and search. """ - logger.debug("Creating Chain of Calls Retriever") - logger.debug("Starting building Chain of Calls Retriever") + logger.debug("Chain of Calls Retriever - creating instance for ecosystem=%s", ecosystem) self.ecosystem = ecosystem - logger.debug("Chain of Calls Retriever - creating dependency tree") + logger.debug("Chain of Calls Retriever - creating dependency tree") # Build dependency tree based on the parameter programming language/package manager. self.dependency_tree = DependencyTree(ecosystem=ecosystem, query=query) - logger.debug("Chain of Calls Retriever - get language parser") + logger.debug("Chain of Calls Retriever - getting language parser") self.language_parser = get_language_function_parser(ecosystem, self.dependency_tree) self.manifest_path = manifest_path # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. @@ -70,148 +289,227 @@ def __init__(self, documents: List[Document], self.tree_dict = dict() # Build a dependency tree using the dependency tree builder logic. + logger.debug("Chain of Calls Retriever - building dependency tree from manifest") tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) for package, parents in tree.items(): parents.extend([package]) - self.tree_dict[package] = list() - self.tree_dict[package].append(parents) - self.tree_dict[package].append([]) # For package exclusions - self.tree_dict[package].append({}) # For methods exclusions + self.tree_dict[package] = parents self.supported_packages = list(self.tree_dict.keys()) - logger.debug("Chain of Calls Retriever - populating functions documents") - allowed_files_extensions = self.language_parser.supported_files_extensions() - - # filter out unsupported files extensions. - documents = [doc for doc in documents - if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) - .endswith(ext)])] - - documents_of_functions, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_functions") - # filter out types and full code documents, retaining only functions/methods documents in this attribute. - if not exist: - self.documents_of_functions = [doc for doc in tqdm(documents, total=len(documents), desc="Filtering documents, leaving functions/methods ...") - if self.language_parser.is_function(doc)] - save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_functions, "documents_of_functions") - else: - self.documents_of_functions = documents_of_functions - - documents_of_types, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_types") - # filter out full code documents and functions/methods docs, retaining only types/classes docs - if not exist: - self.documents_of_types = [doc for doc in tqdm(documents, total=len(documents) , desc="Filtering documents, leaving types/classes ...") - if self.language_parser.is_doc_type(doc)] - save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.documents_of_types, "documents_of_types") - else: - self.documents_of_types = documents_of_types - - # boolean attribute that indicates whether a path was found or not, initially set to False. - self.found_path = False - # Filter out all documents but full source docs. - self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in documents - if doc.metadata.get('content_type') == 'simplified_code'} - - # Filter function list with the function existing in the dependencies - self.documents = [doc for doc in tqdm(self.documents_of_functions, total=len(self.documents_of_functions) , desc="Filtering function list to functions existing in the dependencies") - if ('content_type' in doc.metadata and doc.metadata['content_type'] == 'functions_classes') - and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or - any(parent for parent in self.supported_packages - if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] - - logger.debug("Chain of Calls Retriever - after documents_of_full_sources") - self.last_visited_parent_package_indexes = dict() - - type_inheritance, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "type_inheritance") - if not exist: - self.type_inheritance = create_inheritance_map([fsrc for fsrc in self.documents_of_full_sources.values()]) - save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.type_inheritance, "type_inheritance") - else: - self.type_inheritance = type_inheritance - - types_classes_fields_mapping, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "types_classes_fields_mapping") - # Constructing a map of types and classes to their attributes/members/fields - if not exist: - self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields( - self.documents_of_types, self.type_inheritance) - save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.types_classes_fields_mapping, "types_classes_fields_mapping") - else: - self.types_classes_fields_mapping = types_classes_fields_mapping - - functions_local_variables_index, exist = retrieve_from_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "functions_local_variables_index") - # Create a data structure containing dict of key=(function_name@source_file),value = dict of - # local variables names mapped to (types, (values or expressions)) - if not exist: - self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) - save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, self.functions_local_variables_index, "functions_local_variables_index") - else: - self.functions_local_variables_index = functions_local_variables_index + logger.debug("Chain of Calls Retriever - dependency tree built (%d packages)", len(self.supported_packages)) + + # Get or create shared repo-level data. Multiple retrievers for the + # same (git_repo, ref) share one _JavaRepoData instance, eliminating + # duplicate multi-GB data structures in memory. + logger.debug("Chain of Calls Retriever - getting or creating shared repo data") + self._repo_data = _get_or_create_repo_data(documents, code_source_info, self.language_parser) + + # Set direct attribute references to shared repo data for backward + # compatibility — all existing code reads self.xxx and works unchanged. + self._source_to_fn_docs = self._repo_data.source_to_fn_docs + self.documents_of_types = self._repo_data.documents_of_types + self.documents_of_full_sources = self._repo_data.documents_of_full_sources + self.type_inheritance = self._repo_data.type_inheritance + self.types_classes_fields_mapping = self._repo_data.types_classes_fields_mapping + self.functions_local_variables_index = self._repo_data.functions_local_variables_index + logger.debug("Chain of Calls Retriever - shared repo data linked") + + # Pre-compute which jar_names match any supported package (O(unique_jars × packages) + # instead of O(all_docs × packages)) + logger.debug("Chain of Calls Retriever - computing valid jar names") + prefix_3p = self.language_parser.dir_name_for_3rd_party_packages() + unique_jar_names = {extract_jar_name(doc.metadata['source']) + for doc in self._iter_documents_of_functions() + if doc.metadata.get('content_type') == 'functions_classes' + and doc.metadata['source'].startswith(prefix_3p)} + valid_jar_names = {jar_name for jar_name in unique_jar_names + if any(p for p in self.supported_packages + if p is not None and jar_name in p)} + logger.debug("Chain of Calls Retriever - found %d unique jars, %d valid jars", + len(unique_jar_names), len(valid_jar_names)) + + # Build jar-name index directly from documents_of_functions without + # materializing an intermediate self.documents list. + # Filters the same way: keep docs with content_type 'functions_classes' + # that are either root-level or have a jar_name in valid_jar_names. + logger.debug("Chain of Calls Retriever - building doc index (root_docs + jar_to_docs)") + self._root_docs, self._jar_to_docs = self._build_doc_index_filtered( + self._iter_documents_of_functions(), prefix_3p, valid_jar_names) + logger.debug("Chain of Calls Retriever - doc index built (root_docs=%d, jar_groups=%d)", + len(self._root_docs), len(self._jar_to_docs)) + + # Satisfy base class documents attribute via property (avoids duplicate list) + self._documents_materialized = None + logger.debug("Chain of Calls Retriever - init complete") + + @property + def documents(self): + """Lazily reconstruct the filtered documents list from the pre-built index. + Only materialized if explicitly accessed (e.g. via get_documents()). + """ + if self._documents_materialized is None: + self._documents_materialized = list(self._root_docs) + for docs in self._jar_to_docs.values(): + self._documents_materialized.extend(docs) + return self._documents_materialized + + @documents.setter + def documents(self, value): + self._documents_materialized = value + + def _iter_documents_of_functions(self): + """Iterate over all function documents from the source-keyed index + without materializing a flat list.""" + for docs in self._source_to_fn_docs.values(): + yield from docs + + @property + def documents_of_functions(self): + """Provide the documents_of_functions interface expected by callers + (base class attribute, FL, search_for_called_function) by iterating + the source-keyed index lazily. + + Returns a list for callers that need indexing/len; built on demand + from the memory-efficient _source_to_fn_docs index. + """ + return list(self._iter_documents_of_functions()) + + @documents_of_functions.setter + def documents_of_functions(self, value): + """Allow direct assignment (used by tests that bypass __init__).""" + self._source_to_fn_docs = {} + for doc in (value or []): + src = doc.metadata.get('source', '') + if src in self._source_to_fn_docs: + self._source_to_fn_docs[src].append(doc) + else: + self._source_to_fn_docs[src] = [doc] + + def get_fn_docs_by_source(self, source: str) -> list[Document]: + """Fast O(1) lookup of function documents by source path. + Used by find_function() instead of iterating all documents.""" + return self._source_to_fn_docs.get(source, []) + + def _get_parents(self, package: str, ctx: _JavaSearchCtx) -> list | None: + """Look up parents for a package, checking both the immutable tree_dict + and any packages added during the current search (ctx.tree_additions).""" + parents = self.tree_dict.get(package) + if parents is not None: + return parents + return ctx.tree_additions.get(package) + + @staticmethod + def _build_doc_index_from(documents, language_parser): + """Build root_docs list and jar_to_docs dict from a documents list.""" + root_docs = [] + jar_to_docs: dict[str, list[Document]] = {} + for doc in documents: + if language_parser.is_root_package(doc): + root_docs.append(doc) + else: + jar_name = extract_jar_name(doc.metadata['source']) + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + return root_docs, jar_to_docs + + def _build_doc_index_filtered(self, documents_of_functions, prefix_3p, valid_jar_names): + """Build root_docs and jar_to_docs directly from documents_of_functions, + applying the content_type and valid_jar_names filter inline. - logger.debug("Chain of Calls Retriever - after functions_local_variables_index") + This avoids materializing an intermediate self.documents list. + """ + root_docs = [] + jar_to_docs: dict[str, list[Document]] = {} + for doc in documents_of_functions: + if doc.metadata.get('content_type') != 'functions_classes': + continue + source = doc.metadata['source'] + if source.startswith(prefix_3p): + jar_name = extract_jar_name(source) + if jar_name not in valid_jar_names: + continue + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + else: + if self.language_parser.is_root_package(doc): + root_docs.append(doc) + else: + jar_name = extract_jar_name(source) + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + return root_docs, jar_to_docs - def __find_caller_function(self, document_function: Document, function_package: str) -> Document: + def __find_caller_function(self, document_function: Document, function_package: str, ctx: _JavaSearchCtx) -> Document: """ This method gets function and package as arguments, search and return a caller function of a package, if exists :param document_function: the document containing the function code and signature :param function_package: the package name containing the function + :param ctx: per-search mutable state :return: a document of a function that is calling document_function """ - # Create a copy of the document types list - documents_of_types: list[Document] = self.documents_of_types[:] direct_parents = list() # gets list of all direct parents of function - list_of_packages = self.tree_dict.get(function_package) - if list_of_packages: - direct_parents.extend(list_of_packages[PARENTS_INDEX]) + parents = self._get_parents(function_package, ctx) + if parents: + direct_parents.extend(parents) # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) function_file_name = document_function.metadata.get('source') - relevant_docs_to_search_in = list() - last_visited_package_index = (self.last_visited_parent_package_indexes + last_visited_package_index = (ctx.last_visited_parent_package_indexes .get(calculate_hashable_string_for_function(function_file_name, function_name_to_search), 0)) - func_pack_from_tree = self.tree_dict.get(function_package) - package_exclusions = func_pack_from_tree[EXCLUSIONS_INDEX] - method_exclusions = func_pack_from_tree[METHOD_EXCLUSIONS_INDEX] + package_exclusions = ctx.exclusions[function_package] + method_exclusions = ctx.method_exclusions[function_package] target_class_names: frozenset[str] - class_name = self.language_parser.get_class_name_from_class_function(document_function) - key = (class_name, document_function.metadata['source']) + + fqcn = self.language_parser.get_class_name_from_class_function(document_function) if "dummy" not in function_file_name: + key = (fqcn, function_file_name) target_class_names = get_target_class_names(self.type_inheritance[key]) + # Non-dummy path: use self.documents_of_types directly (read-only, no copy needed) + documents_of_types = self.documents_of_types else: - target_class_names = frozenset([class_name]) - target_type_doc = Document(page_content="public class " + class_name + "{}", - metadata={"source": self.language_parser.dir_name_for_3rd_party_packages() + - "/" + function_package.partition(':')[-1] + - "/" + class_name + ".java", - "ecosystem": self.ecosystem}) - documents_of_types.append(target_type_doc) + fqcn_no_dummy = fqcn.replace(dummy_package_name, "") + target_class_names = frozenset([fqcn_no_dummy]) + target_type_doc = Document(page_content="public class " + fqcn.rpartition('.')[2] + "{}", + metadata={"source": self.language_parser.dir_name_for_3rd_party_packages() + + "/" + fqcn_no_dummy.replace(".", "/") + + ".java", + "ecosystem": self.ecosystem}) + # Dummy path: copy only here since we need to append a synthetic doc + documents_of_types = self.documents_of_types + [target_type_doc] + document_function = target_type_doc # Search for caller functions only at parents according to dependency tree. - for package in direct_parents[last_visited_package_index:]: - sources_location_packages = True - if self.tree_dict.get(package)[PARENTS_INDEX][0] == ROOT_LEVEL_SENTINEL: - sources_location_packages = False - - artifact_id = package.split(":", 2)[1] - possible_docs = self.get_possible_docs(function_name_to_search, - artifact_id, - package_exclusions, - sources_location_packages, - target_class_names, - method_exclusions) - - jar_name = convert_from_maven_artifact(package) - # Collect all potential caller functions - for doc in self.get_functions_for_package(package_name=jar_name, - documents=possible_docs, - sources_location_packages=sources_location_packages, - function_to_search=function_name_to_search, - callee_function_file_name=function_file_name): - relevant_docs_to_search_in.append(doc) - # Perform the search only on the subset of potential caller functions - for doc in relevant_docs_to_search_in: + # Iterate candidate docs lazily via generators instead of collecting into a list first. + def _candidate_docs(): + for package in direct_parents[last_visited_package_index:]: + sources_location_packages = True + if self._get_parents(package, ctx)[0] == ROOT_LEVEL_SENTINEL: + sources_location_packages = False + + artifact_id = package.split(":", 2)[1] + possible_docs = self._get_possible_docs(function_name_to_search, + artifact_id, + sources_location_packages, + target_class_names, + method_exclusions, + ctx.root_docs, ctx.jar_to_docs) + + jar_name = convert_from_maven_artifact(package) + yield from self.get_functions_for_package(package_name=jar_name, + documents=possible_docs, + sources_location_packages=sources_location_packages, + function_to_search=function_name_to_search, + callee_function_file_name=function_file_name) + + # Perform the search on candidate caller functions (lazily generated) + for doc in _candidate_docs(): function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, callee_function_name= function_name_to_search, @@ -229,7 +527,7 @@ def __find_caller_function(self, document_function: Document, function_package: functions_local_variables_index= self.functions_local_variables_index, documents_of_functions= - self.documents_of_functions, + self._source_to_fn_docs, type_inheritance=self.type_inheritance) method_name = extract_method_name_with_params(doc.page_content) @@ -278,32 +576,46 @@ def _is_method_excluded(self, function_name_to_search: str, target_class_names: return key in method_exclusions - # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all - # excluded functions, and all function that their body doesn't contain the target function name to search for. - def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], - sources_location_packages: bool, - target_class_names: frozenset[str], - method_exclusions: dict) -> (list[Document], bool): + def _get_possible_docs(self, function_name_to_search: str, package: str, + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict, + root_docs: list, jar_to_docs: dict) -> list[Document]: + """Core filtering logic used by both get_possible_docs and __find_caller_function.""" if sources_location_packages: - result = [doc for doc in self.documents if package in doc.metadata.get('source') - and self.language_parser.is_function(doc) and - not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + candidates = [] + for jar_name, docs in jar_to_docs.items(): + if package in jar_name: + candidates.extend(docs) + result = [doc for doc in candidates + if not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] else: - result = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and - self.language_parser.is_function(doc) and - not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + result = [doc for doc in root_docs + if not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] return result + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + """ + Filter out irrelevant functions that cannot be caller functions. + Delegates to _get_possible_docs using the instance's pre-built index. + """ + return self._get_possible_docs(function_name_to_search, package, + sources_location_packages, target_class_names, + method_exclusions, self._root_docs, self._jar_to_docs) + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of # calls from application to input function in the input package. def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: """Sync implementations for retriever.""" - self.found_path = False + ctx = _JavaSearchCtx() class_name, method_name, package_name = self.extract_from_query(query) @@ -316,12 +628,15 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: package_name = package found_package = True break - # If it's , then create a document for it. + # If it's found, then create a document for it. if found_package: target_function_doc = self.__find_initial_function(class_name=class_name, method_name=method_name, package_name=package_name, - documents=self.documents) + ctx=ctx) + # Use the pre-built index from __init__ (immutable, shared across calls) + ctx.root_docs = self._root_docs + ctx.jar_to_docs = self._jar_to_docs if target_function_doc: matching_documents.append(target_function_doc) @@ -346,7 +661,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, class_name) - root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value[0]] + root_package = next((key for key, value in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value), None) prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() # find all parents ( all importing packages) of the input package so we'll have candidate pkgs to search in. parents = list({ @@ -359,23 +674,27 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: for doc in importing_docs: if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): - parents.append(root_package[0]) + parents.append(root_package) break - self.tree_dict[package_name] = [parents, [], {}] - - # Create function list with the function existing in the dependencies - self.documents = [doc for doc in tqdm(self.documents_of_functions, total=len(self.documents_of_functions) , desc="Filtering function list to functions existing in the dependencies") if doc.metadata['content_type'] == 'functions_classes' - and (not doc.metadata['source'].startswith(self.language_parser.dir_name_for_3rd_party_packages()) or - any(parent for parent in self.tree_dict[package_name][PARENTS_INDEX] - if parent is not None and extract_jar_name(doc.metadata['source']) in parent))] - + ctx.tree_additions[package_name] = parents + + # Build local doc index for the dummy package search by selecting matching + # entries from the pre-built index (avoids O(N) rescan of all documents) + dummy_parents = ctx.tree_additions[package_name] + ctx.root_docs = list(self._root_docs) + ctx.jar_to_docs = {} + for jar_name, docs in self._jar_to_docs.items(): + if any(parent for parent in dummy_parents + if parent is not None and jar_name in parent): + ctx.jar_to_docs[jar_name] = docs current_package_name = package_name # main loop. while not end_loop: # Find a caller function and containing package in the dependency tree according to hierarchy found_document = self.__find_caller_function(document_function=target_function_doc, - function_package=current_package_name) + function_package=current_package_name, + ctx=ctx) # If found, then add it to path if found_document: matching_documents.append(found_document) @@ -383,14 +702,14 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: # If the function is in the application ( root package), then we finished and found such a path. if self.language_parser.is_root_package(found_document): end_loop = True - self.found_path = True + ctx.found_path = True # Otherwise, we continue to search for callers for the current found function, in order to extend # the chain of calls and potentially find a path from application to the vulnerable # function in input package else: target_function_doc = found_document # extract package name from function document - current_package_name = self.__determine_doc_package_name(target_function_doc) + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) else: # end loop because didn't find a caller for initial function if len(matching_documents) == 1: @@ -401,24 +720,24 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: dead_end_node = matching_documents.pop() logger.info(f"\nmatching_documents size is {len(matching_documents)}") # Excludes dead end function node from future searches, as it led to nowhere. - self.tree_dict.get(current_package_name)[EXCLUSIONS_INDEX].append(dead_end_node) + ctx.exclusions[current_package_name].append(dead_end_node) target_function_doc = matching_documents[-1] - current_package_name = self.__determine_doc_package_name(target_function_doc) + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was # found or not. - return matching_documents, self.found_path + return matching_documents, ctx.found_path def extract_from_query(self, query: str) -> tuple[str, str, str]: - (package_name, function) = tuple( query.splitlines()[0].strip('"\'').replace("#", ".").split(",")) + (package_name, function) = tuple( query.splitlines()[0].strip('"\'\u2018\u2019\u201c\u201d').replace("#", ".").split(",")) class_name = function.rpartition('.')[0] - method_name = function.rpartition('.')[2] + method_name = re.sub(r'\(.*\)$', '', function.rpartition('.')[2]) if not class_name and self.is_java_fqcn(package_name): class_name = package_name - package_name, class_name = self.infer_class_name_and_package_name(method_name, class_name) + package_name, class_name = self.infer_class_name_and_package_name(method_name, class_name, package_name) return class_name, method_name, package_name @@ -436,71 +755,51 @@ def __get_parents(self, importing_docs: list[Document], prefix_of_3rd_parties_li return parents - def __determine_doc_package_name(self, target_function_doc): + def __determine_doc_package_name(self, target_function_doc, ctx: _JavaSearchCtx): """ Determine the logical package/dependency identifier for a given document. - This method inspects `target_function_doc.metadata['source']` and tries to - resolve it to a "package name" or dependency key based on `self.tree_dict` - and the language parser's 3rd-party directory convention. - - Behaviour: - - If the source path contains the directory name returned by - `self.language_parser.dir_name_for_3rd_party_packages()`, the document - is treated as coming from a 3rd-party JAR: - * Extract a `jar_name` from the source path via `extract_jar_name`. - * Iterate over all values in `self.tree_dict`; for each entry look at - `dep_vals[0]` (a collection of dependency identifiers). - * Return the first dependency string (`dep_val`) that contains - `jar_name` as a substring. - - - Otherwise, the document is treated as non-3rd-party (e.g. first-party): - * Extract a `jar_name` from `"/" + src`. - * If `jar_name` exists as a key in `self.tree_dict` and its value is - truthy, return `jar_name`. - - - If no matching dependency/package can be found, return an empty string. - - Parameters - ---------- - target_function_doc : - A document object whose `metadata['source']` field contains the source - path used to infer the package/dependency. - - Returns - ------- - str - The resolved package/dependency identifier, or an empty string if no - match could be determined. + Checks both self.tree_dict (immutable) and ctx.tree_additions (search-local). """ src = target_function_doc.metadata['source'] if self.language_parser.dir_name_for_3rd_party_packages() in src: jar_name = extract_jar_name(src) - for dep_vals in self.tree_dict.values(): - for dep_val in dep_vals[0]: + for dep_vals in itertools.chain(self.tree_dict.values(), ctx.tree_additions.values()): + for dep_val in dep_vals: if jar_name in dep_val: return dep_val else: jar_name = extract_jar_name('/' + src) - if self.tree_dict[jar_name]: + parents = self._get_parents(jar_name, ctx) + if parents: return jar_name return "" - def __find_initial_function(self, class_name: str, method_name: str, package_name: str ,documents: list[Document]) -> Document | None: + def __find_initial_function(self, class_name: str, method_name: str, package_name: str, + ctx: _JavaSearchCtx) -> Document | None: jar_name = convert_from_maven_artifact(package_name) - - relevant_docs = [doc for doc in documents - if (self.language_parser.dir_name_for_3rd_party_packages() in doc.metadata.get('source') - and jar_name in doc.metadata.get('source') or - not self.language_parser.dir_name_for_3rd_party_packages() in doc.metadata.get('source')) and - self.language_parser.get_class_name_from_class_function(doc) == class_name.rpartition('.')[2] and - self.language_parser.get_function_name(doc) == method_name] - package_exclusions = self.tree_dict.get(package_name)[EXCLUSIONS_INDEX] + prefix_3p = self.language_parser.dir_name_for_3rd_party_packages() + + # Iterate over _root_docs + _jar_to_docs values (same docs as old self.documents) + # using a generator to avoid materializing the filtered list. + def _candidates(): + for doc in self._root_docs: + yield doc + for docs in self._jar_to_docs.values(): + for doc in docs: + yield doc + + package_exclusions = ctx.exclusions[package_name] # TODO handle method overloading - for document in relevant_docs: - package_exclusions.append(document) - return document + for doc in _candidates(): + source = doc.metadata.get('source') + if prefix_3p in source and jar_name not in source: + continue + if (self.language_parser.get_class_name_from_class_function(doc) == class_name and + self.language_parser.get_function_name(doc) == method_name): + package_exclusions.append(doc) + return doc return None @@ -675,55 +974,40 @@ def is_java_fqcn(self, s: str) -> bool: return _FQCN_STRICT_RE.match(s) is not None - def infer_class_name_and_package_name(self, method_name: str, class_name: str) -> (str, str): - simple_class_name = class_name - parts = class_name.split(".") - # if fqcn, take the simple class name - if len(parts) > 1: - simple_class_name = parts[-1] + def infer_class_name_and_package_name(self, method_name: str, class_name: str, package_name: str) -> (str, str): + maven_gav = is_maven_gav(package_name) + + # Two-pass approach: prefer exact FQCN match, fall back to simple-name match. + # This prevents substring false positives (e.g., XStream matching XStreamer). + for exact_first in (True, False): + for doc in self._iter_documents_of_functions(): + if method_name in doc.page_content and method_name == self.language_parser.get_function_name(doc): + doc_fqcn = self.language_parser.get_class_name_from_class_function(doc) + + if exact_first: + # Pass 1: exact FQCN match only + if doc_fqcn != class_name: + continue + else: + # Pass 2: simple class name match (for short/partial class names) + doc_simple = doc_fqcn.rsplit('.', 1)[-1] + class_simple = class_name.rsplit('.', 1)[-1] + if doc_simple != class_simple: + continue - for doc in self.documents_of_functions: - if method_name in doc.page_content and method_name == self.language_parser.get_function_name(doc): - if self.language_parser.get_class_name_from_class_function(doc) == simple_class_name: source = doc.metadata['source'] - artifact_name = self.extract_maven_artifact(source) - for val in self.tree_dict: - if artifact_name in val: - return val, self.extract_fqcn(source) - - return "", class_name - - def extract_fqcn(self, text: str) -> str: - """ - Convert a source file path to an FQCN. - Works both when a '-sources/' prefix exists and when it doesn't. - - Examples: - 'dependencies-sources/hibernate-core-6.6.13.Final-sources/org/hibernate/type/descriptor/java/ArrayJavaType.java' - -> 'org.hibernate.type.descriptor.java.ArrayJavaType' - 'org/hibernate/type/descriptor/java/ArrayJavaType.java' - -> 'org.hibernate.type.descriptor.java.ArrayJavaType' - """ - - p = text.replace("\\", "/") - - # If there's a '-sources/' prefix, drop everything up to and including the last occurrence. - marker = "-sources/" - cut = p.rfind(marker) - tail = p[cut + len(marker):] if cut != -1 else p - - # Strip leading slash if any - if tail.startswith("/"): - tail = tail[1:] - - # Drop the .java suffix (case-sensitive per your example; change to lower() if needed) - if tail.endswith(".java"): - tail = tail[:-5] + # If the library is provided make sure the doc belongs to library + if maven_gav and artifact_name in package_name: + return package_name, doc_fqcn + # If no library is provided look for the function + elif not maven_gav: + for val in self.tree_dict: + if artifact_name in val: + return val, doc_fqcn - # Convert path separators to dots to form the FQCN - return tail.replace("/", ".") + return package_name, class_name def extract_maven_artifact(self, path: str) -> str: """ @@ -752,4 +1036,4 @@ def extract_maven_artifact(self, path: str) -> str: if len(pkg_dirs) < 2: return "" - return f"{artifact}:{version}" + return f"{artifact}:{version}" \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py index 6b54c00da..4f95cadbf 100644 --- a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py +++ b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py @@ -1,26 +1,368 @@ +from __future__ import annotations + import bisect +import contextvars import os +import re +from dataclasses import dataclass, field -from typing import List, Tuple, Optional, Dict, Set +from typing import List, Tuple, Optional, Dict, FrozenSet from langchain_community.document_loaders.parsers.language.java import JavaSegmenter +_TEST_METHOD_ANNOTATIONS: FrozenSet[str] = frozenset({ + # JUnit 4/5 + "Test", + "ParameterizedTest", + "RepeatedTest", + "TestFactory", + "TestTemplate", + # TestNG + "org.testng.annotations.Test", # handled by simple-name "Test" too +}) + +_TEST_LIFECYCLE_ANNOTATIONS: FrozenSet[str] = frozenset({ + # JUnit 4 + "Before", + "After", + "BeforeClass", + "AfterClass", + # JUnit 5 + "BeforeEach", + "AfterEach", + "BeforeAll", + "AfterAll", + # TestNG + "BeforeMethod", + "AfterMethod", + "BeforeClass", + "AfterClass", + "BeforeSuite", + "AfterSuite", + "BeforeTest", + "AfterTest", + "BeforeGroups", + "AfterGroups", +}) + +# Class-level "this is a test class" markers (Quarkus + common Spring test slices) +_TEST_CLASS_ANNOTATIONS: FrozenSet[str] = frozenset({ + "QuarkusTest", + + # Spring / Spring Boot test slices (common ones) + "SpringBootTest", + "WebMvcTest", + "WebFluxTest", + "DataJpaTest", + "DataJdbcTest", + "DataMongoTest", + "JdbcTest", + "JooqTest", + "JsonTest", + "RestClientTest", + "GraphQlTest", + "ContextConfiguration", + "SpringJUnitConfig", + "SpringJUnitWebConfig", + "AutoConfigureMockMvc", + "AutoConfigureWebTestClient", + + # Older Spring JUnit 4 runners/config + "RunWith", # handled specially (needs SpringRunner) + "ExtendWith", # handled specially (needs SpringExtension) +}) + +# Scan only these chars; everything else can be skipped in big jumps. +# (We deliberately include '-' because we only care about "->".) +_LAMBDA_SCAN_SPECIAL_RE = re.compile(r'[/"\'-]') + +_pkg_imp_re = re.compile(r"(?m)^[ \t]*(?:package|import)\s+[^;\n]+;\s*\n") class JavaSegmenterWithMethods(JavaSegmenter): def __init__(self, code: str): super().__init__(code) + # ------------------------------ test filtering -------------------------------- + + def _looks_like_template_source(self, src: str) -> bool: + """ + Return True if the source contains template-language markers in *code*. + + This scanner ignores: + - Java // line comments + - Java /* ... */ block/Javadoc comments + - Java string literals "..." + - Java char literals '...' + - Java text blocks \"\"\" ... \"\"\" (Java 15+) + + Therefore, template markers that occur *only* inside strings/comments/text blocks + will NOT cause the file to be skipped. + """ + # Cheap prefilter + if "<" not in src and "$" not in src: + return False + + n = len(src) + i = 0 + + while i < n: + ch = src[i] + + # ---- comments ---- + if ch == '/' and i + 1 < n: + nxt = src[i + 1] + if nxt == '/': + # line comment + i = src.find('\n', i + 2) + if i == -1: + return False + i += 1 + continue + if nxt == '*': + # block/Javadoc comment + end = src.find('*/', i + 2) + if end == -1: + return False + i = end + 2 + continue + + # ---- text blocks / strings / chars ---- + if src.startswith('"""', i): + # text block + i += 3 + while i < n: + j = src.find('"""', i) + if j == -1: + return False + # if escaped \"\"\", keep scanning + k = j - 1 + backslashes = 0 + while k >= 0 and src[k] == '\\': + backslashes += 1 + k -= 1 + if (backslashes % 2) == 1: + i = j + 1 + continue + i = j + 3 + break + continue + + if ch == '"': + # regular string + i += 1 + while i < n: + if src[i] == '\\': + i += 2 + continue + if src[i] == '"': + i += 1 + break + i += 1 + continue + + if ch == "'": + # char literal + i += 1 + while i < n: + if src[i] == '\\': + i += 2 + continue + if src[i] == "'": + i += 1 + break + i += 1 + continue + + # ---- marker detection in code only ---- + # Check the handful of marker prefixes cheaply. + if ch == '$' and i + 1 < n and src[i + 1] == '{': + return True + if ch == '<' and i + 1 < n: + c1 = src[i + 1] + if c1 in ('#', '@'): + return True + if c1 == '/' and i + 2 < n and src[i + 2] == '#': + return True + + i += 1 + + return False + + def _read_qualified_ident_after_at(self, src: str, at_idx: int, end: int) -> Tuple[int, str, str]: + """ + Parse an annotation name starting at '@' (src[at_idx] == '@') within [at_idx, end). + + Returns + ------- + (next_idx, full_name, simple_name) + + - full_name can include dots: org.junit.jupiter.api.Test + - simple_name is the last segment: Test + """ + i = at_idx + 1 + n = min(end, len(src)) + + # Special-case annotation type declaration: '@interface' is not a "use" annotation. + if i + 8 <= n and src.startswith("interface", i): + return i + 9, "interface", "interface" + + # Read first identifier segment + if i >= n or not (_is_ident_start(src[i]) or src[i].isalpha()): + return at_idx + 1, "", "" + + start = i + i += 1 + while i < n and _is_ident_part(src[i]): + i += 1 + + # Read optional ".segment" repeats + while i < n and src[i] == '.': + dot = i + j = dot + 1 + if j < n and (_is_ident_start(src[j]) or src[j].isalpha()): + i = j + 1 + while i < n and _is_ident_part(src[i]): + i += 1 + else: + break + + full_name = src[start:i] + simple_name = full_name.rsplit(".", 1)[-1] + return i, full_name, simple_name + + def _source_is_test_source(self, src: str) -> bool: + """Heuristic test-source detector. + + Returns True iff `src` *appears* to be a test class/source. + + Detects: + - JUnit 4/5 + TestNG method/lifecycle annotations (e.g. @Test, @BeforeEach) + - JUnit3-style `extends TestCase` + - Quarkus / Spring / Spring Boot test classes via `_TEST_CLASS_ANNOTATIONS` + + Notes: + - Ignores tokens inside comments and string/text blocks. + - Designed to be fast: a cheap substring prefilter avoids scanning most sources. + """ + # Fast prefilter: + # - If there is no '@', the only supported positive is JUnit3-style `extends ... TestCase`. + # - If '@' exists, avoid scanning typical non-test sources by requiring at least one + # test-ish keyword to appear somewhere in the raw text. + if "@" not in src: + if "TestCase" not in src or "extends" not in src: + return False + else: + # Important: test-class annotations are often present even if no @Test is. + # `Test` is a broad but capitalized token that strongly correlates with test annotations. + if ("Test" not in src and + "RunWith" not in src and + "ExtendWith" not in src and + "ContextConfiguration" not in src and + "SpringJUnit" not in src and + "AutoConfigureMockMvc" not in src and + "AutoConfigureWebTestClient" not in src and + "org.junit" not in src and + "org.testng" not in src): + return False + + n = len(src) + i = 0 + saw_extends = False + + while i < n: + i = _skip_ws_comments(src, i) + if i >= n: + break + + ch = src[i] + + # Skip any string/char/text-block literal quickly. + if ch == '"' or ch == "'" or (ch == 'r' and i + 1 < n and src[i + 1] == '"'): + i = _skip_java_string_like(src, i) + continue + + if ch == '@': + # Parse annotation name (qualified or simple). + ni, _full, simple = self._read_qualified_ident_after_at(src, i, n) + + # Avoid misclassifying annotation type declarations: `public @interface Foo { ... }` + if simple == "interface": + i = ni + continue + + # JUnit/TestNG method-level / lifecycle annotations. + if simple in _TEST_METHOD_ANNOTATIONS or simple in _TEST_LIFECYCLE_ANNOTATIONS: + return True + + # Quarkus / Spring / Spring Boot class-level test annotations. + if simple in _TEST_CLASS_ANNOTATIONS: + # Only treat generic @ExtendWith/@RunWith as a "Spring test" if they bind + # the Spring extension/runner; otherwise they are too generic. + if simple == "ExtendWith": + j = _skip_ws_comments(src, ni) + if j < n and src[j] == '(': + close = _match_balanced_parens(src, j) + if close != -1 and "SpringExtension" in src[j:close]: + return True + i = ni + continue + + if simple == "RunWith": + j = _skip_ws_comments(src, ni) + if j < n and src[j] == '(': + close = _match_balanced_parens(src, j) + if close != -1 and "SpringRunner" in src[j:close]: + return True + i = ni + continue + + return True + + # Skip annotation argument list (if any) to avoid rescanning big nested blocks. + i = _skip_ws_comments(src, ni) + if i < n and src[i] == '(': + close = _match_balanced_parens(src, i) + i = (close + 1) if close != -1 else (i + 1) + continue + + # Detect JUnit3 style `extends ... TestCase` (ignoring comments/strings). + if _is_ident_start(ch): + j = i + 1 + while j < n and _is_ident_part(src[j]): + j += 1 + ident = src[i:j] + + if ident == "extends": + saw_extends = True + elif saw_extends: + # Allow qualified form: `extends junit.framework.TestCase` + if ident.endswith("TestCase") and ident.rsplit(".", 1)[-1] == "TestCase": + return True + saw_extends = False + + i = j + continue + + i += 1 + + return False + def extract_functions_classes(self) -> List[str]: """ This method extracts all the golang methods and anonymous functions from a source code, and appends them to - the list of all golang regular functions calculated in parent class' method. - : - :return: a list of all golang functions/methods/anonymous functions ( each one with a complete implementation, + the list of all java regular functions calculated in parent class' method. + :return: a list of all java functions/methods/anonymous functions ( each one with a complete implementation, signature + body) """ + if self._looks_like_template_source(self.code): + return [] + + if self._source_is_test_source(self.code): + return [] + function_classes = super().extract_functions_classes() - methods = extract_methods(self.code) # TODO Add constructors to the method list + methods = extract_methods(self.code) inner_classes = extract_inner_classes(self.code) # TODO Add anonymous inner classes support function_classes.extend(methods) function_classes.extend(inner_classes) @@ -493,25 +835,37 @@ def _skip_java_string_like(src: str, i: int) -> int: # --------------- delimiter & ignored-span index (one-pass pre-scan) ---------- -_PAR_O2C: Dict[int, int] = {} -_PAR_C2O: Dict[int, int] = {} -_BR_O2C: Dict[int, int] = {} -_BR_C2O: Dict[int, int] = {} - -# All spans to ignore for scanning (comments, strings, text blocks) -_IGN_SPANS: List[Tuple[int, int]] = [] # [(start, end)], end exclusive -_IGN_STARTS: List[int] = [] -_IGN_ENDS: List[int] = [] - -_DELIM_SRC_ID: Optional[int] = None +@dataclass +class _DelimCache: + """Per-task delimiter index cache. Stored in a ContextVar so concurrent + asyncio tasks each get their own isolated copy.""" + par_o2c: Dict[int, int] = field(default_factory=dict) + par_c2o: Dict[int, int] = field(default_factory=dict) + br_o2c: Dict[int, int] = field(default_factory=dict) + br_c2o: Dict[int, int] = field(default_factory=dict) + ign_spans: List[Tuple[int, int]] = field(default_factory=list) + ign_starts: List[int] = field(default_factory=list) + ign_ends: List[int] = field(default_factory=list) + src_id: Optional[int] = None + +_delim_cache_var: contextvars.ContextVar[_DelimCache] = contextvars.ContextVar('_delim_cache_var') + +def _get_delim_cache() -> _DelimCache: + try: + return _delim_cache_var.get() + except LookupError: + c = _DelimCache() + _delim_cache_var.set(c) + return c def _ignored_span_at(pos: int) -> Optional[Tuple[int, int]]: """Return (start,end) ignored span containing pos, or None.""" - if not _IGN_SPANS: + c = _get_delim_cache() + if not c.ign_spans: return None - idx = bisect.bisect_right(_IGN_STARTS, pos) - 1 - if idx >= 0 and _IGN_STARTS[idx] <= pos < _IGN_ENDS[idx]: - return _IGN_SPANS[idx] + idx = bisect.bisect_right(c.ign_starts, pos) - 1 + if idx >= 0 and c.ign_starts[idx] <= pos < c.ign_ends[idx]: + return c.ign_spans[idx] return None def _build_delim_index(source: str) -> Tuple[ @@ -617,61 +971,26 @@ def _populate_delim_index_cache(java_source: str): """ Build and cache delimiter/ignored-span indexes for a given Java source string. - Parameters - ---------- - java_source : str - The entire Java source text to index. - - Side Effects - ------------ - Populates (overwrites) the following global caches for O(1) delimiter lookups: - - _PAR_O2C : Dict[int, int] # map of '(' byte index -> matching ')' - - _PAR_C2O : Dict[int, int] # map of ')' byte index -> matching '(' - - _BR_O2C : Dict[int, int] # map of '{' byte index -> matching '}' - - _BR_C2O : Dict[int, int] # map of '}' byte index -> matching '{' - - _IGN_SPANS : List[Tuple[int, int]] # ignored spans (comments/strings/text blocks), end-exclusive - - _IGN_STARTS : List[int] # start positions of _IGN_SPANS (for bisect) - - _IGN_ENDS : List[int] # end positions of _IGN_SPANS (for bisect) - - _DELIM_SRC_ID : Optional[int] # identity of the currently indexed source - - Complexity - ---------- - O(len(src)) time and O(k) memory, where k is the number of delimiters and - ignored spans discovered. - - Notes - ----- - The cache is keyed by the *identity* of `src` (via id(src)), not its value. - Rebuild the index if you switch to a different source object. + Stores the result in a per-task ContextVar so concurrent asyncio tasks + each get their own isolated cache. """ - global _PAR_O2C, _PAR_C2O, _BR_O2C, _BR_C2O, _DELIM_SRC_ID, _IGN_SPANS, _IGN_STARTS, _IGN_ENDS - - # Local typed temps (ok to annotate) - par_o2c: Dict[int, int] - par_c2o: Dict[int, int] - br_o2c: Dict[int, int] - br_c2o: Dict[int, int] - spans: List[Tuple[int, int]] - par_o2c, par_c2o, br_o2c, br_c2o, spans = _build_delim_index(java_source) - # Assign to globals (no annotations here) - _PAR_O2C = par_o2c - _PAR_C2O = par_c2o - _BR_O2C = br_o2c - _BR_C2O = br_c2o - - _IGN_SPANS = spans - _IGN_STARTS = [a for (a, _) in spans] - _IGN_ENDS = [b for (_, b) in spans] - - _DELIM_SRC_ID = id(java_source) + c = _get_delim_cache() + c.par_o2c = par_o2c + c.par_c2o = par_c2o + c.br_o2c = br_o2c + c.br_c2o = br_c2o + c.ign_spans = spans + c.ign_starts = [a for (a, _) in spans] + c.ign_ends = [b for (_, b) in spans] + c.src_id = id(java_source) def _clear_delim_index_cache(): - global _PAR_O2C, _PAR_C2O, _BR_O2C, _BR_C2O, _DELIM_SRC_ID, _IGN_SPANS, _IGN_STARTS, _IGN_ENDS - _PAR_O2C.clear(); _PAR_C2O.clear(); _BR_O2C.clear(); _BR_C2O.clear() - _IGN_SPANS.clear(); _IGN_STARTS.clear(); _IGN_ENDS.clear() - _DELIM_SRC_ID = None + c = _get_delim_cache() + c.par_o2c.clear(); c.par_c2o.clear(); c.br_o2c.clear(); c.br_c2o.clear() + c.ign_spans.clear(); c.ign_starts.clear(); c.ign_ends.clear() + c.src_id = None # -------------------------- forward balanced matchers ------------------------- @@ -762,11 +1081,12 @@ def _match_balanced_parens(source: str, open_paren_index: int) -> int: ----- - Time complexity is O(K) where K is the distance from `open_paren_index` to the matching ')', or to the end if unterminated. - - The cache is used only when `_DELIM_SRC_ID == id(source)`. + - The cache is used only when the delim cache was built for this exact source object. """ # Cached O(1) lookup when indices were prebuilt for this exact source object. - if _DELIM_SRC_ID == id(source): - cached_close = _PAR_O2C.get(open_paren_index) + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_close = _dc.par_o2c.get(open_paren_index) if cached_close is not None: return cached_close @@ -836,8 +1156,9 @@ def _match_balanced_braces(source: str, open_brace_index: int) -> int: to its match (or to the end for unterminated input). """ # Cached O(1) close-brace lookup if indexes were built for this exact source object. - if _DELIM_SRC_ID == id(source): - cached_close = _BR_O2C.get(open_brace_index) + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_close = _dc.br_o2c.get(open_brace_index) if cached_close is not None: return cached_close @@ -876,108 +1197,32 @@ def _match_balanced_braces(source: str, open_brace_index: int) -> int: # ------------------------------ backward helpers ------------------------------ -def _prev_word(src: str, pos: int) -> str: - j = pos - 1 - while j >= 0 and src[j] in ' \t\r\n\f': - j -= 1 - if j < 0: - return "" - end = j + 1 - while j >= 0 and _is_ident_part(src[j]): - j -= 1 - return src[j+1:end] - -def _skip_leading_annotations(src: str, i: int) -> int: - """ - From position i (roughly at the beginning of a signature), skip any number of - annotations and comments so that method-level annotations do NOT appear in the - returned snippet. Returns first non-annotation token index. - """ - n = len(src) - i = _skip_ws_comments(src, i) - while i < n: - if src[i] == '@': - i += 1 - while i < n and (src[i].isalnum() or src[i] in ('_', '$', '.')): - i += 1 - i = _skip_ws_comments(src, i) - if i < n and src[i] == '(': - i = _match_balanced_parens(src, i) + 1 - i = _skip_ws_comments(src, i) - continue - j = _skip_ws_comments(src, i) - if j != i: - i = j - continue - break - return i - -def _back_ident(src: str, i: int) -> Tuple[int, Optional[str]]: - """ - From position i (typically the '(' of a parameter list), scan left to find the - simple identifier just before it. Returns (start_index, name) or (i, None). - """ - j = i - 1 - # skip ws - while j >= 0 and src[j] in ' \t\r\n\f': - j -= 1 - if j < 0: - return i, None - end = j + 1 - while j >= 0 and _is_ident_part(src[j]): - j -= 1 - start = j + 1 - name = src[start:end] - if name and _is_ident_start(name[0]): - return start, name - return i, None - -def _find_left_boundary(src: str, before: int) -> int: +def _strip_java_noise(snippet: str) -> str: """ - Return index just after the nearest real boundary (';', '{', '}') occurring before `before`, - ignoring comments/strings and jumping over balanced () for speed. - IMPORTANT: Treat '}', '{', ';' as immediate boundaries BEFORE any jump, so we don't leap - across the previous method body and accidentally glue multiple methods together. + Remove comments, javadocs, annotations, and (defensively) package/import lines. + Designed to be fast: cheap substring checks first, heavy work only if needed. """ - if _DELIM_SRC_ID != id(src): - # simple forward fallback - i = 0 - last = 0 - before = max(0, min(before, len(src))) - while i < before: - j = _skip_ws_comments(src, i) - if j != i: - i = j; continue - if i < before and (src.startswith('"""', i) or src[i] in ('"', "'")): - i = _skip_java_string_like(src, i); continue - if src[i] in (';', '{', '}'): - last = i + 1 - i += 1 - return last - - # fast backward using indexes - i = min(before - 1, len(src) - 1) - while i >= 0: - # hop out of ignored span - span = _ignored_span_at(i) - if span: - i = span[0] - 1 - continue + if not snippet: + return snippet - ch = src[i] + # Fast path: almost all snippets do NOT need stripping. + if ( + "/*" not in snippet + and "//" not in snippet + and "@" not in snippet + and "package" not in snippet + and "import" not in snippet + ): + return snippet - # FIRST: boundary check - if ch in (';', '{', '}'): - return i + 1 + # Reuse existing (previously unused) stripper: removes comments + annotations safely. + snippet = _strip_body_comments_and_annotations(snippet, 0, len(snippet)) - # jump across groups for speed (do NOT jump across '}' – handled above) - if ch == ')': - o = _PAR_C2O.get(i) if _DELIM_SRC_ID == id(src) else None - i = (o - 1) if o is not None else (i - 1) - continue + # Remove package/import lines if present (defensive; inner-class extraction used to prepend imports). + if "package" in snippet or "import" in snippet: + snippet = _pkg_imp_re.sub("", snippet) - i -= 1 - return 0 + return snippet def _match_paren_backwards(source: str, close_paren_index: int) -> int: """ @@ -1012,11 +1257,12 @@ def _match_paren_backwards(source: str, close_paren_index: int) -> int: ----- - Time complexity is O(K), where K is the distance from the closing paren to its match (or to the start if unterminated). - - The cache is used only when `_DELIM_SRC_ID == id(source)`. + - The cache is used only when the delim cache was built for this exact source object. """ # Cached O(1) lookup when indices were built for this exact source object. - if _DELIM_SRC_ID == id(source): - cached_open = _PAR_C2O.get(close_paren_index) + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_open = _dc.par_c2o.get(close_paren_index) if cached_open is not None: return cached_open @@ -1103,75 +1349,6 @@ def __init__(self, name: str, start: int, end: int) -> None: self.start = start self.end = end -def _find_all_named_types(src: str) -> List[_TypeRegion]: - """ - Find all class/interface/enum/record bodies and names, including nested types. - """ - n = len(src) - i = 0 - types: List[_TypeRegion] = [] - while i < n: - i = _skip_ws_comments(src, i) - if i >= n: break - - # skip literals - if src.startswith('"""', i) or src[i] in ('"', "'"): - i = _skip_java_string_like(src, i); continue - - # token - if _is_ident_start(src[i]) or src[i] == '@': - tok_start = i + (1 if src[i] == '@' else 0) - j = tok_start - while j < n and _is_ident_part(src[j]): j += 1 - tok = src[tok_start:j] - - if tok in ('class', 'interface', 'enum', 'record'): - # name - k = _skip_ws_comments(src, j) - name_start = k - while k < n and _is_ident_part(src[k]): k += 1 - if k == name_start: # '.class' or broken - i = j; continue - name = src[name_start:k] - - # type params / record header - k = _skip_ws_comments(src, k) - if k < n and src[k] == '<': - k = _match_balanced_angles(src, k) + 1 - k = _skip_ws_comments(src, k) - if tok == 'record' and k < n and src[k] == '(': - k = _match_balanced_parens(src, k) + 1 - k = _skip_ws_comments(src, k) - - # to opening '{' - while k < n and src[k] != '{': - if src.startswith('"""', k) or src[k] in ('"', "'"): - k = _skip_java_string_like(src, k) - else: - k += 1 - k = _skip_ws_comments(src, k) - - if k < n and src[k] == '{': - end = _match_balanced_braces(src, k) - if name: - types.append(_TypeRegion(name=name, start=k, end=end)) - i = k + 1 - continue - - i = j - continue - - i += 1 - - return types - -def _enclosing_type_name(types: List[_TypeRegion], pos: int) -> Optional[str]: - best: Optional[_TypeRegion] = None - for t in types: - if t.start <= pos <= t.end and (best is None or t.start >= best.start): - best = t - return best.name if best else None - # --------------------------------- lambdas ------------------------------------ def _capture_lambda_at_arrow(src: str, arrow: int) -> Optional[Tuple[int, int]]: @@ -1242,403 +1419,1204 @@ def _capture_lambda_at_arrow(src: str, arrow: int) -> Optional[Tuple[int, int]]: end = k return (param_start, end) -def _extract_lambdas_in_range(src: str, start: int, end: int) -> List[str]: +def _extract_lambdas_in_range(src: str, start: int, end: int) -> List[str]: + """ + Extract Java lambda expressions within [start, end). + + Optimized for huge files: + - Fast early-out if there's no '->' at all in the range. + - Uses a regex "find next interesting char" jump-table so we don't + call helpers per-character. + + Semantics are intended to match the previous implementation: + - Skip // and /*...*/ comments + - Skip strings/chars/text blocks via _skip_java_string_like() + - Capture lambdas via _capture_lambda_at_arrow() + - Strip comments in the captured lambda via _strip_comments_keep_strings_range() + """ out: List[str] = [] - i = start n = len(src) - end = min(end, n) - while i < end: - j = _skip_ws_comments(src, i) - if j != i: - i = j; continue - if i < end and (src.startswith('"""', i) or src[i] in ('"', "'")): - i = _skip_java_string_like(src, i); continue - if i + 1 < end and src[i] == '-' and src[i + 1] == '>': - span = _capture_lambda_at_arrow(src, i) - if span: - s, e = span - if s >= start and e <= end: - clean = _strip_comments_keep_strings_range(src, s, e) - out.append(clean) - i = e + if start < 0: + start = 0 + if end > n: + end = n + if start >= end: + return out + + # Cheap win for big files with no lambdas at all (like many autogenerated sources): + if src.find("->", start, end) == -1: + return out + + i = start + while True: + m = _LAMBDA_SCAN_SPECIAL_RE.search(src, i, end) + if not m: + break + + i = m.start() + ch = src[i] + + # Comments + if ch == '/': + if i + 1 < end: + nxt = src[i + 1] + # line comment + if nxt == '/': + nl = src.find('\n', i + 2, end) + i = end if nl == -1 else nl + 1 + continue + # block / javadoc comment + if nxt == '*': + j = src.find('*/', i + 2, end) + i = end if j == -1 else j + 2 + continue + i += 1 + continue + + # Strings / chars / text blocks + if ch in ('"', "'"): + i = _skip_java_string_like(src, i) + continue + + # Potential lambda arrow + if ch == '-': + if i + 1 < end and src[i + 1] == '>': + span = _capture_lambda_at_arrow(src, i) + if span: + s, e = span + if s >= start and e <= end: + clean = src[s:e] + clean = _strip_java_noise(clean) + out.append(clean) + i = e + continue + i += 2 continue - i += 2 + + i += 1 continue - i += 1 - return out -# ------------------ small helpers for method detection ------------------------ + # Fallback (should be unreachable due to regex character class) + i += 1 -def _header_closes_at_brace(src: str, close_p: int, brace_idx: int) -> bool: - """Return True if src[close_p] == ')' and after optional ws/comments + throws we land exactly on '{'.""" - k = _skip_ws_comments(src, close_p + 1) - k = _skip_throws_clause(src, k) - k = _skip_ws_comments(src, k) - return k == brace_idx + return out # ------------------------------- core extractor ------------------------------- def _extract_methods_anywhere( src: str, start: int = 0, - end: Optional[int] = None, - types: Optional[List[_TypeRegion]] = None, -) -> List[str]: + end: int | None = None, + types: list | None = None, +) -> list[str]: """ Extract Java method definitions (with bodies) anywhere within [start, end), - INCLUDING constructors. Anonymous/inner/local class bodies are recursively scanned. - Method-level annotations preceding a signature are excluded from the returned slice. - Lambdas are NOT returned here (they are gathered in a separate pass). - - Performance: - - Single forward scan with O(1) delimiter lookups when _set_delim_index() cache - is active. - - Avoids quadratic behavior by skipping ignored spans and using balanced matchers. + INCLUDING constructors. Anonymous/inner/local class bodies are scanned. + + Key behavior: + - For named inner types, emitted comment includes ALL nesting levels + EXCLUDING the top-level declaring type: + e.g. "Outer.Inner1.Inner2" -> "Inner1.Inner2", + "CreateMultipartUploadRequest.BuilderImpl" -> "BuilderImpl". + + Annotation behavior: + - Lines that contain ONLY annotations (and whitespace/comments) immediately + preceding a member are excluded from the returned slice. + + Enum behavior (critical fix): + - Do NOT treat enum constant initializers as class members. + Method parsing inside a named enum body is disabled until the FIRST ';' + at that enum body's brace depth (the end of the enum-constant section). + + Notes on performance: + - Single forward scan. + - Balanced matchers are used only for candidates at class-member level. + - No regex usage in hot paths. """ - if end is None: - end = len(src) - if types is None: - types = _find_all_named_types(src) - - # Enclosing named-type lookup (innermost) - types_sorted = sorted(types, key=lambda t: t.start) - type_starts = [t.start for t in types_sorted] - - def _enclosing_region(pos: int) -> Optional[_TypeRegion]: - """ - Return the innermost named type region containing pos. - IMPORTANT: keep scanning left until we either find an enclosing region - or run out of candidates. Do NOT early-return None if the nearest - left type ended before pos; an outer type may still enclose pos. - """ - idx = bisect.bisect_right(type_starts, pos) - 1 - while idx >= 0: - t = types_sorted[idx] - if t.start <= pos <= t.end: - return t - # If this candidate doesn't enclose pos, step left and keep looking. - idx -= 1 - return None - - # Quick enum-constants section end cache (keyed by enum body '{' position) - enum_term_cache: Dict[int, int] = {} - def _region_kind(reg: _TypeRegion) -> str: - if not reg: - return "" - i2 = _find_left_boundary(src, reg.start) - while i2 < reg.start: - j2 = _skip_ws_comments(src, i2); i2 = j2 - if i2 >= reg.start: break - if src.startswith('"""', i2) or src[i2] in ('"', "'"): - i2 = _skip_java_string_like(src, i2); continue - if _is_ident_start(src[i2]) or src[i2] == '@': - tok_start = i2 + (1 if src[i2] == '@' else 0) - j2 = tok_start - while j2 < reg.start and _is_ident_part(src[j2]): j2 += 1 - tok = src[tok_start:j2] - if tok in ('class', 'interface', 'enum', 'record'): - return tok - i2 = j2; continue - i2 += 1 - return "" + # ------------------------- + # Small internal structs + # ------------------------- + class _TypeRegion: + __slots__ = ("kind", "name", "start", "end", "body_start", "body_end", "parent", "qual_name") + + def __init__( + self, + kind: str, + name: str, + start: int, + end: int, + body_start: int, + body_end: int, + parent: int, + qual_name: str, + ) -> None: + self.kind = kind + self.name = name + self.start = start + self.end = end + self.body_start = body_start + self.body_end = body_end + self.parent = parent + self.qual_name = qual_name + + class _ClassBodyFrame: + __slots__ = ("named_type_idx", "body_depth", "member_start", + "is_enum", "enum_constants_done") + + def __init__(self, named_type_idx: int, body_depth: int, member_start: int, is_enum: bool) -> None: + self.named_type_idx = named_type_idx # >=0 for named type, -1 for anonymous class body + self.body_depth = body_depth # brace_depth inside this body + self.member_start = member_start # where next member starts + + self.is_enum = is_enum + # For enums: constants come first. Members start only after the first ';' at body depth. + self.enum_constants_done = (not is_enum) + + # ------------------------- + # Low-level helpers (all local) + # ------------------------- + def _is_id_start(c: str) -> bool: + return c.isalpha() or c == "_" or c == "$" + + def _is_id_part(c: str) -> bool: + return c.isalnum() or c == "_" or c == "$" + + def _skip_ws_and_comments(s: str, i: int, lim: int) -> int: + while i < lim: + c = s[i] + if c.isspace(): + i += 1 + continue + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": # line comment + i += 2 + while i < lim and s[i] != "\n": + i += 1 + continue + if n2 == "*": # block comment + j = s.find("*/", i + 2, lim) + i = lim if j == -1 else (j + 2) + continue + break + return i - def _enum_constants_terminator(type_region: _TypeRegion) -> int: + def _signature_has_initializer_assignment(sig_start: int, name_start: int) -> bool: """ - Find the position of the semicolon that terminates the enum-constants list - inside an enum body, or return -1 if no explicit terminator exists. - - Parameters - ---------- - type_region : _TypeRegion - The region describing the enum body. `type_region.start` is the index - of the opening '{' and `type_region.end` is the index of the matching '}'. - - Returns - ------- - int - The index of the terminating ';' that appears at *brace depth 1* inside - the enum body. If the constants section runs directly into the closing - brace (i.e., no explicit ';'), returns -1. Also returns -1 for - unterminated/degenerate cases. + True if there's an '=' before `name_start` that is NOT inside an annotation's (...) args + and not inside comments/strings. - Caching - ------- - Results are cached in `enum_term_cache` keyed by `type_region.start`. + This is used to reject parsing '(' that belongs to field initializers like: + Foo X = new Foo(...) { ... } """ - # Cache fast-path - cached_pos = enum_term_cache.get(type_region.start) - if cached_pos is not None: - return cached_pos - - scan_pos: int = type_region.start + 1 # first character after '{' - body_end: int = type_region.end # index of matching '}' - brace_depth: int = 1 # already inside the enum body - - while scan_pos < body_end: - next_pos = _skip_ws_comments(src, scan_pos) - if next_pos != scan_pos: - scan_pos = next_pos - continue - - # Skip over literals so braces/semicolons inside them are ignored. - if src.startswith('"""', scan_pos) or src[scan_pos] in ('"', "'"): - scan_pos = _skip_java_string_like(src, scan_pos) - continue - - ch = src[scan_pos] - - if ch == '{': - brace_depth += 1 - elif ch == '}': - brace_depth -= 1 - if brace_depth == 0: - # Reached end of enum body without seeing a terminator. - enum_term_cache[type_region.start] = -1 - return -1 - elif ch == ';' and brace_depth == 1: - # The terminator separating constants from members. - enum_term_cache[type_region.start] = scan_pos - return scan_pos - elif ch == '(': - # Skip (...) blocks to avoid miscounting nested tokens. - scan_pos = _match_balanced_parens(src, scan_pos) + 1 - continue - elif ch == '<': - # Skip generic type argument lists inside declarations. - scan_pos = _match_balanced_angles(src, scan_pos) + 1 - continue - - scan_pos += 1 - - # No explicit terminator found. - enum_term_cache[type_region.start] = -1 - return -1 - - def _in_enum_constants_section(name_pos: int) -> bool: - reg = _enclosing_region(name_pos) - if not reg or _region_kind(reg) != 'enum': + # Fast path: no '=' at all + if s.find("=", sig_start, name_start) == -1: return False - term = _enum_constants_terminator(reg) - return term == -1 or name_pos < term - - def _scan_paren_internals_for_anonymous_classes(open_idx: int, close_idx: int, out_accum: List[str]): - # Look for "new Type(...){ ... }" inside argument lists; recurse into their bodies - k2 = open_idx + 1 - while k2 < close_idx: - kk = _skip_ws_comments(src, k2) - if kk != k2: - k2 = kk; continue - if k2 < close_idx and (src.startswith('"""', k2) or src[k2] in ('"', "'")): - k2 = _skip_java_string_like(src, k2); continue - - pos = src.find('new', k2, close_idx) - if pos == -1: - break - k2 = pos - - # Skip false positives inside ignored spans - sp = _ignored_span_at(k2) if _DELIM_SRC_ID == id(src) else None - if sp and sp[0] <= k2 < sp[1]: - k2 = sp[1]; continue - - pre = src[k2 - 1] if k2 > open_idx + 1 else '' - post = src[k2 + 3] if k2 + 3 < close_idx else '' - if (k2 == open_idx + 1 or not _is_ident_part(pre)) and (k2 + 3 >= close_idx or not _is_ident_part(post)): - t = _skip_ws_comments(src, k2 + 3) - # Qualified type with optional type params - while t < close_idx: - if _is_ident_start(src[t]): - t += 1 - while t < close_idx and _is_ident_part(src[t]): t += 1 - if t < close_idx and src[t] == '.': - t += 1; continue - elif t < close_idx and src[t] == '<': - t = _match_balanced_angles(src, t) + 1 - else: - break - t = _skip_ws_comments(src, t) - - if t < close_idx and src[t] == '(': - ctor_close = _match_balanced_parens(src, t) - u = _skip_ws_comments(src, ctor_close + 1) - if u < close_idx and src[u] == '{': - body_end = _match_balanced_braces(src, u) - if u + 1 <= body_end: - out_accum.extend(_extract_methods_anywhere(src, start=u + 1, end=body_end, types=types)) - k2 = body_end + 1; continue - k2 = ctor_close + 1; continue - - k2 = k2 + 3 # move past this "new" - - def _body_might_have_local_types(beg: int, end_: int) -> bool: - s = src - return (s.find('class', beg, end_) != -1 or - s.find('interface', beg, end_) != -1 or - s.find('enum', beg, end_) != -1 or - s.find('record', beg, end_) != -1) - out: List[str] = [] - i = start - n = end - seen_spans: Set[Tuple[int, int]] = set() + NORMAL3, LINE3, BLOCK3, STRING3, CHAR3, TEXT3 = 0, 1, 2, 3, 4, 5 + st = NORMAL3 + i3 = sig_start + while i3 < name_start: + c3 = s[i3] + + if st == NORMAL3: + if c3 == "/" and i3 + 1 < name_start: + n3 = s[i3 + 1] + if n3 == "/": + st = LINE3 + i3 += 2 + continue + if n3 == "*": + st = BLOCK3 + i3 += 2 + continue - while i < n: - j = _skip_ws_comments(src, i) - if j != i: - i = j; continue - if i >= n: break + if c3 == '"': + if i3 + 2 < name_start and s[i3:i3 + 3] == '"""': + st = TEXT3 + i3 += 3 + continue + st = STRING3 + i3 += 1 + continue - if src.startswith('"""', i) or src[i] in ('"', "'"): - i = _skip_java_string_like(src, i); continue + if c3 == "'": + st = CHAR3 + i3 += 1 + continue - if src[i] == '(': - name_start, name = _back_ident(src, i) - close_p = _match_balanced_parens(src, i) + # Skip an annotation (and its (...) if present) so '=' inside it doesn't count. + if c3 == "@": + i3 += 1 + # qualified annotation name + while i3 < name_start and (_is_id_part(s[i3]) or s[i3] == "."): + i3 += 1 + i3 = _skip_ws_and_comments(s, i3, name_start) + if i3 < name_start and s[i3] == "(": + r3 = _match_balanced_paren(s, i3, name_start) + if r3 == -1: + # malformed; be conservative (don't reject) + return False + i3 = r3 + 1 + continue - if not name: - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - i = close_p + 1 - continue + if c3 == "=": + return True - # Record headers are NOT methods/constructors; skip them explicitly. - if _prev_word(src, name_start) == 'record': - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - i = close_p + 1 + i3 += 1 continue - # Control-flow invocations like `if (...)`, `for (...)` etc. are not methods. - if name in _NON_METHOD_TOKENS: - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - i = close_p + 1 + if st == LINE3: + if c3 == "\n": + st = NORMAL3 + i3 += 1 continue - left_bound = _find_left_boundary(src, name_start) - - # "new" between boundary and name means likely a constructor call or anonymous class literal - # (i.e., not a declaration). If found, optionally recurse into the anonymous class body. - def _has_new_token_between(s: str, left: int, right: int) -> bool: - left = max(0, left); right = min(len(s), right) - i2 = left - while True: - pos = s.find('new', i2, right) - if pos == -1: return False - if _DELIM_SRC_ID == id(s): - sp = _ignored_span_at(pos) - if sp and sp[0] <= pos < sp[1]: - i2 = pos + 3; continue - pre = s[pos - 1] if pos > left else '' - post = s[pos + 3] if pos + 3 < right else '' - if (pos == left or not _is_ident_part(pre)) and (pos + 3 >= right or not _is_ident_part(post)): - return True - i2 = pos + 3 - - if _has_new_token_between(src, left_bound, name_start): - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - kk = _skip_ws_comments(src, close_p + 1) - if kk < n and src[kk] == '{': - # Anonymous inner class: collect methods from its body. - body_end = _match_balanced_braces(src, kk) - if kk + 1 <= body_end: - out.extend(_extract_methods_anywhere(src, start=kk + 1, end=body_end, types=types)) - i = body_end + 1 - else: - i = close_p + 1 + if st == BLOCK3: + if c3 == "*" and i3 + 1 < name_start and s[i3 + 1] == "/": + st = NORMAL3 + i3 += 2 + continue + i3 += 1 continue - # Enum constant bodies (class bodies inside the constants section) — recurse. - k = _skip_ws_comments(src, close_p + 1) - if k < n and src[k] == '{' and _in_enum_constants_section(name_start): - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - body_end = _match_balanced_braces(src, k) - if k + 1 <= body_end: - out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) - i = body_end + 1 + if st == STRING3: + if c3 == "\\": + i3 += 2 + continue + if c3 == '"': + st = NORMAL3 + i3 += 1 continue - # We expect a method/constructor body here. - k = _skip_throws_clause(src, k) - if k >= n or src[k] != '{': - if close_p < n: - _scan_paren_internals_for_anonymous_classes(i, close_p, out) - i = close_p + 1 + if st == CHAR3: + if c3 == "\\": + i3 += 2 + continue + if c3 == "'": + st = NORMAL3 + i3 += 1 continue - sig_guess = _find_left_boundary(src, name_start) - sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) - - # Constructors are now included: if name == enclosing type name, we still emit it. - body_end = _match_balanced_braces(src, k) - sig_start = _skip_ws_comments(src, sig_start) # final guard - - header_no_ann = _strip_annotations_in_range(src, sig_start, k) - clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) - - clean_body = _strip_body_comments_and_annotations(src, k, body_end) - - method_text = clean_header + clean_body - - span = (sig_start, body_end) - if span not in seen_spans: - seen_spans.add(span) - out.append(method_text) - - # Recurse into body for local/anonymous types - if k + 1 <= body_end and _body_might_have_local_types(k + 1, body_end): - out.extend(_extract_methods_anywhere(src, start=k + 1, end=body_end, types=types)) - - i = body_end + 1 - continue + # TEXT3 + if i3 + 2 < name_start and s[i3:i3 + 3] == '"""': + st = NORMAL3 + i3 += 3 + continue + i3 += 1 - # Secondary pattern: brace-first line (signature ended previous line) - if src[i] == '{': - p = src.rfind(')', start, i) - if p != -1: - open_p = _match_paren_backwards(src, p) - if open_p > 0 and _header_closes_at_brace(src, p, i): - name_start, name = _back_ident(src, open_p) - if name and name not in _NON_METHOD_TOKENS: - left_bound = _find_left_boundary(src, name_start) - # Filter out anonymous class/ctor callsites; constructors are INCLUDED. - def _has_new_between(s, a, b): - a = max(0, a); b = min(len(s), b) - pos = s.find('new', a, b) - while pos != -1: - sp = _ignored_span_at(pos) if _DELIM_SRC_ID == id(s) else None - if not sp or not (sp[0] <= pos < sp[1]): - pre = s[pos - 1] if pos > a else '' - post = s[pos + 3] if s and (pos + 3) < b else '' - if (pos == a or not _is_ident_part(pre)) and (pos + 3 >= b or not _is_ident_part(post)): - return True - pos = s.find('new', pos + 3, b) - return False - if not _has_new_between(src, left_bound, name_start) and _prev_word(src, name_start) != 'record': - body_end = _match_balanced_braces(src, i) - sig_guess = _find_left_boundary(src, name_start) - sig_start = _skip_leading_annotations(src, _skip_ws_comments(src, sig_guess)) - sig_start = _skip_ws_comments(src, sig_start) + return False - header_no_ann = _strip_annotations_in_range(src, sig_start, i) - clean_header = _strip_comments_keep_strings_range(header_no_ann, 0, len(header_no_ann)) + def _match_balanced_paren(s: str, lparen: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + i = lparen + 1 + depth = 1 - clean_body = _strip_body_comments_and_annotations(src, i, body_end) + while i < lim: + c = s[i] + if state == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + elif c == '"': + if i + 2 < lim and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + elif c == "'": + state = CHAR + i += 1 + continue + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + return i + elif state == LINE: + if c == "\n": + state = NORMAL + elif state == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + elif state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + elif state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + else: # TEXT + if i + 2 < lim and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return -1 + + def _match_balanced_brace(s: str, lbrace: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + i = lbrace + 1 + depth = 1 + + while i < lim: + c = s[i] + if state == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + elif c == '"': + if i + 2 < lim and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + elif c == "'": + state = CHAR + i += 1 + continue + elif c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + elif state == LINE: + if c == "\n": + state = NORMAL + elif state == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + elif state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + elif state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + else: # TEXT + if i + 2 < lim and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return -1 + + def _line_is_annotation_only(s: str, line_start: int, line_end: int) -> bool: + i = line_start + while i < line_end and s[i].isspace(): + i += 1 + if i >= line_end or s[i] != "@": + return False + + # Parse one-or-more annotations: @Name or @pkg.Name or @Name(...) + while True: + if i >= line_end or s[i] != "@": + break + i += 1 + if i >= line_end or not _is_id_start(s[i]): + return False + i += 1 + while i < line_end and (_is_id_part(s[i]) or s[i] == "."): + i += 1 + + i = _skip_ws_and_comments(s, i, line_end) + if i < line_end and s[i] == "(": + r = _match_balanced_paren(s, i, line_end) + if r == -1: + return False + i = r + 1 + + i = _skip_ws_and_comments(s, i, line_end) + if i < line_end and s[i] == "@": + continue + break + + i = _skip_ws_and_comments(s, i, line_end) + while i < line_end and s[i].isspace(): + i += 1 + return i >= line_end + + def _skip_annotation_only_lines(s: str, i: int, lim: int) -> int: + n_local = len(s) + while i < lim: + line_end = s.find("\n", i, lim) + if line_end == -1: + line_end = min(lim, n_local) + if not _line_is_annotation_only(s, i, line_end): + return i + i = line_end + 1 + return i + + # ------------------------- + # Type discovery (named inner chain computed as qual_name) + # ------------------------- + def _find_all_named_types(s: str) -> list[_TypeRegion]: + n_local = len(s) + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + + class_kinds = {"class", "interface", "enum", "record"} + + out_types: list[_TypeRegion] = [] + type_stack: list[int] = [] # indices of open named types + brace_stack: list[tuple[int, int]] = [] # (open_brace_pos, named_type_idx_or_-1) + + pending_kind: str | None = None + pending_start: int = -1 + pending_name: str | None = None + + def _prev_non_ws_local(pos: int) -> int: + while pos >= 0 and s[pos].isspace(): + pos -= 1 + return pos + + i = 0 + while i < n_local: + c = s[i] + + if state == NORMAL: + if c == "/" and i + 1 < n_local: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + if c == '"': + if i + 2 < n_local and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + if c == "'": + state = CHAR + i += 1 + continue + + if _is_id_start(c): + j = i + 1 + while j < n_local and _is_id_part(s[j]): + j += 1 + tok = s[i:j] + + if pending_kind is None: + if tok in class_kinds: + # ------------------------------ + # ✅ CRITICAL GUARD: + # Ignore ".class" and other member-access uses: + # StandardScheme.class.equals(...) + # Foo.class + # This prevents bogus "class equals" types. + # ------------------------------ + p = _prev_non_ws_local(i - 1) + if p >= 0 and s[p] == ".": + i = j + continue + # Defensive: ignore "class." patterns too + if j < n_local and s[j] == ".": + i = j + continue + + pending_kind = tok + pending_start = i + pending_name = None + else: + if pending_name is None: + pending_name = tok + + i = j + continue + + if c == "{": + brace_pos = i + if pending_kind is not None and pending_name is not None: + parent = type_stack[-1] if type_stack else -1 + qual = pending_name if parent == -1 else (out_types[parent].qual_name + "." + pending_name) + out_types.append( + _TypeRegion( + kind=pending_kind, + name=pending_name, + start=pending_start, + end=-1, + body_start=brace_pos + 1, + body_end=-1, + parent=parent, + qual_name=qual, + ) + ) + t_idx = len(out_types) - 1 + brace_stack.append((brace_pos, t_idx)) + type_stack.append(t_idx) + pending_kind = None + pending_start = -1 + pending_name = None + else: + brace_stack.append((brace_pos, -1)) + i += 1 + continue + + if c == "}": + if brace_stack: + _, t_idx = brace_stack.pop() + if t_idx != -1: + out_types[t_idx].body_end = i + out_types[t_idx].end = i + 1 + if type_stack and type_stack[-1] == t_idx: + type_stack.pop() + else: + # defensive fallback + try: + type_stack.remove(t_idx) + except ValueError: + pass + i += 1 + continue + + i += 1 + continue + + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue + + if state == BLOCK: + if c == "*" and i + 1 < n_local and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue + + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue + + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue + + # TEXT + if i + 2 < n_local and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + # Keep only closed types + closed: list[_TypeRegion] = [] + for t in out_types: + if t.end != -1 and t.body_end != -1: + closed.append(t) + return closed + + def _normalize_types(maybe_types: list) -> list[_TypeRegion]: + norm: list[_TypeRegion] = [] + for t in maybe_types: + # attribute-style + if hasattr(t, "kind") and hasattr(t, "name") and hasattr(t, "body_start") and hasattr(t, "body_end"): + kind = getattr(t, "kind") + name = getattr(t, "name") + tstart = getattr(t, "start", -1) + tend = getattr(t, "end", -1) + bstart = getattr(t, "body_start") + bend = getattr(t, "body_end") + parent = getattr(t, "parent", -1) + qual = getattr(t, "qual_name", "") or name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue - method_text = clean_header + clean_body + # dict-style + if isinstance(t, dict): + kind = t.get("kind", "") + name = t.get("name", "") + tstart = t.get("start", -1) + tend = t.get("end", -1) + bstart = t.get("body_start", -1) + bend = t.get("body_end", -1) + parent = t.get("parent", -1) + qual = t.get("qual_name") or name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue + + # tuple/list positional fallback (kind,name,start,end,body_start,body_end,parent,qual_name) + if isinstance(t, (tuple, list)) and len(t) >= 6: + kind = t[0] + name = t[1] + tstart = t[2] if len(t) > 2 else -1 + tend = t[3] if len(t) > 3 else -1 + bstart = t[4] + bend = t[5] + parent = t[6] if len(t) > 6 else -1 + qual = t[7] if len(t) > 7 else name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue + + # unknown shape -> ignore + return norm + + def _repair_qual_names(types_list: list[_TypeRegion]) -> None: + """ + If caller-provided `types` lack correct qual_name chains, repair them using `parent`. + Safe no-op if already correct. + """ + any_needs = False + for t in types_list: + if t.parent != -1 and ("." not in (t.qual_name or "")): + any_needs = True + break + if not any_needs: + return + + memo: dict[int, str] = {} + + def qual(i: int) -> str: + if i in memo: + return memo[i] + t = types_list[i] + if t.parent == -1: + memo[i] = t.name + else: + p = qual(t.parent) + memo[i] = p + "." + t.name if p else t.name + return memo[i] + + for idx in range(len(types_list)): + types_list[idx].qual_name = qual(idx) + + # ------------------------- + # Main extraction logic + # ------------------------- + s = src + n = len(s) + if end is None: + end = n + if start < 0: + start = 0 + if end > n: + end = n + if start >= end: + return [] + + if types is None: + types2 = _find_all_named_types(s) + else: + types2 = _normalize_types(types) + _repair_qual_names(types2) + + # Map opening brace positions to named type index. + # Named type opening brace is at (body_start - 1). + lbrace_to_type: dict[int, int] = {} + for idx, t in enumerate(types2): + lbrace_to_type[t.body_start - 1] = idx + + # Precompute "inner-type" comment payload per named type idx. + # We strip the top-level declaring type (first segment) and keep the rest. + inner_comment_by_idx: list[str | None] = [None] * len(types2) + for idx, t in enumerate(types2): + if t.parent != -1: + q = t.qual_name or t.name + dot = q.find(".") + inner_comment_by_idx[idx] = q[dot + 1:] if dot != -1 else t.name + + # Scanner state + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + + out: list[str] = [] + + # Brace stack kinds: 0 = other, 1 = named type body, 2 = anonymous class body + brace_stack: list[tuple[int, int]] = [] + brace_depth = 0 + + # Track parentheses and whether they belong to a `new ... ( ... )` object creation + paren_objcreate_stack: list[bool] = [] + in_new_expr = False # True from seeing token "new" until we consume its '(' or cancel + + expecting_anon_class_brace = False + class_bodies: list[_ClassBodyFrame] = [] + + modifiers = { + "public", "protected", "private", "abstract", "final", "static", "native", + "synchronized", "strictfp", "default", "transient", "volatile", "sealed", "non-sealed", + } + ctrl_like = { + "if", "for", "while", "switch", "catch", "do", "try", "return", "throw", "new", + "case", "assert", "break", "continue", "else", "finally", "yield", + "this", "super", + } + + def _current_named_type_name() -> str | None: + if not class_bodies: + return None + idx = class_bodies[-1].named_type_idx + if idx < 0: + return None + return types2[idx].name + + def _current_inner_comment() -> str | None: + if not class_bodies: + return None + idx = class_bodies[-1].named_type_idx + if idx < 0: + return None + return inner_comment_by_idx[idx] + + def _is_constructor_allowed(method_name: str) -> bool: + tn = _current_named_type_name() + return tn is not None and method_name == tn + + def _prev_non_ws(pos: int, floor: int) -> int: + j = pos + while j >= floor and s[j].isspace(): + j -= 1 + return j + + def _read_prev_identifier(pos: int, floor: int) -> tuple[str, int, int]: + j = _prev_non_ws(pos, floor) + if j < floor: + return "", -1, -1 + if not _is_id_part(s[j]): + return "", -1, -1 + k = j + while k >= floor and _is_id_part(s[k]): + k -= 1 + return s[k + 1: j + 1], k + 1, j + 1 + + def _has_return_type_before_name(name_start: int, floor: int) -> bool: + j = name_start - 1 + while j >= floor: + while j >= floor and s[j].isspace(): + j -= 1 + if j < floor: + break + if not _is_id_part(s[j]): + j -= 1 + continue + + tok, ts, _ = _read_prev_identifier(j, floor) + if not tok: + break + + # skip annotation identifiers (@Something) + p = _prev_non_ws(ts - 1, floor) + if p >= floor and s[p] == "@": + j = p - 1 + continue + + if tok in modifiers: + j = ts - 1 + continue + + # ✅ IMPORTANT: keywords like "new"/"return"/"if"/... are NOT return types + if tok in ctrl_like: + j = ts - 1 + continue + + return True + return False + + + def _try_parse_method_at_lparen(lparen: int) -> tuple[int, int] | None: + frame = class_bodies[-1] + member_floor = frame.member_start + + # method/ctor name token before '(' + j = _prev_non_ws(lparen - 1, member_floor) + if j < member_floor or not _is_id_part(s[j]): + return None + + k = j + while k >= member_floor and _is_id_part(s[k]): + k -= 1 + name_start = k + 1 + name = s[name_start: j + 1] + if not name or name in ctrl_like: + return None + + # ------------------------------------------------------------ + # ✅ NEW: Reject annotation argument parens: "@Something(...)". + # This prevents "@SuppressWarnings(...)" from being mis-parsed as a method. + # ------------------------------------------------------------ + p0 = _prev_non_ws(name_start - 1, member_floor) + if p0 >= member_floor and s[p0] == "@": + return None + + # Reject qualified/call expressions: ".name(" or "::name(" + if p0 >= member_floor and s[p0] in (".", ":"): + return None + + # Compute the "real signature start" (skip ws/comments + annotation-only lines) + sig0 = _skip_ws_and_comments(s, member_floor, end) + sig0 = _skip_annotation_only_lines(s, sig0, min(lparen, end)) + + # ------------------------------------------------------------ + # ✅ Field initializer guard: + # If there's an '=' before this '(' in the same member signature, + # we're in an initializer, not a method/ctor declaration. + # ------------------------------------------------------------ + if sig0 < lparen and s.find("=", sig0, lparen) != -1: + if _signature_has_initializer_assignment(sig0, lparen): + return None + + # Reject type declarations that have (...) headers (e.g., "record R(...){...}") + prev_tok, _, _ = _read_prev_identifier(name_start - 1, member_floor) + if prev_tok in ("class", "interface", "enum", "record"): + return None + + # Reject "new Name(" (constructor call) vs declaration + if prev_tok == "new": + return None + + rparen = _match_balanced_paren(s, lparen, end) + if rparen == -1: + return None + + # Find '{' starting the body (skip ws/comments; allow throws etc) + i2 = _skip_ws_and_comments(s, rparen + 1, end) + if i2 >= end: + return None + if i2 + 1 < end and s[i2: i2 + 2] == "->": + return None + if s[i2] == ";": + return None + + # Scan forward until '{' or ';' or '->', respecting strings/comments. + NORMAL2, LINE2, BLOCK2, STRING2, CHAR2, TEXT2 = 0, 1, 2, 3, 4, 5 + st2 = NORMAL2 + j2 = i2 + body_lbrace = -1 + + while j2 < end: + c2 = s[j2] + if st2 == NORMAL2: + if c2 == "/" and j2 + 1 < end: + n2 = s[j2 + 1] + if n2 == "/": + st2 = LINE2 + j2 += 2 + continue + if n2 == "*": + st2 = BLOCK2 + j2 += 2 + continue + if c2 == '"': + if j2 + 2 < end and s[j2: j2 + 3] == '"""': + st2 = TEXT2 + j2 += 3 + continue + st2 = STRING2 + j2 += 1 + continue + if c2 == "'": + st2 = CHAR2 + j2 += 1 + continue + + if j2 + 1 < end and s[j2: j2 + 2] == "->": + return None + if c2 == ";": + return None + if c2 == "{": + body_lbrace = j2 + break + + elif st2 == LINE2: + if c2 == "\n": + st2 = NORMAL2 + elif st2 == BLOCK2: + if c2 == "*" and j2 + 1 < end and s[j2 + 1] == "/": + st2 = NORMAL2 + j2 += 2 + continue + elif st2 == STRING2: + if c2 == "\\": + j2 += 2 + continue + if c2 == '"': + st2 = NORMAL2 + elif st2 == CHAR2: + if c2 == "\\": + j2 += 2 + continue + if c2 == "'": + st2 = NORMAL2 + else: # TEXT2 + if j2 + 2 < end and s[j2: j2 + 3] == '"""': + st2 = NORMAL2 + j2 += 3 + continue + + j2 += 1 + + if body_lbrace == -1: + return None + + # ------------------------------------------------------------ + # ✅ IMPORTANT CHANGE: use sig0 as the "floor" for return-type detection. + # This prevents Javadoc/comments/previous-members from faking a return type. + # ------------------------------------------------------------ + if frame.named_type_idx >= 0: + if not _has_return_type_before_name(name_start, sig0): + if not _is_constructor_allowed(name): + return None + else: + if not _has_return_type_before_name(name_start, sig0): + return None + + rbrace = _match_balanced_brace(s, body_lbrace, end) + if rbrace == -1: + return None + method_end = rbrace + 1 + + # Signature start excluding annotation-only lines + sig_start = _skip_annotation_only_lines(s, sig0, min(body_lbrace, end)) + return sig_start, method_end + + i = start + while i < end: + c = s[i] + + if state == NORMAL: + # comment/string/text transitions + if c == "/" and i + 1 < end: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + + if c == '"': + if i + 2 < end and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue - span = (sig_start, body_end) - if span not in seen_spans: - seen_spans.add(span) - out.append(method_text) + if c == "'": + state = CHAR + i += 1 + continue + + # identifiers: detect 'new' + if _is_id_start(c): + j = i + 1 + while j < end and _is_id_part(s[j]): + j += 1 + tok = s[i:j] + if tok == "new": + in_new_expr = True + i = j + continue - if i + 1 <= body_end and _body_might_have_local_types(i + 1, body_end): - out.extend(_extract_methods_anywhere(src, start=i + 1, end=body_end, types=types)) - i = body_end + 1 + # cancel `new` expectation on array creation or obvious expression terminators + if in_new_expr and c in ("[", "{", ";"): + in_new_expr = False + + # Cancel expecting anon brace if we hit meaningful char that's not '{' + if expecting_anon_class_brace and not c.isspace() and c != "{": + expecting_anon_class_brace = False + + if c == "(": + paren_objcreate_stack.append(in_new_expr) + in_new_expr = False # consumed the object-creation marker for this '(' + + # method parsing at class member level + if class_bodies: + frame = class_bodies[-1] + # Only extract methods for *named* types; skip anonymous-class bodies (new X() { ... }). + if frame.named_type_idx >= 0 and brace_depth == frame.body_depth and frame.enum_constants_done: + parsed = _try_parse_method_at_lparen(i) + if parsed is not None: + sig_start, method_end = parsed + method_src = s[sig_start:method_end] + + # Strip comments/javadocs/annotations/package/imports FIRST + method_src = _strip_java_noise(method_src) + + # Then append the inner-type marker so it survives stripping + inner = _current_inner_comment() + if inner: + method_src = method_src.rstrip() + f"\n/* inner-type: {inner} */" + + out.append(method_src) + frame.member_start = method_end + i = method_end continue + + i += 1 + continue + + if c == ")": + if paren_objcreate_stack and paren_objcreate_stack.pop(): + expecting_anon_class_brace = True + i += 1 + continue + + if c == "{": + brace_depth += 1 + + t_idx = lbrace_to_type.get(i, -1) + if t_idx != -1: + brace_stack.append((i, 1)) + is_enum = (types2[t_idx].kind == "enum") + class_bodies.append(_ClassBodyFrame(t_idx, brace_depth, i + 1, is_enum)) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + if expecting_anon_class_brace: + brace_stack.append((i, 2)) + class_bodies.append(_ClassBodyFrame(-1, brace_depth, i + 1, False)) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + brace_stack.append((i, 0)) + in_new_expr = False + i += 1 + continue + + if c == "}": + if brace_stack: + _, kind = brace_stack.pop() + + # if closing a block directly inside a class body, advance member_start + if class_bodies: + frame = class_bodies[-1] + if kind == 0 and brace_depth == frame.body_depth + 1: + frame.member_start = i + 1 + + if kind == 1: + # closing named type: pop frame, and advance parent's member_start + if class_bodies and class_bodies[-1].named_type_idx != -1: + class_bodies.pop() + if class_bodies: + class_bodies[-1].member_start = i + 1 + elif kind == 2: + # closing anonymous class: pop frame only + if class_bodies and class_bodies[-1].named_type_idx == -1: + class_bodies.pop() + + brace_depth = max(0, brace_depth - 1) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + if c == ";" and class_bodies: + frame = class_bodies[-1] + if brace_depth == frame.body_depth: + # Enum constant section ends at the first semicolon at enum-body depth. + if frame.is_enum and not frame.enum_constants_done: + frame.enum_constants_done = True + frame.member_start = i + 1 + i += 1 + continue + + i += 1 + continue + + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue + + if state == BLOCK: + if c == "*" and i + 1 < end and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue + + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue + + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue + + # TEXT + if i + 2 < end and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue i += 1 return out @@ -1647,383 +2625,610 @@ def _has_new_between(s, a, b): def extract_methods(java_source: str) -> List[str]: """ - Returns a list of Java method bodies/signatures (with bodies) plus all arrow-lambda - forms found in the file. Excludes constructors, comments/Javadoc, and strips - any method-level annotations that start before the signature token. + Returns a list of Java method/constructor bodies/signatures (with bodies) plus all arrow-lambda + forms found in the file. Original comments/Javadoc are excluded and method-level annotations are + stripped during extraction. For methods/constructors declared inside a *nested named type*, a small synthetic + trailing comment is appended: + /* inner-type: */ + (Top-level type members are unchanged.) Order: methods first, then lambdas. Optimized for large files by pre-indexing comments/strings and delimiter pairs (O(1) matching for (), {}). + + Additional filtering: + - Exclude only the *standard* Object overrides: + * boolean equals(Object) + * int hashCode() + * String toString() + (Overloads are kept.) """ + def _is_ident_char(ch: str) -> bool: + return ch.isalnum() or ch in ('_', '$') + + def _read_ident_fwd(s: str, pos: int, end: int) -> Tuple[int, str]: + if pos >= end or not _is_ident_char(s[pos]): + return pos, "" + start = pos + pos += 1 + while pos < end and _is_ident_char(s[pos]): + pos += 1 + return pos, s[start:pos] + + def _read_qualified_ident_fwd(s: str, pos: int, end: int) -> Tuple[int, str]: + pos, seg = _read_ident_fwd(s, pos, end) + if not seg: + return pos, "" + parts = [seg] + while pos < end and s[pos] == '.': + dot = pos + pos += 1 + pos2, seg2 = _read_ident_fwd(s, pos, end) + if not seg2: + # not actually qualified; stop at dot + return dot, ".".join(parts) + parts.append(seg2) + pos = pos2 + return pos, ".".join(parts) + + def _skip_ws(s: str, pos: int, end: int) -> int: + while pos < end and s[pos].isspace(): + pos += 1 + return pos + + def _has_top_level_comma(params: str) -> bool: + # track generic nesting so commas inside Map don't look like param separators + depth = 0 + for ch in params: + if ch == '<': + depth += 1 + elif ch == '>' and depth > 0: + depth -= 1 + elif ch == ',' and depth == 0: + return True + return False + + def _standard_override_kind(method_text: str) -> str: + """ + Return "equals" / "hashCode" / "toString" if this is exactly the standard Object override, + else "". + + Assumptions: method_text is already stripped of comments/Javadoc/annotations by extraction. + """ + open_paren = method_text.find('(') + if open_paren == -1: + return "" + + # Find the first '{' after '('; that's the method/ctor body start in extracted text. + brace = method_text.find('{', open_paren) + if brace == -1: + return "" + + # Close paren must be before the body. + close_paren = method_text.rfind(')', open_paren, brace) + if close_paren == -1: + return "" + + # Method name: scan left from '(' + j = open_paren - 1 + while j >= 0 and method_text[j].isspace(): + j -= 1 + name_end = j + 1 + while j >= 0 and _is_ident_char(method_text[j]): + j -= 1 + name_start = j + 1 + method_name = method_text[name_start:name_end] + if method_name not in ("equals", "hashCode", "toString"): + return "" + + # Return type token: scan left from method_name_start + k = name_start - 1 + while k >= 0 and method_text[k].isspace(): + k -= 1 + rt_end = k + 1 + while k >= 0 and (method_text[k].isalnum() or method_text[k] in ('_', '$', '.')): + k -= 1 + return_type = method_text[k + 1:rt_end] + + params = method_text[open_paren + 1:close_paren] + + # ---- hashCode() ---- + if method_name == "hashCode": + if return_type != "int": + return "" + if params.strip(): + return "" + return "hashCode" + + # ---- toString() ---- + if method_name == "toString": + if return_type not in ("String", "java.lang.String"): + return "" + if params.strip(): + return "" + return "toString" + + # ---- equals(Object) ---- + if method_name == "equals": + if return_type != "boolean": + return "" + p = params.strip() + if not p: + return "" + if _has_top_level_comma(p): + return "" + + # parse: [final] (Object|java.lang.Object) + tmp = p + pos = 0 + end = len(tmp) + + pos = _skip_ws(tmp, pos, end) + pos2, maybe_final = _read_ident_fwd(tmp, pos, end) + if maybe_final == "final": + pos = _skip_ws(tmp, pos2, end) + + pos, type_name = _read_qualified_ident_fwd(tmp, pos, end) + if type_name not in ("Object", "java.lang.Object"): + return "" + + pos = _skip_ws(tmp, pos, end) + + # Reject arrays/varargs/generic suffixes on the parameter type (not the standard override) + if pos < end and (tmp[pos] == '[' or tmp[pos] == '<' or tmp.startswith("...", pos)): + return "" + + pos, param_name = _read_ident_fwd(tmp, pos, end) + if not param_name: + return "" + + # nothing else meaningful should follow in a normal single-param decl + if tmp[_skip_ws(tmp, pos, end):].strip(): + return "" + + return "equals" + + return "" + _populate_delim_index_cache(java_source) try: - # Extract methods = _extract_methods_anywhere(java_source, 0, len(java_source)) - # Lambdas in a separate pass across the whole file (so we don't double-count) + + # Filter only the standard Object overrides (keep overloads). + filtered: List[str] = [m for m in methods if not _standard_override_kind(m)] + lambdas = _extract_lambdas_in_range(java_source, 0, len(java_source)) - return methods + lambdas + return filtered + lambdas finally: _clear_delim_index_cache() -def extract_inner_classes(src: str) -> List[str]: - """ - Return a list of source slices for *inner* named types (class/interface/enum/record), - including nested inner types. Each returned slice is prefixed with the file's - top-level import statements (including 'import static ...;'). +def extract_inner_classes(src: str) -> list[str]: """ + Extract Java *named* inner classes/interfaces/enums/records (with full bodies) + from a compilation unit text. - # ----------------------- - # Tiny local "utilities" - # ----------------------- - def _is_ident_char(ch): - return ch.isalnum() or ch in ('_', '$') + Changes vs prior behavior: + - Does NOT prefix extracted inner types with the file's package/import block. + - Strips from each extracted inner type snippet: + * // line comments + * /* ... */ block comments (incl. Javadocs) + * annotations (e.g. @SuppressWarnings(...), @Nullable, @A.B) + while preserving strings/chars/text blocks. + + Notes: + - Anonymous inner classes are NOT returned here (TODO remains). + - Performance: single forward scan + O(k) balanced-brace matches for each found inner type. + Stripping is O(m) per extracted snippet and is skipped when markers are absent. + + Bugfix: + - Avoids false positives from identifiers like `record` used as a variable name + (e.g. `Object record = ...`) by only treating `class|interface|enum|record` + as declaration keywords at a declaration-start position (after `{`, `}`, `;`, start, + or after modifiers/annotations), and by cancelling pending candidates on `=` / `;` + (and on `(` for non-record kinds). + - Fixes slice start for inner types so we never “pull in” the previous statement line + (e.g. enum constants ending with `;`) when computing the snippet start. + """ + s = src + n = len(s) + if n == 0: + return [] + + # Quick reject + if ("class" not in s and "interface" not in s and "enum" not in s and "record" not in s): + return [] + + # ---------- local helpers ---------- + def _is_ident_start(ch: str) -> bool: + return ch.isalpha() or ch in "_$" + + def _is_ident_part(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + def _match_balanced_brace(lbrace: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + st = NORMAL + i = lbrace + 1 + depth = 1 - def _skip_string(s, i): - i += 1 - n = len(s) - while i < n: + while i < lim: c = s[i] - if c == '\\': - i += 2 - elif c == '"': - return i + 1 - else: - i += 1 - return n + if st == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + st = LINE + i += 2 + continue + if n2 == "*": + st = BLOCK + i += 2 + continue + if c == '"': + if i + 2 < lim and s[i:i + 3] == '"""': + st = TEXT + i += 3 + continue + st = STRING + i += 1 + continue + if c == "'": + st = CHAR + i += 1 + continue - def _skip_char(s, i): - i += 1 - n = len(s) - while i < n: - c = s[i] - if c == '\\': - i += 2 - elif c == "'": - return i + 1 - else: - i += 1 - return n + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + + elif st == LINE: + if c == "\n": + st = NORMAL + elif st == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + st = NORMAL + i += 2 + continue + elif st == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + st = NORMAL + elif st == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + st = NORMAL + else: # TEXT + if i + 2 < lim and s[i:i + 3] == '"""': + st = NORMAL + i += 3 + continue - def _skip_ws_comments(s, i): - """Skip whitespace and //... or /*...*/ comments. Always makes progress or returns.""" - n = len(s) - while i < n: - start = i - # whitespace - while i < n and s[i].isspace(): - i += 1 - # // line comment - if i + 1 < n and s[i] == '/' and s[i + 1] == '/': - i += 2 - while i < n and s[i] != '\n': - i += 1 - continue - # /* block comment */ - if i + 1 < n and s[i] == '/' and s[i + 1] == '*': - i += 2 - end = s.find('*/', i) - if end == -1: - return n - i = end + 2 - continue - if i == start: - break - return i + i += 1 - def _read_ident(s, i): - j, n = i, len(s) - while j < n and _is_ident_char(s[j]): - j += 1 - return s[i:j], j + return -1 - def _word_at(s, i, word): - n = len(s) - end = i + len(word) - if end > n or s[i:end] != word: + def _line_is_annotation_only(line_start: int, line_end: int) -> bool: + """ + True if the line contains only annotations (possibly multiple) plus whitespace/comments. + """ + i = line_start + while i < line_end and s[i].isspace(): + i += 1 + if i >= line_end or s[i] != "@": return False - pre = s[i - 1] if i > 0 else '' - post = s[end] if end < n else '' - return (not _is_ident_char(pre)) and (not _is_ident_char(post)) - - def _skip_parenthesized(s, i): - n = len(s) - if i >= n or s[i] != '(': - return i - depth = 1 - i += 1 - while i < n and depth > 0: - i = _skip_ws_comments(s, i) - if i >= n: + + # Parse one-or-more annotations: @Name or @pkg.Name or @Name(...) + while True: + if i >= line_end or s[i] != "@": break - c = s[i] - if c == '"': - i = _skip_string(s, i); continue - if c == "'": - i = _skip_char(s, i); continue - if c == '(': - depth += 1; i += 1; continue - if c == ')': - depth -= 1; i += 1; continue i += 1 - return i + if i >= line_end or not _is_ident_start(s[i]): + return False + i += 1 + while i < line_end and (_is_ident_part(s[i]) or s[i] == "."): + i += 1 - def _skip_leading_annotations(s, i): - """Skip @Anno, @pkg.Anno, @Anno(...), stacked.""" - n = len(s) - while True: i = _skip_ws_comments(s, i) - if i >= n or s[i] != '@': - return i + if i < line_end and s[i] == "(": + r = _match_balanced_parens(s, i) + if r == -1: + return False + i = r + 1 + + i = _skip_ws_comments(s, i) + if i < line_end and s[i] == "@": + continue + break + + i = _skip_ws_comments(s, i) + while i < line_end and s[i].isspace(): + i += 1 + return i >= line_end + + def _consume_annotations(i: int) -> int: + """ + Consume one-or-more annotations starting at s[i] == '@' and return the first index + after them (skipping whitespace/comments between annotations). + """ + while i < n and s[i] == "@": i += 1 + if i >= n or not _is_ident_start(s[i]): + return i + # qualified name - while i < n and (_is_ident_char(s[i]) or s[i] == '.'): + i += 1 + while i < n and (_is_ident_part(s[i]) or s[i] == "."): i += 1 - i = _skip_ws_comments(s, i) - if i < n and s[i] == '(': - i = _skip_parenthesized(s, i) - # loop to allow many annotations - def _match_brace(s, i_open): - n = len(s) - if i_open >= n or s[i_open] != '{': - return None - depth, i = 1, i_open + 1 - steps, limit = 0, 10_000_000 - while i < n and depth > 0: - steps += 1 - if steps > limit: - return None i = _skip_ws_comments(s, i) - if i >= n: - break - c = s[i] - if c == '"': - i = _skip_string(s, i); continue - if c == "'": - i = _skip_char(s, i); continue - if c == '{': - depth += 1; i += 1; continue - if c == '}': - depth -= 1 - if depth == 0: + if i < n and s[i] == "(": + r = _match_balanced_parens(s, i) + if r == -1: return i - i += 1; continue - i += 1 - return None + i = r + 1 - def _find_left_boundary(s, brace_pos): + i = _skip_ws_comments(s, i) + return i + + def _sig_start_for_type(kind_pos: int) -> int: """ - Find the start position of the type declaration (keyword position). - Backward bounded search. + Start at the beginning of the line containing the kind keyword, and (optionally) + include immediately preceding annotation-only lines. Critically, NEVER walk into + the previous statement line (e.g., enum constants ending with ';'). """ - window_size = 4000 - base = max(0, brace_pos - window_size) - window = s[base:brace_pos] - best = -1 - for kw in ('class', 'interface', 'enum', 'record'): - j = window.rfind(kw) - while j != -1: - pre = window[j - 1] if j > 0 else '' - post = window[j + len(kw)] if j + len(kw) < len(window) else '' - if not _is_ident_char(pre) and not _is_ident_char(post): - best = max(best, j) - break - j = window.rfind(kw, 0, j) - return base + best if best != -1 else max(0, brace_pos - 1) - - # ------------------------------- - # Collect top-level import block - # ------------------------------- - def _collect_top_level_imports(s): - imports = [] - NORMAL, LINE, BLOCK, STRING, CHAR = 0, 1, 2, 3, 4 - state = NORMAL - depth = 0 - i, n = 0, len(s) + # Anchor to the declaration line that actually contains `class|interface|enum|record`. + sig_line_start = s.rfind("\n", 0, kind_pos) + 1 + sig_start = sig_line_start + + # Now look upward for annotation-only lines directly above. + k = sig_line_start + # Trim back over whitespace/newlines to the end of the previous non-empty content. + while k > 0 and s[k - 1].isspace(): + k -= 1 - def _word_at_local(pos, word): - end = pos + len(word) - if end > n or s[pos:end] != word: - return False - pre = s[pos - 1] if pos > 0 else '' - post = s[end] if end < n else '' - return (not _is_ident_char(pre)) and (not _is_ident_char(post)) + while k > 0: + line_start = s.rfind("\n", 0, k - 1) + 1 + if _line_is_annotation_only(line_start, k): + sig_start = line_start + k = line_start + while k > 0 and s[k - 1].isspace(): + k -= 1 + continue + break - saw_top_level_type = False + return sig_start - while i < n and not saw_top_level_type: - ch = s[i] - if state == NORMAL: - if ch == '/': - if i + 1 < n and s[i + 1] == '/': - state = LINE; i += 2; continue - if i + 1 < n and s[i + 1] == '*': - state = BLOCK; i += 2; continue - if ch == '"': state = STRING; i += 1; continue - if ch == "'": state = CHAR; i += 1; continue - - if ch == '{': - depth += 1 - if depth == 1: - saw_top_level_type = True - i += 1 - continue - if ch == '}': - depth = max(0, depth - 1); i += 1; continue + # ---------- scan ---------- + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL - if depth == 0 and ch == 'i' and _word_at_local(i, 'import'): - j = i + 6 - while j < n and s[j] != ';': - j += 1 - if j < n and s[j] == ';': - imports.append(s[i:j + 1].strip()) - i = j + 1 - continue + inner_classes: list[str] = [] + + class_kinds = {"class", "interface", "enum", "record"} + class_mods = { + "public", "protected", "private", + "abstract", "final", "static", "sealed", "strictfp", + } + + brace_depth = 0 + named_type_depth_stack: list[int] = [] + + pending_kind: str | None = None + pending_kind_pos: int = -1 + pending_name: str | None = None + # Only True at a plausible declaration start position. + can_start_decl = True + + i = 0 + while i < n: + c = s[i] + + if state == NORMAL: + # comments + if c == "/" and i + 1 < n: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + + # string/text/char + if c == '"': + if i + 2 < n and s[i:i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + if c == "'": + state = CHAR i += 1 continue - elif state == LINE: - if ch == '\n': state = NORMAL - i += 1; continue + # ---- cancel/advance on delimiters that cannot belong to a type header ---- + if c == ";": + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = True + i += 1 + continue - elif state == BLOCK: - if ch == '*' and i + 1 < n and s[i + 1] == '/': - state = NORMAL; i += 2 - else: - i += 1 + if c == "=": + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = False + i += 1 continue - elif state == STRING: - if ch == '\\': i += 2 - else: - if ch == '"': state = NORMAL - i += 1 + if c == "(" and pending_kind is not None and pending_kind != "record": + # `class|interface|enum` headers never contain '(' + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = False + i += 1 continue - else: # CHAR - if ch == '\\': i += 2 - else: - if ch == "'": state = NORMAL - i += 1 + # annotations at decl-start: consume and remain in decl-start mode + if c == "@" and can_start_decl and pending_kind is None: + # Special-case: allow "@interface Foo { ... }" as an inner "interface" decl. + if (i + 10 <= n and s[i + 1:i + 10] == "interface" and + (i + 10 == n or not _is_ident_part(s[i + 10]))): + i += 1 # next loop sees "interface" + continue + + i = _consume_annotations(i) + can_start_decl = True continue - return ("\n".join(imports) + ("\n" if imports else "")) + # identifier tokenizing + if _is_ident_start(c): + j = i + 1 + while j < n and _is_ident_part(s[j]): + j += 1 + tok = s[i:j] + + # After kind keyword, accept very next identifier as the type name. + if pending_kind is not None and pending_name is None: + pending_name = tok + can_start_decl = False + i = j + continue - # ----------------------------------- - # Scan: find all named type regions - # ----------------------------------- - def _find_all_named_types(s): - kinds = ('class', 'interface', 'enum', 'record') - regions = [] # tuples: (name, kind, start, end) - i, n = 0, len(s) - steps, limit = 0, 10_000_000 + # Only recognize kind keywords at declaration starts. + if pending_kind is None and can_start_decl: + if tok in class_mods: + can_start_decl = True + elif tok in class_kinds: + pending_kind = tok + pending_kind_pos = i + pending_name = None + can_start_decl = False + else: + can_start_decl = False - while i < n: - steps += 1 - if steps > limit: - break + i = j + continue - i = _skip_ws_comments(s, i) - if i >= n: - break + # Otherwise we're in code/header; not a new decl start. + can_start_decl = False + i = j + continue - c = s[i] - if c == '"': - i = _skip_string(s, i); continue - if c == "'": - i = _skip_char(s, i); continue + # brace open + if c == "{": + brace_depth += 1 + can_start_decl = True - i = _skip_leading_annotations(s, i) - if i >= n: - break + if pending_kind is not None and pending_name is not None: + is_inner = bool(named_type_depth_stack) - matched_kind = None - for kw in kinds: - if _word_at(s, i, kw): - matched_kind = kw - break - if not matched_kind: - i += 1 - continue + # FIXED: anchor to the declaration line; do not walk into previous statements. + sig_start = _sig_start_for_type(pending_kind_pos) - # consume keyword - i += len(matched_kind) - i = _skip_ws_comments(s, i) + body_close = _match_balanced_brace(i, n) + if body_close != -1 and is_inner: + raw = s[sig_start:body_close + 1] - # read type name - name, i2 = _read_ident(s, i) - if not name: - i += 1 - continue - i = i2 + cleaned = raw + if "/" in cleaned and (("//" in cleaned) or ("/*" in cleaned)): + cleaned = _strip_comments_keep_strings_range(cleaned, 0, len(cleaned)) + if "@" in cleaned: + cleaned = _strip_annotations_in_range(cleaned, 0, len(cleaned)) - # consume header up to '{' - while i < n: - i = _skip_ws_comments(s, i) - if i >= n or s[i] == '{': - break - if s[i] == '"': - i = _skip_string(s, i); continue - if s[i] == "'": - i = _skip_char(s, i); continue - if _is_ident_char(s[i]): - _, i = _read_ident(s, i); continue - if s[i] == '<': - depth = 1; i += 1 - while i < n and depth > 0: - i = _skip_ws_comments(s, i) - if i >= n: break - ch = s[i] - if ch == '"': i = _skip_string(s, i); continue - if ch == "'": i = _skip_char(s, i); continue - if ch == '<': depth += 1; i += 1; continue - if ch == '>': depth -= 1; i += 1; continue - i += 1 + cleaned = cleaned.strip() + if cleaned: + inner_classes.append(cleaned) + + named_type_depth_stack.append(brace_depth) + + pending_kind = None + pending_kind_pos = -1 + pending_name = None + + i += 1 continue - if s[i] == '(': - i = _skip_parenthesized(s, i); continue - i += 1 - if i >= n or s[i] != '{': + pending_kind = None + pending_kind_pos = -1 + pending_name = None + + i += 1 continue - body_open = i - body_close = _match_brace(s, body_open) - if body_close is None: - break + # brace close + if c == "}": + if brace_depth > 0: + if named_type_depth_stack and named_type_depth_stack[-1] == brace_depth: + named_type_depth_stack.pop() + brace_depth -= 1 - regions.append((name, matched_kind, body_open, body_close)) - i = body_close + 1 + pending_kind = None + pending_kind_pos = -1 + pending_name = None - return regions + can_start_decl = True + i += 1 + continue - # ----------------------------------- - # Assemble results for inner classes - # ----------------------------------- - imports_block = _collect_top_level_imports(src) - imports_prefix = imports_block if (not imports_block or imports_block.endswith("\n")) else imports_block + "\n" + # any other non-ws char likely means we're in an expression/statement + if not c.isspace(): + can_start_decl = False - regions = _find_all_named_types(src) # (name, kind, start, end) - regions.sort(key=lambda t: (t[2], -(t[3] - t[2]))) # by start, longer first + i += 1 + continue - results = [] - stack = [] # elements: (end, sig_start) + # ---------- non-NORMAL states ---------- + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue - for name, kind, start, end in regions: - # pop completed outers - while stack and start >= stack[-1][0]: - stack.pop() + if state == BLOCK: + if c == "*" and i + 1 < n and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue - # compute header/signature start - hdr = _find_left_boundary(src, start) - hdr = _skip_ws_comments(src, hdr) - sig_start = _skip_leading_annotations(src, hdr) - sig_start = _skip_ws_comments(src, sig_start) + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue - is_inner = bool(stack) - if is_inner and end >= sig_start: - type_src = src[sig_start:end + 1] - results.append(imports_prefix + type_src) + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue - stack.append((end, sig_start)) + # TEXT + if i + 2 < n and s[i:i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 - return results \ No newline at end of file + return inner_classes \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py index 47f6c33ab..3230c1e7b 100644 --- a/src/exploit_iq_commons/utils/java_utils.py +++ b/src/exploit_iq_commons/utils/java_utils.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass from functools import lru_cache -from typing import Dict, List, Tuple, Optional, Any, Set +from typing import Collection, Dict, List, Tuple, Optional, Any, Set from langchain_core.documents import Document from tqdm import tqdm @@ -58,6 +58,19 @@ re.X, ) +# Matches real Java type headers like "class Foo", not ".class" +_JAVA_TYPE_HEADER_RE = re.compile(r'(? str: - """ - Return the simple name of the first class/interface/enum/record declared - anywhere in the given Java source (top-level or nested). Makes no assumptions - about leading comments, javadocs, annotations, package/imports, etc. - - - Skips // line comments, /* ... */ (incl. Javadoc) block comments, strings "..." - and character literals '...'. - - Handles 'record' headers (e.g., `record Point(int x, int y) { ... }`). - - Matches annotation type declarations too (e.g., `public @interface Foo {}`), - because the token `interface` is still present. - - Returns the *simple* identifier (e.g., 'Point'). - """ - n = len(src) - i = 0 - - # States - NORMAL, LINE, BLOCK, STRING, CHAR = 0, 1, 2, 3, 4 - state = NORMAL - - KW = ("class", "interface", "enum", "record") - - def is_ws(ch: str) -> bool: - return ch <= " " and ch in " \t\r\n\f\v" - - def is_id_start(ch: str) -> bool: - # Java allows '_' and '$'; also allow unicode letters - return ( - ch == "_" or ch == "$" or - ("A" <= ch <= "Z") or ("a" <= ch <= "z") or - (ch >= "\u0080" and ch.isalpha()) - ) - - def is_id_part(ch: str) -> bool: - return is_id_start(ch) or ("0" <= ch <= "9") - - def word_at(pos: int, word: str) -> bool: - """Return True if 'word' starts at src[pos] with Java-style word boundaries.""" - end = pos + len(word) - if end > n or src[pos:end] != word: - return False - # left boundary: start of file or not an identifier part - if pos > 0 and (src[pos - 1].isalnum() or src[pos - 1] in "_$"): - return False - # right boundary: end of file or not an identifier part - if end < n and (src[end].isalnum() or src[end] in "_$"): - return False - return True - - while i < n: - ch = src[i] - - if state == NORMAL: - # comment/string/char entry - if ch == "/": - if i + 1 < n and src[i + 1] == "/": - state = LINE - i += 2 - continue - if i + 1 < n and src[i + 1] == "*": - state = BLOCK - i += 2 - continue - elif ch == '"': - state = STRING - i += 1 - continue - elif ch == "'": - state = CHAR - i += 1 - continue - - # Try to match a type keyword at this position - for kw in KW: - if word_at(i, kw): - j = i + len(kw) - # skip whitespace between keyword and identifier - while j < n and is_ws(src[j]): - j += 1 - # Next must be a valid Java identifier start (type name) - if j < n and is_id_start(src[j]): - k = j + 1 - while k < n and is_id_part(src[k]): - k += 1 - return src[j:k] - # If not a valid identifier here, keep scanning - i += 1 +@lru_cache(maxsize=100000) +def is_java_method(source: str) -> bool: + s = source - elif state == LINE: - if ch == "\n": - state = NORMAL - i += 1 + # avoid scanning twice for '(' and '->' + has_paren = "(" in s + has_arrow = "->" in s + if not has_paren and not has_arrow: + return False - elif state == BLOCK: - if ch == "*" and i + 1 < n and src[i + 1] == "/": - state = NORMAL - i += 2 - else: - i += 1 + n = len(s) + find_in_source = s.find + startswith = s.startswith + ws = JAVA_METHOD_WS_CHARS - elif state == STRING: - if ch == "\\" and i + 1 < n: - i += 2 # skip escaped + # --- FAST REJECT #1: skip leading whitespace/comments and reject FreeMarker templates --- + # This avoids scanning huge .ftl-like files (your ArrowType.java template input). + j = 0 + while j < n: + ch = s[j] + if ch in ws: + j += 1 + continue + if ch == "/" and j + 1 < n: + c2 = s[j + 1] + if c2 == "/": + nl = find_in_source("\n", j + 2) + j = n if nl == -1 else nl + 1 continue - if ch == '"': - state = NORMAL - i += 1 - - elif state == CHAR: - if ch == "\\" and i + 1 < n: - i += 2 + if c2 == "*": + end = find_in_source("*/", j + 2) + j = n if end == -1 else end + 2 continue - if ch == "'": - state = NORMAL - i += 1 + break - return "" + if j < n and (s.startswith("<#", j) or s.startswith("<@", j)): + return False -@lru_cache(maxsize=2500000) -def is_java_method(source: str) -> bool: - """ - Return True iff the given Java source snippet is a *method or constructor* - declaration, or (fallback) contains a lambda ('->') outside of comments/strings. - - Notes: - - Methods with explicit return type: detected. - - Constructors (no return type) are treated as methods too. - - Control-flow constructs are excluded. - - A lambda arrow outside comments/strings/text-blocks returns True. - """ - s = source - # Safe early reject: no method/ctor is possible without '('; no lambda possible without '->'. - if "(" not in s and "->" not in s: + # --- FAST REJECT #2: compilation unit headers ("package"/"import") --- + # Safe for your prior examples (method/ctor/lambda snippets) and avoids full-file scans. + leading_ident_match = _LEADING_IDENT_RE.match(s, j) + if leading_ident_match and leading_ident_match.group(0) in ("package", "import"): return False - n = len(s) + # Conservative “might be a top-level type decl somewhere” flag: + # Use a real "type header" detector, not substring "class" (to avoid `.class` false positives). + maybe_type_decl = bool(has_arrow and _JAVA_TYPE_HEADER_RE.search(s)) + i = 0 NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 @@ -864,10 +788,6 @@ def is_java_method(source: str) -> bool: brace = 0 saw_lambda = False - find_in_source = s.find - startswith = s.startswith - ws = JAVA_METHOD_WS_CHARS - def is_id_start(ch: str) -> bool: return ch == "_" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) @@ -878,13 +798,9 @@ def is_id_part(ch: str) -> bool: (ch >= "\u0080" and ch.isalnum()) ) - def _skip_quoted(j: int, quote: str) -> int: - """ - Skip to the first unescaped closing quote starting at j (j is after opening quote). - Returns index right after the closing quote, or n if none. - """ - while j < n: - end = find_in_source(quote, j) + def _skip_quoted(jq: int, quote: str) -> int: + while jq < n: + end = find_in_source(quote, jq) if end == -1: return n k = end - 1 @@ -894,92 +810,92 @@ def _skip_quoted(j: int, quote: str) -> int: k -= 1 if back % 2 == 0: return end + 1 - j = end + 1 + jq = end + 1 return n - def skip_paren(j: int) -> int: + def skip_paren(jp: int) -> int: depth = 1 st = NORMAL - while j < n and depth > 0: - ch = s[j] + while jp < n and depth > 0: + chp = s[jp] if st == NORMAL: - if ch == "/": - if j + 1 < n: - c2 = s[j + 1] + if chp == "/": + if jp + 1 < n: + c2 = s[jp + 1] if c2 == "/": st = LINE - j += 2 + jp += 2 continue if c2 == "*": st = BLOCK - j += 2 + jp += 2 continue - elif ch == '"': - if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': + elif chp == '"': + if jp + 2 < n and s[jp + 1] == '"' and s[jp + 2] == '"': st = TEXT - j += 3 + jp += 3 continue st = STRING - j += 1 + jp += 1 continue - elif ch == "'": + elif chp == "'": st = CHAR - j += 1 + jp += 1 continue - elif ch == "(": + elif chp == "(": depth += 1 - j += 1 + jp += 1 continue - elif ch == ")": + elif chp == ")": depth -= 1 - j += 1 + jp += 1 continue else: - j += 1 + jp += 1 continue elif st == LINE: - nl = find_in_source("\n", j) - j = n if nl == -1 else nl + 1 + nl = find_in_source("\n", jp) + jp = n if nl == -1 else nl + 1 st = NORMAL elif st == BLOCK: - end = find_in_source("*/", j) - j = n if end == -1 else end + 2 + end = find_in_source("*/", jp) + jp = n if end == -1 else end + 2 st = NORMAL elif st == STRING: - j = _skip_quoted(j, '"') + jp = _skip_quoted(jp, '"') st = NORMAL elif st == CHAR: - j = _skip_quoted(j, "'") + jp = _skip_quoted(jp, "'") st = NORMAL else: # TEXT while True: - end = find_in_source('"""', j) + end = find_in_source('"""', jp) if end == -1: - j = n + jp = n break k = end - 1 back = 0 while k >= 0 and s[k] == "\\": back += 1 k -= 1 - j = end + 3 + jp = end + 3 if back % 2 == 0: break st = NORMAL - return j + return jp - def read_ident(j: int) -> tuple[Optional[str], int]: - if j >= n or not is_id_start(s[j]): - return None, j - k = j + 1 + def read_ident(jr: int) -> tuple[Optional[str], int]: + if jr >= n or not is_id_start(s[jr]): + return None, jr + k = jr + 1 while k < n and is_id_part(s[k]): k += 1 - return s[j:k], k + return s[jr:k], k - def read_qual_ident(j: int) -> tuple[Optional[str], int]: - name, k = read_ident(j) + def read_qual_ident(jr: int) -> tuple[Optional[str], int]: + name, k = read_ident(jr) if not name: - return None, j + return None, jr while k < n and s[k] == ".": nm2, k2 = read_ident(k + 1) if not nm2: @@ -987,139 +903,132 @@ def read_qual_ident(j: int) -> tuple[Optional[str], int]: k = k2 return name, k - def skip_ws_comments(j: int) -> int: - while j < n: - ch = s[j] - if ch in ws: - j += 1 + def skip_ws_comments(jw: int) -> int: + while jw < n: + chw = s[jw] + if chw in ws: + jw += 1 continue - if ch == "/" and j + 1 < n: - c2 = s[j + 1] + if chw == "/" and jw + 1 < n: + c2 = s[jw + 1] if c2 == "/": - nl = find_in_source("\n", j + 2) - j = n if nl == -1 else nl + 1 + nl = find_in_source("\n", jw + 2) + jw = n if nl == -1 else nl + 1 continue if c2 == "*": - end = find_in_source("*/", j + 2) - j = n if end == -1 else end + 2 + end = find_in_source("*/", jw + 2) + jw = n if end == -1 else end + 2 continue break - return j + return jw - def has_dot_between(j: int, k: int) -> bool: - j = skip_ws_comments(j) - while j < k: - ch = s[j] - if ch == ".": + def has_dot_between(jd: int, kd: int) -> bool: + jd = skip_ws_comments(jd) + while jd < kd: + chd = s[jd] + if chd == ".": return True - if ch in ws: - j += 1 + if chd in ws: + jd += 1 continue - if ch == "/" and j + 1 < n: - c2 = s[j + 1] + if chd == "/" and jd + 1 < n: + c2 = s[jd + 1] if c2 == "/": - nl = find_in_source("\n", j + 2) - j = n if nl == -1 else nl + 1 + nl = find_in_source("\n", jd + 2) + jd = n if nl == -1 else nl + 1 continue if c2 == "*": - end = find_in_source("*/", j + 2) - j = n if end == -1 else end + 2 + end = find_in_source("*/", jd + 2) + jd = n if end == -1 else end + 2 continue - j += 1 + jd += 1 return False def plausible_method_decl(name_start: int) -> bool: - """ - Must see a real return type token before the name (method), - OR see only annotations/modifiers/generics/comments before the name (constructor). - """ - j = name_start - j0 = max(s.rfind("\n", 0, j), s.rfind("{", 0, j), s.rfind(";", 0, j)) + 1 - j = skip_ws_comments(j0) + jx = name_start + j0 = max(s.rfind("\n", 0, jx), s.rfind("{", 0, jx), s.rfind(";", 0, jx)) + 1 + jx = skip_ws_comments(j0) broken = False - while j < name_start: - if s[j] == JAVA_ANNOTATION_SYMBOL: - _, j = read_qual_ident(j + 1) - j = skip_ws_comments(j) - if j < n and s[j] == "(": - j = skip_paren(j + 1) - j = skip_ws_comments(j) + while jx < name_start: + if s[jx] == JAVA_ANNOTATION_SYMBOL: + _, jx = read_qual_ident(jx + 1) + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "(": + jx = skip_paren(jx + 1) + jx = skip_ws_comments(jx) continue - if s[j] == "<": + if s[jx] == "<": depth = 1 - j += 1 - while j < n and depth > 0: - if s[j] == "<": + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": depth += 1 - elif s[j] == ">": + elif s[jx] == ">": depth -= 1 - j += 1 - j = skip_ws_comments(j) + jx += 1 + jx = skip_ws_comments(jx) continue - tok, k = read_ident(j) + tok, k = read_ident(jx) if tok: if tok in JAVA_METHOD_PRIM_TYPES: return True - if tok in JAVA_METHOD_METH_MODS: - j = skip_ws_comments(k) + jx = skip_ws_comments(k) continue - if tok == "non": k2 = skip_ws_comments(k) if k2 < n and s[k2] == "-": k3 = skip_ws_comments(k2 + 1) id2, j2 = read_ident(k3) if id2 == "sealed": - j = skip_ws_comments(j2) + jx = skip_ws_comments(j2) continue - # reference return type start - j = k - j = skip_ws_comments(j) + jx = k + jx = skip_ws_comments(jx) - if j < n and s[j] == "<": + if jx < n and s[jx] == "<": depth = 1 - j += 1 - while j < n and depth > 0: - if s[j] == "<": + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": depth += 1 - elif s[j] == ">": + elif s[jx] == ">": depth -= 1 - j += 1 - j = skip_ws_comments(j) + jx += 1 + jx = skip_ws_comments(jx) - while j + 1 < n and s[j] == "[" and s[j + 1] == "]": - j += 2 - j = skip_ws_comments(j) + while jx + 1 < n and s[jx] == "[" and s[jx + 1] == "]": + jx += 2 + jx = skip_ws_comments(jx) tcount = 0 while tcount < 4: - j = skip_ws_comments(j) - if j < n and s[j] == "&": - j = skip_ws_comments(j + 1) - nm2, k2 = read_ident(j) + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "&": + jx = skip_ws_comments(jx + 1) + nm2, k2 = read_ident(jx) if not nm2: break - j = k2 - j = skip_ws_comments(j) - if j < n and s[j] == "<": + jx = k2 + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "<": depth = 1 - j += 1 - while j < n and depth > 0: - if s[j] == "<": + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": depth += 1 - elif s[j] == ">": + elif s[jx] == ">": depth -= 1 - j += 1 - j = skip_ws_comments(j) - while j + 1 < n and s[j] == "[" and s[j + 1] == "]": - j += 2 - j = skip_ws_comments(j) + jx += 1 + jx = skip_ws_comments(jx) + while jx + 1 < n and s[jx] == "[" and s[jx + 1] == "]": + jx += 2 + jx = skip_ws_comments(jx) tcount += 1 else: break @@ -1129,13 +1038,10 @@ def plausible_method_decl(name_start: int) -> bool: broken = True break - # No return type found, but nothing invalid encountered -> constructor - return not broken + return not broken # constructor while i < n: if state == NORMAL: - # Big win: when inside braces, skip boring characters in C (regex search), - # only landing on characters that can change state or matter for lambda/braces. if brace > 0: m = _BRACE_NORMAL_SPECIAL_RE.search(s, i) if not m: @@ -1143,9 +1049,10 @@ def plausible_method_decl(name_start: int) -> bool: i = m.start() ch = s[i] - # keep ordering consistent with original: lambda before comment/string checks if ch == "-" and i + 1 < n and s[i + 1] == ">": saw_lambda = True + if not maybe_type_decl: + return True i += 2 continue @@ -1164,7 +1071,7 @@ def plausible_method_decl(name_start: int) -> bool: continue if ch == '"': - if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + if i + 2 < n and s[i + 1] == '"' and s[i + 2] == '"': state = TEXT i += 3 continue @@ -1188,15 +1095,15 @@ def plausible_method_decl(name_start: int) -> bool: i += 1 continue - # non-handled special (should not happen) i += 1 continue - # brace == 0: full logic (header-level) ch = s[i] if ch == "-" and i + 1 < n and s[i + 1] == ">": saw_lambda = True + if not maybe_type_decl: + return True i += 2 continue @@ -1212,7 +1119,7 @@ def plausible_method_decl(name_start: int) -> bool: i += 2 continue elif ch == '"': - if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + if i + 2 < n and s[i + 1] == '"' and s[i + 2] == '"': state = TEXT i += 3 continue @@ -1232,12 +1139,10 @@ def plausible_method_decl(name_start: int) -> bool: i += 1 continue - # brace==0 path if ch in ws: i += 1 continue - # skip annotations at top level if ch == JAVA_ANNOTATION_SYMBOL: _, i = read_qual_ident(i + 1) i = skip_ws_comments(i) @@ -1245,43 +1150,46 @@ def plausible_method_decl(name_start: int) -> bool: i = skip_paren(i + 1) continue - # skip modifiers & detect type headers early if is_id_start(ch): - ident, j = read_ident(i) + ident, j2 = read_ident(i) + if ident in JAVA_METHOD_CLASS_MODS: - i = skip_ws_comments(j) + i = skip_ws_comments(j2) continue + if ident == "non": - k = skip_ws_comments(j) + k = skip_ws_comments(j2) if k < n and s[k] == "-": k2 = skip_ws_comments(k + 1) - id2, j2 = read_ident(k2) + id2, j3 = read_ident(k2) if id2 == "sealed": - i = skip_ws_comments(j2) + i = skip_ws_comments(j3) continue + if ident in JAVA_METHOD_CLASS_KINDS: - i2 = skip_ws_comments(j) - name, j2 = read_ident(i2) + i2 = skip_ws_comments(j2) + name, j4 = read_ident(i2) if name: return False - i = j + i = j2 continue - # potential method/ctor header + i = j2 + continue + if ch == "(": - j = i - 1 - while j >= 0 and s[j] in ws: - j -= 1 - end = j + 1 - while j >= 0 and is_id_part(s[j]): - j -= 1 - name = s[j+1:end] if end > (j + 1) else None + jx = i - 1 + while jx >= 0 and s[jx] in ws: + jx -= 1 + end = jx + 1 + while jx >= 0 and is_id_part(s[jx]): + jx -= 1 + name = s[jx + 1:end] if end > (jx + 1) else None if name and (name not in JAVA_METHOD_CTRL_WORDS) and not has_dot_between(end, i): k = skip_paren(i + 1) q = skip_ws_comments(k) - # optional throws if startswith("throws", q) and (q + 6 == n or not is_id_part(s[q + 6])): q = skip_ws_comments(q + 6) while q < n and s[q] not in "{;": @@ -1309,7 +1217,7 @@ def plausible_method_decl(name_start: int) -> bool: q = skip_ws_comments(q) if q < n and s[q] in "{;": - if plausible_method_decl(j + 1): + if plausible_method_decl(jx + 1): return True i += 1 @@ -1378,9 +1286,9 @@ def is_java_type(source: str) -> bool: brace = 0 ws = " \t\r\n\f\v" - class_kinds = {"class", "interface", "enum", "record"} - class_mods = {"public","protected","private","abstract","final","static","sealed","strictfp"} - meth_ctrl = {"if","for","while","switch","catch","synchronized","return","new","case"} + class_kinds = JAVA_METHOD_CLASS_KINDS + class_mods = JAVA_METHOD_CLASS_MODS + meth_ctrl = JAVA_METHOD_CTRL_WORDS def is_ws(ch): return ch in ws def is_id_start(ch): return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or (ch >= '\u0080' and ch.isalpha()) @@ -1395,28 +1303,41 @@ def read_ident(j: int): def skip_ws_comments(j: int) -> int: while j < n: ch = s[j] - if is_ws(ch): j += 1; continue + if is_ws(ch): + j += 1 + continue if ch == '/' and j + 1 < n: c2 = s[j + 1] if c2 == '/': - j = (s.find('\n', j + 2) + 1) or n; continue + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue if c2 == '*': end = s.find("*/", j + 2) - j = n if end == -1 else end + 2; continue + j = n if end == -1 else end + 2 + continue break return j def skip_paren_balanced(j: int) -> int: - depth = 1; st = NORMAL + depth = 1 + st = NORMAL while j < n and depth > 0: ch = s[j] if st == NORMAL: if ch == '/': + # FIX: if it's not // or /*, it's an operator; advance! if j + 1 < n and s[j+1] == '/': - j = (s.find('\n', j + 2) + 1) or n; continue + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue if j + 1 < n and s[j+1] == '*': end = s.find('*/', j + 2) - j = n if end == -1 else end + 2; continue + j = n if end == -1 else end + 2 + continue + j += 1 + continue + elif ch == '"': if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': st = TEXT; j += 3; continue @@ -1429,18 +1350,21 @@ def skip_paren_balanced(j: int) -> int: depth -= 1; j += 1; continue else: j += 1; continue + elif st == STRING: while j < n: c2 = s[j]; j += 1 if c2 == '\\': j += 1 elif c2 == '"': break st = NORMAL + elif st == CHAR: while j < n: c2 = s[j]; j += 1 if c2 == '\\': j += 1 elif c2 == "'": break st = NORMAL + else: # TEXT while True: end = s.find('"""', j) @@ -1450,6 +1374,7 @@ def skip_paren_balanced(j: int) -> int: j = end + 3 if back % 2 == 0: break st = NORMAL + return j def skip_angles_balanced(j: int) -> int: @@ -1460,10 +1385,13 @@ def skip_angles_balanced(j: int) -> int: elif ch == '>': depth -= 1 elif ch == '/': if j + 1 < n and s[j+1] == '/': - j = (s.find('\n', j + 2) + 1) or n; continue + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue if j + 1 < n and s[j+1] == '*': end = s.find('*/', j + 2) - j = n if end == -1 else end + 2; continue + j = n if end == -1 else end + 2 + continue j += 1 return j @@ -1476,10 +1404,13 @@ def has_dot_between(a: int, b: int) -> bool: j += 1; continue if ch == '/' and j + 1 < n: if s[j+1] == '/': - j = (s.find('\n', j + 2) + 1) or n; continue + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue if s[j+1] == '*': end = s.find('*/', j + 2) - j = n if end == -1 else end + 2; continue + j = n if end == -1 else end + 2 + continue j += 1 return False @@ -1515,12 +1446,10 @@ def skip_one_type(p: int) -> int: # NEW: skip comma-separated type list after extends/implements/permits def skip_type_list(p: int) -> int: p = skip_ws_comments(p) - tcount = 0 while p < n: p2 = skip_one_type(p) if p2 == p: break p = skip_ws_comments(p2) - tcount += 1 if p < n and s[p] == ',': p = skip_ws_comments(p + 1) continue @@ -1683,12 +1612,12 @@ def skip_type_list(p: int) -> int: return False -def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: +def create_inheritance_map(full_source_documents: Collection[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: """ Build a jar-aware inheritance/implements map. - Key: (simple_name, source_path) - Value: ordered, deduped list of (simple_name, source_path) including: + Key: (fqcn, source_path) + Value: ordered, deduped list of (fqcn, source_path) including: - self - ancestors (class chain or interface parents) - implemented interfaces (direct, via ancestors, and via super-interfaces) @@ -1701,7 +1630,7 @@ def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], External/unseen types (e.g., JDK classes/interfaces) are INCLUDED in the upward/interface lists as placeholders with the current type's source path: - (simple_name, current_source_of_key) + (fqcn_or_token, current_source_of_key) Placeholders are NOT used to wire reverse edges (subclasses/implementers). Parsing guarantees: @@ -1717,6 +1646,60 @@ def create_inheritance_map(java_types: List[Document]) -> dict[Tuple[str, str], """ # ---------------- helpers ---------------- + + def _balance_body_end(s: str, open_brace_idx: int) -> int: + """Return index just after the matching '}' for body that starts at open_brace_idx.""" + depth = 1 + i = open_brace_idx + 1 + n = len(s) + while i < n and depth: + ch = s[i] + if ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + i += 1 + return i + + def _find_all_type_decls(src: str) -> List[Dict]: + """ + Return ordered list of type decl dicts with fields: + kind, name, header (text up to '{'), hstart, bstart, bend, enclosing (names list). + Uses a single pass nesting stack to compute 'enclosing' chain. + """ + s = strip_comments_preserving_newlines(src) + decls: List[Dict] = [] + + # First pass: collect raw declarations with their body ranges + for m in TYPE_DECL_RE.finditer(s): + kind, name = m.group(1), m.group(2) + brace = s.find("{", m.end()) + if brace == -1: + continue # malformed; ignore + bend = _balance_body_end(s, brace) + header_text = s[m.start():brace] + decls.append({ + "kind": kind, + "name": name, + "header": re.sub(r"\s+", " ", header_text).strip(), + "hstart": m.start(), + "bstart": brace, + "bend": bend, + }) + + if not decls: + return [] + + # Second pass: assign enclosing chain using a simple interval stack + decls.sort(key=lambda d: d["hstart"]) + stack: List[Dict] = [] + for d in decls: + while stack and d["hstart"] >= stack[-1]["bend"]: + stack.pop() + d["enclosing"] = [x["name"] for x in stack] + stack.append(d) + + return decls def strip_comments_preserving_newlines(s: str) -> str: """Remove /* ... */ and // ... comments, preserving newlines for positions.""" def _blk(m): @@ -1817,7 +1800,7 @@ def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: def meta_source(doc) -> str: try: meta = getattr(doc, "metadata", None) or {} - return meta.get("source") or meta.get("path") or "" + return meta.get("source") or "" except Exception: return "" @@ -1957,78 +1940,101 @@ def source_jar_id(path: str) -> str: variants_by_fq: Dict[str, List[Dict]] = {} known_fqcns: set[str] = set() - for doc in tqdm(java_types, total=len(java_types), desc="Building types inheritance map..."): + for doc in tqdm(full_source_documents, total=len(full_source_documents), desc="Building types inheritance map..."): src_all = doc.page_content or "" - # Prefer 'Code for:' header if present - idx = src_all.lower().find("code for:") - if idx != -1: - line_end = src_all.find('\n', idx) - if line_end == -1: - line_end = len(src_all) - header_line = src_all[idx:line_end] - header_text_raw = re.sub(r".*?Code\s*for\s*:\s*", "", header_line, flags=re.I).strip() - else: - header_text_raw = find_header_slice_no_comments(src_all) or "" - - if not header_text_raw: - header_text_raw = drop_leading_annotations(before_brace(src_all)) - - header_text = before_brace(header_text_raw) - header_text = re.sub(r"\s+", " ", header_text).strip() - - m_name = name_kind_re.search(header_text) - + # Parse package/imports once per compilation unit pkg = (pkg_re.search(src_all) or [None, ""])[1] imps = import_re.findall(src_all) src_path = meta_source(doc) + file_simple = simple_name_from_filename(doc) # e.g., "FormEncodedDataDefinition" + s_nc = strip_comments_preserving_newlines(src_all) - if not m_name: - # fallback to filename - file_simple = simple_name_from_filename(doc) + # Discover ALL type declarations (top-level + nested) + decls = _find_all_type_decls(src_all) + + # Fallback: no decls found → keep legacy behavior (top-level by filename) + if not decls: if not file_simple: continue - fqcn = f"{pkg}.{file_simple}" if pkg else file_simple kind = "class" extends_names: List[str] = [] implements_names: List[str] = [] - parsed_name = file_simple - else: - kind, parsed_name = m_name.group(1), m_name.group(2) + declared_simple = file_simple + fqcn = f"{pkg}.{declared_simple}" if pkg else declared_simple + + rec = { + "fqcn": fqcn, + "simple": declared_simple, + "pkg": pkg, + "imports": imps, + "kind": kind, + "extends_raw": extends_names, + "implements_raw": implements_names, + "source": src_path, + "jar": source_jar_id(src_path), + } + variants_by_fq.setdefault(fqcn, []).append(rec) + known_fqcns.add(fqcn) + continue + + # If we're processing a *slice* that only contains an inner type, we may not see the outer. + # Detect presence of the outer by name in this source (to avoid mislabeling top-level siblings). + has_outer_decl = bool(file_simple and re.search( + rf"\b(class|interface|enum|record)\b\s+{re.escape(file_simple)}\b", s_nc)) + + # Build records for every discovered declaration + for d in decls: + kind = d["kind"] + declared_simple = d["name"] + header_text = d["header"] + + # Extract extends/implements from the specific header ext_seg, impl_seg = extract_clauses_after_kindname(header_text) - # LAST RESORT (classes only): scan de-commented full source for "class ... extends ..." - if kind == "class" and (not ext_seg) and ("extends" in src_all): - s_nc = strip_comments_preserving_newlines(src_all) + # LAST RESORT (classes only): scan for "class ... extends ..." in de-commented source slice + if kind == "class" and (not ext_seg) and ("extends" in s_nc): m_ext2 = re.search( - rf"\bclass\s+{re.escape(parsed_name)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", + rf"\bclass\s+{re.escape(declared_simple)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", s_nc ) if m_ext2: ext_seg = m_ext2.group(1).strip() + extends_names = split_type_list_strict(ext_seg) if ext_seg else [] implements_names = split_type_list_strict(impl_seg) if impl_seg else [] - # Defensive cleanup to avoid generic bounds leaking as parents/interfaces - extends_names = [clean_type_token(x) for x in extends_names if clean_type_token(x)] - implements_names = [clean_type_token(x) for x in implements_names if clean_type_token(x)] - - file_simple = simple_name_from_filename(doc) - simple_name = file_simple or parsed_name - fqcn = f"{pkg}.{simple_name}" if pkg else simple_name - - rec = { - "fqcn": fqcn, - "simple": simple_name, - "pkg": pkg, - "imports": imps, - "kind": kind, - "extends_raw": extends_names, - "implements_raw": implements_names, - "source": src_path, - "jar": source_jar_id(src_path), - } - variants_by_fq.setdefault(fqcn, []).append(rec) - known_fqcns.add(fqcn) + # Defensive cleanup + extends_names = [clean_type_token(x) for x in extends_names if clean_type_token(x)] + implements_names = [clean_type_token(x) for x in implements_names if clean_type_token(x)] + + # Compute FQCN with correct nesting + if d["enclosing"]: + # Full chain from discovered parents: pkg.Outer.Inner.Current + prefix = ".".join(d["enclosing"]) + fqcn = f"{pkg}.{prefix}.{declared_simple}" if pkg else f"{prefix}.{declared_simple}" + else: + # No enclosing type discovered. + # If this looks like an inner slice (no outer decl visible in this source) + # and filename suggests the real outer, prefix with file_simple. + if file_simple and declared_simple != file_simple and not has_outer_decl: + outer_fq = f"{pkg}.{file_simple}" if pkg else file_simple + fqcn = f"{outer_fq}.{declared_simple}" + else: + fqcn = f"{pkg}.{declared_simple}" if pkg else declared_simple + + rec = { + "fqcn": fqcn, + "simple": declared_simple, + "pkg": pkg, + "imports": imps, + "kind": kind, + "extends_raw": extends_names, + "implements_raw": implements_names, + "source": src_path, + "jar": source_jar_id(src_path), + } + variants_by_fq.setdefault(fqcn, []).append(rec) + known_fqcns.add(fqcn) # simple -> known FQCNs by_simple: Dict[str, List[str]] = {} @@ -2038,7 +2044,13 @@ def source_jar_id(path: str) -> str: def make_resolver(pkg: str, imports: List[str]): """ Build a simple type resolver for a compilation unit. - See top-level docstring for resolution precedence. + Resolution precedence: + 1) explicit imports + 2) same package + 3) star-import packages (unique) + 4) globally unique simple name + Also handles partially-qualified nested types like + 'FormParserFactory.ParserDefinition' → 'io.undertow.server.handlers.form.FormParserFactory.ParserDefinition'. """ explicit: Dict[str, str] = {} star_import_packages: List[str] = [] @@ -2048,31 +2060,46 @@ def make_resolver(pkg: str, imports: List[str]): else: explicit[simple_name_from_fqcn(imp)] = imp # simple -> explicit FQCN - def resolve(token: str) -> str: - tok = base_token(token) - if not tok: - return "" # empty token - if '.' in tok: - return tok - if tok in explicit: - return explicit[tok] + def _resolve_head(head_simple: str) -> str | None: + """Resolve a top-level simple name to FQCN using the precedence above.""" + if head_simple in explicit: + return explicit[head_simple] if pkg: - candidate = f"{pkg}.{tok}" + candidate = f"{pkg}.{head_simple}" if candidate in known_fqcns: return candidate # Star imports – accept only a unique hit candidates: List[str] = [] - for fqcn in by_simple.get(tok, []): - if any(fqcn.startswith(base + ".") for base in star_import_packages): - candidates.append(fqcn) + for fq_top in by_simple.get(head_simple, []): + if any(fq_top.startswith(base + ".") for base in star_import_packages): + candidates.append(fq_top) if len(candidates) == 1: return candidates[0] # Globally unique simple name across all known types - all_candidates = by_simple.get(tok, []) + all_candidates = by_simple.get(head_simple, []) if len(all_candidates) == 1: return all_candidates[0] - # Unknown or ambiguous → keep simple name - return tok + return None + + def resolve(token: str) -> str: + tok = base_token(token).strip() + if not tok: + return "" # empty token + + # Handle partially-qualified nested types, e.g., "FormParserFactory.ParserDefinition" + if "." in tok: + head, rest = tok.split(".", 1) + # If already looks fully-qualified for a known top-level type in this package, + # prefer composing with the package (avoids leaving it partially-qualified). + fq_head = _resolve_head(head) + if fq_head: + return fq_head + "." + rest + # If we can't resolve the head, leave as-is (may already be a fully-qualified package path) + return tok + + # Simple (non-dotted) type name + fq_simple = _resolve_head(tok) + return fq_simple if fq_simple else tok return resolve @@ -2143,7 +2170,7 @@ def extends_chain_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: """ Upward chain for classes (linear) and interface parents (DFS), jar-aware; placeholders where parents are unseen. - Returns list of (simple, source). + Returns list of (fqcn, source). """ if start_vid in extends_chain_pairs_memo: return extends_chain_pairs_memo[start_vid] @@ -2172,9 +2199,10 @@ def dfs_iface(v: Tuple[str, str]): seen_vids.add(parent_vid) if is_interface_record(parent_rec): dfs_iface(parent_vid) - chain.append((parent_rec["simple"], parent_rec["source"])) + chain.append((parent_rec["fqcn"], parent_rec["source"])) else: - chain.append((simple_name_from_fqcn(parent_fqcn), cur["source"])) + # Keep the best-available token (may be FQCN or simple if unresolved) + chain.append((parent_fqcn, cur["source"])) dfs_iface(start_vid) out = tup_dedup_keep_order(chain) @@ -2198,13 +2226,13 @@ def dfs_iface(v: Tuple[str, str]): if parent_vid in visited_vids or parent_vid == cur_vid: break visited_vids.add(parent_vid) - chain.append((parent_rec["simple"], parent_rec["source"])) + chain.append((parent_rec["fqcn"], parent_rec["source"])) if not is_interface_record(parent_rec): cur_vid = parent_vid else: break else: - chain.append((simple_name_from_fqcn(parent_fqcn), r["source"])) + chain.append((parent_fqcn, r["source"])) break out = tup_dedup_keep_order(chain) @@ -2295,7 +2323,7 @@ def dfs_iface_vid(v: Tuple[str, str]): def iface_super_pairs(iface_vid: Tuple[str, str]) -> List[Tuple[str, str]]: """ For an interface variant vid, return all transitive super-interfaces - as (simple, source) pairs, jar-aware. Memoized. + as (fqcn, source) pairs, jar-aware. Memoized. """ if iface_vid in iface_super_pairs_memo: return iface_super_pairs_memo[iface_vid] @@ -2329,10 +2357,10 @@ def dfs_iface_vid(v: Tuple[str, str]): if parent_rec: pk = (parent_rec["fqcn"], parent_rec["source"]) dfs_iface_vid(pk) - add_pair((parent_rec["simple"], parent_rec["source"])) + add_pair((parent_rec["fqcn"], parent_rec["source"])) else: # unknown external interface -> placeholder bound to current interface's source - add_pair((simple_name_from_fqcn(parent_fqcn), cur_src)) + add_pair((parent_fqcn, cur_src)) visiting.remove(v) dfs_iface_vid(iface_vid) @@ -2349,6 +2377,7 @@ def all_interfaces_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + interfaces from ancestor classes (and their super-interfaces). For an interface: its super-interfaces. Jar-aware; placeholders only for interfaces not present in provided set. + Returns (fqcn, source) pairs. """ if start_vid in all_ifaces_pairs_memo: return all_ifaces_pairs_memo[start_vid] @@ -2374,12 +2403,12 @@ def add_iface_and_supers(iface_fqcn: str): iface_rec = pick_variant_for_jar(iface_fqcn, jar_id) if iface_rec: ik = (iface_rec["fqcn"], iface_rec["source"]) - add_pair((iface_rec["simple"], iface_rec["source"])) + add_pair((iface_rec["fqcn"], iface_rec["source"])) for sup in iface_super_pairs(ik): add_pair(sup) else: # external/unseen -> placeholder (cannot know its supers) - add_pair((simple_name_from_fqcn(iface_fqcn), cur_src)) + add_pair((iface_fqcn, cur_src)) # Direct implements on this type for iface_fqcn in rec.get("impls", []): @@ -2467,7 +2496,7 @@ def all_implementers_of_interface_variant(iface_vid: Tuple[str, str]) -> List[Tu out: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} for vid, rec in rec_by_vid.items(): - self_name = rec["simple"] + self_fqcn = rec["fqcn"] source = rec.get("source", "") up_chain_pairs = extends_chain_pairs(vid) # may include placeholders @@ -2477,20 +2506,20 @@ def all_implementers_of_interface_variant(iface_vid: Tuple[str, str]) -> List[Tu if is_interface_record(rec): impl_vids = all_implementers_of_interface_variant(vid) # vids only down_pairs.extend( - (rec_by_vid[v]["simple"], rec_by_vid[v]["source"]) for v in impl_vids + (rec_by_vid[v]["fqcn"], rec_by_vid[v]["source"]) for v in impl_vids ) else: subs_vids = all_subclasses_variant(vid) down_pairs.extend( - (rec_by_vid[v]["simple"], rec_by_vid[v]["source"]) for v in subs_vids + (rec_by_vid[v]["fqcn"], rec_by_vid[v]["source"]) for v in subs_vids ) - vals = tup_dedup_keep_order([(self_name, source)] + up_chain_pairs + iface_pairs + down_pairs) - out[(self_name, source)] = vals + vals = tup_dedup_keep_order([(self_fqcn, source)] + up_chain_pairs + iface_pairs + down_pairs) + out[(self_fqcn, source)] = vals return out -@lru_cache(maxsize=2500000) +@lru_cache(maxsize=100000) def extract_method_name_with_params(java_src: str) -> str | None: """ Return 'methodName()' from a Java method header string. @@ -2694,19 +2723,28 @@ def match_paren_close(start: int) -> int: params_norm = ', '.join(filter(None, cleaned)) return f"{name}({params_norm})" -def find_function(caller_src:str, method_name:str, documents_of_functions:list[Document]) -> Document | None: +def find_function(caller_src:str, method_name:str, documents_of_functions) -> Document | None: + """Find a function document by source path and method name. + + documents_of_functions can be either: + - list[Document]: iterates all documents (original behavior) + - dict[str, list[Document]]: O(1) lookup by source path (from _source_to_fn_docs) + """ # TODO make it smarter, also use the method parameters to check if this is the actual method - for doc in documents_of_functions: - if caller_src == doc.metadata["source"]: - extracted_method_name = extract_method_name_with_params(doc.page_content) + if isinstance(documents_of_functions, dict): + candidates = documents_of_functions.get(caller_src, []) + else: + candidates = (doc for doc in documents_of_functions if caller_src == doc.metadata["source"]) - if extracted_method_name and method_name in extracted_method_name: - return doc + for doc in candidates: + extracted_method_name = extract_method_name_with_params(doc.page_content) + if extracted_method_name and method_name in extracted_method_name: + return doc def get_target_class_names(inheritance_list: List[Tuple[str, str]]) -> frozenset[str]: return frozenset([class_name for (class_name, source) in inheritance_list]) -@lru_cache(maxsize=80000) +@lru_cache(maxsize=50000) def strip_java_generics(type_str: str) -> str: """ Remove all balanced <...> generic sections from a Java type string. @@ -2754,4 +2792,42 @@ def strip_java_generics(type_str: str) -> str: res = _PUNCT_SPACE_RE.sub(r'\1', res) # "Map []" -> "Map[]", "Map . Entry" -> "Map.Entry" res = _VARARGS_SPACE_RE.sub(r'\1', res) # "List ..." -> "List..." - return res \ No newline at end of file + return res + +def extract_fqcn(text: str) -> str: + """ + Convert a source file path to an FQCN. + Supported forms: + 1) dependencies-sources/--sources//.java + 2) .../src/main/java//.java + 3) /.java (no prefix) + + Examples: + 'dependencies-sources/hibernate-core-6.6.13.Final-sources/org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + 'org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + '.../src/main/java/org/keycloak/.../VerifyEmailActionTokenHandler.java' + -> 'org.keycloak....VerifyEmailActionTokenHandler' + """ + p = (text or "").replace("\\", "/").strip() + + # Prefer the standard Maven/Gradle source-root marker when present. + src_marker = "/src/main/java/" + if src_marker in p: + tail = p.split(src_marker, 1)[1] + else: + # Otherwise, if there's a '-sources/' prefix, drop everything up to and including + # the last occurrence (handles dependencies-sources/...-sources/...). + marker = "-sources/" + cut = p.rfind(marker) + tail = p[cut + len(marker):] if cut != -1 else p + + if tail.startswith("/"): + tail = tail[1:] + + # Drop the .java suffix (case-sensitive per your examples) + if tail.endswith(".java"): + tail = tail[:-5] + + return tail.replace("/", ".") \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py index 1b909141f..12c3455a3 100644 --- a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py +++ b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py @@ -19,20 +19,8 @@ from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase from exploit_iq_commons.utils.dep_tree import ( - GOLANG_MANIFEST, - JAVA_MANIFEST, - JS_MANIFEST, - MANIFESTS_TO_ECOSYSTEMS, - PYPROJECT_TOML, - PYTHON_MANIFEST, - SETUP_PY, - Ecosystem, + detect_ecosystem, get_dependency_tree_builder, - C_CPLUSPLUS_MANIFEST_1, - C_CPLUSPLUS_MANIFEST_2, - C_CPLUSPLUS_MANIFEST_3, - C_CPLUSPLUS_MANIFEST_4, - C_CPLUSPLUS_EXTENSIONS, ) from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE @@ -54,35 +42,6 @@ class TransitiveCodeSearcher: def __init__(self, chain_of_calls_retriever: ChainOfCallsRetrieverBase): self.chain_of_calls_retriever = chain_of_calls_retriever - @staticmethod - def _has_c_cpp_sources(git_repo_path: Path) -> bool: - """Check for the presence of C/C++ source files in the repository. - - This method scans the directory tree rooted at `git_repo_path` to verify if it contains - actual C/C++ source code (files with extensions defined in C_CPLUSPLUS_EXTENSIONS). - This is used to filter out projects that might have a build file (like Makefile) but - no actual C/C++ code. - - The search is optimized to: - - skip common non-source directories (all starting with '.') - - return True immediately upon finding the first matching file. - - Parameters - ---------- - git_repo_path : Path - The root directory of the git repository to scan. - - Returns - ------- - bool - True if at least one C/C++ source file is found, False otherwise. - """ - for root, dirs, files in os.walk(git_repo_path): - dirs[:] = [d for d in dirs if not d.startswith('.')] - if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files): - return True - return False - @staticmethod def download_dependencies(git_repo_path: Path) -> bool: """ Download all dependencies according to manifest file in the Git repository @@ -93,41 +52,10 @@ def download_dependencies(git_repo_path: Path) -> bool: Returns whether dependencies were downloaded or not. """ - ecosystem: Ecosystem - # Check the root dir of the repo for existence of manifests, the precedence of which manifest file t o check is - # according to the order from top to bottom - if os.path.isfile(git_repo_path / GOLANG_MANIFEST): - ecosystem = MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] - elif ( - os.path.isfile(git_repo_path / PYTHON_MANIFEST) - or os.path.isfile(git_repo_path / PYPROJECT_TOML) - or os.path.isfile(git_repo_path / SETUP_PY) - ): - ecosystem = MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] - elif os.path.isfile(git_repo_path / JS_MANIFEST): - ecosystem = MANIFESTS_TO_ECOSYSTEMS[JS_MANIFEST] - elif os.path.isfile(git_repo_path / JAVA_MANIFEST): - ecosystem = MANIFESTS_TO_ECOSYSTEMS[JAVA_MANIFEST] - else: - # 1. Direct checks - candidates = [ - Path(git_repo_path / C_CPLUSPLUS_MANIFEST_1), - Path(git_repo_path / C_CPLUSPLUS_MANIFEST_2), - Path(git_repo_path / C_CPLUSPLUS_MANIFEST_3), - Path(git_repo_path / C_CPLUSPLUS_MANIFEST_4), - Path(git_repo_path / "auto" / - C_CPLUSPLUS_MANIFEST_4) - ] - found = [p for p in candidates if p.is_file()] - if found: - if TransitiveCodeSearcher._has_c_cpp_sources(git_repo_path): - ecosystem = MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] - else: - logger.info(f"Generic build file found but no C/C++ sources detected in {git_repo_path}. Skipping C ecosystem.") - return False - else: - logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") - return False + ecosystem = detect_ecosystem(git_repo_path) + if ecosystem is None: + logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") + return False try: logger.info(f"Started installing packages for {ecosystem.value}") diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index c613e6866..d6321ae61 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -56,6 +56,8 @@ functions: enable_functions_usage_search: true Function Locator: _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder @@ -87,6 +89,7 @@ functions: - Call Chain Analyzer - Function Caller Finder - Function Locator + - Function Library Version Finder max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 4e6ca79f7..a67e18c1b 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -63,6 +63,8 @@ functions: enable_functions_usage_search: true Function Locator: _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder @@ -94,6 +96,7 @@ functions: - Call Chain Analyzer - Function Caller Finder - Function Locator + - Function Library Version Finder max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 331289e92..397258e62 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -67,6 +67,8 @@ functions: enable_functions_usage_search: true Function Locator: _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder @@ -98,6 +100,7 @@ functions: - Call Chain Analyzer - Function Caller Finder - Function Locator + - Function Library Version Finder max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 7916989c1..779b8efe5 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -60,6 +60,8 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Function Library Version Finder: + _type: calling_function_library_version_finder cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -68,6 +70,7 @@ functions: - Docs Semantic Search # - Code Keyword Search # Uncomment to enable keyword search - CVE Web Search + - Function Library Version Finder max_concurrency: null max_iterations: 10 prompt_examples: false @@ -141,7 +144,7 @@ llms: base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 - max_tokens: 2000 + max_tokens: 1000 top_p: 0.01 generate_cvss_llm: _type: nim diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 4f1e168d1..f140850d4 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -12,8 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import asyncio +import json +from pathlib import Path from vuln_analysis.runtime_context import ctx_state import typing from aiq.builder.builder import Builder @@ -31,11 +33,27 @@ from pydantic import Field from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.tools.transitive_code_search import package_name_from_locator_query from vuln_analysis.utils.error_handling_decorator import ToolRaisedException from vuln_analysis.utils.prompting import get_agent_prompt from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from vuln_analysis.functions.react_internals import build_system_prompt, build_classification_prompt, build_package_filter_prompt, AgentState, Thought, Observation, Classification, PackageSelection, CodeFindings, SystemRulesTracker, _build_tool_arguments, FORCED_FINISH_PROMPT, COMPREHENSION_PROMPT, MEMORY_UPDATE_PROMPT, AGENT_SYS_PROMPT_NON_REACHABILITY, AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY, AGENT_THOUGHT_INSTRUCTIONS_GO +from vuln_analysis.utils.prompting import build_tool_descriptions +from vuln_analysis.utils.prompt_factory import TOOL_SELECTION_STRATEGY, TOOL_SELECTION_STRATEGY_NON_REACHABILITY, TOOL_ECOSYSTEM_REGISTRY, FEW_SHOT_EXAMPLES +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_from_osv, filter_context_to_package +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY +from langgraph.graph import StateGraph, END, START +from langgraph.prebuilt import ToolNode +from langchain_core.messages import SystemMessage, HumanMessage, AIMessage, RemoveMessage + +import uuid +import tiktoken +from nat.builder.context import Context + logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): """ @@ -65,15 +83,14 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): cve_web_search_enabled: bool = Field(default=True, description="Whether to enable CVE Web Search tool or not.") verbose: bool = Field(default=False, description="Set to true for verbose output") + context_window_token_limit: int = Field( + default=5000, + description="Estimated token threshold for pruning old messages in observation node." + ) - -async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, - state: AgentMorpheusEngineState) -> AgentExecutor: - from vuln_analysis.utils.prompting import build_tool_descriptions +async def common_build_tools(config: CVEAgentExecutorToolConfig, builder: Builder, state: AgentMorpheusEngineState) -> tuple[list[typing.Any], list[str], list[str]]: tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - # Filter tools that are not available based on state tools = [ tool for tool in tools @@ -89,13 +106,524 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, state.code_index_path is None)) ) ] - # Get tool names after filtering for dynamic guidance enabled_tool_names = [tool.name for tool in tools] - + tool_descriptions_list = [t.name + ": " + t.description for t in tools] # Build tool selection guidance with strategic context tool_descriptions = build_tool_descriptions(enabled_tool_names) + return tools, tool_descriptions, tool_descriptions_list + +def _validate_go_vendor_packages( + source_info: list, + candidate_packages: list[dict], +) -> tuple[list[dict], list[str]]: + """Check which Go candidate packages actually exist in the vendor directory.""" + code_si = next((si for si in source_info if si.type == "code"), None) + if code_si is None: + return candidate_packages, [] + + repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) + vendor_path = repo_path / "vendor" + if not vendor_path.is_dir(): + return candidate_packages, [] + + validated = [] + removed = [] + for pkg in candidate_packages: + pkg_name = pkg.get("name", "") + if (vendor_path / pkg_name).is_dir(): + validated.append(pkg) + else: + removed.append(pkg_name) + + if validated: + return validated, removed + return candidate_packages, [] + + +async def _enrich_go_candidates( + cve_intel: list, + source_info: list, + critical_context: list[str], + candidate_packages: list[dict], + vulnerable_functions_set: set[str], +) -> tuple[list[dict], list[str]]: + """Enrich Go candidates via OSV and validate against vendor directory.""" + ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) + if not ghsa_has_packages or not vulnerable_functions_set: + intel = cve_intel[0] if cve_intel else None + if intel: + await enrich_go_from_osv(intel, critical_context, candidate_packages, vulnerable_functions_set) + + if candidate_packages: + candidate_packages, removed_pkgs = _validate_go_vendor_packages( + source_info, candidate_packages + ) + if removed_pkgs: + logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) + + return candidate_packages, sorted(vulnerable_functions_set) + + +async def _create_graph_agent(config: CVEAgentExecutorToolConfig, builder: Builder, state: AgentMorpheusEngineState): + + tools, tool_guidance_list, tool_descriptions_list = await common_build_tools(config, builder, state) + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + thought_llm = llm.with_structured_output(Thought) + comprehension_llm = llm.with_structured_output(CodeFindings) + observation_llm = llm.with_structured_output(Observation) + reachability_llm = llm.with_structured_output(Classification) + package_filter_llm = llm.with_structured_output(PackageSelection) + tool_guidance = "\n".join(tool_guidance_list) + descriptions = "\n".join(tool_descriptions_list) + default_system_prompt = build_system_prompt(descriptions, tool_guidance) + tool_node = ToolNode(tools, handle_tool_errors=True) + TOOL_NODE = "tool_node" + THOUGHT_NODE = "thought_node" + FORCED_FINISH_NODE = "forced_finish_node" + PRE_PROCESS_NODE = "pre_process_node" + OBSERVATION_NODE = "observation_node" + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + def _count_tokens(text: str) -> int: + """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + def _truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400) -> str: + """Truncate tool output to fit observation prompt within completion token budget. + + Code Keyword Search is the primary source of bloat (3700+ chars of redundant + source snippets). CCA and FL outputs are typically small. + Preserves the most informative parts based on tool type. + """ + token_count = _count_tokens(tool_output) + if token_count <= max_tokens: + return tool_output + + lines = tool_output.split('\n') + + if tool_name == ToolNames.CALL_CHAIN_ANALYZER: + head = '\n'.join(lines[:3]) + remaining = token_count - _count_tokens(head) + return f"{head}\n[... truncated {remaining} tokens ...]" + + if tool_name == ToolNames.CODE_KEYWORD_SEARCH: + kept_lines = [] + kept_tokens = 0 + for line in lines: + is_header = line.startswith("---") or "Main application" in line or "library dependencies" in line or line.strip() == "" + line_tokens = _count_tokens(line) + if is_header or kept_tokens + line_tokens <= max_tokens: + kept_lines.append(line) + kept_tokens += line_tokens + if kept_tokens >= max_tokens: + break + kept_lines.append(f"[... truncated {token_count - kept_tokens} tokens ...]") + return '\n'.join(kept_lines) + + # Default: head (70%) + tail (30%) + head_budget = int(max_tokens * 0.7) + tail_budget = max_tokens - head_budget + head_lines = [] + head_tokens = 0 + for line in lines: + lt = _count_tokens(line) + if head_tokens + lt > head_budget: + break + head_lines.append(line) + head_tokens += lt + tail_lines = [] + tail_tokens = 0 + for line in reversed(lines): + lt = _count_tokens(line) + if tail_tokens + lt > tail_budget: + break + tail_lines.insert(0, line) + tail_tokens += lt + truncated = token_count - head_tokens - tail_tokens + return '\n'.join(head_lines) + f"\n[... truncated {truncated} tokens ...]\n" + '\n'.join(tail_lines) + + def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: + """Estimate the token count thought_node will send to the LLM.""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return _count_tokens("\n".join(parts)) + + def _build_tool_guidance_for_ecosystem(ecosystem: str, available_tools: list, is_reachability: str = "yes") -> tuple[str, str]: + """Build tool guidance using language-specific strategies when available.""" + filtered_tools = [ + t for t in available_tools + if (t.name != ToolNames.FUNCTION_CALLER_FINDER or ecosystem == "go") and + (t.name != ToolNames.FUNCTION_LIBRARY_VERSION_FINDER or ecosystem == "java") + ] + list_of_tool_names = [t.name for t in filtered_tools] + list_of_tool_descriptions = [t.name + ": " + t.description for t in filtered_tools] + + strategy = TOOL_SELECTION_STRATEGY if is_reachability == "yes" else TOOL_SELECTION_STRATEGY_NON_REACHABILITY + lang = ecosystem.lower() if ecosystem else "" + if lang in strategy: + tool_guidance_local = strategy[lang] + if lang == "java": + tool_guidance_local += ( + " Use Function Library Version Finder to verify the installed version of a library " + "before concluding exploitability (e.g., input 'commons-beanutils')." + ) + if is_reachability == "yes": + hint = FEW_SHOT_EXAMPLES.get(lang, "") + if hint: + tool_guidance_local += f"\nHint: {hint}" + else: + tool_guidance_list_local = build_tool_descriptions(list_of_tool_names) + tool_guidance_local = "\n".join(tool_guidance_list_local) + + descriptions_local = "\n".join(list_of_tool_descriptions) + return tool_guidance_local, descriptions_local + + async def pre_process_node(state: AgentState) -> AgentState: + workflow_state = ctx_state.get() + ecosystem = workflow_state.original_input.input.image.ecosystem.value if workflow_state.original_input.input.image.ecosystem else "" + with AGENT_TRACER.push_active_function("pre_process node", input_data=f"ecosystem:{ecosystem}") as span: + try: + critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + vulnerable_functions_set = set(vulnerable_functions) + + if ecosystem == "go": + candidate_packages, vulnerable_functions = await _enrich_go_candidates( + workflow_state.cve_intel, + workflow_state.original_input.input.image.source_info, + critical_context, + candidate_packages, + vulnerable_functions_set, + ) + + selected_package = None + app_package = None + if len(candidate_packages) > 1: + image_input = workflow_state.original_input.input.image + image_name = image_input.name + source_repos = image_input.source_info + image_repo = source_repos[0].git_repo if source_repos else None + filter_prompt = build_package_filter_prompt( + ecosystem, candidate_packages, + image_name=image_name, image_repo=image_repo, + critical_context=critical_context, + ) + selection: PackageSelection = await package_filter_llm.ainvoke([HumanMessage(content=filter_prompt)]) + selected_package = selection.selected_package + app_package = selected_package + logger.info("Package filter selected '%s' from %d candidates (reason: %s)", + selected_package, len(candidate_packages), selection.reason) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + elif len(candidate_packages) == 1: + selected_package = candidate_packages[0].get("name") + app_package = selected_package + logger.info("Single candidate package after validation: '%s'", selected_package) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + + critical_context.append( + "TASK: Investigate usage and reachability of the vulnerable function/module in the container. " + "Use the vulnerable module name from GHSA as primary investigation target." + ) + + question = state.get("input") or "" + context_block = "\n".join(critical_context) + classification_prompt = build_classification_prompt(context_block, question) + classification_result: Classification = await reachability_llm.ainvoke([HumanMessage(content=classification_prompt)]) + span.set_output({ + "critical_context": critical_context, + "candidate_packages": candidate_packages, + "selected_package": selected_package, + "app_package": app_package if selected_package else None, + "reachability_question": classification_result.is_reachability, + }) + + is_reachability = classification_result.is_reachability + + if is_reachability == "yes": + tool_guidance_local, descriptions_local = _build_tool_guidance_for_ecosystem(ecosystem, tools) + go_instructions = {"instructions": AGENT_THOUGHT_INSTRUCTIONS_GO} if ecosystem == "go" else {} + runtime_prompt = build_system_prompt(descriptions_local, tool_guidance_local, **go_instructions) + active_tool_names = [t.name for t in tools] + else: + reachability_tool_names = { ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER} + non_reach_tools = [t for t in tools if t.name not in reachability_tool_names] + tool_guidance_local, descriptions_local = _build_tool_guidance_for_ecosystem(ecosystem, non_reach_tools, is_reachability="no") + runtime_prompt = build_system_prompt( + descriptions_local, tool_guidance_local, + instructions=AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY, + sys_prompt=AGENT_SYS_PROMPT_NON_REACHABILITY, + ) + active_tool_names = [t.name for t in non_reach_tools] + logger.info("Non-reachability question detected; removed reachability tools from prompt") + rules_tracker = state.get("rules_tracker") + app_package = app_package if selected_package else None + rules_tracker.set_target_package(app_package) + rules_tracker.set_allowed_tools(active_tool_names) + if is_reachability == "yes": + rules_tracker.set_target_functions(vulnerable_functions) + return { + "ecosystem": ecosystem, + "runtime_prompt": runtime_prompt, + "is_reachability": is_reachability, + "observation": Observation(memory=critical_context, results=[]), + "critical_context": critical_context, + "app_package": app_package if selected_package else None, + } + except Exception as e: + logger.exception("pre_process_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + + async def thought_node(state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function("thought node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") or default_system_prompt + messages = [SystemMessage(content=active_prompt)] + state["messages"] + obs = state.get("observation", None) + if obs is not None: + memory_list = obs.memory if obs.memory else ["No prior knowledge."] + recent_findings = obs.results if obs.results else ["No recent findings."] + memory_context = "\n".join(f"- {m}" for m in memory_list) + findings_context = "\n".join(f"- {f}" for f in recent_findings) + context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" + messages.append(SystemMessage(content=context_block)) + response: Thought = await thought_llm.ainvoke(messages) + + final_answer = "waiting for the agent to respond" + if response.mode == "finish": + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + elif response.actions is None: + logger.warning("LLM returned mode='act' but actions is None, forcing finish") + ai_message = AIMessage(content=response.thought or "No actions provided, finishing.") + response = Thought( + thought=response.thought or "No actions provided", + mode="finish", + actions=None, + final_answer=response.thought or "Insufficient evidence to provide a definitive answer." + ) + final_answer = response.final_answer + else: + tool_name = response.actions.tool + arguments = _build_tool_arguments(response.actions) + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{ + "name": tool_name, + "args": arguments, + "id": tool_call_id + }] + ) + + span.set_output({"mode": response.mode, "step": step_num + 1}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": config.max_iterations, + "output": final_answer + } + except Exception as e: + logger.exception("thought_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def should_continue(state: AgentState) -> str: + thought = state.get("thought", None) + if thought is not None and thought.mode == "finish": + return END + if state.get("step", 0) >= state.get("max_steps", config.max_iterations): + return FORCED_FINISH_NODE + return TOOL_NODE + + async def forced_finish_node(state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function("forced_finish node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") or default_system_prompt + messages = [SystemMessage(content=active_prompt)] + state["messages"] + messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) + obs = state.get("observation", None) + if obs is not None and obs.memory: + memory_context = "\n".join(f"- {m}" for m in obs.memory) + messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) + response: Thought = await thought_llm.ainvoke(messages) + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Failed to generate a final answer within the maximum allowed steps." + ai_message = AIMessage(content=final_answer) + response = Thought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer + ) + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", config.max_iterations), + "observation": state.get("observation", None), + "output": final_answer + } + except Exception as e: + logger.exception("forced_finish_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def observation_node(state: AgentState) -> AgentState: + tool_message = state["messages"][-1] + last_thought_text = state["thought"].thought if state.get("thought") else "No previous thought." + tool_used = state["thought"].actions.tool if state.get("thought") and state["thought"].actions else "Unknown" + tool_input_detail = "" + if state.get("thought") and state["thought"].actions: + actions = state["thought"].actions + if actions.package_name and actions.function_name: + tool_input_detail = f"{actions.package_name},{actions.function_name}" + elif actions.query: + tool_input_detail = actions.query + elif actions.tool_input: + tool_input_detail = actions.tool_input + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + rules_tracker = state.get("rules_tracker") + with AGENT_TRACER.push_active_function("observation node", input_data=f"tool used:{tool_used}") as span: + try: + tool_output_for_llm = tool_message.content + result, error_message = rules_tracker.check_thought_behavior(tool_used, tool_input_detail, tool_output_for_llm) + if result: + span.set_output({"rule_error": error_message}) + return {"messages": [HumanMessage(content=error_message)]} + + if state.get("ecosystem", "").lower() == "java": + truncated_output = _truncate_tool_output(tool_output_for_llm, tool_used) + else: + truncated_output = tool_output_for_llm + + # Step 1: Comprehension -- reads raw tool output, produces compact findings + ctx_lines = state.get("critical_context", []) + critical_context_text = "\n".join(ctx_lines) if ctx_lines else "N/A" + comp_prompt = COMPREHENSION_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + critical_context=critical_context_text, + tool_used=tool_used, + tool_input_detail=tool_input_detail, + last_thought_text=last_thought_text, + tool_output=truncated_output, + ) + code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + + findings_text = "\n".join(f"- {f}" for f in code_findings.findings) + + # Step 2: Memory update -- merges compressed findings into cumulative memory + mem_prompt = MEMORY_UPDATE_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + previous_memory=previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") or default_system_prompt + estimated = _estimate_tokens(active_prompt, messages, new_observation) + prune_messages = [] + orig_estimated = estimated + + span_trace_dict = {"comprehension_findings": code_findings.findings, "tool_outcome": code_findings.tool_outcome} + + if estimated > config.context_window_token_limit and len(messages) > 3: + prunable = messages[1:-2] + for msg in prunable: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, config.context_window_token_limit, + ) + span_trace_dict["orig_estimated"] = orig_estimated + span_trace_dict["estimated"] = estimated + span.set_output(span_trace_dict) + cca_results = list(state.get("cca_results", [])) + if tool_used == ToolNames.CALL_CHAIN_ANALYZER: + stripped = tool_output_for_llm.strip().lstrip("([") + first_token = stripped.split(",", 1)[0].strip().lower() + if first_token == "true": + cca_results.append(True) + elif first_token == "false": + cca_results.append(False) + package_validated = state.get("package_validated") + if tool_used == ToolNames.FUNCTION_LOCATOR and state.get("is_reachability") == "yes": + input_pkg = package_name_from_locator_query(tool_input_detail) + target_pkg = (state.get("app_package") or "").strip().lower() + if target_pkg and input_pkg == target_pkg: + if "Package is valid" in tool_output_for_llm: + package_validated = True + elif "Package is not valid" in tool_output_for_llm and package_validated is None: + package_validated = False + return { + "messages": prune_messages, + "observation": new_observation, + "step": state.get("step", 0), + "cca_results": cca_results, + "package_validated": package_validated, + } + except Exception as e: + logger.exception("observation_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + async def create_graph(): + flow = StateGraph(AgentState) + flow.add_node(THOUGHT_NODE, thought_node) + flow.add_node(TOOL_NODE, tool_node) + flow.add_node(FORCED_FINISH_NODE, forced_finish_node) + flow.add_node(PRE_PROCESS_NODE, pre_process_node) + flow.add_node(OBSERVATION_NODE, observation_node) + flow.add_edge(START, PRE_PROCESS_NODE) + flow.add_edge(PRE_PROCESS_NODE, THOUGHT_NODE) + flow.add_conditional_edges( + THOUGHT_NODE, + should_continue, + {END: END, TOOL_NODE: TOOL_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE} + ) + flow.add_edge(TOOL_NODE, OBSERVATION_NODE) + flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) + flow.add_edge(FORCED_FINISH_NODE, END) + + app = flow.compile() + if config.verbose: + app.get_graph().draw_mermaid_png(output_file_path="flow.png") + return app + return await create_graph() +async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, + state: AgentMorpheusEngineState) -> AgentExecutor: + + tools, tool_descriptions,_ = await common_build_tools(config, builder, state) + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) tool_guidance = "\n".join(tool_descriptions) + # Get prompt template prompt_template_str = get_agent_prompt(config.prompt, config.prompt_examples) @@ -129,13 +657,38 @@ async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, return agent_executor -async def _process_steps(agent, steps, semaphore): +async def _process_steps(agent, steps, semaphore, max_iterations: int = 10): + + async def _process_step(step): - if semaphore: - async with semaphore: - return await agent.ainvoke({"input": step}) - else: - return await agent.ainvoke({"input": step}) + async def call_agent(initial_state,config=None): + if config: + return await agent.ainvoke(initial_state,config=config) + else: + return await agent.ainvoke(initial_state) + + initial_state = {"input": step} + config = None + if not isinstance(agent, AgentExecutor): + initial_state = { + "input": step, + "messages": [HumanMessage(content=step)], + "step": 0, + "max_steps": max_iterations, + "thought": None, + "observation": None, + "output": "waiting for the agent to respond", + "rules_tracker": SystemRulesTracker(), + } + config = { + "recursion_limit": 50 + } + with AGENT_TRACER.push_active_function("checklist_question", input_data=step[:80]): + if semaphore: + async with semaphore: + return await call_agent(initial_state, config) + else: + return await call_agent(initial_state, config) return await asyncio.gather(*(_process_step(step) for step in steps), return_exceptions=True) @@ -171,7 +724,8 @@ def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, re # If the agent encounters a parsing error or a server error after retries, replace the error # with default values to prevent the pipeline from crashing outputs[i].append({"input": checklist_questions[i][j], "output": replace_exceptions_value, - "intermediate_steps": None}) + "intermediate_steps": None, "cca_results": [], + "package_validated": None}) if isinstance(answer, ToolRaisedException): tool_raised_exception: ToolRaisedException = answer logger.warning(f"An exception encountered during tool execution, in result [{i}][{j}]. for " @@ -199,7 +753,9 @@ def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, re results[i][j]["intermediate_steps"] = None outputs[i].append({"input": answer["input"], "output": answer["output"], - "intermediate_steps": results[i][j]["intermediate_steps"]}) + "intermediate_steps": results[i][j]["intermediate_steps"], + "cca_results": answer.get("cca_results", []), + "package_validated": answer.get("package_validated")}) return outputs @@ -210,12 +766,23 @@ async def cve_agent(config: CVEAgentExecutorToolConfig, builder: Builder): async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: trace_id.set(state.original_input.input.scan.id) ctx_state.set(state) - agent = await _create_agent(config, builder, state) - results = await asyncio.gather(*(_process_steps(agent, steps, semaphore) - for steps in state.checklist_plans.values()), return_exceptions=True) + + checklist_plans = state.checklist_plans + + agent = await _create_graph_agent(config, builder, state) + + results = await asyncio.gather(*(_process_steps(agent, steps, semaphore, config.max_iterations) + for steps in checklist_plans.values()), return_exceptions=True) results = _postprocess_results(results, config.replace_exceptions, config.replace_exceptions_value, - list(state.checklist_plans.values())) - state.checklist_results = dict(zip(state.checklist_plans.keys(), results)) + list(checklist_plans.values())) + state.checklist_results = dict(zip(checklist_plans.keys(), results)) + + with AGENT_TRACER.push_active_function("agent_finish", input_data={ + "cve_count": len(checklist_plans), + "scan_id": state.original_input.input.scan.id, + }): + pass + return state yield FunctionInfo.from_fn( diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 696c7fb9b..ab6f3c985 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -58,13 +58,14 @@ async def cve_checklist(config: CVEChecklistToolConfig, builder: Builder): agent_config = builder.get_function_config(config.agent_name) agent_tool_names = agent_config.tool_names if hasattr(agent_config, 'tool_names') else None - async def generate_checklist_for_cve(cve_intel): + async def generate_checklist_for_cve(cve_intel, ecosystem: str = ""): checklist = await generate_checklist(prompt=config.prompt, llm=llm, input_dict=cve_intel, tool_names=agent_tool_names, - enable_llm_list_parsing=False) + enable_llm_list_parsing=False, + ecosystem=ecosystem) checklist = await _parse_list([checklist]) @@ -72,10 +73,11 @@ async def generate_checklist_for_cve(cve_intel): async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: trace_id.set(state.original_input.input.scan.id) + ecosystem = state.original_input.input.image.ecosystem.value if state.original_input and state.original_input.input.image.ecosystem else "" intel_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe(state.cve_intel) workflow_cve_intel = intel_df.to_dict(orient='records') - results = await asyncio.gather(*(generate_checklist_for_cve(cve_intel) for cve_intel in workflow_cve_intel)) + results = await asyncio.gather(*(generate_checklist_for_cve(cve_intel, ecosystem=ecosystem) for cve_intel in workflow_cve_intel)) state.checklist_plans = dict(results) return state diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index 63e4cccb4..df3e9c542 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -29,6 +29,7 @@ from exploit_iq_commons.data_models.common import AnalysisType from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id from exploit_iq_commons.utils.credential_client import credential_context +from exploit_iq_commons.utils.dep_tree import detect_ecosystem from vuln_analysis.tools.tool_names import ToolNames logger = LoggingFactory.get_agent_logger(__name__) @@ -70,7 +71,7 @@ async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig from exploit_iq_commons.utils.document_embedding import DocumentEmbedding from vuln_analysis.utils.full_text_search import FullTextSearch - from exploit_iq_commons.utils.git_utils import get_repo_from_path + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager from exploit_iq_commons.data_models.input import ManualSBOMInfoInput from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache @@ -229,18 +230,15 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: "Ensure the source repositories are setup correctly"), base_image) - # Replace ref with specific commit hash for each source info - for si in source_infos: - try: - # Get the sanitized path from the embedder instance - repo_path = embedder.get_repo_path(si) - repo = get_repo_from_path(repo_path.parent, repo_path.name) - si.ref = repo.commit().hexsha - except ValueError as e: - logger.warning("Failed to get commit hash for repo defined in %s: %s", si, e) - continue - except GitCloneError as e: - failure_reason = f"Error: {e}" + # Detect ecosystem from cloned repo manifests if not provided by the client + if message.image.ecosystem is None: + code_sources = [si for si in source_infos if si.type == 'code'] + if code_sources: + repo_path = embedder.get_repo_path(code_sources[0]) + detected = detect_ecosystem(repo_path) + if detected is not None: + message.image.ecosystem = detected + logger.info("Detected ecosystem '%s' from repo manifests", detected.value) except Exception as e: failure_reason = f"Failure to build VDB , Error: {e}" diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index db7fba0b0..3bd61f794 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -36,6 +36,30 @@ class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"): llm_name: str = Field(description="The LLM model to use") +def _all_cca_not_reachable(checklist_items: list[dict]) -> bool: + """Return True if all Call Chain Analyzer results are False (not reachable). + Returns False if no CCA calls were made (no signal to gate on). + """ + all_cca = [] + for item in checklist_items: + all_cca.extend(item.get("cca_results", [])) + return len(all_cca) > 0 and not any(all_cca) + + +def _package_not_found(checklist_items: list[dict]) -> bool: + """Return True if Function Locator confirmed the target package is absent. + Fires when any thread has package_validated=False and no thread has True. + """ + has_false = False + for item in checklist_items: + val = item.get("package_validated") + if val is True: + return False + if val is False: + has_false = True + return has_false + + @register_function(config_type=CVESummarizeToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder): @@ -45,19 +69,52 @@ async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder): from vuln_analysis.utils.prompting import SUMMARY_PROMPT llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - prompt = PromptTemplate(input_variables=["response"], template=SUMMARY_PROMPT) - chain = prompt | llm - async def summarize_cve(results): + async def summarize_cve(results, ecosystem: str = ""): + checklist_items = results[1] response = '\n'.join( - [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])]) + [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(checklist_items)]) + + if _package_not_found(checklist_items): + response = ( + "PACKAGE PRESENCE GATE: Function Locator confirmed the vulnerable package " + "is NOT present in this container. A CVE cannot be exploitable if the " + "vulnerable package does not exist. Code matches from other packages are " + "irrelevant. The verdict MUST be 'not exploitable'.\n\n" + + response + ) + logger.info("Package presence gate activated: target package not found") + elif _all_cca_not_reachable(checklist_items): + response = ( + "REACHABILITY GATE: Call Chain Analyzer confirmed the vulnerable function " + "is NOT reachable from application code in ALL reachability checks performed. " + "An unreachable function cannot be exploited regardless of other findings " + "(missing mitigations, absent protections, etc. are irrelevant if the code " + "path is never executed). The verdict MUST be 'not exploitable'.\n\n" + + response + ) + logger.info("Reachability gate activated: all CCA results are negative") + + summary_prompt = SUMMARY_PROMPT + if ecosystem.lower() == "java": + summary_prompt = summary_prompt.replace( + "3. FOCUS: Use only definitive checklist results; ignore inconclusive items", + "3. FOCUS: Use only definitive checklist results; ignore inconclusive items\n\n" + "4. CRITICAL: If ANY checklist item reports that Call Chain Analyzer confirmed a vulnerable\n" + " function is REACHABLE (True) from application code, the verdict MUST be \"exploitable\"\n" + " unless another item provides definitive contrary evidence (e.g., version check confirmed\n" + " the installed version is fixed). Negative results from other items using different function\n" + " names or wrong inputs do NOT override a confirmed positive reachability finding." + ) + prompt = PromptTemplate(input_variables=["response"], template=summary_prompt) + chain = prompt | llm final_summary = await chain.ainvoke({"response": response}) - return final_summary.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: trace_id.set(state.original_input.input.scan.id) - results = await asyncio.gather(*(summarize_cve(results) for results in state.checklist_results.items())) + ecosystem = state.original_input.input.image.ecosystem.value if state.original_input and state.original_input.input.image.ecosystem else "" + results = await asyncio.gather(*(summarize_cve(results, ecosystem=ecosystem) for results in state.checklist_results.items())) state.final_summaries = dict(zip(state.checklist_results.keys(), results)) return state diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py new file mode 100644 index 000000000..7ec52bb27 --- /dev/null +++ b/src/vuln_analysis/functions/react_internals.py @@ -0,0 +1,536 @@ +from pydantic import BaseModel, Field +from typing import Any +from typing import Literal +from langgraph.graph import MessagesState +#---- REACT Schemas ----# + +class ToolCall(BaseModel): + tool: str = Field(description="Exact tool name from AVAILABLE_TOOLS") + #tool_input: str = Field(description="The input for the tool. Example: Code Keyword Search: PQescapeLiteral") + package_name: str | None = Field( + default=None, + description="Package/module name. REQUIRED when using Function Locator, Function Caller Finder, or Call Chain Analyzer. E.g. libpq, urllib, github.com/org/pkg" + ) + function_name: str | None = Field( + default=None, + description="Function or method name with optional args. REQUIRED with package_name for code path tools. E.g. PQescapeLiteral(), parse(), errors.New(\"x\")" + ) + # For search tools (Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search) + query: str | None = Field( + default=None, + description="Search query. Use for search tools when package_name/function_name don't apply" + ) + # Fallback: if LLM uses tool_input for simple query-only tools + tool_input: str | None = Field( + default=None, + description="Legacy/fallback input. Prefer package_name+function_name or query." + ) + reason: str = Field(description="Briefly explain why this specific tool/input helps the investigation") + +class Thought(BaseModel): + thought: str = Field( + description="Brief reasoning about next step (max 3-4 sentences)", + max_length=3000, + ) + mode: Literal["act", "finish"] = Field(description="'act' to call tools (when more information is needed), 'finish' to return the final answer (when you have sufficient evidence)") + + actions: ToolCall | None = Field(default=None, description="When mode is 'act', the tool to execute") + + final_answer: str | None = Field( + default=None, + description="When mode is 'finish', concise answer (3-5 sentences) with key evidence", + max_length=3000, + ) + +class CodeFindings(BaseModel): + """Compressed code comprehension output from raw tool results.""" + findings: list[str] = Field( + description="3-5 key technical facts from tool output. Each describes what the code DOES " + "and how it relates to the investigation goal, not just that it was found." + ) + tool_outcome: str = Field( + description="One-line summary: CALLED: [tool] with [input] -> [brief outcome]" + ) + + +class Observation(BaseModel): + results: list[str] = Field( + description="3-5 key technical facts from this tool output. Each fact must describe what the code DOES and how it relates to the investigation goal, not just that it was found." + ) + + memory: list[str] = Field( + description="Cumulative factual findings. Each item is a single, discrete technical fact that records functional behavior and context (e.g. what a function does, which package it belongs to, whether it is semantically relevant to the goal)." + ) + + +class Classification(BaseModel): + """Structured output for reachability-question classification.""" + is_reachability: Literal["yes", "no"] = Field( + description="Answer 'yes' if the question is a reachability question, 'no' otherwise." + ) + + +class PackageSelection(BaseModel): + """Structured output for selecting the most relevant package from multiple candidates.""" + selected_package: str = Field( + description="The exact name of the single most relevant package to investigate, copied verbatim from the candidates list." + ) + reason: str = Field( + description="One-sentence justification for why this package is the best investigation target." + ) + +class SystemRulesTracker: + def __init__(self): + self.action_history = {} + self.target_package = None + self.allowed_tools = [] + self.target_functions: dict[str, bool] = {} + def set_allowed_tools(self, allowed_tools: list[str]): + self.allowed_tools = allowed_tools + def set_target_package(self, target_package: str): + self.target_package = target_package + def set_target_functions(self, functions: list[str]): + self.target_functions = {f: False for f in functions} + + @staticmethod + def _is_empty_result(output) -> bool: + """Check if a tool output represents an empty/no-results response. + + Handles both string ('[]', '') and list ([]) formats since + ToolMessage.content can be either type. + """ + if isinstance(output, list): + return len(output) == 0 + if isinstance(output, str): + return output.strip() in ("[]", "") + return False + + def add_action(self, action: str, action_input: str, output): + entry = {"input": action_input, "output": output} + if action not in self.action_history: + self.action_history[action] = [entry] + else: + self.action_history[action].append(entry) + + def _rule_number_7(self, action: str, action_input: str, output) -> bool: + if action != "Code Keyword Search": + return False + if "." not in action_input: + return False + if not self._is_empty_result(output): + return False + if action not in self.action_history: + return False + prev = self.action_history[action][-1] + if "." in prev["input"] and self._is_empty_result(prev["output"]): + return True + return False + + @staticmethod + def _normalize_package_name(name: str) -> str: + return name.strip().lower().replace("-", "_") + + def _rule_number_8(self, action: str, action_input: str, output) -> bool: + if self.target_package is None: + return False + if action not in ("Function Locator", "Call Chain Analyzer", "Function Caller Finder"): + return False + if action not in self.action_history: + input_pkg = self._normalize_package_name(action_input.split(",")[0]) + target_pkg = self._normalize_package_name(self.target_package) + # Allow Java GAV with version suffix: input "group:artifact:version" + # should match target "group:artifact" + if input_pkg != target_pkg and not input_pkg.startswith(target_pkg + ":"): + return True + return False + def _rule_use_allowed_tools(self, action: str) -> bool: + if action not in self.allowed_tools: + return True + return False + + def _rule_number_9(self, action: str, action_input: str) -> tuple[bool, str]: + if not self.target_functions: + return False, "" + if action not in ("Call Chain Analyzer", "Function Caller Finder"): + return False, "" + input_function = action_input.split(",", 1)[-1].strip() if "," in action_input else "" + if not input_function: + return False, "" + input_short = input_function.rsplit(".", 1)[-1].lower() + for fn in self.target_functions: + if fn.lower() == input_short: + self.target_functions[fn] = True + return False, "" + pending = [fn for fn, checked in self.target_functions.items() if not checked] + if pending: + return True, ( + f"You are NOT following Rule 9. The CVE lists specific vulnerable functions " + f"that you MUST investigate first: {', '.join(pending)}. " + f"Check these functions before investigating other functions." + ) + return False, "" + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + if self._rule_number_7(action, action_input, output): + return True, ("You are NOT following Rule 7. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules.") + if self._rule_number_8(action, action_input, output): + return True, (f"You are NOT following Rule 8. You are using the wrong package name. You MUST use the target package name {self.target_package} see KNOWLEDGE as the package_name before trying alternative packages. Follow the rules.") + if self._rule_use_allowed_tools(action): + return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") + rule9, msg9 = self._rule_number_9(action, action_input) + if rule9: + return True, msg9 + self.add_action(action, action_input, output) + return False, "" + +class AgentState(MessagesState): + input: str = "" + step: int = 0 + max_steps: int = 10 + #memory: str | None = None + thought: Thought | None = None + observation: Observation | None = None + output: str = "" + ecosystem: str | None = None + runtime_prompt: str | None = None + app_package: str | None = None + is_reachability: str = "yes" + rules_tracker: SystemRulesTracker = SystemRulesTracker() + critical_context: list[str] = [] + cca_results: list[bool] = [] + package_validated: bool | None = None + +### --- End of REACT Schemas ----# +#---- REACT Prompt Templates ----# +AGENT_SYS_PROMPT = ( + "You are a security analyst investigating CVE exploitability in container images.\n" + "MANDATORY STEPS (follow in order, do NOT skip any):\n" + "1. IDENTIFY the vulnerable component/function from the CVE description.\n" + "2. SEARCH for its presence using Code Keyword Search.\n" + "3. TRACE reachability using Call Chain Analyzer. " + " - Use the Function Locator to verify the package name and find the function name." + " - For Go: use Function Caller Finder to identify which application functions call the vulnerable library function, BEFORE running Call Chain Analyzer." + " - Keyword search alone is NOT sufficient -- you must trace the call chain.\n" + "4. ASSESS: only after completing reachability checks, determine exploitability.\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the code is absent.\n" + "- Do NOT claim a function is used unless a tool confirmed it.\n" + "- Code Keyword Search proves code PRESENCE in the container, NOT reachability.\n" + "- Function Locator validates package/function NAMES, NOT reachability. It confirms the name exists, not that it is called.\n" + "- Only Call Chain Analyzer can confirm reachability. The application may contain code it never calls.\n" + "- When the question asks whether a function is called or reachable, do NOT conclude based on Code Keyword Search or Function Locator alone -- you MUST use Call Chain Analyzer.\n" + "STOPPING RULES:\n" + "- POSITIVE reachability (Call Chain Analyzer returns True): you MAY conclude exploitable and finish.\n" + "- NEGATIVE reachability (Call Chain Analyzer returns False): record the result. " + "You may only conclude 'not exploitable' after Call Chain Analyzer has confirmed the function is not reachable.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Never give bare assertions (e.g. 'not exploitable'). Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- Distinguish between code being PRESENT (exists in container), REACHABLE (called from application code), and EXPLOITABLE (attacker-controlled input can trigger it). Do not conflate these.\n" + "- If tool results conflict with each other, state the conflict explicitly rather than silently picking one side.\n" + "- Finding that a security check is ABSENT is potential evidence of vulnerability, not evidence of safety.\n" + "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." +) + +# Update LANGGRAPH_SYSTEM_PROMPT_TEMPLATE in react_internals.py +LANGGRAPH_SYSTEM_PROMPT_TEMPLATE = """{sys_prompt} + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +AGENT_THOUGHT_INSTRUCTIONS = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. +4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +5. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +7. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +8. When using Function Locator, Call Chain Analyzer, or Function Caller Finder, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +9. If KNOWLEDGE lists "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)", you MUST investigate those specific functions FIRST before checking any other functions in the same package. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before Call Chain Analyzer"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now trace reachability with Call Chain Analyzer", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} +""" + +AGENT_THOUGHT_INSTRUCTIONS_GO = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. +4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +5. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +7. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +8. When using Function Locator, Call Chain Analyzer, or Function Caller Finder, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +9. If KNOWLEDGE lists "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)", you MUST investigate those specific functions FIRST before checking any other functions in the same package. +10. After Function Locator validates the package, use Function Caller Finder to identify callers. If FCF finds callers: the function IS reachable -- you MAY conclude and finish. If FCF returns empty: this does NOT mean unreachable. You MUST proceed to Call Chain Analyzer. Do NOT finish or conclude without calling Call Chain Analyzer when FCF returned empty. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before tracing callers"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now find which app functions call this library function", "mode": "act", "actions": {{"tool": "Function Caller Finder", "package_name": "", "function_name": ".()", "query": null, "tool_input": null, "reason": "Identify application functions that call the vulnerable library function"}}, "final_answer": null}} + + + +{{"thought": "Function Caller Finder returned no callers, but this does not prove unreachable. MUST call Call Chain Analyzer to confirm reachability", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} +""" + +AGENT_SYS_PROMPT_NON_REACHABILITY = ( + "You are a security analyst investigating CVE exploitability in container images.\n" + "This is NOT a reachability question -- do NOT trace call chains.\n" + "MANDATORY STEPS (follow in order, do NOT skip any):\n" + "1. IDENTIFY the vulnerable component/function from the CVE description.\n" + "2. SEARCH for its presence using Code Keyword Search.\n" + "3. DISTINGUISH where the code was found: main application code vs. package dependencies. " + " Results from Code Keyword Search are grouped into 'Main application' and 'Application library dependencies'. " + " Pay close attention to which group contains the match.\n" + "4. ASSESS: determine the answer based on code presence, version info, configuration, " + " or any other evidence relevant to the question. Use Code Semantic Search or Docs Semantic Search " + " for deeper understanding when needed.\n" + "CRITICAL RULE:\n" + "- If a vulnerable function is found ONLY in 'Application library dependencies' " + " and NOT in 'Main application', this means the code exists in the dependency " + " tree but the main application does NOT directly use it. " + " Do NOT conclude the application is vulnerable based solely on presence in " + " dependency libraries. State clearly: 'found in dependency libraries but not " + " directly referenced by the main application.'\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the code is absent.\n" + "- Do NOT claim a function is used unless a tool confirmed it.\n" + "- Use CVE Web Search to gather additional vulnerability context if needed.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Never give bare assertions (e.g. 'not exploitable'). Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- Distinguish between code being PRESENT (exists in container) and EXPLOITABLE (attacker-controlled input can trigger it). Do not conflate these.\n" + "- If tool results conflict with each other, state the conflict explicitly rather than silently picking one side.\n" + "- Finding that a security check is ABSENT is potential evidence of vulnerability, not evidence of safety.\n" + "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." +) + +AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Function Locator: MUST set package_name AND function_name. Do NOT use query. +4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +5. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +6. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +7. When Code Keyword Search results are grouped, note whether matches are in "Main application" or "Application library dependencies" -- this distinction is important for your analysis. +8. Function Locator validates package/function NAMES only. It confirms the name exists, not that it is called or reachable. +9. When using Function Locator, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present in the container"}}, "final_answer": null}} + + +{{"thought": "Found the function in dependencies. Use Function Locator to validate the package and function name", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name"}}, "final_answer": null}} + + +{{"thought": "Have enough evidence about the vulnerable component presence and context", "mode": "finish", "actions": null, "final_answer": "The vulnerable function was found in the container's dependency libraries under . The main application code does not directly reference it. Based on the CVE description and the evidence gathered, ..."}} +""" + +CLASSIFICATION_PROMPT_TEMPLATE = """You are classifying a CVE investigation question. + +Context (CVE / vulnerable packages): +{context_block} + +Question: {question} + +A reachability question asks whether the vulnerable code/symbol is called or reachable in the codebase, or whether untrusted data can reach it. Is this a reachability question? Answer only yes or no.""" + +PACKAGE_FILTER_PROMPT_TEMPLATE = """The container runs "{image_context}". +{image_match_note} +Candidate packages: +{candidates} + +Ecosystem: {ecosystem} +{critical_context_section} +Rules: +1. If a candidate was pre-identified as matching the container image/repo (see note above), you MUST select it. This OVERRIDES all other rules. Do NOT select a sub-library or component package. +2. Otherwise, pick the package whose ecosystem matches "{ecosystem}". +3. Discard ecosystem-repackaging wrappers (e.g. maven webjars wrapping an npm library) in favour of the native package. +4. Return the package name exactly as it appears in the candidates list above.""" + +FORCED_FINISH_PROMPT = """Maximum steps reached. You MUST set mode="finish" and provide final_answer NOW. +Do NOT call any more tools. Summarize your evidence in 3-5 sentences. +RESPONSE: +{{""" + +COMPREHENSION_PROMPT = """Analyze the tool output and extract key technical findings. +GOAL: {goal} +TARGET PACKAGE (vulnerability): {selected_package} +VULNERABILITY CONTEXT: +{critical_context} +TOOL USED: {tool_used} +TOOL INPUT: {tool_input_detail} +THOUGHT: {last_thought_text} +NEW OUTPUT: +{tool_output} + +CODE COMPREHENSION RULES: +1. READ the actual code snippets in NEW OUTPUT. Do NOT just check whether something was "found" or "not found." +2. For each code snippet or function returned, determine: + - What does this code actually DO? (its functional purpose, not just its name) + - What data does it operate on and in what context? + - Is its purpose semantically related to the GOAL, or is it merely a keyword match? +3. DISTINGUISH same-named entities: a function with the same name in a different package, module, or namespace is a DIFFERENT function. Always record the specific package/module context. +4. CHECK technology fit: if a finding comes from a different programming language, framework, or platform than the target under investigation, note the mismatch explicitly. Do NOT treat it as equivalent. +5. RECORD functional behavior, not just location. Findings must describe what the code DOES (e.g., "rate_limit() in X throttles I/O bandwidth for read/write operations"), not just where it was found (e.g., "rate_limit found in X"). +6. SEPARATE keyword presence from investigation relevance: a security-related function existing in the codebase is NOT automatically a mitigation or evidence for the vulnerability being investigated. Explain HOW the finding relates to the specific GOAL. +7. GROUND TO VENDOR MITIGATIONS: If VULNERABILITY CONTEXT contains "KNOWN MITIGATIONS:", compare the code and behavior in NEW OUTPUT against those mitigations. Record whether the codebase implements, partially implements, or contradicts the vendor's recommended mitigations (e.g. configuration settings, patches, workarounds). Do not treat unrelated security-related code as evidence of mitigation for this CVE. + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty, contains an error, or indicates tool failure, findings must only state: "FAILED: {tool_used} [{tool_input_detail}] - [reason]". Do NOT infer or fabricate positive findings. +- Reachability tags apply ONLY to Call Chain Analyzer results. Code Keyword Search and Function Locator do NOT determine reachability. +- If Call Chain Analyzer returned NEGATIVE (False), include: "NOT reachable via [package]." +- If Call Chain Analyzer returned POSITIVE (True), include: "REACHABLE via [package] - sufficient evidence." +- For Function Locator results, include: "VALIDATED: [package],[function] exists" -- this is NOT reachability. +- When Code Keyword Search results are grouped into "Main application" and "Application library dependencies", prioritize findings from the main application group. +- findings: 3-5 key technical facts from this OUTPUT only. Each fact must reflect code comprehension (what the code does), not just keyword presence. +- tool_outcome: a single line "CALLED: [tool] with [input] -> [brief outcome]". +- Keep only CVE-exploitability-relevant information. +RESPONSE: +{{""" + +MEMORY_UPDATE_PROMPT = """Merge new findings into the investigation memory. +GOAL: {goal} +TARGET PACKAGE (vulnerability): {selected_package} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS (from tool analysis): +{findings} +TOOL CALL RECORD: {tool_outcome} + +RULES: +- memory: Start from PREVIOUS MEMORY. Append new facts from NEW FINDINGS. Record absence if nothing was found. No duplicates. Factual only. +- Add TOOL CALL RECORD verbatim to memory so future steps know what was already tried. +- If NEW FINDINGS report a failure (starts with "FAILED:"), add the failure to memory. Do NOT infer or fabricate positive findings. +- Reachability tags apply ONLY to Call Chain Analyzer results. Code Keyword Search and Function Locator do NOT determine reachability. +- If NEW FINDINGS mention "NOT reachable", add to memory: "NOT reachable via [package]." +- If NEW FINDINGS mention "REACHABLE", add: "REACHABLE via [package] - sufficient evidence." +- For Function Locator validation, record: "VALIDATED: [package],[function] exists" -- this is NOT reachability. +- When findings mention "Main application" vs "Application library dependencies", preserve this distinction in memory. +- If PREVIOUS MEMORY contains "KNOWN MITIGATIONS:" and findings are relevant (e.g. config, code path, or version), add to memory whether the codebase aligns with or contradicts those mitigations (e.g. "Mitigation check: follow_symlinks=False not set" or "Vendor mitigation (patch X) applied in version Y"). +- results: copy the NEW FINDINGS as-is. +- Keep only CVE-exploitability-relevant information. +RESPONSE: +{{""" + +# Legacy prompt kept for backwards compatibility with older traces +OBSERVATION_NODE_PROMPT = COMPREHENSION_PROMPT + +### --- End of REACT Prompt Templates ----# +def build_system_prompt( + tool_descriptions: str, + tool_guidance: str, + instructions: str = AGENT_THOUGHT_INSTRUCTIONS, + sys_prompt: str | None = None, +) -> str: + sys_prompt = sys_prompt or AGENT_SYS_PROMPT + return LANGGRAPH_SYSTEM_PROMPT_TEMPLATE.format( + sys_prompt=sys_prompt, + tools=tool_descriptions, + tool_instructions=instructions, + tool_selection_strategy=tool_guidance, + ) + + +def build_classification_prompt(context_block: str, question: str) -> str: + """Build the reachability-question classification prompt from context and user question.""" + return CLASSIFICATION_PROMPT_TEMPLATE.format( + context_block=context_block, + question=question, + ) + + +def _find_image_matching_candidate( + candidates: list[dict], + image_name: str | None, + image_repo: str | None, +) -> str | None: + """Return the candidate name that appears in the image/repo string, or None.""" + context = " ".join( + part.lower() for part in (image_name, image_repo) if part + ) + if not context: + return None + for c in candidates: + name = c["name"].lower() + if len(name) >= 3 and name in context: + return c["name"] + return None + + +def build_package_filter_prompt( + ecosystem: str, + candidates: list[dict], + image_name: str | None = None, + image_repo: str | None = None, + critical_context: list[str] | None = None, +) -> str: + """Build the package-selection prompt from ecosystem, candidate packages, and container image info.""" + candidate_lines = "\n".join( + f"- {c['name']} (source: {c.get('source', 'unknown')}, ecosystem: {c.get('ecosystem', 'N/A')})" + for c in candidates + ) + parts = [] + if image_name: + parts.append(image_name) + if image_repo: + parts.append(f"repo: {image_repo}") + image_context = ", ".join(parts) if parts else "unknown" + + matched = _find_image_matching_candidate(candidates, image_name, image_repo) + if matched: + image_match_note = f'MATCH DETECTED: candidate "{matched}" matches the container image/repo. Select it (Rule 1).' + critical_context_section = "" + else: + image_match_note = "NO MATCH: no candidate package name was found in the image/repo identifier. Rule 1 does not apply — use Rule 2." + if critical_context: + context_block = "\n".join(critical_context) + critical_context_section = f"\nVulnerability context (use to disambiguate candidates):\n{context_block}\n" + else: + critical_context_section = "" + + return PACKAGE_FILTER_PROMPT_TEMPLATE.format( + ecosystem=ecosystem, + candidates=candidate_lines, + image_context=image_context, + image_match_note=image_match_note, + critical_context_section=critical_context_section, + ) + + +def _build_tool_arguments(actions: ToolCall)->dict[str, Any]: + pkg_tools = {"Function Locator", "Function Caller Finder", "Call Chain Analyzer"} + if actions.tool in pkg_tools and actions.package_name and actions.function_name: + return {"query": f"{actions.package_name},{actions.function_name}"} + if actions.query: + return {"query": actions.query} + if actions.tool_input: + return {"query": actions.tool_input} # fallback + raise ValueError(f"Tool {actions.tool} requires package_name+function_name or query/tool_input") + + + diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 7166b5e5a..0b24fcc1e 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -41,7 +41,7 @@ async def lexical_search(config: LexicalSearchToolConfig, builder: Builder): # from vuln_analysis.utils.full_text_search import FullTextSearch @catch_tool_errors(LEXICAL_CODE_SEARCH) - async def _arun(query: str) -> list: + async def _arun(query: str) -> str: workflow_state = ctx_state.get() code_index_path = workflow_state.code_index_path full_text_search = FullTextSearch(cache_path=code_index_path) diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py new file mode 100644 index 000000000..0bfc98694 --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -0,0 +1,464 @@ +"""Concurrency tests for transitive code search infrastructure. + +Tests that retrievers are immutable after __init__ (mutable state lives in per-search +context objects), per-repo build serialization, and ecosystem-aware cache keys. +""" +import asyncio +import time +import threading + +import pytest +from unittest.mock import MagicMock, patch +from langchain_core.documents import Document + +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever +from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher + +from vuln_analysis.tools.transitive_code_search import ( + _build_or_get_cached, + _searcher_cache, + _searcher_building, + _repo_build_locks, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _build_fake_java_retriever(): + """Build a minimal JavaChainOfCallsRetriever (bypassing __init__) with + the immutable structure used after Strategy C refactor. + tree_dict stores parents directly (not [parents, exclusions, method_exclusions]). + No mutable per-search state on the instance. + """ + r = JavaChainOfCallsRetriever.__new__(JavaChainOfCallsRetriever) + + # Immutable after __init__ + r.tree_dict = { + "com.example:lib-a:1.0": ["root:app:1.0"], + "com.example:lib-b:2.0": ["root:app:1.0"], + "root:app:1.0": ["__root__"], + } + r.documents = [ + Document(page_content="public void foo(){}", metadata={"source": "lib-a-1.0-sources/Foo.java", "content_type": "functions_classes"}), + Document(page_content="public void bar(){}", metadata={"source": "lib-b-2.0-sources/Bar.java", "content_type": "functions_classes"}), + Document(page_content="public void main(){}", metadata={"source": "src/main/java/App.java", "content_type": "functions_classes"}), + ] + r._root_docs = [r.documents[2]] + r._jar_to_docs = { + "lib-a:1.0": [r.documents[0]], + "lib-b:2.0": [r.documents[1]], + } + r.supported_packages = list(r.tree_dict.keys()) + + # Read-only shared state + r.ecosystem = MagicMock() + r.dependency_tree = MagicMock() + r.language_parser = MagicMock() + r.manifest_path = "/fake" + r.documents_of_functions = r.documents[:] + r.documents_of_types = [] + r.documents_of_full_sources = {} + r.type_inheritance = {} + r.types_classes_fields_mapping = {} + r.functions_local_variables_index = {} + + return r + + +def _build_fake_go_retriever(): + """Build a minimal ChainOfCallsRetriever (non-Java, bypassing __init__) with + the immutable structure used after Strategy C refactor. + tree_dict stores parents directly (not [parents, exclusions]). + No mutable per-search state on the instance. + """ + r = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + + r.tree_dict = { + "crypto/x509": ["__root__"], + "net/http": ["__root__"], + } + r.documents = [ + Document(page_content="func ParsePKCS1PrivateKey(){}", metadata={"source": "vendor/crypto/x509/pkcs1.go", "content_type": "functions_classes"}), + Document(page_content="func ListenAndServe(){}", metadata={"source": "vendor/net/http/server.go", "content_type": "functions_classes"}), + Document(page_content="func main(){}", metadata={"source": "cmd/main.go", "content_type": "functions_classes"}), + ] + r.supported_packages = list(r.tree_dict.keys()) + + # Read-only shared state + r.ecosystem = MagicMock() + r.dependency_tree = MagicMock() + r.language_parser = MagicMock() + r.manifest_path = "/fake" + r.documents_of_functions = r.documents[:] + r.documents_of_types = [] + r.documents_of_full_sources = {} + r.types_classes_fields_mapping = {} + r.functions_local_variables_index = {} + + return r + + +def _make_si(git_repo: str, ref: str = "main"): + """Create a minimal source_info list for cache key derivation.""" + return [SourceDocumentsInfo(type="code", git_repo=git_repo, ref=ref, + include=["**/*.java"], exclude=[])] + + +def _clear_caches(): + """Reset module-level caches between tests.""" + _searcher_cache.clear() + _searcher_building.clear() + _repo_build_locks.clear() + + +def _make_java_searcher(): + """Create a mock TransitiveCodeSearcher with a Java retriever.""" + mock = MagicMock() + mock.chain_of_calls_retriever = MagicMock(spec=JavaChainOfCallsRetriever) + return mock + + +def _make_nonjava_searcher(): + """Create a mock TransitiveCodeSearcher with a non-Java retriever.""" + mock = MagicMock() + mock.chain_of_calls_retriever = MagicMock(spec=ChainOfCallsRetriever) + return mock + + +def _make_slow_builder(build_log, sleep_secs=0.2, java=True): + """Create a synchronous _build_searcher replacement that records timing. + + Runs inside asyncio.to_thread (in a real thread), so uses time.sleep. + """ + lock = threading.Lock() + + def slow_build(si, query): + tag = query.split(",")[0] + start = time.monotonic() + time.sleep(sleep_secs) + end = time.monotonic() + with lock: + build_log.append((tag, start, end)) + return _make_java_searcher() if java else _make_nonjava_searcher() + + return slow_build + + +# --------------------------------------------------------------------------- +# Immutability tests — Java (no mutable per-search state on instance) +# --------------------------------------------------------------------------- + +def test_java_retriever_has_no_mutable_search_state(): + """JavaChainOfCallsRetriever should not have found_path, + last_visited_parent_package_indexes, or exclusion lists on the instance.""" + r = _build_fake_java_retriever() + assert not hasattr(r, 'found_path'), "found_path should not be on instance" + assert not hasattr(r, 'last_visited_parent_package_indexes'), \ + "last_visited_parent_package_indexes should not be on instance" + + +def test_java_tree_dict_stores_parents_directly(): + """tree_dict values should be parent lists, not [parents, exclusions, method_exclusions].""" + r = _build_fake_java_retriever() + for pkg, value in r.tree_dict.items(): + assert isinstance(value, list), f"tree_dict[{pkg}] should be a list of parents" + # Parents are strings, not nested lists + assert all(isinstance(p, str) for p in value), \ + f"tree_dict[{pkg}] should contain parent strings, got {value}" + + +def test_java_retriever_shared_instance_is_safe(): + """Two references to the same Java retriever should be safe since + mutable state lives in _JavaSearchCtx, not on the instance.""" + original = _build_fake_java_retriever() + # Both references point to same object — this is safe now + ref_a = original + ref_b = original + assert ref_a is ref_b + assert ref_a.tree_dict is ref_b.tree_dict + assert ref_a.documents is ref_b.documents + + +# --------------------------------------------------------------------------- +# Immutability tests — non-Java (no mutable per-search state on instance) +# --------------------------------------------------------------------------- + +def test_nonjava_retriever_has_no_mutable_search_state(): + """ChainOfCallsRetriever should not have found_path, + last_visited_parent_package_indexes, or last_visited on the instance.""" + r = _build_fake_go_retriever() + assert not hasattr(r, 'found_path'), "found_path should not be on instance" + assert not hasattr(r, 'last_visited_parent_package_indexes'), \ + "last_visited_parent_package_indexes should not be on instance" + assert not hasattr(r, 'last_visited'), "last_visited should not be on instance" + + +def test_nonjava_tree_dict_stores_parents_directly(): + """tree_dict values should be parent lists, not [parents, exclusions].""" + r = _build_fake_go_retriever() + for pkg, value in r.tree_dict.items(): + assert isinstance(value, list), f"tree_dict[{pkg}] should be a list of parents" + assert all(isinstance(p, str) for p in value), \ + f"tree_dict[{pkg}] should contain parent strings, got {value}" + + +def test_nonjava_retriever_shared_instance_is_safe(): + """Two references to the same non-Java retriever should be safe since + mutable state lives in _SearchCtx, not on the instance.""" + original = _build_fake_go_retriever() + ref_a = original + ref_b = original + assert ref_a is ref_b + assert ref_a.tree_dict is ref_b.tree_dict + assert ref_a.documents is ref_b.documents + + +# --------------------------------------------------------------------------- +# _build_or_get_cached concurrency tests — Java (per-package cache keys) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_java_same_repo_different_packages_are_serialized(): + """Two Java builds for different packages on the same repo must NOT run + concurrently and must each produce a separate cache entry.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + build_log = [] + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=_make_slow_builder(build_log, java=True)): + task1 = asyncio.create_task(_build_or_get_cached(si, "pkg-a:art-a:1.0,ClassA.foo")) + task2 = asyncio.create_task(_build_or_get_cached(si, "pkg-b:art-b:2.0,ClassB.bar")) + await asyncio.gather(task1, task2) + + assert len(build_log) == 2, f"Expected 2 Java builds (different packages), got {len(build_log)}" + # Builds must NOT overlap + (tag1, start1, end1) = build_log[0] + (tag2, start2, end2) = build_log[1] + assert start2 >= end1 or start1 >= end2, ( + f"Builds overlapped! {tag1}: [{start1:.4f}, {end1:.4f}], " + f"{tag2}: [{start2:.4f}, {end2:.4f}]" + ) + + +@pytest.mark.asyncio +async def test_different_repos_can_build_concurrently(): + """Builds for different repos should NOT be serialized.""" + _clear_caches() + si_a = _make_si("https://github.com/example/repo-a") + si_b = _make_si("https://github.com/example/repo-b") + build_log = [] + + def slow_build(si, query): + tag = si[0].git_repo.split("/")[-1] + start = time.monotonic() + time.sleep(0.2) + end = time.monotonic() + build_log.append((tag, start, end)) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=slow_build): + task1 = asyncio.create_task(_build_or_get_cached(si_a, "pkg-a:art-a:1.0,Foo.bar")) + task2 = asyncio.create_task(_build_or_get_cached(si_b, "pkg-b:art-b:2.0,Baz.qux")) + await asyncio.gather(task1, task2) + + assert len(build_log) == 2 + (_, start1, end1) = build_log[0] + (_, start2, end2) = build_log[1] + assert start2 < end1, ( + f"Different-repo builds should run concurrently but were serialized: " + f"build1=[{start1:.4f}, {end1:.4f}], build2=[{start2:.4f}, {end2:.4f}]" + ) + + +@pytest.mark.asyncio +async def test_same_key_deduplicates_build(): + """Two concurrent requests for the exact same query should only build once.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + build_count = 0 + count_lock = threading.Lock() + + def counting_build(build_si, q): + nonlocal build_count + with count_lock: + build_count += 1 + time.sleep(0.1) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + task1 = asyncio.create_task(_build_or_get_cached(si, query)) + task2 = asyncio.create_task(_build_or_get_cached(si, query)) + results = await asyncio.gather(task1, task2) + + assert build_count == 1, f"Expected 1 build (deduplicated), got {build_count}" + assert results[0] is results[1], "Both tasks should return the same cached searcher" + + +@pytest.mark.asyncio +async def test_cache_hit_skips_build(): + """If the cache already has an entry for the repo key, no build should happen.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + # Non-Java cache uses repo-level key (git_repo, ref) + repo_key = ("https://github.com/example/repo", "main") + + pre_cached = _make_nonjava_searcher() + _searcher_cache[repo_key] = pre_cached + + build_count = 0 + + def counting_build(build_si, q): + nonlocal build_count + build_count += 1 + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + result = await _build_or_get_cached(si, query) + + assert build_count == 0, "Build should not run when cache hit exists" + assert result is pre_cached, "Should return the pre-cached searcher" + + +@pytest.mark.asyncio +async def test_java_cache_hit_skips_build(): + """If the cache already has a Java entry for the full key, no build should happen.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") + + pre_cached = _make_java_searcher() + _searcher_cache[full_key] = pre_cached + + build_count = 0 + + def counting_build(build_si, q): + nonlocal build_count + build_count += 1 + return _make_java_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + result = await _build_or_get_cached(si, query) + + assert build_count == 0, "Build should not run when Java cache hit exists" + assert result is pre_cached, "Should return the pre-cached Java searcher" + + +@pytest.mark.asyncio +async def test_build_failure_cleans_up_building_marker(): + """If _build_searcher raises, the building marker must be cleaned up.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") + repo_key = ("https://github.com/example/repo", "main") + + def failing_build(build_si, q): + raise RuntimeError("Maven failed") + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=failing_build): + with pytest.raises(RuntimeError, match="Maven failed"): + await _build_or_get_cached(si, query) + + assert full_key not in _searcher_building, "Building marker not cleaned up after failure" + assert full_key not in _searcher_cache, "Failed build should not be cached" + assert repo_key not in _searcher_cache, "Failed build should not be cached" + + +@pytest.mark.asyncio +async def test_java_repo_lock_recheck_avoids_redundant_build(): + """When Java task B waits for the repo lock while task A builds, task B should + recheck the cache after acquiring the lock and skip its own build if A + already cached a result for B's key.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query_a = "pkg-a:art-a:1.0,ClassA.foo" + query_b = "pkg-b:art-b:2.0,ClassB.bar" + full_key_b = ("https://github.com/example/repo", "main", "pkg-b:art-b:2.0") + + build_count = 0 + count_lock = threading.Lock() + + def build_that_precaches_b(build_si, q): + nonlocal build_count + with count_lock: + build_count += 1 + current = build_count + if current == 1: + time.sleep(0.1) + _searcher_cache[full_key_b] = _make_java_searcher() + return _make_java_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=build_that_precaches_b): + task1 = asyncio.create_task(_build_or_get_cached(si, query_a)) + await asyncio.sleep(0.01) + task2 = asyncio.create_task(_build_or_get_cached(si, query_b)) + await asyncio.gather(task1, task2) + + assert build_count == 1, ( + f"Expected 1 build (B should hit cache after repo lock), got {build_count}" + ) + + +# --------------------------------------------------------------------------- +# Non-Java cache sharing tests +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_nonjava_same_repo_different_packages_share_cache(): + """For non-Java ecosystems, different packages on the same repo should + share a single cache entry (repo_key), avoiding redundant builds.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + build_count = 0 + count_lock = threading.Lock() + + def counting_build(build_si, q): + nonlocal build_count + with count_lock: + build_count += 1 + time.sleep(0.1) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + task1 = asyncio.create_task(_build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey")) + task2 = asyncio.create_task(_build_or_get_cached(si, "net/http,ListenAndServe")) + results = await asyncio.gather(task1, task2) + + # Non-Java: both should resolve to the same (repo, ref) cache entry. + # Second task either deduplicates via _searcher_building or finds the + # repo_key in cache after the repo lock. + assert build_count == 1, ( + f"Expected 1 build for non-Java (shared repo_key), got {build_count}" + ) + assert results[0] is results[1], "Both tasks should return the same cached searcher" + + +@pytest.mark.asyncio +async def test_nonjava_caches_under_repo_key(): + """Non-Java builds should cache under (git_repo, ref), not (git_repo, ref, package).""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + repo_key = ("https://github.com/example/repo", "main") + full_key = ("https://github.com/example/repo", "main", "crypto/x509") + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + return_value=_make_nonjava_searcher()): + await _build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey") + + assert repo_key in _searcher_cache, "Non-Java should cache under repo_key" + assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" \ No newline at end of file diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 766dbf3f3..b432fc257 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -8,7 +8,7 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig from exploit_iq_commons.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, - ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput + ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput , SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) from vuln_analysis.runtime_context import ctx_state from vuln_analysis.tools.tests.mock_documents import (python_script_example, python_init_function_example, @@ -216,6 +216,7 @@ def mock_file_open(*args, **kwargs): "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file] } ]) +@pytest.mark.skip @patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) @patch('builtins.open', side_effect=mock_file_open) async def test_transitive_search_python_parameterized(mock_open, mock_run_command,test_case): @@ -231,7 +232,7 @@ async def test_transitive_search_python_parameterized(mock_open, mock_run_comman ) with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', - return_value=(test_case["mock_documents"], True)): + return_value=(test_case["mock_documents"], True)): result = await transitive_code_search_runner_coroutine(test_case["search_query"]) (path_found, list_path) = result @@ -297,11 +298,11 @@ async def test_c_transitive_search(): #create sample sbom packages sbom_list = [ - SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), - SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), - SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') - ] - + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + set_input_for_next_run( git_repository='https://github.com/postgres/postgres', git_ref='REL_13_14', @@ -321,7 +322,7 @@ async def test_c_transitive_search(): print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") assert len(list_path) == 2 - assert path_found == True + assert path_found == True @pytest.mark.asyncio @@ -332,11 +333,11 @@ async def test_c_transitive_search_2(): #create sample sbom packages sbom_list = [ - SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), - SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), - SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') - ] - + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + set_input_for_next_run( git_repository='https://github.com/postgres/postgres', git_ref='REL_13_14', @@ -375,7 +376,7 @@ async def test_transitive_search_java_1(): (path_found, list_path) = result print(result) assert path_found is False - assert len(list_path) in [0,1] + assert len(list_path) is 1 @pytest.mark.asyncio async def test_transitive_search_java_2(): @@ -399,9 +400,9 @@ async def test_transitive_search_java_2(): print(result) assert path_found is True assert len(list_path) is 2 - document = list_path[1] - assert 'src/main/java/io/cryostat' in document.metadata['source'] - assert 'StringUtils.isBlank(' in document.page_content + # list_path contains concise strings: "source_path :: function_signature" + assert 'src/main/java/io/cryostat' in list_path[-1] + assert 'StringUtils' in list_path[0] and 'isBlank(' in list_path[0] @pytest.mark.skip @pytest.mark.asyncio @@ -426,8 +427,7 @@ async def test_transitive_search_java_3(): print(result) assert path_found is True assert len(list_path) > 1 - document = list_path[-1] - assert 'src/main/java/io/cryostat' in document.metadata['source'] + assert 'src/main/java/io/cryostat' in list_path[-1] @pytest.mark.asyncio async def test_transitive_search_java_4(): @@ -451,32 +451,7 @@ async def test_transitive_search_java_4(): print(result) assert path_found is True assert len(list_path) > 1 - document = list_path[-1] - assert 'src/main/java/io/cryostat' in document.metadata['source'] - -# Test method reference -@pytest.mark.asyncio -async def test_transitive_search_java_5(): - transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() - set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", - git_ref="8f753753379e9381429b476aacbf6890ef101438", - included_extensions=["**/*.java"], - excluded_extensions=["target/**/*", - "build/**/*", - "*.class", - ".gradle/**/*", - ".mvn/**/*", - ".gitignore", - "test/**/*", - "tests/**/*", - "src/test/**/*", - "pom.xml", - "build.gradle"]) - result = await transitive_code_search_runner_coroutine("commons-io:commons-io:2.16.1,org.apache.commons.io.FileUtils.forceDeleteOnExit") - (path_found, list_path) = result - print(result) - assert path_found is False - assert len(list_path) is 1 + assert 'src/main/java/io/cryostat' in list_path[-1] @pytest.mark.asyncio async def test_java_script_transitive_search_1(): @@ -719,7 +694,7 @@ async def test_javascript_class_transitive_call(mock_open, mock_npm_ls): - func_a: vulnerable function - func_b: triggers func_a - Source code imports the class and calls func_b - + Expected: The vulnerable func_a should be reachable via the call chain: app.js -> VulnerableClass.func_b -> VulnerableClass.func_a """ @@ -799,7 +774,7 @@ async def test_javascript_class_transitive_call(mock_open, mock_npm_ls): print(f"DEBUG: path_found = {path_found}") print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") - + # Expected call chain: func_a -> func_b -> processUserData (3 functions) assert path_found == True, "func_a should be reachable through func_b -> processUserData" assert len(list_path) == 3, f"Should have call chain of 3 functions (func_a -> func_b -> processUserData), got {len(list_path)}" @@ -817,7 +792,7 @@ async def test_javascript_class_inheritance_transitive_call(mock_open, mock_npm_ - Source code has a child class that extends the base class: - CustomHandler extends VulnerableBase - Another source file creates an instance of child class and calls the inherited method - + Expected: The vulnerable executeCommand should be reachable via the call chain: app.js -> processRequest -> CustomHandler (instance) -> VulnerableBase.executeCommand """ @@ -899,7 +874,7 @@ async def test_javascript_class_inheritance_transitive_call(mock_open, mock_npm_ print(f"DEBUG: path_found = {path_found}") print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") - + # Expected call chain: executeCommand (from VulnerableBase) -> processRequest (2 functions) # Note: The child class CustomHandler doesn't redefine the method, so it's inherited assert path_found == True, "executeCommand should be reachable through CustomHandler instance in processRequest" @@ -917,7 +892,7 @@ async def test_javascript_object_literal_transitive_call(mock_open, mock_npm_ls) - vulnerableMethod: vulnerable function using eval - wrapperMethod: triggers vulnerableMethod via this.vulnerableMethod - Source code imports the object and calls wrapperMethod - + Expected: The vulnerable method should be reachable via the call chain: app.js -> dangerousUtils.wrapperMethod -> dangerousUtils.vulnerableMethod """ @@ -995,7 +970,7 @@ async def test_javascript_object_literal_transitive_call(mock_open, mock_npm_ls) print(f"DEBUG: path_found = {path_found}") print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") - + # Expected call chain: vulnerableMethod -> wrapperMethod -> processUserInput (3 functions) assert path_found == True, "vulnerableMethod should be reachable through wrapperMethod -> processUserInput" assert len(list_path) == 3, f"Should have call chain of 3 functions (vulnerableMethod -> wrapperMethod -> processUserInput), got {len(list_path)}" @@ -1010,7 +985,7 @@ async def test_javascript_object_method_direct_call(mock_open, mock_npm_ls): Test transitive search for direct JavaScript object method call where: - A dependency package contains an object literal with a vulnerable method - Source code directly calls the vulnerable method - + Expected: The vulnerable method should be reachable directly from source: app.js -> apiUtils.unsafeEval """ @@ -1072,7 +1047,31 @@ async def test_javascript_object_method_direct_call(mock_open, mock_npm_ls): print(f"DEBUG: path_found = {path_found}") print(f"DEBUG: list_path = {list_path}") print(f"DEBUG: len(list_path) = {len(list_path)}") - + # Expected call chain: unsafeEval -> executeCode (2 functions) assert path_found == True, "unsafeEval should be reachable directly from executeCode" assert len(list_path) == 2, f"Should have call chain of 2 functions (unsafeEval -> executeCode), got {len(list_path)}" + + +@pytest.mark.parametrize("raw_query, expected_cleaned", [ + # Standard ASCII quotes + ("'commons-beanutils:commons-beanutils:1.9.4'", "commons-beanutils:commons-beanutils:1.9.4"), + ('"commons-beanutils:commons-beanutils:1.9.4"', "commons-beanutils:commons-beanutils:1.9.4"), + # Unicode smart quotes (left/right single) + ("\u2018commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), + # Unicode smart quotes (left/right double) + ("\u201ccommons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), + # Mixed: ASCII left, unicode right + ("'commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), + ("\"commons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), + # No quotes + ("commons-beanutils:commons-beanutils:1.9.4", "commons-beanutils:commons-beanutils:1.9.4"), + # Whitespace + quotes + (" 'commons-beanutils:commons-beanutils:1.9.4' ", "commons-beanutils:commons-beanutils:1.9.4"), + # Trailing newline junk from LLM + ("'commons-beanutils:commons-beanutils:1.9.4'\nPlease wait...", "commons-beanutils:commons-beanutils:1.9.4"), +]) +def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): + """Test that query cleaning handles both ASCII and Unicode smart quotes.""" + cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() + assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" diff --git a/src/vuln_analysis/tools/tool_names.py b/src/vuln_analysis/tools/tool_names.py index 248fec7fd..6f46faa61 100644 --- a/src/vuln_analysis/tools/tool_names.py +++ b/src/vuln_analysis/tools/tool_names.py @@ -44,6 +44,9 @@ class ToolNames: CONTAINER_ANALYSIS_DATA = "Container Analysis Data" """Retrieves pre-analyzed data from earlier container scan steps""" + FUNCTION_LIBRARY_VERSION_FINDER = "Function Library Version Finder" + """Checks in which library version the function is used""" + # Export as module-level constants CODE_SEMANTIC_SEARCH = ToolNames.CODE_SEMANTIC_SEARCH @@ -54,6 +57,7 @@ class ToolNames: FUNCTION_CALLER_FINDER = ToolNames.FUNCTION_CALLER_FINDER CVE_WEB_SEARCH = ToolNames.CVE_WEB_SEARCH CONTAINER_ANALYSIS_DATA = ToolNames.CONTAINER_ANALYSIS_DATA +FUNCTION_LIBRARY_VERSION_FINDER = ToolNames.FUNCTION_LIBRARY_VERSION_FINDER @@ -66,5 +70,6 @@ class ToolNames: 'FUNCTION_CALLER_FINDER', 'CVE_WEB_SEARCH', 'CONTAINER_ANALYSIS_DATA', - 'FUNCTION_LOCATOR' -] + 'FUNCTION_LOCATOR', + 'FUNCTION_LIBRARY_VERSION_FINDER', +] \ No newline at end of file diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 6106b3178..fa50e6fd1 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -12,7 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import os +from collections import OrderedDict from vuln_analysis.runtime_context import ctx_state from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher @@ -35,16 +37,90 @@ from ..utils.function_name_locator import FunctionNameLocator from exploit_iq_commons.logging.loggers_factory import LoggingFactory -from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever, _release_repo_data PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME = "package_and_function_locator" FUNCTION_NAME_EXTRACTOR_TOOL_NAME = "calling_function_name_extractor" +FUNCTION_LIBRARY_VERSION_FINDER_TOOL_NAME = "calling_function_library_version_finder" + TRANSITIVE_CODE_SEARCH_TOOL_NAME = "transitive_code_search" logger = LoggingFactory.get_agent_logger(__name__) +# Maximum number of entries in the searcher cache. +# Evicts oldest entry when exceeded to bound memory consumption. +_SEARCHER_CACHE_MAX_SIZE = 6 + +# Module-level cache: avoids rebuilding the TransitiveCodeSearcher for the same +# repo+ref(+package for Java) across concurrent requests within the same process. +# Key is (git_repo, ref) for non-Java, (git_repo, ref, package_name) for Java +# because JavaDependencyTreeBuilder.build_tree uses the query's package to filter the dep graph. +_searcher_cache: OrderedDict[tuple, TransitiveCodeSearcher] = OrderedDict() +_searcher_cache_lock = asyncio.Lock() +# Per-key build coordination: tasks waiting for the same key await its Event +# instead of holding the global lock during the entire build. +_searcher_building: dict[tuple, asyncio.Event] = {} +# Per-repo build serialization: builds for the same (git_repo, ref) share filesystem +# resources (git checkout, install_dependencies, document creation). +# Concurrent builds on the same repo corrupt each other, so we serialize them. +_repo_build_locks: dict[tuple, asyncio.Lock] = {} +_repo_locks_lock = asyncio.Lock() + +_QUERY_FORMAT_ERROR = ( + "Invalid input format. The query must contain exactly one comma separating package and function. " + "Expected format 1: 'package_name,function_name' (e.g., 'urllib,parse'). " + "Expected format 2 (java): 'maven_gav,class_name.function_name' or 'maven_gav,fqcn.function_name' (preferred) " + "(e.g., 'commons-beanutils:commons-beanutils:x.y.z,a.b.c.ClassA.foo'). " + "Each tool call must contain exactly ONE query. Do NOT combine multiple queries with 'and'. " + "Please retry with the correct format." +) + + +def _summarize_call_chain(call_hierarchy_list: list[Document]) -> list[str]: + """Summarize a call chain into concise strings for the agent scratchpad. + + Extracts source file path and the first line of each function (the signature) + instead of returning full Document objects with complete source code. + Preserves the original list order (library function first, application code last). + """ + summary = [] + for doc in call_hierarchy_list: + source = doc.metadata.get('source', 'unknown') + # Extract just the function signature (first line of page_content) + content = doc.page_content.strip() + first_line = content.split('\n')[0].strip() if content else '' + # Truncate very long signatures + if len(first_line) > 150: + first_line = first_line[:150] + '...' + summary.append(f"{source} :: {first_line}") + return summary + + +def _validate_query_format(query: str) -> tuple[bool, str]: + """Validate that query matches expected format and return (is_valid, error_message_or_cleaned_query).""" + cleaned = query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() + # Remove surrounding backticks if present + cleaned = cleaned.strip("`") + parts = cleaned.split(",") + if len(parts) != 2 or not parts[0].strip() or not parts[1].strip(): + return False, f"{_QUERY_FORMAT_ERROR} Received: '{query}'" + return True, cleaned + + +def package_name_from_locator_query(query: str) -> str | None: + """Return the package segment from a validated ``package,function`` query. + + Uses the same cleaning and validation rules as the locator / transitive tools. + Returns ``None`` if ``query`` does not match the expected format. + """ + is_valid, cleaned = _validate_query_format(query) + if not is_valid: + return None + package = cleaned.split(",", 1)[0].strip().lower() + return package or None + class TransitiveCodeSearchToolConfig(FunctionBaseConfig, name=("%s" % TRANSITIVE_CODE_SEARCH_TOOL_NAME)): """ @@ -63,6 +139,13 @@ class PackageAndFunctionLocatorToolConfig(FunctionBaseConfig, name=("%s" % PACKA Package and function locator tool used to validate package names and find function names using fuzzy matching. """ + +class FunctionLibraryVersionFinderToolConfig(FunctionBaseConfig, name=FUNCTION_LIBRARY_VERSION_FINDER_TOOL_NAME): + """ + Checks in which library version the function is used. + """ + + def get_call_of_chains_retriever(documents_embedder, si, query: str): documents: list[Document] git_repo = None @@ -82,16 +165,189 @@ def get_call_of_chains_retriever(documents_embedder, si, query: str): manifest_path=git_repo, query=query, code_source_info=code_source_info) + # Release the raw documents list — the retriever has classified and indexed + # them into its own structures. Dropping this reference allows the GC to + # reclaim any Document objects not retained by the retriever's indexes. + del documents return coc_retriever -def get_transitive_code_searcher(query: str): + +def _get_repo_key(si) -> tuple | None: + """Derive a repo-level key (git_repo, ref) for filesystem serialization.""" + for source_info in si: + if source_info.type == "code": + return (source_info.git_repo, source_info.ref) + return None + + +async def _get_repo_lock(si) -> asyncio.Lock: + """Get or create a per-repo asyncio.Lock for serializing builds on the same repo.""" + repo_key = _get_repo_key(si) + async with _repo_locks_lock: + if repo_key not in _repo_build_locks: + _repo_build_locks[repo_key] = asyncio.Lock() + return _repo_build_locks[repo_key] + + +def _get_cache_keys(si, query: str) -> tuple[tuple | None, tuple | None]: + """Derive cache keys from source_info list. + + Returns (repo_key, full_key): + - repo_key = (git_repo, ref) — used for non-Java ecosystems where + build_tree produces the same result regardless of package. + - full_key = (git_repo, ref, package_name) — used for Java where + JavaDependencyTreeBuilder.build_tree uses -DtargetIncludes to + filter the dependency graph per package. + """ + for source_info in si: + if source_info.type == "code": + repo_key = (source_info.git_repo, source_info.ref) + package_name = query.split(",")[0].strip() if query else "" + full_key = (source_info.git_repo, source_info.ref, package_name) + return repo_key, full_key + return None, None + + +def _build_searcher(si, query: str) -> TransitiveCodeSearcher: + """Synchronous helper that builds a TransitiveCodeSearcher. + + Separated so it can be offloaded to a thread via asyncio.to_thread(), + keeping the event loop free for other tasks. + For Java, repo-level data (documents, type maps) is shared across + package-specific retrievers via _JavaRepoData (in-memory cache keyed + by (git_repo, ref), backed by pickle sub-caches on disk). + """ + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query) + return TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) + + +async def _build_or_get_cached(si, query: str) -> TransitiveCodeSearcher: + """Build a TransitiveCodeSearcher, or return a cached one. + + Cache keys: + - Non-Java: (git_repo, ref) — build_tree is package-independent. + - Java: (git_repo, ref, package_name) — build_tree uses -DtargetIncludes. + + Since the ecosystem isn't known until after the first build, lookups check + both the repo-level key and the full (package-inclusive) key. After building, + the result is cached under the appropriate key based on the retriever type. + + Coordination: + - Same-key dedup via asyncio.Event (avoids redundant builds). + - Per-repo serialization via asyncio.Lock (protects shared filesystem: + git checkout, install_dependencies, document creation). + - The expensive build runs in a thread via asyncio.to_thread(). + """ + repo_key, full_key = _get_cache_keys(si, query) + + while True: + async with _searcher_cache_lock: + # Check both cache keys: repo-level (non-Java) and full (Java) + if repo_key and repo_key in _searcher_cache: + logger.info("Reusing cached TransitiveCodeSearcher for %s", repo_key) + _searcher_cache.move_to_end(repo_key) + return _searcher_cache[repo_key] + if full_key and full_key in _searcher_cache: + logger.info("Reusing cached TransitiveCodeSearcher for %s", full_key) + _searcher_cache.move_to_end(full_key) + return _searcher_cache[full_key] + + # Check if another task is already building this exact key + if full_key and full_key in _searcher_building: + event = _searcher_building[full_key] + else: + event = None + if full_key: + _searcher_building[full_key] = asyncio.Event() + + if event: + # Another task is building this key; await it, then re-check cache + logger.info("Waiting for another task to build TransitiveCodeSearcher for %s", full_key) + await event.wait() + continue + + # This task is responsible for building. + # Acquire the per-repo lock to serialize builds that share the same + # filesystem (git checkout, install_dependencies, document creation). + repo_lock = await _get_repo_lock(si) + try: + async with repo_lock: + # Re-check cache after acquiring repo lock — another build for a + # different package on the same repo may have finished while we waited. + async with _searcher_cache_lock: + if repo_key and repo_key in _searcher_cache: + logger.info("Reusing cached TransitiveCodeSearcher for %s (after repo lock)", repo_key) + _searcher_cache.move_to_end(repo_key) + if full_key in _searcher_building: + _searcher_building[full_key].set() + del _searcher_building[full_key] + return _searcher_cache[repo_key] + if full_key and full_key in _searcher_cache: + logger.info("Reusing cached TransitiveCodeSearcher for %s (after repo lock)", full_key) + _searcher_cache.move_to_end(full_key) + if full_key in _searcher_building: + _searcher_building[full_key].set() + del _searcher_building[full_key] + return _searcher_cache[full_key] + + # Build outside _searcher_cache_lock but inside repo_lock — + # other tasks can read/write the cache while this build runs, + # but no concurrent build on the same repo's filesystem. + logger.info("Building TransitiveCodeSearcher for %s", full_key) + searcher = await asyncio.to_thread(_build_searcher, si, query) + + async with _searcher_cache_lock: + # Cache under the appropriate key based on ecosystem + if isinstance(searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): + store_key = full_key + else: + store_key = repo_key + if store_key: + if len(_searcher_cache) >= _SEARCHER_CACHE_MAX_SIZE: + evicted_key, evicted_searcher = _searcher_cache.popitem(last=False) + logger.info("Evicted oldest TransitiveCodeSearcher cache entry: %s", evicted_key) + if (hasattr(evicted_searcher, 'chain_of_calls_retriever') + and isinstance(evicted_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever) + and hasattr(evicted_searcher.chain_of_calls_retriever, '_repo_data')): + _release_repo_data(evicted_searcher.chain_of_calls_retriever._repo_data) + _searcher_cache[store_key] = searcher + if full_key and full_key in _searcher_building: + _searcher_building[full_key].set() + del _searcher_building[full_key] + + return searcher + except BaseException: + # Clean up the building marker so waiting tasks don't hang forever. + # BaseException catches KeyboardInterrupt, SystemExit, etc. + async with _searcher_cache_lock: + if full_key and full_key in _searcher_building: + _searcher_building[full_key].set() + del _searcher_building[full_key] + raise + + +async def get_transitive_code_searcher(query: str): state: AgentMorpheusEngineState = ctx_state.get() - if state.transitive_code_searcher is None or isinstance(state.transitive_code_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): - si = state.original_input.input.image.source_info - documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query) - transitive_code_searcher = TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) - state.transitive_code_searcher = transitive_code_searcher + si = state.original_input.input.image.source_info + + if state.transitive_code_searcher is None: + state.transitive_code_searcher = await _build_or_get_cached(si, query) + elif isinstance(state.transitive_code_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): + # Java: different queries produce different dep trees (build_tree uses + # -DtargetIncludes for GAV queries), so rebuild when the package changes. + # Only re-check the cache when the package actually changed to avoid + # unnecessary lock acquisition on every tool call within the same request. + _, full_key = _get_cache_keys(si, query) + async with _searcher_cache_lock: + cached = _searcher_cache.get(full_key) + if cached is not None and cached is state.transitive_code_searcher: + pass # Same searcher, no change needed + else: + state.transitive_code_searcher = await _build_or_get_cached(si, query) + + # Both Java and non-Java retrievers use per-search context objects (_JavaSearchCtx / _SearchCtx) + # for mutable state, so the retriever instance is immutable after __init__ and needs no deep copy. return state.transitive_code_searcher @@ -108,30 +364,36 @@ def get_transitive_code_searcher(query: str): @register_function(config_type=TransitiveCodeSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def transitive_search(config: TransitiveCodeSearchToolConfig, - builder: Builder): # pylint: disable=unused-argument + builder: Builder, verbose_mode: bool = False): # pylint: disable=unused-argument """ Call Chain Analyzer tool used to search source code function reachability. """ @catch_tool_errors(TRANSITIVE_CODE_SEARCH_TOOL_NAME) async def _arun(query: str) -> tuple: + is_valid, validation_result = _validate_query_format(query) + if not is_valid: + return False, [validation_result] transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher(query) - result = transitive_code_searcher.search(query) - return result + transitive_code_searcher = await get_transitive_code_searcher(validation_result) + found_path, call_hierarchy_list = transitive_code_searcher.search(validation_result) + # Return concise call chain summary instead of full Document objects + # to avoid blowing up the agent's context window with source code. + path_summary = _summarize_call_chain(call_hierarchy_list) + return found_path, path_summary yield FunctionInfo.from_fn( _arun, description=(""" Checks if a function from a package is reachable from application code through the call chain. Make sure the input format is matching exactly one of the following formats: - + Input format 1: 'package_name,function_name'. Example 1: 'urllib,parse'. - - Input format 2(java): 'maven_gav,class_name.function_name'. - Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'. - - Returns: (is_reachable: bool, call_hierarchy_path: list). + + Input format 2(java): 'maven_gav,class_name.function_name' or 'maven_gav,fqcn.function_name' (preferred). + Example 2(java): 'commons-beanutils:commons-beanutils:x.y.z,a.b.c.ClassA.foo'. + + Returns: (is_reachable: bool, call_chain_path: list[str]). """)) @@ -146,7 +408,7 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, async def _arun(query: str) -> list: coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher(query) + transitive_code_searcher = await get_transitive_code_searcher(query) coc_retriever = transitive_code_searcher.chain_of_calls_retriever function_name_extractor = FunctionNameExtractor(coc_retriever) result = function_name_extractor.fetch_list(query) @@ -164,7 +426,7 @@ async def _arun(query: str) -> list: @register_function(config_type=PackageAndFunctionLocatorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def package_and_function_locator(config: PackageAndFunctionLocatorToolConfig, - builder: Builder): # pylint: disable=unused-argument + builder: Builder): # pylint: disable=unused-argument """ Function Locator tool used to validate package names and find function names using fuzzy matching. Mandatory first step for code path analysis. @@ -172,12 +434,15 @@ async def package_and_function_locator(config: PackageAndFunctionLocatorToolConf @catch_tool_errors(PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME) async def _arun(query: str) -> dict: + is_valid, validation_result = _validate_query_format(query) + if not is_valid: + return {"error": validation_result} coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = get_transitive_code_searcher(query) + transitive_code_searcher = await get_transitive_code_searcher(validation_result) coc_retriever = transitive_code_searcher.chain_of_calls_retriever locator = FunctionNameLocator(coc_retriever) - result = await locator.locate_functions(query) + result = await locator.locate_functions(validation_result) pkg_msg = "Package is valid." if not locator.is_package_valid and not locator.is_std_package: pkg_msg = "Package is not valid." @@ -197,8 +462,67 @@ async def _arun(query: str) -> dict: Input format 1: 'package_name,function_name' or 'package_name,class_name.method_name'. Example 1: 'libxml2,xmlParseDocument'. - Input format 2(java): 'maven_gav,class_name.method_name'. - Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'. + Input format 2(java): 'maven_gav,class_name.method_name' or 'maven_gav,fqcn.method_name' (preferred). + Example 2(java): 'commons-beanutils:commons-beanutils:x.y.z,a.b.c.ClassA.foo'. Returns: {'ecosystem': str, 'package_msg': str, 'result': [function_names]}. """)) + + +@register_function(config_type=FunctionLibraryVersionFinderToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def library_version_finder(config: FunctionLibraryVersionFinderToolConfig, + builder: Builder): # pylint: disable=unused-argument + """ + Function Library Version Finder tool used to check which version of a package/library + is installed in the application's dependency tree. + """ + + @catch_tool_errors(FUNCTION_LIBRARY_VERSION_FINDER_TOOL_NAME) + async def _arun(query: str) -> dict: + transitive_code_searcher = await get_transitive_code_searcher(query) + coc_retriever = transitive_code_searcher.chain_of_calls_retriever + + # Clean the query: strip whitespace, trailing junk after newlines, then quotes (including unicode smart quotes) + cleaned_query = query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() + # Search for matching packages in the dependency tree + search_term = cleaned_query.lower() + matching_packages = [] + for package in coc_retriever.supported_packages: + # Match against full GAV or individual segments + # Java GAV format: "groupId:artifactId:version" + package_lower = package.lower() + parts = package_lower.split(":") + if (search_term in package_lower or + any(search_term == part for part in parts) or + any(search_term in part for part in parts)): + matching_packages.append(package) + + if not matching_packages: + return { + "ecosystem": coc_retriever.ecosystem.name, + "found": False, + "message": f"No package matching '{cleaned_query}' found in the dependency tree.", + "matching_packages": [] + } + + return { + "ecosystem": coc_retriever.ecosystem.name, + "found": True, + "message": f"Found {len(matching_packages)} matching package(s) in the dependency tree.", + "matching_packages": matching_packages + } + + yield FunctionInfo.from_fn( + _arun, + description=(""" + Finds the exact version of a library/package installed in the application's dependency tree. + Use this tool to determine the EXACT version of a library before interpreting version-specific + information from CVE Web Search results. + + Input: library or package name to search for. + Example (java): 'commons-beanutils' + Example (python): 'requests' + + Returns: {'ecosystem': str, 'found': bool, 'message': str, 'matching_packages': list}. + For Java, matching_packages contains Maven GAV coordinates like 'groupId:artifactId:version'. +""")) \ No newline at end of file diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 3ed21b159..ff6d83b9d 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -115,7 +115,8 @@ async def generate_checklist(prompt: str | None, llm: BaseLanguageModel, input_dict: dict, tool_names: list[str] | None = None, - enable_llm_list_parsing: bool = False) -> str: + enable_llm_list_parsing: bool = False, + ecosystem: str = "") -> str: from vuln_analysis.utils.prompting import build_tool_descriptions @@ -133,21 +134,36 @@ async def generate_checklist(prompt: str | None, else: tool_descriptions = "Analysis tools will be used to investigate these questions." + if ecosystem.lower() == "java": + version_guidance = ( + "\n- Verify the installed library version matches the vulnerable version range " + "using available tools" + ) + function_inference = ( + "\n- If no specific function named, infer the entry point method from the attack " + "vector and library purpose, and name it in the first checklist item" + ) + else: + version_guidance = ( + "\n- Vulnerable package version is already confirmed installed; focus on other " + "exploitability factors" + ) + function_inference = "" + # Add tool_descriptions to input_dict for Jinja2 rendering - # This treats it as a Jinja2 variable, consistent with all CVE fields input_dict_with_tools = { **input_dict, 'tool_descriptions': tool_descriptions } - + intel = ( additional_intel_prompting + "\n" "\n\n" "\n- If CVE describes a vulnerable function/method, first checklist item MUST " "check if code calls it" - "\n- Vulnerable package version is already confirmed installed; focus on other " - "exploitability factors" + + function_inference + + version_guidance + "\n- Each item must be answerable with available analysis tools (code/doc search, " "dependency checks)" "\n- Use specific technical names from CVE details (functions, components, configurations)" @@ -155,10 +171,29 @@ async def generate_checklist(prompt: str | None, "\n" "\n\nGenerate checklist:" ) - + cve_prompt1 = (prompt + intel) + + # For Java, replace MOD_FEW_SHOT content priorities with Java-specific guidance + if ecosystem.lower() == "java": + _original_content = ( + "If the CVE mentions a specific vulnerable function or method in a given package or library, the FIRST\n" + " checklist item must verify whether that function in that package or library is called or imported\n" + " in the codebase - function should be specified together with the package name,\n" + " for example : 'Is the function1 function from the package1 package called in the codebase?'" + ) + _java_content = ( + "If the CVE mentions a specific vulnerable function or method, the FIRST checklist item must use\n" + " the EXACT function/method name from the CVE description (preserve Class.method format if present).\n" + " Example: 'Is the Class.method function from the package called in the codebase?'\n" + " - If NO specific function is named, infer the entry point method from the attack vector and library\n" + " purpose (e.g., serialization library + input manipulation → fromXML/parse/unmarshal). Always\n" + " name the specific method in the first checklist item." + ) + cve_prompt1 = cve_prompt1.replace(_original_content, _java_content) + try: - # Jinja2 renders {tool_descriptions} along with all CVE fields + # Jinja2 renders {tool_descriptions} and {version_guidance} along with all CVE fields format_cve_intel = await format_jinja_prompt(cve_prompt1, input_dict_with_tools) gen_checklist = await llm.ainvoke(format_cve_intel) diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 3f09349bf..0a02ab991 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re from pathlib import Path @@ -84,6 +85,13 @@ def replace_quoted(match): return input_query +ECOSYSTEM_DEP_DIRS = { + "c": "rpm_libs/", + "go": "vendor/", + "javascript": "node_modules/", + "python": "transitive_env/", + "java": "dependencies-sources/", +} class FullTextSearch: INDEX_TYPE = "tantivy" @@ -120,7 +128,7 @@ def add_documents(self, documents: Iterable): def is_empty(self): return self.index.searcher().num_docs == 0 - def search_index(self, query: str, top_k: int = 10) -> list[dict] | str: + def search_index(self, query: str, top_k: int = 10) -> str: self.index.reload() try: @@ -129,10 +137,41 @@ def search_index(self, query: str, top_k: int = 10) -> list[dict] | str: query = clean_query(query) query, _ = self.index.parse_query_lenient(query) searcher = self.index.searcher() - results = searcher.search(query, limit=top_k).hits - return [{ - "source": searcher.doc(doc_id)["file_path"][0], "content": searcher.doc(doc_id)["content"][0] - } for _, doc_id in results] + results = searcher.search(query, limit=50).hits + app_docs = [] + dep_docs = [] + + + vendors_list = list(ECOSYSTEM_DEP_DIRS.values()) + for _, doc_id in results: + raw = searcher.doc(doc_id) + doc = {"source": raw["file_path"][0], "content": raw["content"][0]} + if any(doc["source"].startswith(vendor) for vendor in vendors_list): + dep_docs.append(doc) + else: + app_docs.append(doc) + + total_app = len(app_docs) + total_dep = len(dep_docs) + + if total_app == 0 and total_dep == 0: + return "[]" + + app_docs = app_docs[:top_k] + remaining = top_k - len(app_docs) + dep_docs = dep_docs[:max(remaining, 0)] + + app_header = f"Main application ({len(app_docs)} of {total_app} results)" + dep_header = f"Application library dependencies ({len(dep_docs)} of {total_dep} results)" + + parts = [app_header] + if app_docs: + parts.append(json.dumps(app_docs)) + parts.append(dep_header) + if dep_docs: + parts.append(json.dumps(dep_docs)) + + return "\n".join(parts) except Exception as e: logger.exception(e) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 13f3adbc8..047113396 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -22,6 +22,9 @@ from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager +from exploit_iq_commons.data_models.input import SBOMPackage + logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") @@ -39,8 +42,27 @@ def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): self.stdlib_cache = StandardLibraryCache.get_instance() self.short_go_package_name = {} if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: - self.short_go_package_name = self.build_short_go_package_name() + self.short_go_package_name = self.build_short_go_package_name() + self.sbom_dict = self.build_sbom_dict() + def build_sbom_dict(self) -> dict: + sbom_dict = {} + for package in RPMDependencyManager.get_instance().sbom: + sbom_dict[package.name] = package.version + return sbom_dict + def check_package_in_sbom(self, package: str) -> tuple[bool, str]: + if package in self.sbom_dict: + return True, self.sbom_dict[package] + else: + return False, None + + def check_fuzzy_match_in_sbom(self, package: str) -> tuple[bool, str]: + fuzzy_matches = difflib.get_close_matches(package, self.sbom_dict.keys(), n=3, cutoff=0.6) + if fuzzy_matches: + return True, ", ".join(fuzzy_matches) + else: + return False, None + def build_short_go_package_name(self) -> dict: short_go_package_name = {} for package in self.coc_retriever.supported_packages: @@ -48,6 +70,56 @@ def build_short_go_package_name(self) -> dict: short_go_package_name[short_name] = package return short_go_package_name + def _resolve_go_fqdn_to_module(self, package: str) -> str | None: + """Find the longest supported_package that is a prefix of the input package.""" + best_match = None + for supported_package in self.coc_retriever.supported_packages: + if package.lower().startswith(supported_package.lower() + "/") or \ + package.lower() == supported_package.lower(): + if best_match is None or len(supported_package) > len(best_match): + best_match = supported_package + return best_match + + @staticmethod + def _is_java_gav_version_mismatch(requested: str, match: str) -> bool: + """Check if requested and match are the same Java GAV but with different versions.""" + req_parts = requested.split(":") + match_parts = match.split(":") + if len(req_parts) == 3 and len(match_parts) == 3: + return req_parts[0] == match_parts[0] and req_parts[1] == match_parts[1] and req_parts[2] != match_parts[2] + return False + + @staticmethod + def _find_java_gav_auto_resolve(requested: str, matches: list[str]) -> str | None: + """If requested is group:artifact (no version), find a match with the same group:artifact. + Returns the full GAV if exactly one match found, None otherwise.""" + req_parts = requested.split(":") + if len(req_parts) != 2: + return None + resolved = [m for m in matches if m.startswith(requested + ":") and len(m.split(":")) == 3] + if len(resolved) == 1: + return resolved[0] + return None + + def _build_java_package_error(self, package: str, close_matches: list[str]) -> str: + """Build error message for Java when package is not found (after auto-resolve already failed). + Detects version mismatches and returns a CONCLUSION or generic error.""" + version_mismatches = [m for m in close_matches if self._is_java_gav_version_mismatch(package, m)] + if version_mismatches: + installed_versions = ', '.join(version_mismatches) + return ( + f"CONCLUSION: The requested version '{package}' is NOT installed. " + f"The installed version is: {installed_versions}. " + f"Since the vulnerable version is not present, the application is NOT affected by this CVE. " + f"Do NOT search with the installed version — report that the vulnerable version is absent." + ) + return ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: JAVA" + ) + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: logger.info("package %s is not in supported packages", package) if self.coc_retriever.supported_packages is None: @@ -105,6 +177,31 @@ def python_flow_control(self, input_function: str, package_docs) -> list[str]: close_matches = difflib.get_close_matches(input_function, list_of_matching_combinations, n=10, cutoff=0.3) return close_matches + def java_flow_control(self, input_function: str, package_docs) -> list[str]: + """ + Java flow control - uses fuzzy matching on ClassName.methodName candidates. + Handles class queries, method queries, and full ClassName.method queries uniformly. + """ + candidates = [] + for doc in package_docs: + if self.lang_parser: + cn = self.lang_parser.get_class_name_from_class_function(doc) + fn = self.lang_parser.get_function_name(doc) + if cn and fn: + simple_cn = cn.rsplit(".", 1)[-1] + candidates.append(f"{simple_cn}.{fn}") + candidates = list(dict.fromkeys(candidates)) + + matches = difflib.get_close_matches(input_function, candidates, n=10, cutoff=0.6) + if not matches: + matches = difflib.get_close_matches(input_function, candidates, n=10, cutoff=0.3) + if matches: + matches.append( + "INFO: For Call Chain Analyzer, use ClassName.methodName format " + "(e.g. PropertyUtilsBean.getProperty), not just the class name." + ) + return matches + def common_flow_control(self,input_function: str, package_docs) -> list[str]: """ C/Go flow control have the same logic for finding close matches for function names in the package @@ -138,11 +235,16 @@ def search_in_third_party_packages(self, package: str) -> bool: Returns: True if package is found in third party packages, False otherwise """ - + def is_same_package(package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() == package_name_from_tree.lower() # Check in supported_packages list (handles None case) long_path = False - if self.coc_retriever.supported_packages is not None: - long_path = package in self.coc_retriever.supported_packages + + for supported_package in self.coc_retriever.supported_packages: + if is_same_package(package, supported_package): + long_path = True + break + # Check in short_go_package_name dict (keys) short_path = package in (self.short_go_package_name or {}) @@ -150,6 +252,10 @@ def search_in_third_party_packages(self, package: str) -> bool: return True if short_path: return True + if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + fqdn_match = self._resolve_go_fqdn_to_module(package) + if fqdn_match: + return True return False async def locate_functions(self, query: str) -> list[str]: @@ -183,11 +289,35 @@ async def locate_functions(self, query: str) -> list[str]: function = parts_input[1] try: + is_error = False logger.info("Package and Function Locator: searching for (%s,%s)", package, function) # First validate package name exists in supported packages if (not self.search_in_third_party_packages(package)): logger.info("Package '%s' not found in supported packages", package) + + in_sbom, sbom_version = self.check_package_in_sbom(package) + if in_sbom: + self.is_package_valid = True + logger.info("Package '%s' (version %s) found in SBOM but no source code available", package, sbom_version) + return [ + ( + f"INFO: Package '{package}' (version {sbom_version}) is present in the container SBOM, " + f"however no source code is available so package code content and function names cannot be checked." + ) + ] + + fuzzy_in_sbom, fuzzy_matches = self.check_fuzzy_match_in_sbom(package) + if fuzzy_in_sbom: + logger.info("Package '%s' not found exactly in SBOM, but fuzzy matches found: %s", package, fuzzy_matches) + return [ + ( + f"WARNING: Package '{package}' was NOT found in the container. " + f"Similar SBOM packages exist: {fuzzy_matches}. The function name could NOT be verified. Do NOT treat this as a confirmed match." + + ) + ] + is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) if is_standard_lib: self.is_std_package = True @@ -199,36 +329,52 @@ async def locate_functions(self, query: str) -> list[str]: ) ] else: - is_standard_lib_api = await quick_standard_lib_check(package, self.coc_retriever.ecosystem) - if is_standard_lib_api: - self.is_std_package = True - self.is_package_valid = True - self.stdlib_cache.add_to_cache(package, self.coc_retriever.ecosystem) - return [ - ( - f"INFO: Package '{package}' is a standard library package. " - f"make call with the Transitive code search tool" - )] + if (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value != Ecosystem.JAVA.value): + is_standard_lib_api,is_error = await quick_standard_lib_check(package, self.coc_retriever.ecosystem) + if is_standard_lib_api: + self.is_std_package = True + self.is_package_valid = True + self.stdlib_cache.add_to_cache(package, self.coc_retriever.ecosystem) + return [ + ( + f"INFO: Package '{package}' is a standard library package. " + f"make call with the Transitive code search tool" + )] close_package_matches = self.handle_package_not_in_supported_packages(package) + error_msg = "" if close_package_matches: - error_msg = ( - f"ERROR: Package '{package}' not found in available packages. " - f"Close matches are: {', '.join(close_package_matches)}. " - f"Please use one of the suggested package names. " - f"Available ecosystem: " - f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" - ) + is_java = (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.JAVA.value) + if is_java: + resolved_package = self._find_java_gav_auto_resolve(package, close_package_matches) + if resolved_package: + logger.info("Auto-resolved '%s' to '%s'", package, resolved_package) + package = resolved_package + self.is_package_valid = True + else: + error_msg = self._build_java_package_error(package, close_package_matches) + else: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) else: - error_msg = ( - f"ERROR: Package '{package}' not found in available packages. " - f"No close matches found. " - f"Available ecosystem: " - f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" - ) - logger.error(error_msg) - return [error_msg] # Return error message that LLM can see - # Fallback: on cache miss, sparsely verify via API - + if not is_error: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"No close matches found. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) + else: + error_msg = "UNKNOWN could not determine if package is standard library" + if not self.is_package_valid: + logger.error(error_msg) + return [error_msg] # Package is valid, proceed with function search self.is_package_valid = True @@ -240,6 +386,10 @@ async def locate_functions(self, query: str) -> list[str]: # package_docs = self.coc_retriever.documents_of_functions if package in self.short_go_package_name: package = self.short_go_package_name[package] + elif self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + resolved = self._resolve_go_fqdn_to_module(package) + if resolved: + package = resolved package_docs = [ doc for doc in (self.coc_retriever.documents_of_functions or []) @@ -249,6 +399,14 @@ async def locate_functions(self, query: str) -> list[str]: ) ] + if not package_docs: + logger.warning("Package '%s' is valid but no source code is indexed", package) + return [ + f"INFO: Package '{package}' is in the dependency tree but no source code is indexed for it. " + f"Function names cannot be verified. Proceed directly with Call Chain Analyzer using " + f"the function name from the CVE description." + ] + match self.coc_retriever.ecosystem.value: case Ecosystem.PYTHON.value: return self.python_flow_control(function, package_docs) @@ -257,7 +415,7 @@ async def locate_functions(self, query: str) -> list[str]: case Ecosystem.GO.value: return self.common_flow_control(function, package_docs) case Ecosystem.JAVA.value: - return self.common_flow_control(function.rsplit(".")[-1], package_docs) + return self.java_flow_control(function, package_docs) case _: return self.common_flow_control(function, package_docs) @@ -281,7 +439,7 @@ async def locate_functions(self, query: str) -> list[str]: logger.error("Error locating functions in package '%s': %s", package, e) return [] -async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> bool: +async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> tuple[bool, bool]: """Quick check if package is standard library Args: package_name: The package name to check @@ -289,23 +447,25 @@ async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> b Returns: True if package is standard library, False otherwise """ + try: + search = MorpheusSerpAPIWrapper(max_retries=2) + result = await search.arun(f"Is '{package_name}' part of the {ecosystem.value} standard library?") + logger.info("quick_standard_lib_check Standard library check result: %s", result) + # Normalize result: if list, join into single string + if isinstance(result, list): + text = " ".join(result).lower() + elif isinstance(result, str): + text = result.lower() + else: + text = str(result).lower() - search = MorpheusSerpAPIWrapper(max_retries=2) - result = await search.arun(f"Is '{package_name}' part of the {ecosystem.value} standard library?") - logger.info("quick_standard_lib_check Standard library check result: %s", result) -# Normalize result: if list, join into single string - if isinstance(result, list): - text = " ".join(result).lower() - elif isinstance(result, str): - text = result.lower() - else: - text = str(result).lower() - - # Basic positive signals and avoidance of negative phrasing - if "part of the standard library" in text or \ - "is a standard library" in text or \ - ("standard library" in text and "not" not in text): - return True - - return False - + # Basic positive signals and avoidance of negative phrasing + if "part of the standard library" in text or \ + "is a standard library" in text or \ + ("standard library" in text and "not" not in text): + return True, False + + return False, False + except Exception as e: + logger.error("Error checking if package is standard library: %s", e) + return False, True diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index ef7e7efb4..32e6942f2 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from packaging.version import InvalidVersion from packaging.version import parse as parse_version from pydpkg import Dpkg @@ -20,6 +22,10 @@ from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + def update_version(incoming_version, current_version, compare): """ @@ -154,3 +160,232 @@ def parse(configurations: list): version_info.append(obj) return version_info + + +# --------------------------------------------------------------------------- +# Critical-context helpers (used by pre_process_node in cve_agent.py) +# --------------------------------------------------------------------------- + +def build_critical_context(cve_intel_list) -> tuple[list[str], list[dict], list[str]]: + """Extract key facts from all available intel sources into a compact context. + + Returns (critical_context, candidate_packages, vulnerable_functions) where + candidate_packages contains dicts with 'name', 'source', and optional + 'ecosystem' keys, and vulnerable_functions is a deduplicated list of short + function names from GHSA. + """ + critical_context = [] + candidate_packages: list[dict] = [] + seen_packages: set[str] = set() + vulnerable_functions: set[str] = set() + + for cve_intel in cve_intel_list: + if cve_intel.nvd is not None: + if cve_intel.nvd.cve_description: + critical_context.append(f"CVE Description: {cve_intel.nvd.cve_description[:400]}") + if cve_intel.nvd.cwe_name: + critical_context.append(f"CWE: {cve_intel.nvd.cwe_name}") + + if cve_intel.ghsa is not None: + if cve_intel.ghsa.vulnerabilities: + for v in cve_intel.ghsa.vulnerabilities[:3]: + vuln = v if isinstance(v, dict) else (v.__dict__ if hasattr(v, '__dict__') else {}) + vf = vuln.get('vulnerable_functions', []) if isinstance(vuln, dict) else getattr(v, 'vulnerable_functions', []) + pkg = vuln.get('package', None) if isinstance(vuln, dict) else getattr(v, 'package', None) + + if vf: + critical_context.append(f"Vulnerable functions (GHSA): {', '.join(vf)}") + short_names = [f.rsplit('.', 1)[-1] for f in vf if '.' in f] + if short_names: + critical_context.append(f"Search keywords: {', '.join(short_names)}") + for f in vf: + vulnerable_functions.add(f.rsplit('.', 1)[-1]) + if pkg: + if isinstance(pkg, dict): + pkg_name = pkg.get("name", "") + pkg_eco = pkg.get("ecosystem", "") + if pkg_name: + critical_context.append(f"Vulnerable module ({pkg_eco}): {pkg_name}") + if pkg_name not in seen_packages: + seen_packages.add(pkg_name) + candidate_packages.append({"name": pkg_name, "source": "ghsa", "ecosystem": pkg_eco}) + elif isinstance(pkg, str): + critical_context.append(f"Affected package: {pkg}") + if pkg not in seen_packages: + seen_packages.add(pkg) + candidate_packages.append({"name": pkg, "source": "ghsa"}) + if cve_intel.ghsa.description and not any("CVE Description" in c for c in critical_context): + critical_context.append(f"CVE Description: {cve_intel.ghsa.description[:400]}") + + if cve_intel.rhsa is not None: + if cve_intel.rhsa.statement: + critical_context.append(f"RHSA Statement: {cve_intel.rhsa.statement[:300]}") + mitigation = getattr(cve_intel.rhsa, 'mitigation', None) + if mitigation: + mit_text = mitigation.get('value', '') if isinstance(mitigation, dict) else str(mitigation) + if mit_text: + critical_context.append(f"KNOWN MITIGATIONS: {mit_text[:500]}") + if cve_intel.rhsa.package_state: + pkgs = list(set(p.package_name for p in cve_intel.rhsa.package_state if p.package_name)) + for p in pkgs: + if p not in seen_packages: + seen_packages.add(p) + candidate_packages.append({"name": p, "source": "rhsa"}) + if len(pkgs) > 5: + critical_context.append( + f"Affected across {len(pkgs)} Red Hat products (sample: {', '.join(pkgs[:5])}). " + "Focus investigation on the vulnerable library/module, not individual products." + ) + elif pkgs: + numbered = ", ".join(f"{i+1}) {p}" for i, p in enumerate(pkgs)) + critical_context.append(f"INVESTIGATE EACH package: {numbered}.") + + if cve_intel.ubuntu is not None: + if cve_intel.ubuntu.ubuntu_description: + critical_context.append(f"Ubuntu note: {cve_intel.ubuntu.ubuntu_description[:200]}") + + if cve_intel.plugin_data: + for pd in cve_intel.plugin_data[:2]: + critical_context.append(f"{pd.label}: {pd.description[:200]}") + + if not critical_context: + critical_context = ["No CVE intel available. Investigate using tools."] + return critical_context, candidate_packages, sorted(vulnerable_functions) + + +_GO_VULN_RE = re.compile(r"pkg\.go\.dev/vuln/(GO-\d{4}-\d+)") +_OSV_API_URL = "https://api.osv.dev/v1/vulns/" +_OSV_TIMEOUT_SECONDS = 5 + + +async def enrich_go_from_osv( + cve_intel, + critical_context: list[str], + candidate_packages: list[dict], + vulnerable_functions: set[str] | None = None, +) -> None: + """Query the OSV API for Go module paths when GHSA has no package data. + + Looks for a pkg.go.dev/vuln/GO-XXXX-XXXX link in GHSA or NVD + references, fetches the advisory from OSV, and injects module + paths and vulnerable symbols into critical_context and + candidate_packages. Fails silently on any error. + """ + refs: list[str] = [] + if cve_intel.ghsa is not None: + refs.extend(getattr(cve_intel.ghsa, "references", None) or []) + if cve_intel.nvd is not None: + refs.extend(cve_intel.nvd.references or []) + + go_id = None + for ref in refs: + m = _GO_VULN_RE.search(ref if isinstance(ref, str) else "") + if m: + go_id = m.group(1) + break + + if not go_id: + return + + try: + import aiohttp + timeout = aiohttp.ClientTimeout(total=_OSV_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(f"{_OSV_API_URL}{go_id}") as resp: + if resp.status != 200: + logger.warning("OSV lookup for %s returned status %d", go_id, resp.status) + return + osv_data = await resp.json() + + seen_paths: set[str] = set() + all_symbols: list[str] = [] + for affected in osv_data.get("affected", []): + eco_specific = affected.get("ecosystem_specific", {}) + for imp in eco_specific.get("imports", []): + path = imp.get("path", "") + if path and path not in seen_paths: + seen_paths.add(path) + candidate_packages.append( + {"name": path, "source": "osv", "ecosystem": "Go"}, + ) + critical_context.append(f"Vulnerable module (Go): {path}") + symbols = imp.get("symbols", []) + all_symbols.extend(symbols) + + if all_symbols: + critical_context.append( + f"Vulnerable functions (Go vuln DB): {', '.join(all_symbols)}" + ) + short_names = [s.rsplit(".", 1)[-1] for s in all_symbols if "." in s] + unique_keywords = list(dict.fromkeys(all_symbols + short_names)) + critical_context.append(f"Search keywords: {', '.join(unique_keywords)}") + if vulnerable_functions is not None: + for s in all_symbols: + vulnerable_functions.add(s.rsplit(".", 1)[-1]) + vulnerable_functions.add(s) + + if seen_paths: + logger.info("OSV enrichment for %s added Go modules: %s", go_id, seen_paths) + except Exception: + logger.warning("OSV enrichment failed for %s", go_id, exc_info=True) + + +# Characters treated as part of a package / module / coordinate token for boundary +# detection (avoids stripping short names like "com" from inside "github.com"). +_PACKAGE_TOKEN_CHARS = r"A-Za-z0-9_.\/:@+\-" + + +def _package_token_boundary_pattern(rn: str) -> str: + esc = re.escape(rn) + c = _PACKAGE_TOKEN_CHARS + return rf"(?:^|(? bool: + if not rn: + return False + return re.search(_package_token_boundary_pattern(rn), entry) is not None + + +def _strip_rejected_package_token(entry: str, rn: str) -> str: + if not rn: + return entry + return re.sub(_package_token_boundary_pattern(rn), "", entry) + + +def filter_context_to_package(critical_context: list[str], selected: str, all_candidates: list[dict]) -> list[str]: + """Narrow CVE intel strings to the user-selected package after disambiguation. + Intel lines are built from GHSA, RHSA... and may mention several modules; this removes + references to *other* candidates so downstream classification / reachability + prompts are not biased toward the wrong package. + + Behavior: + * Replaces the RHSA-style ``INVESTIGATE EACH package:`` line with + ``Target package: ``. + * Drops entire ``Vulnerable module (...)`` / ``Affected package:`` lines when a + *rejected* candidate appears as a **whole token** (not a substring inside a + longer path like ``com`` in ``github.com``). + * For all other lines, removes each rejected name wherever it appears as a + whole token, then collapses runs of whitespace. + + This is best-effort text cleanup, not full natural-language redaction. + """ + rejected_names = {c["name"] for c in all_candidates if c["name"] != selected} + # Longest names first so a shorter rejected string cannot leave a suffix of a longer one. + rejected_ordered = sorted(rejected_names, key=len, reverse=True) + filtered = [] + for entry in critical_context: + if entry.startswith("INVESTIGATE EACH package:"): + filtered.append(f"Target package: {selected}") + continue + if entry.startswith("Vulnerable module (") or entry.startswith("Affected package:"): + if any(_rejected_package_present_as_token(entry, rn) for rn in rejected_ordered): + continue + for rn in rejected_ordered: + entry = _strip_rejected_package_token(entry, rn) + # Token removal leaves gaps; normalize spaces, merge ", ," from list removals, then collapse again. + entry = re.sub(r"\s+", " ", entry).strip() + entry = re.sub(r",\s*,", ", ", entry) + entry = re.sub(r"\s+", " ", entry).strip() + filtered.append(entry) + return filtered diff --git a/src/vuln_analysis/utils/prompt_factory.py b/src/vuln_analysis/utils/prompt_factory.py new file mode 100644 index 000000000..84b5fc69d --- /dev/null +++ b/src/vuln_analysis/utils/prompt_factory.py @@ -0,0 +1,303 @@ +from vuln_analysis.utils.prompting import AGENT_SYS_PROMPT +from vuln_analysis.tools.tool_names import ToolNames + +# Structure: (tool_name, language) -> language-specific input format and call instructions +TOOL_ECOSYSTEM_REGISTRY: dict[tuple[str, str], str] = { + # Code Semantic Search + (ToolNames.CODE_SEMANTIC_SEARCH, "python"): ( + "Input: natural language query. Use Python identifiers: e.g. 'urllib.parse url handling', " + "'PIL.Image image processing', 'asyncio usage'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "go"): ( + "Input: natural language query. Use Go identifiers: e.g. 'encoding/json unmarshaling', " + "'net/http handler', 'context.Context propagation'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "java"): ( + "Input: natural language query. Use Java identifiers: e.g. 'ObjectInputStream deserialization', " + "'javax.servlet request handling', 'java.util collections'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "javascript"): ( + "Input: natural language query. Use JS identifiers: e.g. 'require() imports', " + "'express middleware', 'JSON.parse usage'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "c"): ( + "Input: natural language query. Use C/C++ identifiers: e.g. 'EVP_EncryptInit encryption', " + "'strcpy buffer handling', 'malloc memory allocation'." + ), + # Docs Semantic Search + (ToolNames.DOCS_SEMANTIC_SEARCH, "python"): ( + "Input: natural language question. Phrase for Python docs: e.g. 'How does this app handle image uploads?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "go"): ( + "Input: natural language question. Phrase for Go docs: e.g. 'How is HTTP routing configured?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "java"): ( + "Input: natural language question. Phrase for Java docs: e.g. 'Application architecture and dependencies'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "javascript"): ( + "Input: natural language question. Phrase for JS docs: e.g. 'How does the API handle requests?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "c"): ( + "Input: natural language question. Phrase for C/C++ docs: e.g. 'Cryptographic library usage'" + ), + # Code Keyword Search + (ToolNames.CODE_KEYWORD_SEARCH, "python"): ( + "Input: exact text. Python patterns: 'urllib.parse', 'from X import Y', 'import asyncio'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "go"): ( + "Input: exact text. Go patterns: 'import \"', 'pkg.FunctionName', 'encoding/json'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "java"): ( + "Input: exact text. Java patterns: 'import com.', 'ClassName.method', 'javax.servlet'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "javascript"): ( + "Input: exact text. JS patterns: 'require(', 'import { ', 'process.env'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "c"): ( + "Input: exact text. C/C++ patterns: 'EVP_EncryptInit_ex2', 'OSSL_PARAM', function names." + ), + # Function Locator + (ToolNames.FUNCTION_LOCATOR, "python"): ( + "Input: 'package_name,function_name'. Example: 'urllib,parse' or 'PIL.Image,open'. " + "Call first before Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "go"): ( + "Input: 'package_path,FunctionName'. Example: 'github.com/pkg/errors,New'. " + "Call first before Function Caller Finder or Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "java"): ( + "Input format 2(java): 'maven_gav,class_name.method_name'." + "Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'." + ), + (ToolNames.FUNCTION_LOCATOR, "javascript"): ( + "Input: 'package_name,function_name'. Example: 'express,Router' or npm scope paths. " + "Call first before Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "c"): ( + "Input: 'library_name,function_name'. Example: 'libxml2,xmlParseDocument' or 'openssl,EVP_EncryptInit_ex2'. " + "Call first before Call Chain Analyzer." + ), + # Call Chain Analyzer + (ToolNames.CALL_CHAIN_ANALYZER, "python"): ( + "Input: use validated 'package,function' from Function Locator. Example: 'urllib,parse'." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "go"): ( + "Input format 1: 'package_name,function_name' Example: 'github.com/pkg/errors,New'." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "java"): ( + "Input: use validated maven_gav,class.method from Function Locator." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "javascript"): ( + "Input: use validated package,function from Function Locator." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "c"): ( + "Input format 1: 'package_name,function_name'. Example: 'openssl,EVP_EncryptInit_ex2'." + ), + # Function Caller Finder (Go only) + (ToolNames.FUNCTION_CALLER_FINDER, "go"): ( + "Input: 'package_name,lib.function(args)'. Example: 'github.com/myapp,errors.New(\"text\")'. " + "Go-only. Finds callers of std/lib functions." + ), + # CVE Web Search (language-agnostic) + (ToolNames.CVE_WEB_SEARCH, "python"): " vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "go"): " vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "java"): "vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "javascript"): "vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "c"): "vulnerability search terms.", + # Container Analysis Data (language-agnostic) + (ToolNames.CONTAINER_ANALYSIS_DATA, "python"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "go"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "java"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "javascript"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "c"): "Input: query string for pre-analyzed container findings.", +} + +# Optional: Add language-specific usage hints +FEW_SHOT_EXAMPLES: dict[str, str] = { + "python": "Example: Function Locator input 'urllib,parse'; Code Keyword Search for 'urllib.parse'.", + "go": "Example: Function Caller Finder 'pkg,errors.New(\"x\")'; Function Locator 'github.com/pkg,New'.", + "java": "Example: Function Locator 'group:artifact:ver,Class.method'; Code Keyword Search 'import com.'.", + "javascript": "Example: Code Keyword Search 'require('; Function Locator 'package_name,exported'.", + "c": "Example: Function Locator 'openssl,EVP_EncryptInit_ex2'; Code Keyword Search 'EVP_aes_'.", +} + +# Per-language strategy for when to use which tools +TOOL_SELECTION_STRATEGY: dict[str, str] = { + "python": ( + "Use Code Keyword Search first for exact import/function lookups (e.g. urllib.parse, PIL.Image). " + "For reachability: call Function Locator first with package,function (e.g. urllib,parse), then Call Chain Analyzer with validated names. " + "Use Docs Semantic Search for architecture questions; Code Semantic Search for 'how is X used' patterns." + ), + "go": ( + "For std/lib function reachability: call Function Locator first to validate package paths (e.g. github.com/pkg/errors,New), " + "then use Function Caller Finder to find which app functions call the library method, then Call Chain Analyzer with validated names. " + "Use Code Keyword Search for import paths; Docs/Code Semantic Search for handler and middleware patterns." + ), + "java": ( + "Use Function Locator first with maven GAV format (group:artifact:version,ClassName.methodName). " + "For reachability, use Call Chain Analyzer with validated names from Function Locator. " + "Use Code Keyword Search for import com. and javax. patterns; Docs Semantic Search for Spring/servlet architecture." + ), + "javascript": ( + "Use Code Keyword Search first for require(, import {, and package patterns. " + "For reachability: Function Locator with package_name,exported, then Call Chain Analyzer. " + "Use Docs/Code Semantic Search for middleware, API, and npm package usage patterns." + ), + "c": ( + "Use Code Keyword Search first for exact function name lookups (e.g. PQescapeLiteral, EVP_EncryptInit_ex2). " + "For reachability: call Function Locator first with library_name,function_name (e.g. libpq,PQescapeLiteral) to validate the package and function, " + "then Call Chain Analyzer with validated names to check if it is reachable from application code." + ), +} + +TOOL_SELECTION_STRATEGY_NON_REACHABILITY: dict[str, str] = { + "python": ( + "Use Code Keyword Search first for exact import/function lookups " + "(e.g. urllib.parse, PIL.Image). " + "Pay attention to whether results appear in the main application code " + "or in package dependencies (transitive_env/). " + "Use Function Locator to validate package and function names " + "(e.g. urllib,parse). " + "Use Code Semantic Search to understand how a component is used; " + "Docs Semantic Search for architecture questions. " + "Use CVE Web Search for additional vulnerability context." + ), + "go": ( + "Use Code Keyword Search for import paths and function names. " + "Pay attention to whether results appear in the main application code " + "or in vendor/ dependencies. " + "Use Function Locator to validate package paths and function names " + "(e.g. github.com/pkg/errors,New). " + "Use Code Semantic Search to understand usage patterns; " + "Docs Semantic Search for architecture. " + "Use CVE Web Search for additional vulnerability context." + ), + "java": ( + "Use Code Keyword Search for import statements and class/method patterns " + "(e.g. import com., javax.servlet). " + "Pay attention to whether results appear in the main application code " + "or in library dependencies. " + "Use Function Locator with maven GAV format to validate names " + "(e.g. group:artifact:version,ClassName.methodName). " + "Use Docs Semantic Search for Spring/servlet architecture; " + "Code Semantic Search for usage patterns. " + "Use CVE Web Search for additional vulnerability context." + ), + "javascript": ( + "Use Code Keyword Search first for require(, import {, and package patterns. " + "Pay attention to whether results appear in the main application code " + "or in node_modules/ dependencies. " + "Use Function Locator to validate package and function names " + "(e.g. lodash,defaultsDeep). " + "Use Code/Docs Semantic Search for middleware, API, and npm package " + "usage patterns. " + "Use CVE Web Search for additional vulnerability context." + ), + "c": ( + "Use Code Keyword Search first for exact function name lookups " + "(e.g. PQescapeLiteral, EVP_EncryptInit_ex2). " + "Pay attention to whether results appear in the main application code " + "or in rpm_libs/ dependencies. " + "Use Function Locator to validate library and function names " + "(e.g. libpq,PQescapeLiteral). " + "Use Code Semantic Search to understand usage patterns; " + "Docs Semantic Search for library documentation. " + "Use CVE Web Search for additional vulnerability context." + ), +} + +TOOL_GENERAL_DESCRIPTIONS: dict[str, str] = { + ToolNames.CODE_SEMANTIC_SEARCH: "Searches container source code using semantic search. " + "Finds how functions, libraries, or components are used in the codebase. " + "Answers questions about code implementation and dependencies.", + ToolNames.DOCS_SEMANTIC_SEARCH: "Searches container documentation using semantic search. " + "Answers questions about application purpose, architecture, and features.", + ToolNames.CODE_KEYWORD_SEARCH: "Performs keyword search on container source code for exact text matches. " + "Input should be a function name, class name, or code pattern. " + "Use this first before semantic search tools for precise lookups.", + ToolNames.FUNCTION_LOCATOR: "Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, and provides ecosystem type" + "Make sure the input format is matching exactly one of the following formats:", + + ToolNames.CALL_CHAIN_ANALYZER: "Checks if a function from a package is reachable from application code through the call chain." + "Make sure the input format is matching exactly one of the following formats:", + ToolNames.FUNCTION_CALLER_FINDER: "Finds functions in a package that call a specific library function.", + ToolNames.CVE_WEB_SEARCH: "Searches the web for information about CVEs, vulnerabilities, libraries, " + "and security advisories not available in the container.", + ToolNames.CONTAINER_ANALYSIS_DATA: "Retrieves pre-analyzed container image source code analysis results for the current CVE. " + "Returns a list of findings with 'response' (summary) and 'intermediate_steps' (reasoning) fields. " + "No input required - uses context automatically.", +} + + +class PromptFactory: + def __init__( + self, + template: str, + registry: dict, + examples: dict, + general_descriptions: dict[str, str], + tool_selection_strategy: dict[str, str], + ): + self.template = template + self.registry = registry + self.examples = examples + self.general_descriptions = general_descriptions + self.tool_selection_strategy = tool_selection_strategy + + def build_prompt( + self, + sys_prompt: str, + enabled_tools: list[str], + language: str, + tool_instructions: str | None = None, + ) -> str: + # Build AVAILABLE_TOOLS in format: tool_name(*args, **kwargs) - description + available_tools_lines = [] + for tool_name in enabled_tools: + desc = self.general_descriptions.get(tool_name, "Standard analysis tool.") + available_tools_lines.append(f"{tool_name}(*args, **kwargs) - {desc}") + tools_section = "\n".join(available_tools_lines) + + if tool_instructions is None: + # Fallback: build from language-specific registry + tool_instructions = "" + + #lang_examples = self.examples.get(language, "") + #disable examples for now + lang_examples = "" + tool_selection_strategy = self.tool_selection_strategy.get( + language, f"Optimize for {language} ecosystem patterns." + ) + + return self.template.format( + sys_prompt=sys_prompt, + tools=tools_section, + tool_selection_strategy=tool_selection_strategy, + tool_instructions=f"{tool_instructions}\n\n{lang_examples}" + ) + +# --- Usage --- +LANGGRAPH_SYSTEM_PROMPT_TEMPLATE = """{sys_prompt} + +Answer the investigation question using the available tools. If the input is not a question, formulate it into a question first. A Tool Selection Strategy is provided to help you decide which tools to use. Focus on answering the question. Include your intermediate reasoning in the final answer. + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +factory = PromptFactory( + LANGGRAPH_SYSTEM_PROMPT_TEMPLATE, + TOOL_ECOSYSTEM_REGISTRY, + FEW_SHOT_EXAMPLES, + TOOL_GENERAL_DESCRIPTIONS, + TOOL_SELECTION_STRATEGY, +) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index 45837e0c3..0f6fdfd10 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -77,6 +77,12 @@ def build_tool_descriptions(tool_names: list[str]) -> list[str]: descriptions.append( f"{ToolNames.CONTAINER_ANALYSIS_DATA}: Retrieves findings from earlier container scan analysis" ) + + if ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in tool_names: + descriptions.append( + f"{ToolNames.FUNCTION_LIBRARY_VERSION_FINDER}: Finds installed version of a library/package. " + f"Input: package name (e.g., 'commons-beanutils'). Returns matching packages with versions from the dependency tree" + ) return descriptions diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py index 9a939aa52..0eb30de51 100644 --- a/tests/test_base_tool_descriptions.py +++ b/tests/test_base_tool_descriptions.py @@ -22,9 +22,9 @@ def test_base_returns_list(): """Test that base function returns a list, not a string.""" tool_names = [ToolNames.CODE_SEMANTIC_SEARCH] - + result = build_tool_descriptions(tool_names) - + assert isinstance(result, list) print("✓ Base function returns list") @@ -35,9 +35,9 @@ def test_base_descriptions_format(): ToolNames.CODE_SEMANTIC_SEARCH, ToolNames.CODE_KEYWORD_SEARCH ] - + result = build_tool_descriptions(tool_names) - + # Each description should have format: "Tool Name: Description" for desc in result: assert ":" in desc @@ -45,7 +45,7 @@ def test_base_descriptions_format(): assert len(parts) == 2 assert len(parts[0].strip()) > 0 # Tool name assert len(parts[1].strip()) > 0 # Description - + print("✓ Base descriptions have consistent format") @@ -58,14 +58,15 @@ def test_base_all_tools(): ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER, ToolNames.CVE_WEB_SEARCH, - ToolNames.CONTAINER_ANALYSIS_DATA + ToolNames.CONTAINER_ANALYSIS_DATA, + ToolNames.FUNCTION_LIBRARY_VERSION_FINDER ] - + result = build_tool_descriptions(tool_names) - + # Should have 7 descriptions assert len(result) == 7 - + # Verify all tools are present all_text = " ".join(result) assert "Code Semantic Search" in all_text @@ -75,19 +76,20 @@ def test_base_all_tools(): assert "Function Caller Finder" in all_text assert "CVE Web Search" in all_text assert "Container Analysis Data" in all_text - + assert "Function Library Version Finder" in all_text + print("✓ Base function includes all tools") def test_base_empty_list(): """Test that base function returns empty list when no tools.""" tool_names = [] - + result = build_tool_descriptions(tool_names) - + assert result == [] assert isinstance(result, list) - + print("✓ Base function returns empty list for no tools") @@ -97,29 +99,29 @@ def test_checklist_formats_descriptions(): ToolNames.CODE_SEMANTIC_SEARCH, ToolNames.DOCS_SEMANTIC_SEARCH ] - + # Simulate what checklist_prompt_generator.py does tool_descs = build_tool_descriptions(tool_names) - + if tool_descs: formatted_descs = ["- " + desc for desc in tool_descs] tool_descriptions = "The following tools can be used to answer checklist questions:\n " + "\n ".join(formatted_descs) else: tool_descriptions = "Analysis tools will be used to investigate these questions." - + # Verify checklist formatting assert "The following tools can be used to answer checklist questions:" in tool_descriptions assert "Code Semantic Search" in tool_descriptions assert "Docs Semantic Search" in tool_descriptions assert " - " in tool_descriptions # Indented bullet points - + print("✓ Checklist formats descriptions correctly") def test_mod_few_shot_structure(): """Test that MOD_FEW_SHOT has required structure and placeholders.""" from vuln_analysis.utils.prompting import MOD_FEW_SHOT - + # Check for required section markers assert "" in MOD_FEW_SHOT assert "" in MOD_FEW_SHOT @@ -128,28 +130,59 @@ def test_mod_few_shot_structure(): assert "" in MOD_FEW_SHOT assert "" in MOD_FEW_SHOT assert "" in MOD_FEW_SHOT - + # Check for placeholders assert "{tool_descriptions}" in MOD_FEW_SHOT assert "{examples}" in MOD_FEW_SHOT - + # Check for key instructions assert "3-5 checklist items" in MOD_FEW_SHOT assert "FIRST" in MOD_FEW_SHOT or "first" in MOD_FEW_SHOT assert "vulnerable function" in MOD_FEW_SHOT - + print("✓ MOD_FEW_SHOT structure validated") +def test_cve_web_search_description_warns_about_versions(): + """Test that CVE Web Search description includes version warning.""" + # Without Version Finder: generic version warning + tool_names = [ToolNames.CVE_WEB_SEARCH] + result = build_tool_descriptions(tool_names) + assert len(result) == 1 + desc = result[0] + assert "MULTIPLE versions" in desc + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in desc + + # With Version Finder (Java): references the tool + tool_names = [ToolNames.CVE_WEB_SEARCH, ToolNames.FUNCTION_LIBRARY_VERSION_FINDER] + result = build_tool_descriptions(tool_names) + descs = " ".join(result) + assert "MULTIPLE versions" in descs + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in descs + + print("✓ CVE Web Search description includes version warning") + + +def test_agent_prompt_contains_version_awareness_instructions(): + """Test that agent prompt template contains version awareness instructions.""" + from vuln_analysis.utils.prompting import get_agent_prompt + + prompt = get_agent_prompt() + assert "VERSION" in prompt + + print("✓ Agent prompt contains version awareness instructions") + + if __name__ == "__main__": print("Running Base Tool Descriptions tests...\n") - + test_base_returns_list() test_base_descriptions_format() test_base_all_tools() test_base_empty_list() test_checklist_formats_descriptions() test_mod_few_shot_structure() - - print("\n✅ All base tool descriptions tests passed!") + test_cve_web_search_description_warns_about_versions() + test_agent_prompt_contains_version_awareness_instructions() + print("\n✅ All base tool descriptions tests passed!") From 4a7943da23eb9830d890b51a18e934e9fa709f56 Mon Sep 17 00:00:00 2001 From: etsien Date: Mon, 27 Apr 2026 10:33:28 -0400 Subject: [PATCH 226/286] add apache 2.0 licenses to react_internals and prompt_factory --- src/vuln_analysis/functions/react_internals.py | 15 +++++++++++++++ src/vuln_analysis/utils/prompt_factory.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 7ec52bb27..9561c349c 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pydantic import BaseModel, Field from typing import Any from typing import Literal diff --git a/src/vuln_analysis/utils/prompt_factory.py b/src/vuln_analysis/utils/prompt_factory.py index 84b5fc69d..93c70ec16 100644 --- a/src/vuln_analysis/utils/prompt_factory.py +++ b/src/vuln_analysis/utils/prompt_factory.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from vuln_analysis.utils.prompting import AGENT_SYS_PROMPT from vuln_analysis.tools.tool_names import ToolNames From 49d6145a0645de952a14fe14483ed78ed87b3ef9 Mon Sep 17 00:00:00 2001 From: etsien Date: Mon, 27 Apr 2026 10:55:10 -0400 Subject: [PATCH 227/286] Add Apache 2.0 license boilerplate to utils Python files and readme files --- SECURITY.md | 19 ++++++++++++++++++- kustomize/README.md | 17 +++++++++++++++++ .../utils/error_handling_decorator.py | 15 +++++++++++++++ .../utils/function_name_extractor.py | 15 +++++++++++++++ .../vex/implementations/csaf_generator.py | 15 +++++++++++++++ .../utils/vex/vex_generator_base.py | 15 +++++++++++++++ .../utils/vex/vex_generator_loader.py | 15 +++++++++++++++ src/vuln_analysis/utils/vex/vex_utils.py | 15 +++++++++++++++ 8 files changed, 125 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 351809626..e9348936c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,4 +1,21 @@ - ## Security + + +## Security NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization. diff --git a/kustomize/README.md b/kustomize/README.md index 0fd2d460b..f9da468f7 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,3 +1,20 @@ + + # Procedure to Run and Deploy ## Install and Run Locally diff --git a/src/vuln_analysis/utils/error_handling_decorator.py b/src/vuln_analysis/utils/error_handling_decorator.py index 3114b8764..2bd63b9b3 100644 --- a/src/vuln_analysis/utils/error_handling_decorator.py +++ b/src/vuln_analysis/utils/error_handling_decorator.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools import traceback diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py index 64f734cda..6d789561a 100644 --- a/src/vuln_analysis/utils/function_name_extractor.py +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index d0ebb7d7d..605c37192 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations import json diff --git a/src/vuln_analysis/utils/vex/vex_generator_base.py b/src/vuln_analysis/utils/vex/vex_generator_base.py index 98e8bfcf7..fa8d3c452 100644 --- a/src/vuln_analysis/utils/vex/vex_generator_base.py +++ b/src/vuln_analysis/utils/vex/vex_generator_base.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations from abc import ABC, abstractmethod diff --git a/src/vuln_analysis/utils/vex/vex_generator_loader.py b/src/vuln_analysis/utils/vex/vex_generator_loader.py index aeb47e3b9..c8b46ff1c 100644 --- a/src/vuln_analysis/utils/vex/vex_generator_loader.py +++ b/src/vuln_analysis/utils/vex/vex_generator_loader.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations from .vex_generator_base import VexGenerator diff --git a/src/vuln_analysis/utils/vex/vex_utils.py b/src/vuln_analysis/utils/vex/vex_utils.py index 4a6668d08..e27bf227d 100644 --- a/src/vuln_analysis/utils/vex/vex_utils.py +++ b/src/vuln_analysis/utils/vex/vex_utils.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations from functools import lru_cache From bdb131e22ac82635f8a7618367ff5f865b93374d Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Sun, 3 May 2026 15:44:56 +0300 Subject: [PATCH 228/286] fix(kustomize): replace argilla pull secret with exploit-iq-pull-secret and update image --- kustomize/README.md | 9 +-------- kustomize/base/argilla/deployment.yaml | 4 ++-- kustomize/overlays/tests/kustomization.yaml | 5 ----- kustomize/overlays/tests/user-feedback-ips.secret | 15 --------------- 4 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 kustomize/overlays/tests/user-feedback-ips.secret diff --git a/kustomize/README.md b/kustomize/README.md index a61731f19..f83c99a90 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -111,16 +111,10 @@ argilla_api_key=your_argilla_api_key EOF ``` -4. Create an image pull secret to authorize pulling the `ExploitIQ` container image: +4. Create an image pull secret to authorize pulling the `ExploitIQ` and `Argilla` container images: ```shell oc create secret generic exploit-iq-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson -``` - - Create an image pull secret to authorize pulling the `Argilla` container image: - -```shell -oc create secret generic argilla-user-feedback-ips --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` 5. Edit the `image-registry-credentials.env` and provide image registry credentials required by product scanning to access and pull component images. @@ -363,7 +357,6 @@ sops -d oauth-secrets.env2 > secrets/oauth-secrets.env sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml sops -d secrets.env2 > secrets/secrets.env sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml -sops -d user-feedback-ips.secret > secrets/user-feedback-ips.json sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env diff --git a/kustomize/base/argilla/deployment.yaml b/kustomize/base/argilla/deployment.yaml index f31b3d2a5..0e1c6601d 100644 --- a/kustomize/base/argilla/deployment.yaml +++ b/kustomize/base/argilla/deployment.yaml @@ -18,13 +18,13 @@ spec: spec: restartPolicy: Always imagePullSecrets: - - name: argilla-user-feedback-ips + - name: exploit-iq-pull-secret serviceAccountName: argilla securityContext: fsGroup: 1000 containers: - name: flask-service - image: quay.io/exploit-iq/user-feedback-api:latest + image: quay.io/ecosystem-appeng/exploit-iq-feedback-api:latest imagePullPolicy: Always ports: - containerPort: 5001 diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml index 31b3337c4..39e29f0d7 100644 --- a/kustomize/overlays/tests/kustomization.yaml +++ b/kustomize/overlays/tests/kustomization.yaml @@ -14,11 +14,6 @@ resources: - secrets/exploit-iq-automation-token.yaml secretGenerator: - - name: argilla-user-feedback-ips - files: - - .dockerconfigjson=secrets/user-feedback-ips.json - type: kubernetes.io/dockerconfigjson - - name: exploit-iq-pull-secret files: - .dockerconfigjson=secrets/exploit-iq-ips.json diff --git a/kustomize/overlays/tests/user-feedback-ips.secret b/kustomize/overlays/tests/user-feedback-ips.secret deleted file mode 100644 index e0f44fb90..000000000 --- a/kustomize/overlays/tests/user-feedback-ips.secret +++ /dev/null @@ -1,15 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:DTCFG/YxGdv2PUvFy4ZpUcn71vTKVE9Pq6OdqGrhThVqW5yIZFdIlEutthH9q/AbnzZZalBxf7nb7T/muOnJd+pE+uN1jlrv6kSdaEcwgViK9CQCRVhqIfWCZNFYyGVFpo90TlD8dTrvk0f1LqbnTQGbuhBIscuKmYzJB+FZuUYzBP+7YgLD34lwhRwOd3gM6IwYyptkDYL7BnZB8t7bZSeliuQwmjwJZ49oDVtG8+CZN7UlUw1hxVJPM96BR6Ob,iv:VE3imWMy9VK5IEQ0ZV+WYbHtRNNAUXPZZY3mA52P258=,tag:2UFpjo65+QUs9fvcTueiQQ==,type:str]", - "sops": { - "lastmodified": "2026-01-25T12:11:26Z", - "mac": "ENC[AES256_GCM,data:PpNa9n8COE+xLG2Kmb8U18dAUT7iqhUfsquUqJC+42DdGC5eWU9t+nxJjy2Hgpf+Gj9hZErRQDYn6LZufKl87YcOix5V8TlBx078LIw1ZFch5xvPIoRef47uedpnPZM6UDAYFljDj6VWUXj6zdqC+GypArYYt5c50iIpY31Fd0c=,iv:N/8Ie16mcQ+QnGjg9c4rXK0OSJn+KujWA7S2WFFa4jc=,tag:69iiCf7FIVmm//LrtCixcw==,type:str]", - "pgp": [ - { - "created_at": "2026-01-25T12:11:26Z", - "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdA/VGmVQPa0NNd5tNIIEunjIeYbcNYFAytX7GStmGIWzow\nzGLovmXwgN/5IQAdqZiRovpI5nURr49xCgCjiHfvui0eI0mm05+Zhph7GlSG0FwV\n0lwB7q/kruiHG/tWCtqESzJYMfun7Jh0CWH/yl3K7N3JiwREqUsSUVG4xOdhJEEs\nus/Ov0GJd6isSwb1uLjtCNszmiQXss6Rw/tutDmRwR4nE0J26CujVwqkuSDbbg==\n=RFOe\n-----END PGP MESSAGE-----", - "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" - } - ], - "version": "3.11.0" - } -} From 4b2295f44e58233db995746266d56e341f240e71 Mon Sep 17 00:00:00 2001 From: Heather Zhang <111881174+heatherzh01@users.noreply.github.com> Date: Thu, 26 Mar 2026 01:23:47 -0400 Subject: [PATCH 229/286] adapted stub fallback mechanism --- src/exploit_iq_commons/utils/dep_tree.py | 466 +++++++++++++---------- 1 file changed, 275 insertions(+), 191 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 68a42657d..a74ee8d15 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -13,37 +13,48 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ast +import configparser +import glob as glob_module +import json import os import re import shutil -import time import subprocess -from abc import ABC, abstractmethod +import tarfile +import tempfile +import time +import tomllib +import urllib.error +import urllib.request +import zipfile +from abc import ABC +from abc import abstractmethod from collections import defaultdict +from contextlib import contextmanager from enum import Enum from pathlib import Path -from typing import Any, List, Dict, Tuple, Optional +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple from packaging.specifiers import SpecifierSet -from tqdm import tqdm - -import ast -import configparser -import json -import tomllib -import zipfile from packaging.version import Version as _PkgVersion +from tqdm import tqdm from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser -from contextlib import contextmanager # C_DEP_LIBS_NAME moved here to avoid circular import C_DEP_LIBS_NAME = "rpm_libs" -from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager -from exploit_iq_commons.logging.loggers_factory import LoggingFactory -from exploit_iq_commons.utils.java_utils import add_missing_jar_string, is_maven_gav from collections import deque +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_utils import add_missing_jar_string +from exploit_iq_commons.utils.java_utils import is_maven_gav +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + logger = LoggingFactory.get_agent_logger(__name__) ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' @@ -55,9 +66,23 @@ README_MD = 'README.md' _WALK_EXCLUDE_DIRS = frozenset({ - ".venv", "venv", "env", ".env", "node_modules", "vendor", - "dist", "build", "__pycache__", ".git", "tests", "test", - "fixtures", "docs", "site-packages", ".tox", ".nox", + ".venv", + "venv", + "env", + ".env", + "node_modules", + "vendor", + "dist", + "build", + "__pycache__", + ".git", + "tests", + "test", + "fixtures", + "docs", + "site-packages", + ".tox", + ".nox", }) @@ -148,7 +173,6 @@ class DependencyTreeBuilder(ABC): def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: pass - @abstractmethod # A Method that knows how to install the app dependencies based on the # manifest path and ecosystem. @@ -158,27 +182,30 @@ def install_dependencies(self, manifest_path: Path): class CCppDependencyTreeBuilder(DependencyTreeBuilder): # Pre-compiled regex patterns (optimization: compile once, use many times) - INCLUDE_COMBINED_RE = re.compile( - r'#include\s*([<"])([^>"]+)[>"]' - ) # Combined pattern + INCLUDE_COMBINED_RE = re.compile(r'#include\s*([<"])([^>"]+)[>"]') # Combined pattern # Common system headers for fast-path resolution COMMON_SYSTEM_HEADERS = { - 'stdio.h', 'stdlib.h', 'string.h', 'math.h', 'unistd.h', - 'sys/types.h', 'sys/stat.h', 'errno.h', 'fcntl.h', 'time.h' + 'stdio.h', + 'stdlib.h', + 'string.h', + 'math.h', + 'unistd.h', + 'sys/types.h', + 'sys/stat.h', + 'errno.h', + 'fcntl.h', + 'time.h' } #kernel lib - need only for bpf projects KERNEL_LIBS = ["libbpf"] - KERNEL_LIBS_PATH_MAP = { - "libbpf": "tools/lib/bpf/" - } + KERNEL_LIBS_PATH_MAP = {"libbpf": "tools/lib/bpf/"} def __init__(self): self.include_paths = [] self.root_dir = None self.prj_name: str = "dummy_project" self.exclude_patterns = [ - "test", "tests", "doc", "docs", "example", "examples", - "bench", "benchmark", "demo", "sample" + "test", "tests", "doc", "docs", "example", "examples", "bench", "benchmark", "demo", "sample" ] self.C_STANDARD_LIB = "glibc" self.RPM_LIBS_DIR = C_DEP_LIBS_NAME @@ -194,7 +221,6 @@ def should_exclude_directory(self, dir_path: str) -> bool: path_lower = dir_path.lower() return any(pattern in path_lower for pattern in self.exclude_patterns) - def relative_matches_end(self, full_path, rel_path): # Normalize to avoid redundant slashes or ../ full_path = os.path.normpath(full_path) @@ -206,9 +232,7 @@ def relative_matches_end(self, full_path, rel_path): return full_parts[-len(rel_parts):] == rel_parts def find_all_files(self, root_dir): - exclude_keywords = [ - "test", "tests", "doc", "docs", "example", "examples" - ] + exclude_keywords = ["test", "tests", "doc", "docs", "example", "examples"] files = [] for dirpath, _, filenames in os.walk(root_dir): # Skip any directory that contains excluded keywords @@ -238,10 +262,7 @@ def find_all_files(self, root_dir): try: full_path.rename(new_path) except Exception as e: - logger.warning( - "Rename failed: %s → %s: %s", - full_path, new_path, e - ) + logger.warning("Rename failed: %s → %s: %s", full_path, new_path, e) if not kernel_pkg: for fname in filenames: @@ -263,22 +284,11 @@ def extract_includes_from_file(self, file_path: str) -> list[dict]: if match: bracket_type = match.group(1) header = match.group(2) - include_type = ( - 'local' if bracket_type == '"' else 'system' - ) - if (include_type == 'system' and - header in self.COMMON_SYSTEM_HEADERS): - includes.append({ - 'header': header, - 'line': line_num, - 'type': 'system_common' - }) + include_type = ('local' if bracket_type == '"' else 'system') + if (include_type == 'system' and header in self.COMMON_SYSTEM_HEADERS): + includes.append({'header': header, 'line': line_num, 'type': 'system_common'}) else: - includes.append({ - 'header': header, - 'line': line_num, - 'type': include_type - }) + includes.append({'header': header, 'line': line_num, 'type': include_type}) except (OSError, UnicodeDecodeError): pass return includes @@ -293,21 +303,11 @@ def extract_includes_from_content(self, content: list[str]) -> list[dict]: if match: bracket_type = match.group(1) header = match.group(2) - include_type = ('local' if bracket_type == '"' - else 'system') - if (include_type == 'system' and - header in self.COMMON_SYSTEM_HEADERS): - includes.append({ - 'header': header, - 'line': line_num, - 'type': 'system_common' - }) + include_type = ('local' if bracket_type == '"' else 'system') + if (include_type == 'system' and header in self.COMMON_SYSTEM_HEADERS): + includes.append({'header': header, 'line': line_num, 'type': 'system_common'}) else: - includes.append({ - 'header': header, - 'line': line_num, - 'type': include_type - }) + includes.append({'header': header, 'line': line_num, 'type': include_type}) return includes def module_from_path(self, path: str) -> str: @@ -316,9 +316,7 @@ def module_from_path(self, path: str) -> str: """ if not self.root_dir: return "unknown" - rel_path = Path(os.path.relpath( - os.path.dirname(path), self.root_dir - )) + rel_path = Path(os.path.relpath(os.path.dirname(path), self.root_dir)) parts = rel_path.parts if self.RPM_LIBS_DIR in parts: try: @@ -339,9 +337,7 @@ def read_file(self, file_path: str): except FileNotFoundError: yield None - def build_dependency_tree_enhanced( - self, root_dir: Path, max_workers: int = 1 - ) -> tuple[dict[str, list[str]], dict]: + def build_dependency_tree_enhanced(self, root_dir: Path, max_workers: int = 1) -> tuple[dict[str, list[str]], dict]: """ Build a C/C++ dependency tree for a project directory. @@ -414,14 +410,11 @@ def build_dependency_tree_enhanced( cache_hits = 0 cache_misses = 0 for header, _, including_module in all_includes: - cache_key = ((header, including_module) - if header in ambiguous_headers else header) + cache_key = ((header, including_module) if header in ambiguous_headers else header) if cache_key in batch_header_cache: cache_hits += 1 continue - resolved = self._batch_resolve_header( - header, including_module, filename_to_paths - ) + resolved = self._batch_resolve_header(header, including_module, filename_to_paths) batch_header_cache[cache_key] = resolved cache_misses += 1 # Build the dependency tree using the resolved headers @@ -435,36 +428,26 @@ def build_dependency_tree_enhanced( # Skip system headers and standard library if include_type == 'system_common': continue - if (include_type == 'system' and - os.path.basename(header) in standard_lib_headers): + if (include_type == 'system' and os.path.basename(header) in standard_lib_headers): continue - cache_key = ((header, current_module) - if header in ambiguous_headers else header) + cache_key = ((header, current_module) if header in ambiguous_headers else header) if cache_key in batch_header_cache: resolved_module = batch_header_cache[cache_key] else: - resolved_module = self._batch_resolve_header( - header, current_module, filename_to_paths - ) - if (resolved_module and - resolved_module != current_module): - if (resolved_module != self.C_STANDARD_LIB and - current_module != self.C_STANDARD_LIB): + resolved_module = self._batch_resolve_header(header, current_module, filename_to_paths) + if (resolved_module and resolved_module != current_module): + if (resolved_module != self.C_STANDARD_LIB and current_module != self.C_STANDARD_LIB): tree[current_module].add(resolved_module) tree_as_list = {k: list(v) for k, v in tree.items()} print(f"[dep_tree] batch_header_cache: {len(batch_header_cache)} " f"entries, hits: {cache_hits}, misses: {cache_misses}") cache_stats = { - 'batch_header_cache_size': len(batch_header_cache), - 'cache_hits': cache_hits, - 'cache_misses': cache_misses + 'batch_header_cache_size': len(batch_header_cache), 'cache_hits': cache_hits, 'cache_misses': cache_misses } return tree_as_list, cache_stats - def _batch_resolve_header( - self, header: str, including_module: str, - filename_to_paths: dict[str, list[str]] - ) -> Optional[str]: + def _batch_resolve_header(self, header: str, including_module: str, + filename_to_paths: dict[str, list[str]]) -> Optional[str]: """ Header resolution logic for batch/bulk mode (same as before, but no caching here) Adds debug output for ambiguous headers (limited to 10 @@ -569,7 +552,6 @@ def install_dependencies(self, manifest_path: Path): #we want to use this before the creation of documents by the segmenter class self.find_all_files(self.root_dir) - def _install_dependencies_from_container(self, manifest_path: Path, download_path: str): """Install dependencies from container sources using skopeo""" # Validate environment variables @@ -581,8 +563,7 @@ def _install_dependencies_from_container(self, manifest_path: Path, download_pat password = os.getenv('REGISTRY_REDHAT_PASSWORD') if not username or not password: raise ValueError( - "REGISTRY_REDHAT_USERNAME and REGISTRY_REDHAT_PASSWORD must be set when USE_CONTAINER_SOURCES=true" - ) + "REGISTRY_REDHAT_USERNAME and REGISTRY_REDHAT_PASSWORD must be set when USE_CONTAINER_SOURCES=true") logger.info(f"Downloading container sources for: {container_image}") @@ -593,7 +574,7 @@ def _install_dependencies_from_container(self, manifest_path: Path, download_pat container_sources_dir = self._download_container_sources(container_image, manifest_path) # Extract and organize RPMs - self._extract_and_organize_rpms(container_sources_dir, download_path,container_image) + self._extract_and_organize_rpms(container_sources_dir, download_path, container_image) # Cleanup temporary files self._cleanup_temp_files(container_sources_dir) @@ -602,11 +583,7 @@ def _setup_skopeo_auth(self, username: str, password: str): """Setup skopeo authentication for registry.redhat.io""" logger.info("Setting up skopeo authentication...") - cmd = [ - 'skopeo', 'login', 'registry.redhat.io', - '--username', username, - '--password', password - ] + cmd = ['skopeo', 'login', 'registry.redhat.io', '--username', username, '--password', password] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: @@ -628,11 +605,7 @@ def _download_container_sources(self, container_image: str, manifest_path: Path) logger.info(f"Downloading container source: {source_image}") - cmd = [ - 'skopeo', 'copy', - f'docker://{source_image}', - f'dir:{container_sources_dir}' - ] + cmd = ['skopeo', 'copy', f'docker://{source_image}', f'dir:{container_sources_dir}'] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: @@ -662,10 +635,10 @@ def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: continue logger.info(f"Extracting: {tar_file.name}") try: - result = subprocess.run( - ['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], - capture_output=True, text=True, check=False - ) + result = subprocess.run(['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], + capture_output=True, + text=True, + check=False) if result.returncode != 0: logger.warning(f"Failed to extract {tar_file}: {result.stderr}") except Exception as e: @@ -740,30 +713,22 @@ def find_project_name(self, root_dir="."): if os.path.exists(cmake_path): with open(cmake_path, "r", encoding="utf-8", errors="ignore") as f: for line in f: - m = re.search( - r'project\s*\(\s*([A-Za-z0-9_\-]+)', line, - re.IGNORECASE - ) + m = re.search(r'project\s*\(\s*([A-Za-z0-9_\-]+)', line, re.IGNORECASE) if m: return m.group(1).lower() meson_path = os.path.join(root_dir, "meson.build") if os.path.exists(meson_path): with open(meson_path, "r", encoding="utf-8", errors="ignore") as f: for line in f: - m = re.search( - r'project\s*\(\s*["\']?([A-Za-z0-9_\-]+)', line, - re.IGNORECASE - ) + m = re.search(r'project\s*\(\s*["\']?([A-Za-z0-9_\-]+)', line, re.IGNORECASE) if m: return m.group(1).lower() makefile_path = os.path.join(root_dir, "Makefile") if os.path.exists(makefile_path): with open(makefile_path, "r", encoding="utf-8", errors="ignore") as f: for line in f: - m = re.search( - r'^(PROJECT_NAME|PACKAGE_NAME|PROJECT|PACKAGE)\s*[:=]\s*' - r'([A-Za-z0-9_\-]+)', line - ) + m = re.search(r'^(PROJECT_NAME|PACKAGE_NAME|PROJECT|PACKAGE)\s*[:=]\s*' + r'([A-Za-z0-9_\-]+)', line) if m: return m.group(2).lower() configure_ac_path = os.path.join(root_dir, "configure.ac") @@ -778,10 +743,7 @@ def find_project_name(self, root_dir="."): if os.path.exists(configure_path): with open(configure_path, "r", encoding="utf-8", errors="ignore") as f: for line in f: - m = re.search( - r"^PACKAGE_NAME\s*[:=]\s*['\"]?([A-Za-z0-9_\-]+)['\"]?", - line - ) + m = re.search(r"^PACKAGE_NAME\s*[:=]\s*['\"]?([A-Za-z0-9_\-]+)['\"]?", line) if m: return m.group(1).lower() return "dummy_project" @@ -853,26 +815,24 @@ def get_go_mod_graph_tree(manifest_path) -> str: try: process_object = subprocess.run( ["bash", "-c", f" export GOWORK=off ; go mod graph " - f"-modfile {manifest_path}/go.mod"], - capture_output=True, text=True) + f"-modfile {manifest_path}/go.mod"], + capture_output=True, + text=True) if process_object.returncode > 0: - process_object = subprocess.run([ - "bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" - f"go mod graph" - ], capture_output=True, text=True) + process_object = subprocess.run( + ["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" + f"go mod graph"], + capture_output=True, + text=True) if process_object.returncode > 0: - formatted_error_msg = ( - f"Failed to generate dependencies tree from go.mod " - f"manifest at {manifest_path}, error details => " - f"{process_object.stderr}" - ) + formatted_error_msg = (f"Failed to generate dependencies tree from go.mod " + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}") raise Exception(formatted_error_msg) except Exception as e: - error_message_exception = ( - f"Failed to generate dependencies tree from because go.mod " - f"manifest wasn't found at {manifest_path}, error details => " - f"{repr(e)}" - ) + error_message_exception = (f"Failed to generate dependencies tree from because go.mod " + f"manifest wasn't found at {manifest_path}, error details => " + f"{repr(e)}") logger.error(error_message_exception) raise e return process_object.stdout @@ -884,18 +844,17 @@ def download_go_mod_vendor(self, manifest_path): it downloads it to git repo root dir where the go.mod file located. :param manifest_path: path to go.mod manifest in repo directory. """ - subprocess.run([ - "bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; " - f"go mod vendor" - ]) + subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; " + f"go mod vendor"]) def extract_package_name(self, package_name: str) -> str: if package_name.__contains__("@"): version_start = package_name.index("@") - return package_name[: version_start] + return package_name[:version_start] else: return package_name + class JavaDependencyTreeBuilder(DependencyTreeBuilder): def __init__(self, query: str): @@ -942,7 +901,7 @@ def install_dependencies(self, manifest_path: Path): ) raise Exception(formatted_error_msg) - full_source_path = manifest_path / source_path + full_source_path = manifest_path / source_path for jar in full_source_path.glob("*-sources.jar"): if jar.stat().st_size > 0: dest = full_source_path / jar.stem # folder named after jar @@ -1017,9 +976,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: return graph - def __build_upside_down_dependency_graph( - self, dependency_lines: List[str] - ) -> Tuple[str, Dict[str, List[str]]]: + def __build_upside_down_dependency_graph(self, dependency_lines: List[str]) -> Tuple[str, Dict[str, List[str]]]: root: str = "" stack: List[str] = [] graph_sets: Dict[str, set[str]] = {} # coord -> set of direct parents @@ -1233,11 +1190,7 @@ def extract_version_from_specifier(self, specifier_str: str) -> str | None: if spec.operator in ("<", "<=") and _PkgVersion(spec.version) <= _PkgVersion("3.0"): return "2.7" - lower_bounds = [ - _PkgVersion(spec.version) - for spec in specifier_set - if spec.operator == ">=" - ] + lower_bounds = [_PkgVersion(spec.version) for spec in specifier_set if spec.operator == ">="] if lower_bounds: return str(max(lower_bounds)) @@ -1293,12 +1246,7 @@ def extract_version_from_pyproject_toml(self, content: str) -> str | None: logger.debug("Found requires-python in pyproject.toml: %r", pep621) return self.extract_version_from_specifier(pep621) - poetry = ( - data.get("tool", {}) - .get("poetry", {}) - .get("dependencies", {}) - .get("python") - ) + poetry = (data.get("tool", {}).get("poetry", {}).get("dependencies", {}).get("python")) if poetry: logger.debug("Found tool.poetry.dependencies.python: %r", poetry) normalized = re.sub( @@ -1331,15 +1279,13 @@ def extract_version_from_setup_py(self, content: str) -> str | None: return None for node in ast.walk(tree): - if not (isinstance(node, ast.Call) and - isinstance(node.func, (ast.Name, ast.Attribute)) and - getattr(node.func, "id", None) == "setup"): + if not (isinstance(node, ast.Call) and isinstance(node.func, (ast.Name, ast.Attribute)) + and getattr(node.func, "id", None) == "setup"): continue for kw in node.keywords: - if (kw.arg == "python_requires" and - isinstance(kw.value, ast.Constant) and - isinstance(kw.value.value, str)): + if (kw.arg == "python_requires" and isinstance(kw.value, ast.Constant) + and isinstance(kw.value.value, str)): logger.debug("Found python_requires in setup.py: %r", kw.value.value) return self.extract_version_from_specifier(kw.value.value) @@ -1487,12 +1433,12 @@ def determine_python_version(self, git_repo_path: str) -> str | None: priority_chain = [ (".python-version", self.extract_version_from_python_version_file), - ("pyproject.toml", self.extract_version_from_pyproject_toml), - ("setup.cfg", self.extract_version_from_setup_cfg), - ("setup.py", self.extract_version_from_setup_py), - ("Pipfile", self.extract_version_from_pipfile), - ("README.md", self.extract_version_from_readme_md), - ("README.rst", self.extract_version_from_readme_md), + ("pyproject.toml", self.extract_version_from_pyproject_toml), + ("setup.cfg", self.extract_version_from_setup_cfg), + ("setup.py", self.extract_version_from_setup_py), + ("Pipfile", self.extract_version_from_pipfile), + ("README.md", self.extract_version_from_readme_md), + ("README.rst", self.extract_version_from_readme_md), ] def _try_file(path: Path, extractor) -> str | None: @@ -1513,10 +1459,7 @@ def _try_file(path: Path, extractor) -> str | None: return version for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - d for d in dirnames - if d not in _WALK_EXCLUDE_DIRS and not d.startswith(".") - ] + dirnames[:] = [d for d in dirnames if d not in _WALK_EXCLUDE_DIRS and not d.startswith(".")] if Path(dirpath) == root: continue for filename, extractor in priority_chain: @@ -1544,15 +1487,17 @@ def install_dependencies(self, manifest_path: Path): logger.info("Creating transitive_env with Python %s using uv", python_version) cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" else: - logger.warning( - "Python version undetermined for %s; using uv default interpreter", manifest_path - ) + logger.warning("Python version undetermined for %s; using uv default interpreter", manifest_path) cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME}" run_command(cmd) + site_packages = self._find_site_packages(manifest_path) with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: for line in tqdm(manifest): if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): self.install_dependency(line, manifest_path) + if site_packages: + package_name = re.split(r'[=>< \n]', line.strip())[0] + self._fallback_if_stub_only(package_name, site_packages) def install_dependency(self, dependency, repo_path): dependency = dependency.strip() @@ -1564,6 +1509,148 @@ def install_dependency(self, dependency, repo_path): if not res: logger.warning('Failed to install dependency %s', dependency) + def _find_module_dirs(self, package_name: str, site_packages: Path) -> list[str]: + candidates = [] + try: + from importlib_metadata import distribution + dist = distribution(package_name) + top_level = dist.read_text('top_level.txt') + if top_level: + candidates.extend(top_level.strip().split('\n')) + if candidates: + return candidates + except Exception: + pass + + normalized = package_name.replace('-', '_') + for dist_dir in site_packages.glob(f'{normalized}-*.dist-info'): + top_level_file = dist_dir / 'top_level.txt' + if top_level_file.exists(): + content = top_level_file.read_text().strip() + if content: + candidates.extend(content.split('\n')) + if candidates: + return candidates + + if package_name.startswith('types-'): + base = package_name[6:] + candidates = [f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()] + elif package_name.startswith('mypy-boto3-'): + base = package_name[11:] + candidates = [f'mypy_boto3_{base}'] + elif package_name.endswith('-stubs'): + candidates = [package_name] + else: + candidates = [package_name.replace('-', '_')] + + existing = [c for c in candidates if (site_packages / c).exists()] + return existing if existing else candidates + + def _find_site_packages(self, git_repo_path) -> Path: + pattern = str(git_repo_path / TRANSITIVE_ENV_NAME / 'lib' / '*' / 'site-packages') + matches = glob_module.glob(pattern) + if matches: + return Path(matches[0]) + logger.warning("Could not find site-packages directory") + return None + + def _is_stub_only_package(self, package_name: str, site_packages: Path) -> bool: + pkg_dir = site_packages / package_name + if not pkg_dir.exists(): + return False + py_files = glob_module.glob(str(pkg_dir / '**' / '*.py'), recursive=True) + pyi_files = glob_module.glob(str(pkg_dir / '**' / '*.pyi'), recursive=True) + so_files = glob_module.glob(str(pkg_dir / '**' / '*.so'), recursive=True) + return len(py_files) == 0 and (len(pyi_files) > 0 or len(so_files) > 0) + + def _fallback_if_stub_only(self, package_name: str, site_packages: Path): + """ + check package if stub-only, if so download from PyPI + """ + module_dirs = self._find_module_dirs(package_name, site_packages) + + if not module_dirs: + logger.debug(f"Could not find module directories for package '{package_name}'") + return + + for module_name in module_dirs: + if not self._is_stub_only_package(module_name, site_packages): + continue + + logger.info( + f"Package '{package_name}' (module: '{module_name}') is stub-only, fetching source from PyPI...") + + actual_package = package_name + if package_name.startswith('types-'): + actual_package = package_name[6:] + logger.info( + f"Detected stub package '{package_name}', will fetch source for actual package '{actual_package}'") + elif package_name.endswith('-stubs'): + actual_package = package_name[:-6] + logger.info( + f"Detected stub package '{package_name}', will fetch source for actual package '{actual_package}'") + + possible_names = [ + actual_package, + actual_package.replace('-', '_'), + actual_package.replace('_', '-'), + ] + + data = None + for name in possible_names: + try: + url = f'https://pypi.org/pypi/{name}/json' + with urllib.request.urlopen(url) as r: + data = json.loads(r.read()) + logger.info(f"Found package on PyPI as '{name}'") + break + except urllib.error.HTTPError: + continue + + if not data: + logger.warning(f"Package '{package_name}' not found on PyPI with any variant, skipping fallback") + return + + sdist = next((u for u in data['urls'] if u['packagetype'] == 'sdist'), None) + if not sdist: + logger.warning(f"No sdist found on PyPI for '{package_name}', skipping fallback") + return + + sdist_url = sdist['url'] + + try: + with tempfile.TemporaryDirectory() as tmpdir: + tarball_path = Path(tmpdir) / 'sdist.tar.gz' + urllib.request.urlretrieve(sdist_url, tarball_path) + + with tarfile.open(tarball_path, 'r:gz') as tar: + tar.extractall(tmpdir) + + extracted = [d for d in Path(tmpdir).iterdir() if d.is_dir() and d.name != '__pycache__'] + if not extracted: + return + + src_root = extracted[0] + + candidates = [ + src_root / 'src' / actual_package, + src_root / actual_package, + src_root / 'src' / actual_package.replace('-', '_'), + src_root / actual_package.replace('-', '_'), + ] + + src_dir = next((c for c in candidates if c.exists()), None) + + if src_dir: + dest = site_packages / module_name + shutil.copytree(str(src_dir), str(dest), dirs_exist_ok=True) + logger.info(f"Fallback source copied for '{package_name}' to {dest}") + else: + logger.warning(f"Could not locate source dir for '{package_name}' in sdist") + except Exception as e: + logger.warning(f"Fallback failed for '{package_name}': {e}") + + class JavaScriptDependencyTreeBuilder(DependencyTreeBuilder): def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: @@ -1575,14 +1662,14 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: if not package_json_path.exists(): logger.error(f"package.json not found at {package_json_path}") raise FileNotFoundError(f"package.json not found at {package_json_path}") - + result = run_command(f'cd {manifest_path} && npm ls --json --all') try: npm_tree = json.loads(result) except json.JSONDecodeError as e: raise RuntimeError(f"Failed to parse npm ls JSON output: {e}") self._parse_npm_tree(npm_tree, tree, ROOT_PROJECT) - + return {k: list(v) for k, v in tree.items()} def _parse_npm_tree(self, node: dict, tree: dict, parent: str): @@ -1595,7 +1682,7 @@ def _parse_npm_tree(self, node: dict, tree: dict, parent: str): parent: Name of the parent package """ dependencies = node.get('dependencies', {}) - + for dep_name, dep_info in dependencies.items(): tree[dep_name].add(parent) if isinstance(dep_info, dict): @@ -1611,7 +1698,6 @@ def install_dependencies(self, manifest_path: Path): logger.info("JavaScript dependencies installed successfully") - def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "") -> DependencyTreeBuilder: """ A Factory method function that gets an programming language as argument, @@ -1633,10 +1719,8 @@ def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "" elif programming_language.value == Ecosystem.C_CPP.value: return CCppDependencyTreeBuilder() else: - raise NotImplementedError( - f'Unsupported Programming Language for transitive search -> ' - f'{programming_language}' - ) + raise NotImplementedError(f'Unsupported Programming Language for transitive search -> ' + f'{programming_language}') class DependencyTree: From c498b8db8f453a648d8abe199205c588a315d642 Mon Sep 17 00:00:00 2001 From: Heather Zhang <111881174+heatherzh01@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:47:33 -0400 Subject: [PATCH 230/286] Update src/exploit_iq_commons/utils/dep_tree.py Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --- src/exploit_iq_commons/utils/dep_tree.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index a74ee8d15..200b1dbc0 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1622,6 +1622,11 @@ def _fallback_if_stub_only(self, package_name: str, site_packages: Path): with tempfile.TemporaryDirectory() as tmpdir: tarball_path = Path(tmpdir) / 'sdist.tar.gz' urllib.request.urlretrieve(sdist_url, tarball_path) + expected_hash = sdist.get('digests', {}).get('sha256') + if expected_hash: + actual_hash = hashlib.sha256(tarball_path.read_bytes()).hexdigest() + if actual_hash != expected_hash: + raise ValueError(f"SHA256 mismatch for {package_name}: expected {expected_hash}, got {actual_hash}") with tarfile.open(tarball_path, 'r:gz') as tar: tar.extractall(tmpdir) From dbfce3d6e40b1bd9fe5a8608bef2ba0be3e7edba Mon Sep 17 00:00:00 2001 From: Heather Zhang <111881174+heatherzh01@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:47:41 -0400 Subject: [PATCH 231/286] Update src/exploit_iq_commons/utils/dep_tree.py Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --- src/exploit_iq_commons/utils/dep_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 200b1dbc0..91d456654 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1629,7 +1629,7 @@ def _fallback_if_stub_only(self, package_name: str, site_packages: Path): raise ValueError(f"SHA256 mismatch for {package_name}: expected {expected_hash}, got {actual_hash}") with tarfile.open(tarball_path, 'r:gz') as tar: - tar.extractall(tmpdir) + tar.extractall(tmpdir, filter='data') extracted = [d for d in Path(tmpdir).iterdir() if d.is_dir() and d.name != '__pycache__'] if not extracted: From 2517d03f6e2d96f00980954088a681991967706c Mon Sep 17 00:00:00 2001 From: Heather Zhang <111881174+heatherzh01@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:47:49 -0400 Subject: [PATCH 232/286] Update src/exploit_iq_commons/utils/dep_tree.py Co-authored-by: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> --- src/exploit_iq_commons/utils/dep_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 91d456654..612510f86 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1546,7 +1546,7 @@ def _find_module_dirs(self, package_name: str, site_packages: Path) -> list[str] existing = [c for c in candidates if (site_packages / c).exists()] return existing if existing else candidates - def _find_site_packages(self, git_repo_path) -> Path: + def _find_site_packages(self, git_repo_path) -> Optional[Path]: pattern = str(git_repo_path / TRANSITIVE_ENV_NAME / 'lib' / '*' / 'site-packages') matches = glob_module.glob(pattern) if matches: From f97ef06c2479a2886ab73f4b8fae31b886828703 Mon Sep 17 00:00:00 2001 From: Heather Zhang <111881174+heatherzh01@users.noreply.github.com> Date: Sun, 10 May 2026 11:51:51 -0400 Subject: [PATCH 233/286] solve indentation error; add warning --- src/exploit_iq_commons/utils/dep_tree.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 612510f86..76421ef3f 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -16,6 +16,7 @@ import ast import configparser import glob as glob_module +import hashlib import json import os import re @@ -1623,11 +1624,15 @@ def _fallback_if_stub_only(self, package_name: str, site_packages: Path): tarball_path = Path(tmpdir) / 'sdist.tar.gz' urllib.request.urlretrieve(sdist_url, tarball_path) expected_hash = sdist.get('digests', {}).get('sha256') - if expected_hash: - actual_hash = hashlib.sha256(tarball_path.read_bytes()).hexdigest() - if actual_hash != expected_hash: - raise ValueError(f"SHA256 mismatch for {package_name}: expected {expected_hash}, got {actual_hash}") - + if expected_hash: + actual_hash = hashlib.sha256(tarball_path.read_bytes()).hexdigest() + if actual_hash != expected_hash: + raise ValueError(f"SHA256 mismatch for {package_name}: expected {expected_hash}, got {actual_hash}") + else: + logger.warning( + f"Expected SHA256 hash is missing for package '{package_name}' " + f"at {sdist_url}. Skipping integrity check." + ) with tarfile.open(tarball_path, 'r:gz') as tar: tar.extractall(tmpdir, filter='data') From 3e4feba083ef1b62128ef86eacedc10dc577edf3 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 12 May 2026 10:04:07 +0300 Subject: [PATCH 234/286] chore: config and deploy nim embedding model into tests CI build env Signed-off-by: Zvi Grinberg --- kustomize/README.md | 4 +- .../tests/server-model-config-enc.yaml | 43 ++++++++++--------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index f6ad5d920..1f9be713b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -501,9 +501,9 @@ kustomize build . | oc apply -f - To restrict MCP access to specific groups, edit the `EXPLOITIQ_ALLOWED_GROUPS` value in `base/exploitiq_mcp_server.yaml` (defaults to `exploitiq-users`). To allow any authenticated user, remove the env var entirely. -11. Deploy Self hosted LLM for the automation tests ( Integration tests and Confusion matrix runner): +11. Deploy Self hosted LLM + nim Embedding model for the automation tests ( Integration tests and Confusion matrix runner): ```shell -helm upgrade --install --set nim_embed.enabled=false --set llama3_1_70b_instruct_4bit.storageClass.name=gp3-csi-throughput-2000 --set llama3_1_70b_instruct_4bit.readinessProbe.initialDelaySeconds=25 --set llama3_1_70b_instruct_4bit.readinessProbe.periodSeconds=10 --set global.tolerationsKey=p4d-gpu exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models +helm upgrade --install --set nim_embed.enabled=false --set llama3_1_70b_instruct_4bit.storageClass.name=gp3-csi-throughput-2000 --set llama3_1_70b_instruct_4bit.readinessProbe.initialDelaySeconds=25 --set llama3_1_70b_instruct_4bit.readinessProbe.periodSeconds=10 --set global.tolerationsKey=p4d-gpu --set nim-embed.ngcSecret.apiKey=your_nvidia_ngc_api_key exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models ``` 12. Remove untracked decrypted secrets files diff --git a/kustomize/overlays/tests/server-model-config-enc.yaml b/kustomize/overlays/tests/server-model-config-enc.yaml index f978b3c54..55777b61b 100644 --- a/kustomize/overlays/tests/server-model-config-enc.yaml +++ b/kustomize/overlays/tests/server-model-config-enc.yaml @@ -1,31 +1,32 @@ -apiVersion: ENC[AES256_GCM,data:GqM=,iv:X9ZA6MF+bye5xH0o8LL/4XFDfiVzgDeL/LJUVGNiJnA=,tag:jlRvjVgKLC66t+n/9UPxeQ==,type:str] +apiVersion: ENC[AES256_GCM,data:UPY=,iv:qbLj3gBhjSfcnwa0ataze8hm5lctPL+5xAiqWe7zIx8=,tag:mkhVfM8yQAav1BLWlaSShQ==,type:str] data: - CHECKLIST_MODEL_NAME: ENC[AES256_GCM,data:7OrZQrchux0+jlBPEuDhsyAv1Ix75Yp1/uPKdjy2g2lziSXhdwBrEtSF0U4dQWBITTYp,iv:TAd+SIFStCUd1qQDtQpD9ejlH8R6oCa4mLrwBYPh5nE=,tag:wGSMp+pww/4Zt7mvqVvwrg==,type:str] - CODE_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:4D444ciNBzQPnhYXT0niQZWF12OtQ/stcNpdEbCXCCRbf/jUfkYoai+0u08EsYgmh+TC,iv:SehqcrF7ZA5itfSeEDMyeuFTOBrufGLlGcNxO+yKdp4=,tag:4jJ05Bitj0iVNDvSl2NeaQ==,type:str] - CVE_AGENT_EXECUTOR_MODEL_NAME: ENC[AES256_GCM,data:xtooeRKaRb2uXvEmhcq6W6M9UX1g2fk2WC9jf0cj0ReMlNZ0waj7JOY0KVfWM4n9eBEg,iv:mYGS+QzlUv3aUKQlGF3Qj++h9kg73Btpmi+mf3GUJ3w=,tag:BWLyR3yhAqEdpLE1T4lYpg==,type:str] - DOC_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:g9DV3/7vINF2g4MPUScMedRhx4EzUn6aOHztFmAmhdqU7iaOYAMLQaGZkkMdiXsSqziu,iv:0swptAt4Hwg0/y3f28btcv2Mtq0yYB/mCTfFXjWe00k=,tag:vdgiZOimYt+5uluRtyP+cQ==,type:str] - GENERATE_CVSS_MODEL_NAME: ENC[AES256_GCM,data:uDTbwkxuuohDKJ9X+uAYHE2lJgb8BKEBkTGH5xL1YWy82R6OpWE53l9d9/KAk8JL3COi,iv:bWsmW1J1U0zmWlhj5cvLz1dFYG/NPF3PiZhWS5duT+Q=,tag:w6PCNZhOysrHW7l74X8EiQ==,type:str] - JUSTIFY_MODEL_NAME: ENC[AES256_GCM,data:ar93nbJy18x6Nb++YGHSuh6QGFnmhiyAZtq6hpRsDT6g4XSAmFAsCO5IvBEIZ7dpNprZ,iv:ks3khWQCRnz8mUrvK7B9SVz+wKOF49JU817dWglaDrM=,tag:OB3ULv2x4Ref63j/nlp8DQ==,type:str] - NVIDIA_API_BASE: ENC[AES256_GCM,data:58mO8y0qF6FBOXcc4E3jppjo3ym6A48wGZL9OxTPlHcYptR/AW6KvGdSA8TMk5DG9RE10Edx0ChXB9XsMFvxfQnfA+9OiBnCNxRUXw==,iv:OUlIO/XHdqkvpOM35JfEQSke6TmyCuMOEcvJqLSUkzE=,tag:kifU31FSIu3//q9pc7u40A==,type:str] - PYTHON_VERSION: ENC[AES256_GCM,data:OmHFTw==,iv:qsd4oNKf/oYCSpICfSE7cuUlIfRmI7VrZ+Wth/nNPvM=,tag:Hy0YNQ9lmcbqrXHEjtzfxg==,type:str] - SUMMARIZE_MODEL_NAME: ENC[AES256_GCM,data:vO8ztZTgcvg5tv5FQaKlKBsTs8ms/78Qe9OtmDS7LnibP6K9yljNbz9ai73Ph3CIn4Wu,iv:ak/afG2kXG1rvsriN21zNsjttc2Vt+VoYhoauwWA6ZI=,tag:lf0QVlghAuin8l4keWkY3A==,type:str] -immutable: ENC[AES256_GCM,data:7VHk5CM=,iv:9GuqxkAjk0XtcAHP1xJ7OjfHRDEydV2XCrNzXMzNZho=,tag:J261qozJtzdv+tOzIZYmrA==,type:bool] -kind: ENC[AES256_GCM,data:+XvGoKaqZt3d,iv:ipjdNZIC5ucrWo7Cpa3I64nKLd2eOslQjwe6R0LXsXU=,tag:ZvTEtcfLxIBMvqEw7mLqWA==,type:str] + CHECKLIST_MODEL_NAME: ENC[AES256_GCM,data:K1KWmLWRU3XK7Yy7+F56LCMoxGDZz8WgKJF0M8vXe5z+ScnRpTOJvbpe2fJy0x3g7zXr,iv:nDaOKFCkmm2uoD4+xZRFIWpy8nfcijEuONUvtleq+KI=,tag:rlUb3+DqRwpc+wvHQTrEfQ==,type:str] + CODE_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:cnt51hBxh7tIId6GJYBwcyaoX5J5WUaPV0o999Sb9Z6mGIrfNnC2Vf/5bxXxs4l9I3Bb,iv:PJ4AKPDF2xXGNHneBZkgaRVZiiSet1o4QshSCB1comM=,tag:0KKYFR54GpWz9IGbsKRRxA==,type:str] + CVE_AGENT_EXECUTOR_MODEL_NAME: ENC[AES256_GCM,data:qxaF7Pd5OrCGzRhKleVbbwp3IawXBPRSxNMmeq9yOh4c8PfE2qcg2mB8Kh9kMQJCUjx9,iv:zEQZkdsu7+JBP/r1VO+nCsEZkloAsK7N80c4N+E4A60=,tag:T7dRB9NUnsNrEiRzYbgvow==,type:str] + DOC_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:4NJlINBhf27A+nVu8Fx9ugIjnpDX4/V4kOyk7lWIpomi6UJkiaIDSo5raW2NpZfeOapC,iv:3N5YEc2+Gc/fwR+ecHm/XX/xwHu3RMMbziNibVLLXtA=,tag:SwYHQQg4fx/l/QEp3YJ2lA==,type:str] + GENERATE_CVSS_MODEL_NAME: ENC[AES256_GCM,data:Umwds/0JtexEGjeIliWL9RVbN0K2+gT1FimqFt+tPrEfe1D3wGaPYEAkQAacI/8XCzao,iv:HzvcW4+fvkzL/+Gid37X0tbrq9dP/QKvKwZ2mjOLsyA=,tag:IlTJed0cVSF2G7FMShHn9g==,type:str] + JUSTIFY_MODEL_NAME: ENC[AES256_GCM,data:CIhBKCjWHZbvbYslDyqP6NAEbuouDHxHPx9BAer52CnUUzaq7Lg4ko7F4YYv5o7TPl64,iv:y893vaZq0CTEZFUSFKbb+TEh4xKgGqarVGYosMG1/qw=,tag:IMMugbUBzLUukcw6KSqe0w==,type:str] + NVIDIA_API_BASE: ENC[AES256_GCM,data:G35BJ2UOsB3dLjndxSBNybQbYyeUjIDVU/EcJJLbSFqrClWXOiFiy/MhQ+PJRzmoHdntv9gCWHfUPKhRfBgsyB64wD8DrGDohhJzKQ==,iv:KUDMIRSpb/LQoXBMKTU/Kd59r/ukjdJ5zh6Kh4GLfKQ=,tag:gnWiEqpWHICAmI+PgzSFDA==,type:str] + NIM_EMBED_BASE_URL: ENC[AES256_GCM,data:8ryNEDwuzcwJNXFaUrXCIrUk/oBvDhFH/9yLtn5BUNocbAcJyeQUv5I+YwUXOVQk0fW+2JykfvBn+qQ=,iv:w5e9UNdh3wBbzd4Qc2v6DogAA627+5Fmw3fVsfjzsUs=,tag:D7fa7a1ZSwXSOXPQiE0zqA==,type:str] + PYTHON_VERSION: ENC[AES256_GCM,data:UiXJIQ==,iv:ToEOGsQBAYLE5BDd3zQPoqnH6do+WAggEFjzYMl1qB0=,tag:24jMH7zQoSFImxdy5Ocfnw==,type:str] + SUMMARIZE_MODEL_NAME: ENC[AES256_GCM,data:Nj12r4hMkvCWTTaEYg/EGBNnCulWf3zDTPDmSk3Eqn6ZvrTljNUT1bDDS5t69Y2IfdAK,iv:jIa5PFqd1O/HQ5vN15gOVduEpxs4+b1f4Z7W7cgRYl4=,tag:jycJcuUXjBt2Z2zcOWVi1w==,type:str] +immutable: ENC[AES256_GCM,data:orL33S8=,iv:/VLfTG9b1F0ftvqZcUxGWTH2cfn6dm39+1GUL6AGkJc=,tag:15FjfRmbLTFxxzogbjQL+g==,type:bool] +kind: ENC[AES256_GCM,data:+7INBDEI2vJK,iv:SkJaalGfQKME3FoazUvgnA8FQkjHUxPdOxdMHlhfiYE=,tag:2fn2v9XgCny1WUcNl++mpg==,type:str] metadata: - name: ENC[AES256_GCM,data:bIyOig0niHOw8aT3r5yg6wSZzg==,iv:FNtbtOzmrKkz4Ocb7ujgsBBg72P0ACMlS5DfT8DP4rk=,tag:beE9XVfmH2CzYoCkYJMnuQ==,type:str] + name: ENC[AES256_GCM,data:qPvsckwg4xUthetbcj9+tyqkVA==,iv:8Mdy+qqHwY71YUeyUTKORupwiBTgxk2alimz7vJk4aY=,tag:Y/nOmTT1OPrcqKFxNlKk+g==,type:str] sops: - lastmodified: "2026-01-26T09:27:06Z" - mac: ENC[AES256_GCM,data:xMMv5Lrb9cRGYoEjwLHHf74gadMJfm17697zJn/UI+2bZceSBVlJZAN+4Q8yOtTDM0uC1uuGbwZC82yJuguMNuDUFo8yQJ1evU45A8WPKJXLW9GkLCIuh/8302kH08EI9GLcBR5eXuQCl1N1uKRx56rNCiu/WCFJN3WUS/gM/8A=,iv:C1KHyp68xbcOSCDpDjvoHMsPGB4YsnQ+2dHkyOo+l8g=,tag:0aFf97vxoXupUQX8kQAkRA==,type:str] + lastmodified: "2026-05-12T06:36:55Z" + mac: ENC[AES256_GCM,data:Usc6qyfTA99yewHcWarGo1B2KuKMH0nD/ybcTHgjdkHRJ7ic6Me5RmFAdqu3VJ5hpnSN3fqZiZNaRDW38lnpoovhYn9pWc93CLbcRuvh9hIMe5G+cZMcrLfhqXkt/zRag/EV7MTKiIxyNU8Lxv1YbbB4xh3U/WnhmrHA3ZhKXAo=,iv:aYY7t0Zss2TSwuIrzyukPoYvKZva6qZn4JJvSyvOols=,tag:xkaiBNutE6+115Zsrlyrrw==,type:str] pgp: - - created_at: "2026-01-26T09:27:06Z" + - created_at: "2026-05-12T06:36:55Z" enc: |- -----BEGIN PGP MESSAGE----- - hF4Dy77zzNMwU0sSAQdAANcbJdqeIUK1mUDG/+GglQPLSlMcvS0xt0aP0eHsakEw - EU5sQytLTMNUYH6iRp7BacACVQ8T/1rllgno7lGN5nLoEUDbg/wOtFH3ZHFus6kj - 0l4Bwe9pK+UsHDqy6I38YWHU42AsEECCZjs5fPHUnKBR2oBBpf4S/Eg9/FVnRIOY - QFkPgGIyQIOPbCVzaIo+tzuIwwT0RjQQAcnPZM3eM0ie1q82eD9UstkwAV6Upzqv - =mxQm + hF4Dy77zzNMwU0sSAQdA3lqmunI4yr3ONiEDe0MCZBYScWoagdmILOjylvclZQIw + Ki9ZcnxRh5s2K90HDGpwMX7/PQoDH0jTfRMSH7YmW+pADy6yJxVWmx3tVQFgsKuG + 0lwB9iyP1HoEhAEW3C+6gjZ0QKUQm0gsEWLu7LvP+YHBfGXK9NeiK22eJ/iu39xb + 0qTLHzdR9vkQObEgBkh2l/GBSO9w1ceDKYgL5DxD1R0rHncbk6cNkqvA+QJEOQ== + =b+dX -----END PGP MESSAGE----- fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 unencrypted_suffix: _unencrypted From ffca2d728445dc8a44bd965007ca2a56598c90f2 Mon Sep 17 00:00:00 2001 From: Gal Netanel Date: Wed, 13 May 2026 16:42:29 +0300 Subject: [PATCH 235/286] APPENG-4146: Fix checklist parsing issue caused by unescaped quotes (#232) * APPENG-4146: Fix checklist parsing issue caused by unescaped quotes Checklist generation was failing during parsing due to unescaped quote characters. This change updates the regex used to find unescaped quote characters in the LLM checklist response and add unit tests to verify the fix --- src/exploit_iq_commons/utils/string_utils.py | 20 ++-- .../tests/test_checklist_prompt_generator.py | 93 +++++++++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py diff --git a/src/exploit_iq_commons/utils/string_utils.py b/src/exploit_iq_commons/utils/string_utils.py index 5ba6873d3..ad5061978 100644 --- a/src/exploit_iq_commons/utils/string_utils.py +++ b/src/exploit_iq_commons/utils/string_utils.py @@ -18,14 +18,20 @@ # Find all substrings that start and end with quotes, allowing for spaces before a comma or closing bracket re_quote_capture = re.compile( r""" - (['"]) # Opening quote - ( # Start capturing the quoted content - (?:\\.|[^\\])*? # Non-greedy match for any escaped character or non-backslash character - ) # End capturing the quoted content - \1 # Matching closing quote - (?=\s*,|\s*\]) # Lookahead for whitespace followed by a comma or closing bracket, without including it in the match + (['"]) # Group 1: Opening quote + ( # Group 2: Content + (?: + # Match anything UNLESS it is the specific outer list separator. + # Outer separator = quote + comma + space + quote + CAPITAL LETTER. + # OR it is the closing bracket at the very end of the string. + (?!\1\s*,\s*\1\s*[A-Z]|\1\s*\]\s*$) + . + )* + ) + \1 # Closing quote + (?=\s*[,\]]) # Lookahead for comma or closing bracket """, - re.VERBOSE) + re.VERBOSE | re.DOTALL) # Using the ASCII flag to stricly match numbers 0-9 and not on unicode numbers which include characters like ٤. # https://cve.mitre.org/cve/identifiers/syntaxchange.html diff --git a/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py b/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py new file mode 100644 index 000000000..76d17a893 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py @@ -0,0 +1,93 @@ +import pytest +import logging +logger = logging.getLogger(__name__) + +from vuln_analysis.utils.checklist_prompt_generator import _parse_list + +line1 = '"Is the `USER $USERNAME` Dockerfile instruction used in the container image, potentially allowing an attacker to manipulate supplementary group access?",' +line2 = '"Are there any instances of `ENTRYPOINT ["su", "-", "user"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?",' +line3 = '"Does the container image run with a user that has elevated privileges, potentially increasing the risk of exploiting supplementary group mishandling?",' +line4 = '"Are there any sensitive files or directories within the container that could be accessed by an attacker if they were to bypass primary group restrictions using supplementary group access?",' +line5 = '"Does the container\'s configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability\'s exploitability?"' +list1_as_string = f"[{line1}{line2}{line3}{line4}{line5}]" +list2_as_string_with_newlines = f"[{line1}\n{line2}\n{line3}\n{line4}\n{line5}]" + +expected_line1 = "Is the `USER $USERNAME` Dockerfile instruction used in the container image, potentially allowing an attacker to manipulate supplementary group access?" +expected_line2 = "Are there any instances of `ENTRYPOINT [\"su\", \"-\", \"user\"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?" +expected_line3 = "Does the container image run with a user that has elevated privileges, potentially increasing the risk of exploiting supplementary group mishandling?" +expected_line4 = "Are there any sensitive files or directories within the container that could be accessed by an attacker if they were to bypass primary group restrictions using supplementary group access?" +expected_line5 = "Does the container\'s configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability\'s exploitability?" + +@pytest.mark.asyncio +async def test_parse_single_with_none_escaped_quotes_without_newlines(): + result = await _parse_list([list1_as_string]) + assert result[0][0] == expected_line1 + assert result[0][1] == expected_line2 + assert result[0][2] == expected_line3 + assert result[0][3] == expected_line4 + assert result[0][4] == expected_line5 + +@pytest.mark.asyncio +async def test_parse_single_with_none_escaped_quotes_with_newlines(): + result = await _parse_list([list2_as_string_with_newlines]) + assert result[0][0] == expected_line1 + assert result[0][1] == expected_line2 + assert result[0][2] == expected_line3 + assert result[0][3] == expected_line4 + assert result[0][4] == expected_line5 + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes(): + input_backslashes = r""" + [ + "Are there any instances of `ENTRYPOINT ["su", "-", "user"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?", + "Does \ the container's configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability's exploitability?" + ] + """ + expected_line2 = "Does \\ the container's configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability's exploitability?" + result = await _parse_list([input_backslashes]) + assert result[0][1] == expected_line2 + + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_n(): + input_backslashes_with_newline = """ + [ + "line1 with \nnewlines" + ] + """ + expected_line1 = "line1 with newlines" + result = await _parse_list([input_backslashes_with_newline]) + assert result[0][0] == expected_line1 + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_t(): + char_line = r"line1 with \tchar" + char_line_none_raw = "line1 with \tchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_f(): + char_line = r"line1 with \fchar" + char_line_none_raw = "line1 with \fchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_b(): + char_line = r"line1 with \bchar" + char_line_none_raw = "line1 with \bchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_backslash(): + char_line = r"line1 with \ char" + char_line_none_raw = "line1 with \\ char" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) From 9ed3e225cab906252c4f5ca9bc7a3d5d46db646d Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Tue, 19 May 2026 13:55:43 +0300 Subject: [PATCH 236/286] Added Code Understanding sub agent (#231) * Added Code Understanding sub agent Signed-off-by: Theodor Mihalache --- .tekton/on-cm-runner.yaml | 2 +- Makefile | 4 +- src/exploit_iq_commons/utils/dep_tree.py | 33 +- .../utils/document_embedding.py | 64 +- .../c_lang_function_parsers.py | 6 + .../golang_functions_parsers.py | 9 +- .../java_functions_parsers.py | 93 +- .../javascript_functions_parser.py | 8 + .../lang_functions_parsers.py | 5 +- .../python_functions_parser.py | 7 + src/vuln_analysis/configs/config-http-nim.yml | 9 + .../configs/config-http-openai.yml | 9 + src/vuln_analysis/configs/config-tracing.yml | 9 + src/vuln_analysis/configs/config.yml | 9 + src/vuln_analysis/functions/agent_registry.py | 41 + .../functions/base_graph_agent.py | 504 +++++++++++ .../functions/code_understanding_agent.py | 189 ++++ .../functions/code_understanding_internals.py | 91 ++ src/vuln_analysis/functions/cve_agent.py | 754 ++-------------- src/vuln_analysis/functions/dispatcher.py | 107 +++ .../functions/reachability_agent.py | 338 +++++++ .../functions/react_internals.py | 157 +++- src/vuln_analysis/register.py | 8 +- src/vuln_analysis/runtime_context.py | 7 +- .../tools/configuration_scanner.py | 249 +++++ .../tools/import_usage_analyzer.py | 158 ++++ .../tools/lexical_full_search.py | 12 +- src/vuln_analysis/tools/local_vdb.py | 50 +- .../tests/test_transitive_code_search.py | 24 + src/vuln_analysis/tools/tool_names.py | 58 +- .../code_understanding_prompt_factory.py | 85 ++ src/vuln_analysis/utils/full_text_search.py | 96 +- src/vuln_analysis/utils/intel_utils.py | 13 + .../utils/source_classification.py | 67 ++ src/vuln_analysis/utils/token_utils.py | 99 ++ tests/agent_test_helpers.py | 61 ++ tests/test_agent_registry.py | 102 +++ tests/test_base_graph_agent.py | 848 ++++++++++++++++++ tests/test_base_tool_descriptions.py | 39 +- tests/test_build_code_understanding_tools.py | 276 ++++++ tests/test_code_understanding_internals.py | 176 ++++ .../test_code_understanding_prompt_factory.py | 62 ++ tests/test_configuration_scanner.py | 420 +++++++++ tests/test_dispatcher.py | 160 ++++ tests/test_import_usage_analyzer.py | 278 ++++++ tests/test_intel_utils.py | 69 ++ tests/test_process_steps.py | 294 ++++++ tests/test_python_segmenter.py | 2 +- tests/test_reachability_agent.py | 221 +++++ tests/test_react_internals_rules.py | 418 +++++++++ tests/test_source_classification.py | 141 +++ tests/test_transitive_detection.py | 130 ++- 52 files changed, 6215 insertions(+), 856 deletions(-) create mode 100644 src/vuln_analysis/functions/agent_registry.py create mode 100644 src/vuln_analysis/functions/base_graph_agent.py create mode 100644 src/vuln_analysis/functions/code_understanding_agent.py create mode 100644 src/vuln_analysis/functions/code_understanding_internals.py create mode 100644 src/vuln_analysis/functions/dispatcher.py create mode 100644 src/vuln_analysis/functions/reachability_agent.py create mode 100644 src/vuln_analysis/tools/configuration_scanner.py create mode 100644 src/vuln_analysis/tools/import_usage_analyzer.py create mode 100644 src/vuln_analysis/utils/code_understanding_prompt_factory.py create mode 100644 src/vuln_analysis/utils/source_classification.py create mode 100644 src/vuln_analysis/utils/token_utils.py create mode 100644 tests/agent_test_helpers.py create mode 100644 tests/test_agent_registry.py create mode 100644 tests/test_base_graph_agent.py create mode 100644 tests/test_build_code_understanding_tools.py create mode 100644 tests/test_code_understanding_internals.py create mode 100644 tests/test_code_understanding_prompt_factory.py create mode 100644 tests/test_configuration_scanner.py create mode 100644 tests/test_dispatcher.py create mode 100644 tests/test_import_usage_analyzer.py create mode 100644 tests/test_intel_utils.py create mode 100644 tests/test_process_steps.py create mode 100644 tests/test_reachability_agent.py create mode 100644 tests/test_react_internals_rules.py create mode 100644 tests/test_source_classification.py diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 27d0f6672..6630f8f27 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -269,7 +269,7 @@ spec: resources: requests: memory: "1Gi" - cpu: "500m" + cpu: "300m" readinessProbe: tcpSocket: port: 9092 diff --git a/Makefile b/Makefile index 74b2a19c2..cf7335721 100644 --- a/Makefile +++ b/Makefile @@ -71,8 +71,8 @@ test: test-unit test-llm-metrics ## Run all tests. @echo "All tests have been run." test-unit: ## Run unit tests. - @echo "Running unit tests in $(SRC_DIR)..." - @python -m pytest $(SRC_DIR) $(PYTEST_OPTS) + @echo "Running unit tests in $(SRC_DIR) and tests/..." + @python -m pytest $(SRC_DIR) tests/ $(PYTEST_OPTS) test-llm-metrics: ## Run LLM metrics tests. @echo "Running LLM metrics tests..." diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 76421ef3f..f32d8bb12 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -168,6 +168,10 @@ class DependencyTreeBuilder(ABC): supported ecosystem. """ + # Directory name where dependency source files are stored (e.g. "vendor", "node_modules"). + # Each subclass sets this to its ecosystem's convention. + DEP_SOURCE_DIR: str = "" + @abstractmethod # Build a sort of "upside down" tree - a dict containing mapping of each # package to a list of all consuming packages @@ -182,6 +186,8 @@ def install_dependencies(self, manifest_path: Path): class CCppDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = C_DEP_LIBS_NAME + # Pre-compiled regex patterns (optimization: compile once, use many times) INCLUDE_COMBINED_RE = re.compile(r'#include\s*([<"])([^>"]+)[>"]') # Combined pattern # Common system headers for fast-path resolution @@ -209,7 +215,7 @@ def __init__(self): "test", "tests", "doc", "docs", "example", "examples", "bench", "benchmark", "demo", "sample" ] self.C_STANDARD_LIB = "glibc" - self.RPM_LIBS_DIR = C_DEP_LIBS_NAME + self.RPM_LIBS_DIR = self.DEP_SOURCE_DIR self.output_json_path = None self.ccp_dep_tree = None @@ -751,6 +757,7 @@ def find_project_name(self, root_dir="."): class GoDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "vendor" def install_dependencies(self, manifest_path: Path): self.download_go_mod_vendor(manifest_path) @@ -857,6 +864,7 @@ def extract_package_name(self, package_name: str) -> str: class JavaDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "dependencies-sources" def __init__(self, query: str): self._query = query @@ -871,7 +879,7 @@ def __check_file_exists(self, dir_path: str | Path, filename: str) -> bool: def install_dependencies(self, manifest_path: Path): mvn_command = "mvn" settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml') - source_path = "dependencies-sources" + source_path = self.DEP_SOURCE_DIR if self.__check_file_exists(manifest_path, "mvnw"): mvn_command = "./mvnw" @@ -1126,6 +1134,7 @@ def looks_like_version(v: str) -> bool: return depth, coord class PythonDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = TRANSITIVE_ENV_NAME def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' @@ -1662,6 +1671,7 @@ def _fallback_if_stub_only(self, package_name: str, site_packages: Path): class JavaScriptDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "node_modules" def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: @@ -1733,6 +1743,25 @@ def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "" f'{programming_language}') +# Maps each ecosystem to its builder class — used to build ECOSYSTEM_DEP_DIRS +# from class-level DEP_SOURCE_DIR without instantiating builders. +_ECOSYSTEM_BUILDER_MAP: dict[Ecosystem, type[DependencyTreeBuilder]] = { + Ecosystem.C_CPP: CCppDependencyTreeBuilder, + Ecosystem.GO: GoDependencyTreeBuilder, + Ecosystem.JAVA: JavaDependencyTreeBuilder, + Ecosystem.PYTHON: PythonDependencyTreeBuilder, + Ecosystem.JAVASCRIPT: JavaScriptDependencyTreeBuilder, +} + +# Dynamic mapping of ecosystem → dependency source directory prefix. +# Built from each builder's DEP_SOURCE_DIR class attribute. +ECOSYSTEM_DEP_DIRS: dict[str, str] = { + eco.value: cls.DEP_SOURCE_DIR + "/" + for eco, cls in _ECOSYSTEM_BUILDER_MAP.items() + if cls.DEP_SOURCE_DIR +} + + class DependencyTree: """ A class that represents a dependency tree to access an appropriate diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index 035e86157..642757b2f 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -217,6 +217,52 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) +class _LoggingEmbeddingProxy: + """Wraps an Embeddings instance to log per-batch progress during VDB creation. + + FAISS calls embed_documents once with all texts; the NIM SDK loops + internally in batches of max_batch_size calling _embed per batch. + This proxy intercepts embed_documents and does the batching itself + so it can log progress between batches. + + NOTE: Tightly coupled with langchain_nvidia_ai_endpoints.NVIDIAEmbeddings. + Calls the private _embed(texts, model_type="passage") method directly. + Other Embeddings implementations (langchain ABC) don't expose _embed, + so this proxy will break if the embedding type changes or if NVIDIAEmbeddings + renames/removes _embed in a future version. + """ + + def __init__(self, embedding, total_chunks: int, start_time: float): + self._embedding = embedding + self._total_chunks = total_chunks + self._start_time = start_time + self._embedded = 0 + + def embed_documents(self, texts): + batch_size = getattr(self._embedding, "max_batch_size", 128) + all_embeddings = [] + for i in range(0, len(texts), batch_size): + batch = texts[i:i + batch_size] + batch_start = time.time() + all_embeddings.extend(self._embedding._embed(batch, model_type="passage")) + self._embedded += len(batch) + elapsed = time.time() - self._start_time + rate = self._embedded / elapsed if elapsed > 0 else 0 + remaining_min = ((self._total_chunks - self._embedded) / rate / 60) if rate > 0 else 0 + logger.info("Embedding progress: %d / %d chunks (%.1f%%) - batch took %.2fs - ETA %.1f min", + self._embedded, self._total_chunks, + self._embedded / self._total_chunks * 100, + time.time() - batch_start, + remaining_min) + return all_embeddings + + def embed_query(self, text): + return self._embedding.embed_query(text) + + def __getattr__(self, name): + return getattr(self._embedding, name) + + class DocumentEmbedding: """ A class to create a FAISS database from a list of source documents. The source documents are collected from git @@ -374,8 +420,10 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: """ repo_path = self.get_repo_path(source_info) + cache_name = source_info.type if source_info.type != "code" else "" documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, - source_info.git_repo, source_info.ref) + source_info.git_repo, source_info.ref, + documents_name=cache_name) if documents_were_in_cache or len(documents) > 0: return documents @@ -387,7 +435,8 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: with repo_lock: # Re-check cache — another thread may have populated it while we waited. documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, - source_info.git_repo, source_info.ref) + source_info.git_repo, source_info.ref, + documents_name=cache_name) if documents_were_in_cache or len(documents) > 0: return documents @@ -403,7 +452,8 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: documents = loader.load() logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) - save_to_cache(self._pickle_cache_directory, source_info.git_repo, source_info.ref, documents) + save_to_cache(self._pickle_cache_directory, source_info.git_repo, source_info.ref, documents, + documents_name=cache_name) return documents def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathLike): @@ -465,8 +515,12 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL embedding_start_time = time.time() + # Wrap embedding in a proxy that logs batch progress + total_chunks = len(chunked_documents) + logging_embedding = _LoggingEmbeddingProxy(self._embedding, total_chunks, embedding_start_time) + # Create the FAISS database - db = FAISS.from_documents(chunked_documents, self._embedding) + db = FAISS.from_documents(chunked_documents, logging_embedding) logger.info("Completed embedding in %.2f seconds for '%s'", time.time() - embedding_start_time, output_path) @@ -513,7 +567,7 @@ def build_vdbs(self, # Create embeddings for each source type for source_type in ["code", "doc"]: - if ignore_code_embedding: + if ignore_code_embedding and source_type == "code": continue # Filter the source documents diff --git a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py index 37b73af18..7289e39ca 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py @@ -763,3 +763,9 @@ def is_call_allowed(self, pkg_docs: list[Document], caller_function: Document, c return False return True + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf'#include\s*[<"]({escaped}[^>"]*)[>"]', re.IGNORECASE | re.MULTILINE), + ] diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index 788050625..ed7e94703 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -598,4 +598,11 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package package_name = import_package_line.split(r"\s")[1] if package_name.strip().lower() == callee_package.strip().lower(): return True - return False \ No newline at end of file + return False + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf'import\s+"({escaped}[^"]*)"', re.IGNORECASE | re.MULTILINE), + re.compile(rf'import\s+\(\s*[^)]*"({escaped}[^"]*)"', re.IGNORECASE | re.MULTILINE), + ] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index a5aff80fa..3a2a1d2c9 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -1570,17 +1570,21 @@ def _iter_fqcn_candidates(raw_type_token: str): yield t return - # Simple token: try target allow-list by simple name + # Explicit import is the definitive binding for this simple name. + # Yield only if it resolves to a target type; either way, block + # the _target_by_simple fallthrough. + imp = _explicit_imports.get(t) + if imp: + if imp in target_class_names: + yield imp + return + + # Target allow-list by simple name (only reached when no explicit import) cands = _target_by_simple.get(t) if cands: for fq in cands: yield fq - # Explicit import disambiguation - imp = _explicit_imports.get(t) - if imp: - yield imp - # Wildcard imports: only usable if we can cheaply construct candidates # (we only build candidates that are already in target_class_names to avoid work) if _wild_import_pkgs and cands: @@ -1748,7 +1752,9 @@ def _extract_ctor_type(expr: str) -> str: caller_function_index=_caller_key(), target_class_names=target_class_names, function=function, - code_documents=code_documents + code_documents=code_documents, + caller_explicit_imports=_explicit_imports, + caller_package=_caller_pkg, ) if traced_cast: return True @@ -1807,7 +1813,9 @@ def _extract_ctor_type(expr: str) -> str: caller_function_index=caller_key, target_class_names=target_class_names, function=function, - code_documents=code_documents + code_documents=code_documents, + caller_explicit_imports=_explicit_imports, + caller_package=_caller_pkg, ) if traced: return True @@ -1850,6 +1858,8 @@ def __trace_down_package( target_class_names: frozenset[str], function: Document, code_documents: dict[str, Document], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", ) -> bool: variables_mappings = functions_local_variables_index.get(caller_function_index, {}) # CHANGED: safe fallback parts = expression.split(".") @@ -1880,6 +1890,22 @@ def _fqcn_candidates_from_token(type_token: str) -> list[str]: # Derive simple name and match against allow-list by suffix simple = t.rsplit(".", 1)[-1] + + if caller_explicit_imports: + imported_fqcn = caller_explicit_imports.get(simple) + if imported_fqcn and imported_fqcn not in target_class_names: + return [] + + if (not caller_explicit_imports or simple not in caller_explicit_imports) and caller_package: + same_pkg_fqcn = f"{caller_package}.{simple}" + if same_pkg_fqcn not in target_class_names: + for td in type_documents: + td_src = td.metadata.get('source', '') + if simple in td_src and td_src.endswith(f"/{simple}.java"): + pkg_path = caller_package.replace('.', '/') + if pkg_path in td_src: + return [] + out: list[str] = [] dot_suffix = "." + simple dollar_suffix = "$" + simple @@ -1916,7 +1942,9 @@ def _has_matching_type(type_token: str) -> bool: struct_initializer_expression=struct_initializer_expression, type_documents=type_documents, value_list=value_list, - target_class_names=target_class_names + target_class_names=target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, ) # Property/member is not in function, check if it's member/property of a type @@ -3312,11 +3340,18 @@ def _simple_to_fqcns_index(target_class_names: frozenset[str]) -> dict[str, tupl idx.setdefault(simple, []).append(fq) return {k: tuple(v) for k, v in idx.items()} - def _fqcn_candidates_from_token(self, type_token: str, target_class_names: frozenset[str]) -> tuple[str, ...]: + def _fqcn_candidates_from_token(self, type_token: str, target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", + type_documents: list | None = None) -> tuple[str, ...]: """ Map a possibly-simple type token to FQCN candidates strictly within `target_class_names`. - If token already equals an allowed FQCN => (token,) - Else => all allowed FQCNs that share the same simple name + - If caller_explicit_imports maps the simple name to a FQCN NOT in target_class_names, + the caller's type is definitively different => () + - If the caller's own package contains a class with this simple name and that FQCN + is NOT in target_class_names, the caller's type is the same-package one => () """ t = self._normalize_type_token(type_token) if not t: @@ -3326,6 +3361,19 @@ def _fqcn_candidates_from_token(self, type_token: str, target_class_names: froze simple = self._simple_name_from_type_token(t) if not simple: return () + if caller_explicit_imports: + imported_fqcn = caller_explicit_imports.get(simple) + if imported_fqcn and imported_fqcn not in target_class_names: + return () + if (not caller_explicit_imports or simple not in caller_explicit_imports) and caller_package and type_documents: + same_pkg_fqcn = f"{caller_package}.{simple}" + if same_pkg_fqcn not in target_class_names: + pkg_path = caller_package.replace('.', '/') + suffix = f"/{simple}.java" + for td in type_documents: + td_src = td.metadata.get('source', '') + if td_src.endswith(suffix) and pkg_path in td_src: + return () return self._simple_to_fqcns_index(target_class_names).get(simple, ()) def _has_matching_type_in_package( @@ -3334,11 +3382,16 @@ def _has_matching_type_in_package( type_token: str, type_documents: list[Document], target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", ) -> bool: """ Calls __get_type_docs_matched_with_callee_type with FQCN candidates only. """ - for fq in self._fqcn_candidates_from_token(type_token, target_class_names): + for fq in self._fqcn_candidates_from_token(type_token, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, + type_documents=type_documents): if self.__get_type_docs_matched_with_callee_type(callee_package, fq, type_documents, target_class_names): return True return False @@ -3459,11 +3512,15 @@ def __lookup_package( type_documents, value_list, target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", ) -> bool: if not struct_initializer_expression and resolved_type not in JAVA_METHOD_PRIM_TYPES: if resolved_type and resolved_type not in JAVA_METHOD_PRIM_TYPES: if self._has_matching_type_in_package( - callee_package, resolved_type, type_documents, target_class_names + callee_package, resolved_type, type_documents, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, ): return True @@ -3473,7 +3530,9 @@ def __lookup_package( elif struct_initializer_expression: struct_type = struct_initializer_expression.group(0) # TODO list of expressions if self._has_matching_type_in_package( - callee_package, struct_type, type_documents, target_class_names + callee_package, struct_type, type_documents, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, ): return True @@ -3652,4 +3711,10 @@ def get_package_name(self, function: Document, package_name: str) -> str: artifact_version = f"{parts[1]}:{parts[2]}" else: artifact_version = package_name - return package_name if jar_name == artifact_version else '' \ No newline at end of file + return package_name if jar_name == artifact_version else '' + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"import\s+(?:static\s+)?({escaped}[\w.]*)\s*;", re.IGNORECASE | re.MULTILINE), + ] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py index 0275b64f9..3c6017d77 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -1011,6 +1011,14 @@ def document_imports_package(self, documents: dict[str, Document], package_name: importing_docs.append(doc) return importing_docs + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"""require\s*\(\s*['"]({escaped}[^'"]*)['"]\s*\)""", re.IGNORECASE | re.MULTILINE), + re.compile(rf"""import\s+.*?\s+from\s+['"]({escaped}[^'"]*)['"]\s*""", re.IGNORECASE | re.MULTILINE), + re.compile(rf"""import\s+['"]({escaped}[^'"]*)['"]\s*""", re.IGNORECASE | re.MULTILINE), + ] + def is_a_package(self, package_name: str, doc: Document) -> bool: return package_name in self.get_package_names(doc) diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py index f7160593c..ab1da4064 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -198,4 +198,7 @@ def create_dummy_for_standard_lib(self, package_name: str) -> bool: # is this call allowed by the language? def is_call_allowed(self,pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: - return True \ No newline at end of file + return True + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + return [re.compile(re.escape(package_name), re.IGNORECASE)] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index f019cbbf3..fe7eeec7d 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -417,6 +417,13 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package return True return False + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"import\s+({escaped}[\w.]*)", re.IGNORECASE | re.MULTILINE), + re.compile(rf"from\s+({escaped}[\w.]*)\s+import\s+", re.IGNORECASE | re.MULTILINE), + ] + def is_a_package(self, package_name: str, doc: Document) -> bool: return (not self.is_root_package(doc) and self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index d6321ae61..1d3ae4c23 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -78,6 +78,13 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -90,6 +97,8 @@ functions: - Function Caller Finder - Function Locator - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index a67e18c1b..80c16d7b5 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -85,6 +85,13 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -97,6 +104,8 @@ functions: - Function Caller Finder - Function Locator - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 397258e62..a4d89ff9d 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -89,6 +89,13 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -101,6 +108,8 @@ functions: - Function Caller Finder - Function Locator - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 779b8efe5..9ba5f23be 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -62,6 +62,13 @@ functions: _type: container_image_analysis_data Function Library Version Finder: _type: calling_function_library_version_finder + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -71,6 +78,8 @@ functions: # - Code Keyword Search # Uncomment to enable keyword search - CVE Web Search - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false diff --git a/src/vuln_analysis/functions/agent_registry.py b/src/vuln_analysis/functions/agent_registry.py new file mode 100644 index 000000000..4c12f81a0 --- /dev/null +++ b/src/vuln_analysis/functions/agent_registry.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vuln_analysis.functions.base_graph_agent import BaseGraphAgent + +_AGENT_REGISTRY: dict[str, type[BaseGraphAgent]] = {} + + +def register_agent(agent_type: str): + """Class decorator that registers a BaseGraphAgent subclass under the given type name.""" + def wrapper(cls): + _AGENT_REGISTRY[agent_type] = cls + return cls + return wrapper + + +def get_agent_class(agent_type: str) -> type[BaseGraphAgent]: + if agent_type not in _AGENT_REGISTRY: + raise KeyError(f"Unknown agent type '{agent_type}'. Registered: {list(_AGENT_REGISTRY.keys())}") + return _AGENT_REGISTRY[agent_type] + + +def get_all_agent_types() -> list[str]: + return list(_AGENT_REGISTRY.keys()) diff --git a/src/vuln_analysis/functions/base_graph_agent.py b/src/vuln_analysis/functions/base_graph_agent.py new file mode 100644 index 000000000..51bbf4f7d --- /dev/null +++ b/src/vuln_analysis/functions/base_graph_agent.py @@ -0,0 +1,504 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import uuid +from abc import ABC, abstractmethod + +from langchain_core.messages import SystemMessage, HumanMessage, AIMessage, RemoveMessage +from langgraph.graph import StateGraph, END, START +from langgraph.prebuilt import ToolNode + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.functions.react_internals import ( + AgentState, + BaseRulesTracker, + Thought, + Observation, + CodeFindings, + PackageSelection, + _build_tool_arguments, + FORCED_FINISH_PROMPT, + COMPREHENSION_PROMPT, + MEMORY_UPDATE_PROMPT, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.utils.token_utils import count_tokens, estimate_tokens, truncate_tool_output + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + +_TOOL_AVAILABILITY = { + ToolNames.CODE_SEMANTIC_SEARCH: lambda config, state: state.code_vdb_path is not None, + ToolNames.DOCS_SEMANTIC_SEARCH: lambda config, state: state.doc_vdb_path is not None, + ToolNames.CODE_KEYWORD_SEARCH: lambda config, state: state.code_index_path is not None, + ToolNames.IMPORT_USAGE_ANALYZER: lambda config, state: state.code_index_path is not None, + ToolNames.CVE_WEB_SEARCH: lambda config, state: config.cve_web_search_enabled, + ToolNames.CALL_CHAIN_ANALYZER: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, + ToolNames.FUNCTION_CALLER_FINDER: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, + ToolNames.FUNCTION_LOCATOR: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, +} + + +def _is_tool_available(tool_name, config, state): + check = _TOOL_AVAILABILITY.get(tool_name) + return check(config, state) if check else True + + +class BaseGraphAgent(ABC): + """Template for LangGraph-based CVE investigation agents. + + Subclasses must implement: + - pre_process_node: initialize agent state (prompts, package selection, etc.) + - get_tools: load and select which tools this agent uses + - create_rules_tracker: return the appropriate RulesTracker instance + + Shared graph nodes (thought, observation, forced_finish, should_continue) + and the graph wiring are provided by this base class. + """ + + def __init__(self, tools: list, llm, config): + self.tools = tools + self.config = config + self.thought_llm = llm.with_structured_output(Thought) + self.comprehension_llm = llm.with_structured_output(CodeFindings) + self.observation_llm = llm.with_structured_output(Observation) + self.package_filter_llm = llm.with_structured_output(PackageSelection) + + @property + def agent_type(self) -> str: + """Short identifier for tracing spans (e.g. 'reachability', 'cu').""" + return "base" + + @staticmethod + def _load_all_tools(builder, config) -> list: + from aiq.builder.framework_enum import LLMFrameworkEnum + return builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + async def _select_package( + self, ecosystem: str, candidate_packages: list[dict], critical_context: list[str], + workflow_state, + ) -> tuple[list[str], str | None]: + """Select target package from candidates using LLM filter. + Returns (filtered_critical_context, selected_package). + """ + from vuln_analysis.functions.react_internals import build_package_filter_prompt, _find_image_matching_candidate + from vuln_analysis.utils.intel_utils import filter_context_to_package + + selected_package = None + if len(candidate_packages) > 1: + image_input = workflow_state.original_input.input.image + image_name = image_input.name + image_repo = image_input.source_info[0].git_repo if image_input.source_info else None + + matched = _find_image_matching_candidate(candidate_packages, image_name, image_repo) + if matched: + selected_package = matched + logger.info("Package filter matched '%s' from image/repo (no LLM call needed, %d candidates)", + selected_package, len(candidate_packages)) + else: + filter_prompt = build_package_filter_prompt( + ecosystem, candidate_packages, + image_name=image_name, image_repo=image_repo, + critical_context=critical_context, + ) + selection = await self.package_filter_llm.ainvoke([HumanMessage(content=filter_prompt)]) + selected_package = selection.selected_package + logger.info("Package filter selected '%s' from %d candidates (reason: %s)", + selected_package, len(candidate_packages), selection.reason) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + elif len(candidate_packages) == 1: + selected_package = candidate_packages[0].get("name") + logger.info("Single candidate package: '%s'", selected_package) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + return critical_context, selected_package + + # --- Template methods (subclasses MUST override) --- + + @abstractmethod + async def pre_process_node(self, state: AgentState) -> AgentState: + """Initialize agent state: select package, build prompts, set rules.""" + ... + + @staticmethod + @abstractmethod + def get_tools(builder, config, state) -> list: + """Load tools from the builder and select those this agent needs.""" + ... + + @staticmethod + @abstractmethod + def create_rules_tracker() -> BaseRulesTracker: + """Return a fresh RulesTracker instance for a single question.""" + ... + + # --- Optional hooks (subclasses MAY override) --- + + def build_comprehension_context(self, state: AgentState) -> str: + """Return the vulnerability context string for the comprehension prompt. + Override in subclasses to reduce context and prevent hallucination.""" + ctx_lines = state.get("critical_context", []) + return "\n".join(ctx_lines) if ctx_lines else "N/A" + + def sanitize_findings(self, findings: list[str], state: AgentState) -> list[str]: + """Replace hallucinated CVE IDs in comprehension findings. + Only CVE IDs present in the current investigation are kept.""" + workflow_state = ctx_state.get() + allowed_ids = {intel.vuln_id for intel in workflow_state.cve_intel} + sanitized = [] + for finding in findings: + def _replace(m): + return m.group(0) if m.group(0) in allowed_ids else "the investigated vulnerability" + sanitized.append(re.sub(r"CVE-\d{4}-\d+", _replace, finding)) + return sanitized + + def post_observation(self, state: AgentState, tool_used: str, + tool_output: str, tool_input_detail: str) -> dict: + """Per-agent post-processing after observation_node core logic. + Returns dict to merge into observation_node output. Default: empty.""" + return {} + + def should_truncate_tool_output(self, state: AgentState, tool_used: str) -> bool: + """Whether to apply Java-specific truncation to tool output. Default: False.""" + return False + + def check_finish_allowed(self, state: AgentState) -> tuple[bool, str]: + """Hook to block premature finish. Returns (allowed, error_message). + Override in subclasses to enforce rules before allowing the agent to finish.""" + return True, "" + + # --- Shared helpers --- + + def _prune_messages_to_fit(self, messages: list, keep_tail: int, step_num: int, caller: str) -> None: + """Remove oldest conversation messages to fit within the token limit. + + Preserves messages[0] (system prompt) and the last ``keep_tail`` messages. + Mutates the list in place. + """ + max_tokens = self.config.context_window_token_limit + total = sum( + count_tokens(m.content) for m in messages + if hasattr(m, "content") and isinstance(m.content, str) + ) + min_count = 1 + keep_tail + if total <= max_tokens or len(messages) <= min_count: + return + orig_len = len(messages) + prunable = messages[1:-keep_tail] + for msg in list(prunable): + if total <= max_tokens: + break + total -= count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + messages.remove(msg) + removed = orig_len - len(messages) + if removed: + logger.info( + "%s pruning: removed %d messages, estimated tokens now ~%d (limit %d) at step %d", + caller, removed, total, max_tokens, step_num, + ) + + # --- Shared concrete graph nodes --- + + async def thought_node(self, state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function(f"{self.agent_type}_thought", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + obs = state.get("observation", None) + context_block = self._build_observation_context(obs) + if context_block is None and obs is not None: + context_block = "KNOWLEDGE:\n- No prior knowledge.\nLATEST FINDINGS:\n- No recent findings." + if context_block: + messages.append(SystemMessage(content=context_block)) + + # Keep system prompt (messages[0]) and KNOWLEDGE block (messages[-1]). + # ToolMessages are prunable because observation_node already distilled + # their content into the KNOWLEDGE block via comprehension. + self._prune_messages_to_fit(messages, keep_tail=1, step_num=step_num, caller="thought_node") + + response: Thought = await self.thought_llm.ainvoke(messages) + + final_answer = "waiting for the agent to respond" + if response.mode == "finish": + finish_ok, finish_msg = self.check_finish_allowed(state) + if not finish_ok: + logger.info("%s finish blocked by rules at step %d: %s", self.agent_type, step_num, finish_msg) + span.set_output({"mode": "finish_blocked", "step": step_num + 1}) + blocked_ai = AIMessage(content=response.final_answer or response.thought or "I want to finish.") + return { + "messages": [blocked_ai, HumanMessage(content=finish_msg)], + "thought": None, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": "waiting for the agent to respond", + } + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + elif response.actions is None: + logger.warning("%s LLM returned mode='act' but actions is None, forcing finish", self.agent_type) + ai_message = AIMessage(content=response.thought or "No actions provided, finishing.") + response = Thought( + thought=response.thought or "No actions provided", + mode="finish", + actions=None, + final_answer=response.thought or "Insufficient evidence to provide a definitive answer." + ) + final_answer = response.final_answer + else: + tool_name = response.actions.tool + try: + arguments = _build_tool_arguments(response.actions) + except ValueError as e: + logger.warning( + "%s bad tool arguments at step %d: %s", self.agent_type, step_num, e, + ) + span.set_output({"error": str(e), "step": step_num + 1}) + error_ai = AIMessage(content=response.thought or "I want to call a tool.") + return { + "messages": [error_ai, HumanMessage( + content=f"ERROR: {e}. Provide the required arguments and try again." + )], + "thought": None, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": "waiting for the agent to respond", + } + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{ + "name": tool_name, + "args": arguments, + "id": tool_call_id + }] + ) + + span.set_output({"mode": response.mode, "step": step_num + 1}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": final_answer + } + except Exception as e: + logger.exception("%s thought_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def should_continue(self, state: AgentState) -> str: + if state.get("step", 0) >= state.get("max_steps", self.config.max_iterations): + return "forced_finish_node" + thought = state.get("thought", None) + if thought is None: + return "thought_node" + if thought.mode == "finish": + return END + return "tool_node" + + @staticmethod + def _build_observation_context(obs) -> str | None: + """Format observation memory and recent findings into a context block.""" + if obs is None: + return None + memory_list = obs.memory if obs.memory else [] + recent_findings = obs.results if obs.results else [] + if not memory_list and not recent_findings: + return None + parts = [] + if memory_list: + parts.append("KNOWLEDGE:\n" + "\n".join(f"- {m}" for m in memory_list)) + if recent_findings: + parts.append("LATEST FINDINGS:\n" + "\n".join(f"- {f}" for f in recent_findings)) + return "\n".join(parts) + + async def forced_finish_node(self, state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function(f"{self.agent_type}_forced_finish", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + context_block = self._build_observation_context(state.get("observation", None)) + if context_block: + messages.append(SystemMessage(content=context_block)) + question = state.get("input", "") + finish_prompt = f"QUESTION: {question}\n\n{FORCED_FINISH_PROMPT}" if question else FORCED_FINISH_PROMPT + messages.append(HumanMessage(content=finish_prompt)) + keep_tail = 2 if context_block else 1 + self._prune_messages_to_fit(messages, keep_tail=keep_tail, step_num=step_num, caller="forced_finish_node") + response: Thought = await self.thought_llm.ainvoke(messages) + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Failed to generate a final answer within the maximum allowed steps." + ai_message = AIMessage(content=final_answer) + response = Thought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer + ) + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", self.config.max_iterations), + "observation": state.get("observation", None), + "output": final_answer + } + except Exception as e: + logger.exception("%s forced_finish_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def observation_node(self, state: AgentState) -> AgentState: + tool_message = state["messages"][-1] + last_thought_text = state["thought"].thought if state.get("thought") else "No previous thought." + tool_used = state["thought"].actions.tool if state.get("thought") and state["thought"].actions else "Unknown" + tool_input_detail = "" + if state.get("thought") and state["thought"].actions: + actions = state["thought"].actions + if actions.package_name and actions.function_name: + tool_input_detail = f"{actions.package_name},{actions.function_name}" + elif actions.query: + tool_input_detail = actions.query + elif actions.tool_input: + tool_input_detail = actions.tool_input + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + rules_tracker = state.get("rules_tracker") + with AGENT_TRACER.push_active_function(f"{self.agent_type}_observation", input_data=f"tool:{tool_used}") as span: + try: + tool_output_for_llm = tool_message.content + result, error_message = rules_tracker.check_thought_behavior(tool_used, tool_input_detail, tool_output_for_llm) + if result: + span.set_output({"rule_error": error_message}) + return {"messages": [HumanMessage(content=error_message)]} + + if self.should_truncate_tool_output(state, tool_used): + truncated_output = truncate_tool_output(tool_output_for_llm, tool_used) + else: + truncated_output = tool_output_for_llm + + critical_context_text = self.build_comprehension_context(state) + comp_prompt = COMPREHENSION_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + critical_context=critical_context_text, + tool_used=tool_used, + tool_input_detail=tool_input_detail, + last_thought_text=last_thought_text, + tool_output=truncated_output, + ) + max_input_tokens = self.config.context_window_token_limit - 2000 + prompt_tokens = count_tokens(comp_prompt) + if prompt_tokens > max_input_tokens: + overflow = prompt_tokens - max_input_tokens + output_tokens = count_tokens(truncated_output) + safe_budget = max(200, output_tokens - overflow) + truncated_output = truncate_tool_output(truncated_output, tool_used, max_tokens=safe_budget) + comp_prompt = COMPREHENSION_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + critical_context=critical_context_text, + tool_used=tool_used, + tool_input_detail=tool_input_detail, + last_thought_text=last_thought_text, + tool_output=truncated_output, + ) + logger.info( + "Comprehension prompt truncated: %d -> %d tokens (limit %d)", + prompt_tokens, count_tokens(comp_prompt), max_input_tokens, + ) + code_findings: CodeFindings = await self.comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + + sanitized = self.sanitize_findings(code_findings.findings, state) + findings_text = "\n".join(f"- {f}" for f in sanitized) + + mem_prompt = MEMORY_UPDATE_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + previous_memory=previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await self.observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") + estimated = estimate_tokens(active_prompt, messages, new_observation) + prune_messages = [] + orig_estimated = estimated + + span_trace_dict = {"comprehension_findings": sanitized, "tool_outcome": code_findings.tool_outcome} + + if estimated > self.config.context_window_token_limit and len(messages) > 3: + prunable = messages[1:-2] + for msg in prunable: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= self.config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, self.config.context_window_token_limit, + ) + span_trace_dict["orig_estimated"] = orig_estimated + span_trace_dict["estimated"] = estimated + + # Agent-specific post-processing hook + extra = self.post_observation(state, tool_used, tool_output_for_llm, tool_input_detail) + + span.set_output(span_trace_dict) + base_result = { + "messages": prune_messages, + "observation": new_observation, + "step": state.get("step", 0), + } + base_result.update(extra) + return base_result + except Exception as e: + logger.exception("%s observation_node failed", self.agent_type) + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + # --- Graph wiring --- + # If overridden, should_continue must also be updated to match the new node names. + + async def build_graph(self): + tool_node = ToolNode(self.tools, handle_tool_errors=True) + + flow = StateGraph(AgentState) + flow.add_node("thought_node", self.thought_node) + flow.add_node("tool_node", tool_node) + flow.add_node("forced_finish_node", self.forced_finish_node) + flow.add_node("pre_process_node", self.pre_process_node) + flow.add_node("observation_node", self.observation_node) + flow.add_edge(START, "pre_process_node") + flow.add_edge("pre_process_node", "thought_node") + flow.add_conditional_edges( + "thought_node", + self.should_continue, + {END: END, "tool_node": "tool_node", "forced_finish_node": "forced_finish_node", "thought_node": "thought_node"} + ) + flow.add_edge("tool_node", "observation_node") + flow.add_edge("observation_node", "thought_node") + flow.add_edge("forced_finish_node", END) + + return flow.compile() diff --git a/src/vuln_analysis/functions/code_understanding_agent.py b/src/vuln_analysis/functions/code_understanding_agent.py new file mode 100644 index 000000000..7368dcd28 --- /dev/null +++ b/src/vuln_analysis/functions/code_understanding_agent.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.functions.agent_registry import register_agent +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent, _is_tool_available + +from vuln_analysis.functions.react_internals import ( + AgentState, + Observation, +) +from vuln_analysis.functions.code_understanding_internals import ( + CU_AGENT_SYS_PROMPT, + CU_AGENT_THOUGHT_INSTRUCTIONS, + CodeUnderstandingRulesTracker, +) +from vuln_analysis.utils.code_understanding_prompt_factory import ( + CU_TOOL_SELECTION_STRATEGY, + CU_TOOL_GENERAL_DESCRIPTIONS, +) +from vuln_analysis.utils.intel_utils import build_critical_context +from vuln_analysis.runtime_context import ctx_state, cu_source_scope +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +def _build_cu_system_prompt(descriptions: str, tool_guidance: str) -> str: + return f"""{CU_AGENT_SYS_PROMPT} + + +{descriptions} + + + +{tool_guidance} + + +{CU_AGENT_THOUGHT_INSTRUCTIONS} + +RESPONSE: +{{""" + + +def _build_cu_tool_guidance(ecosystem: str, available_tools: list) -> tuple[str, str]: + tool_names = [t.name for t in available_tools] + tool_desc_lines = [f"{t.name}: {t.description}" for t in available_tools] + + lang = ecosystem.lower() if ecosystem else "" + if lang in CU_TOOL_SELECTION_STRATEGY: + logger.debug("Using %s-specific tool strategy for Code Understanding agent", lang) + guidance = CU_TOOL_SELECTION_STRATEGY[lang] + else: + logger.debug("No ecosystem-specific strategy for '%s', using generic Code Understanding tool guidance", lang) + guidance_list = [] + for name in tool_names: + if name in CU_TOOL_GENERAL_DESCRIPTIONS: + guidance_list.append(f"{name}: {CU_TOOL_GENERAL_DESCRIPTIONS[name]}") + guidance = "\n".join(guidance_list) if guidance_list else "Use the available tools to investigate the question." + + descriptions = "\n".join(tool_desc_lines) + return guidance, descriptions + + +@register_agent("code_understanding") +class CodeUnderstandingAgent(BaseGraphAgent): + + @property + def agent_type(self) -> str: + return "cu" + + _CU_TOOLS = frozenset({ + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + }) + + @staticmethod + def get_tools(builder, config, state) -> list: + all_tools = BaseGraphAgent._load_all_tools(builder, config) + return [ + t for t in all_tools + if t.name in CodeUnderstandingAgent._CU_TOOLS + and _is_tool_available(t.name, config, state) + ] + + @staticmethod + def create_rules_tracker() -> CodeUnderstandingRulesTracker: + return CodeUnderstandingRulesTracker() + + def build_comprehension_context(self, state: AgentState) -> str: + """Return minimal grounding context to prevent CVE ID hallucination. + + The CU agent's comprehension LLM only needs the CVE ID and package name + to ground its findings. Feeding the full GHSA/NVD context activates + parametric knowledge and causes the model to inject CVE IDs from training. + """ + workflow_state = ctx_state.get() + vuln_id = workflow_state.cve_intel[0].vuln_id if workflow_state.cve_intel else "unknown" + package = state.get("app_package") or "unknown" + return ( + f"Investigating {vuln_id} in package {package}.\n" + "Only extract facts explicitly stated in the tool output. " + "Do not add CVE IDs, vulnerability names, or advisory details " + "from your own knowledge." + ) + + async def pre_process_node(self, state: AgentState) -> AgentState: + workflow_state = ctx_state.get() + ecosystem = ( + workflow_state.original_input.input.image.ecosystem.value + if workflow_state.original_input.input.image.ecosystem + else "" + ) + with AGENT_TRACER.push_active_function("cu_pre_process", input_data=f"ecosystem:{ecosystem}") as span: + try: + precomputed = state.get("precomputed_intel") + if precomputed is not None: + critical_context = list(precomputed[0]) + candidate_packages = [dict(p) for p in precomputed[1]] + else: + critical_context, candidate_packages, _ = build_critical_context( + workflow_state.cve_intel + ) + + critical_context, selected_package = await self._select_package( + ecosystem, candidate_packages, critical_context, workflow_state, + ) + + critical_context.append( + "TASK: Investigate usage, configuration, and presence of the vulnerable " + "component in the container. Focus on how the component is used, " + "not on call-chain reachability." + ) + + scope_parts = [] + image_input = workflow_state.original_input.input.image + if image_input.source_info: + for si in image_input.source_info: + if si.git_repo: + repo_name = si.git_repo.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] + scope_parts.append(repo_name) + if selected_package: + scope_parts.append(selected_package) + cu_source_scope.set(scope_parts if scope_parts else None) + logger.debug("Code Understanding source scope set to: %s", scope_parts) + + tool_guidance, descriptions = _build_cu_tool_guidance(ecosystem, self.tools) + runtime_prompt = _build_cu_system_prompt(descriptions, tool_guidance) + active_tool_names = [t.name for t in self.tools] + + rules_tracker = state.get("rules_tracker") + rules_tracker.set_target_package(selected_package) + rules_tracker.set_allowed_tools(active_tool_names) + + span.set_output({ + "selected_package": selected_package, + "agent_type": "code_understanding", + }) + + return { + "ecosystem": ecosystem, + "runtime_prompt": runtime_prompt, + "is_reachability": "no", + "observation": Observation(memory=critical_context, results=[]), + "critical_context": critical_context, + "app_package": selected_package, + } + except Exception as e: + logger.exception("cu_pre_process_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise diff --git a/src/vuln_analysis/functions/code_understanding_internals.py b/src/vuln_analysis/functions/code_understanding_internals.py new file mode 100644 index 000000000..e6fc2bd2a --- /dev/null +++ b/src/vuln_analysis/functions/code_understanding_internals.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.react_internals import BaseRulesTracker + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CodeUnderstandingRulesTracker(BaseRulesTracker): + """Behavioral rules for the Code Understanding sub-agent.""" + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + """Check all Code Understanding rules in priority order. + + Returns (True, error_message) if a rule is violated, (False, '') otherwise. + Rule check order: duplicate call -> Rule 7 (dotted keywords) -> allowed tools. + """ + if self._rule_duplicate_call(action, action_input): + logger.debug("CU duplicate call rule triggered: '%s' with same input", action) + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) + if self._rule_number_7(action, action_input, output): + logger.debug("Code Understanding Rule 7 triggered: dotted query with empty results for tool '%s'", action) + return True, ( + "You are NOT following Rule 7. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules." + ) + if self._rule_use_allowed_tools(action): + logger.debug("Code Understanding allowed-tools rule triggered: '%s' not in %s", action, self.allowed_tools) + return True, ( + f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools " + f"{self.allowed_tools}. Follow the rules." + ) + self.add_action(action, action_input, output) + return False, "" + + +CU_AGENT_SYS_PROMPT = ( + "You are a security analyst investigating a code understanding question about a CVE vulnerability.\n" + "Your goal is to collect evidence about how the codebase uses, configures, or depends on " + "the affected component. This is NOT a reachability question -- do NOT trace call chains.\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the component is absent or not configured.\n" + "- DISTINGUISH where findings come from: main application code vs. dependency libraries.\n" + "- A component being present in dependencies does NOT mean the application uses it.\n" + "- Configuration in framework dependencies (e.g., Spring defaults) IS relevant evidence.\n" + "- Import in an intermediate library (e.g., Spring imports XStream, app imports Spring) IS relevant.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- If tool results conflict, state the conflict explicitly.\n" + "- When citing evidence, explain HOW it relates to the question." +) + +CU_AGENT_THOUGHT_INSTRUCTIONS = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Docs Semantic Search, Code Keyword Search: use query field. +4. Configuration Scanner: use query field with keywords describing what to look for. +5. Import Usage Analyzer: use query field with the package/module name. +6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. +7. If Code Keyword Search returns no results and the query contains dots, retry with just the final component. +8. Before concluding, if the question involves a specific library or package, you MUST use Import Usage Analyzer to check how it is imported and used across sources. +9. GUIDELINE: Synthesize findings across documentation, code, configuration, and imports before drawing conclusions. + + +{{"thought": "Check configuration files for security settings related to the vulnerability", "mode": "act", "actions": {{"tool": "Configuration Scanner", "package_name": null, "function_name": null, "query": "deserialization allowlist denylist security", "tool_input": null, "reason": "Check if vulnerable feature is enabled or disabled in config"}}, "final_answer": null}} + + +{{"thought": "Search source code for imports of the vulnerable library", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "import vulnerable_library", "tool_input": null, "reason": "Find where the package is imported in code"}}, "final_answer": null}} + + +{{"thought": "Check how the vulnerable library is imported and used across the codebase", "mode": "act", "actions": {{"tool": "Import Usage Analyzer", "package_name": null, "function_name": null, "query": "vulnerable_library", "tool_input": null, "reason": "Find all imports and usage sites of the vulnerable package"}}, "final_answer": null}} +""" \ No newline at end of file diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index f140850d4..d1fa7d4d7 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -12,49 +12,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os + import asyncio -import json -from pathlib import Path -from vuln_analysis.runtime_context import ctx_state -import typing + from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -from langchain.agents import AgentExecutor -from langchain.agents import create_react_agent -from langchain.agents.agent import RunnableAgent -from langchain.agents.mrkl.output_parser import MRKLOutputParser from langchain_core.exceptions import OutputParserException -from langchain_core.prompts import PromptTemplate +from langchain_core.messages import HumanMessage from pydantic import Field + from vuln_analysis.data_models.state import AgentMorpheusEngineState -from vuln_analysis.tools.tool_names import ToolNames -from vuln_analysis.tools.transitive_code_search import package_name_from_locator_query from vuln_analysis.utils.error_handling_decorator import ToolRaisedException -from vuln_analysis.utils.prompting import get_agent_prompt +from vuln_analysis.utils.intel_utils import build_critical_context +from vuln_analysis.runtime_context import ctx_state from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id - -from vuln_analysis.functions.react_internals import build_system_prompt, build_classification_prompt, build_package_filter_prompt, AgentState, Thought, Observation, Classification, PackageSelection, CodeFindings, SystemRulesTracker, _build_tool_arguments, FORCED_FINISH_PROMPT, COMPREHENSION_PROMPT, MEMORY_UPDATE_PROMPT, AGENT_SYS_PROMPT_NON_REACHABILITY, AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY, AGENT_THOUGHT_INSTRUCTIONS_GO -from vuln_analysis.utils.prompting import build_tool_descriptions -from vuln_analysis.utils.prompt_factory import TOOL_SELECTION_STRATEGY, TOOL_SELECTION_STRATEGY_NON_REACHABILITY, TOOL_ECOSYSTEM_REGISTRY, FEW_SHOT_EXAMPLES -from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_from_osv, filter_context_to_package -from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path -from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY -from langgraph.graph import StateGraph, END, START -from langgraph.prebuilt import ToolNode -from langchain_core.messages import SystemMessage, HumanMessage, AIMessage, RemoveMessage - -import uuid -import tiktoken from nat.builder.context import Context +from vuln_analysis.functions.agent_registry import get_agent_class, get_all_agent_types +from vuln_analysis.functions.dispatcher import QuestionRouting, dispatch_question +# Import agent modules to trigger @register_agent decorators +import vuln_analysis.functions.reachability_agent # noqa: F401 +import vuln_analysis.functions.code_understanding_agent # noqa: F401 + logger = LoggingFactory.get_agent_logger(__name__) AGENT_TRACER = Context.get() + class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): """ Defines a function that iterates through checklist items using provided tools and gathered intel. @@ -88,672 +75,74 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): description="Estimated token threshold for pruning old messages in observation node." ) -async def common_build_tools(config: CVEAgentExecutorToolConfig, builder: Builder, state: AgentMorpheusEngineState) -> tuple[list[typing.Any], list[str], list[str]]: - - tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - # Filter tools that are not available based on state - tools = [ - tool for tool in tools - if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or - (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or - (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None) or - (tool.name == ToolNames.CVE_WEB_SEARCH and not config.cve_web_search_enabled) or - (tool.name == ToolNames.CALL_CHAIN_ANALYZER and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) or - (tool.name == ToolNames.FUNCTION_CALLER_FINDER and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) or - (tool.name == ToolNames.FUNCTION_LOCATOR and (not config.transitive_search_tool_enabled or - state.code_index_path is None)) - ) - ] - # Get tool names after filtering for dynamic guidance - enabled_tool_names = [tool.name for tool in tools] - tool_descriptions_list = [t.name + ": " + t.description for t in tools] - # Build tool selection guidance with strategic context - tool_descriptions = build_tool_descriptions(enabled_tool_names) - return tools, tool_descriptions, tool_descriptions_list - -def _validate_go_vendor_packages( - source_info: list, - candidate_packages: list[dict], -) -> tuple[list[dict], list[str]]: - """Check which Go candidate packages actually exist in the vendor directory.""" - code_si = next((si for si in source_info if si.type == "code"), None) - if code_si is None: - return candidate_packages, [] - - repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) - vendor_path = repo_path / "vendor" - if not vendor_path.is_dir(): - return candidate_packages, [] - - validated = [] - removed = [] - for pkg in candidate_packages: - pkg_name = pkg.get("name", "") - if (vendor_path / pkg_name).is_dir(): - validated.append(pkg) - else: - removed.append(pkg_name) - - if validated: - return validated, removed - return candidate_packages, [] - - -async def _enrich_go_candidates( - cve_intel: list, - source_info: list, - critical_context: list[str], - candidate_packages: list[dict], - vulnerable_functions_set: set[str], -) -> tuple[list[dict], list[str]]: - """Enrich Go candidates via OSV and validate against vendor directory.""" - ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) - if not ghsa_has_packages or not vulnerable_functions_set: - intel = cve_intel[0] if cve_intel else None - if intel: - await enrich_go_from_osv(intel, critical_context, candidate_packages, vulnerable_functions_set) - - if candidate_packages: - candidate_packages, removed_pkgs = _validate_go_vendor_packages( - source_info, candidate_packages - ) - if removed_pkgs: - logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) - - return candidate_packages, sorted(vulnerable_functions_set) - - -async def _create_graph_agent(config: CVEAgentExecutorToolConfig, builder: Builder, state: AgentMorpheusEngineState): - - tools, tool_guidance_list, tool_descriptions_list = await common_build_tools(config, builder, state) - llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - thought_llm = llm.with_structured_output(Thought) - comprehension_llm = llm.with_structured_output(CodeFindings) - observation_llm = llm.with_structured_output(Observation) - reachability_llm = llm.with_structured_output(Classification) - package_filter_llm = llm.with_structured_output(PackageSelection) - tool_guidance = "\n".join(tool_guidance_list) - descriptions = "\n".join(tool_descriptions_list) - default_system_prompt = build_system_prompt(descriptions, tool_guidance) - tool_node = ToolNode(tools, handle_tool_errors=True) - TOOL_NODE = "tool_node" - THOUGHT_NODE = "thought_node" - FORCED_FINISH_NODE = "forced_finish_node" - PRE_PROCESS_NODE = "pre_process_node" - OBSERVATION_NODE = "observation_node" - - _tiktoken_enc = tiktoken.get_encoding("cl100k_base") - - def _count_tokens(text: str) -> int: - """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" - try: - return len(_tiktoken_enc.encode(text)) - except Exception: - return len(text) // 4 - - def _truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400) -> str: - """Truncate tool output to fit observation prompt within completion token budget. - - Code Keyword Search is the primary source of bloat (3700+ chars of redundant - source snippets). CCA and FL outputs are typically small. - Preserves the most informative parts based on tool type. - """ - token_count = _count_tokens(tool_output) - if token_count <= max_tokens: - return tool_output - - lines = tool_output.split('\n') - - if tool_name == ToolNames.CALL_CHAIN_ANALYZER: - head = '\n'.join(lines[:3]) - remaining = token_count - _count_tokens(head) - return f"{head}\n[... truncated {remaining} tokens ...]" - - if tool_name == ToolNames.CODE_KEYWORD_SEARCH: - kept_lines = [] - kept_tokens = 0 - for line in lines: - is_header = line.startswith("---") or "Main application" in line or "library dependencies" in line or line.strip() == "" - line_tokens = _count_tokens(line) - if is_header or kept_tokens + line_tokens <= max_tokens: - kept_lines.append(line) - kept_tokens += line_tokens - if kept_tokens >= max_tokens: - break - kept_lines.append(f"[... truncated {token_count - kept_tokens} tokens ...]") - return '\n'.join(kept_lines) - - # Default: head (70%) + tail (30%) - head_budget = int(max_tokens * 0.7) - tail_budget = max_tokens - head_budget - head_lines = [] - head_tokens = 0 - for line in lines: - lt = _count_tokens(line) - if head_tokens + lt > head_budget: - break - head_lines.append(line) - head_tokens += lt - tail_lines = [] - tail_tokens = 0 - for line in reversed(lines): - lt = _count_tokens(line) - if tail_tokens + lt > tail_budget: - break - tail_lines.insert(0, line) - tail_tokens += lt - truncated = token_count - head_tokens - tail_tokens - return '\n'.join(head_lines) + f"\n[... truncated {truncated} tokens ...]\n" + '\n'.join(tail_lines) - - def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: - """Estimate the token count thought_node will send to the LLM.""" - parts = [runtime_prompt] - for msg in messages: - if hasattr(msg, "content") and isinstance(msg.content, str): - parts.append(msg.content) - if observation is not None: - for item in (observation.memory or []): - parts.append(item) - for item in (observation.results or []): - parts.append(item) - return _count_tokens("\n".join(parts)) - - def _build_tool_guidance_for_ecosystem(ecosystem: str, available_tools: list, is_reachability: str = "yes") -> tuple[str, str]: - """Build tool guidance using language-specific strategies when available.""" - filtered_tools = [ - t for t in available_tools - if (t.name != ToolNames.FUNCTION_CALLER_FINDER or ecosystem == "go") and - (t.name != ToolNames.FUNCTION_LIBRARY_VERSION_FINDER or ecosystem == "java") - ] - list_of_tool_names = [t.name for t in filtered_tools] - list_of_tool_descriptions = [t.name + ": " + t.description for t in filtered_tools] - - strategy = TOOL_SELECTION_STRATEGY if is_reachability == "yes" else TOOL_SELECTION_STRATEGY_NON_REACHABILITY - lang = ecosystem.lower() if ecosystem else "" - if lang in strategy: - tool_guidance_local = strategy[lang] - if lang == "java": - tool_guidance_local += ( - " Use Function Library Version Finder to verify the installed version of a library " - "before concluding exploitability (e.g., input 'commons-beanutils')." - ) - if is_reachability == "yes": - hint = FEW_SHOT_EXAMPLES.get(lang, "") - if hint: - tool_guidance_local += f"\nHint: {hint}" - else: - tool_guidance_list_local = build_tool_descriptions(list_of_tool_names) - tool_guidance_local = "\n".join(tool_guidance_list_local) - - descriptions_local = "\n".join(list_of_tool_descriptions) - return tool_guidance_local, descriptions_local - - async def pre_process_node(state: AgentState) -> AgentState: - workflow_state = ctx_state.get() - ecosystem = workflow_state.original_input.input.image.ecosystem.value if workflow_state.original_input.input.image.ecosystem else "" - with AGENT_TRACER.push_active_function("pre_process node", input_data=f"ecosystem:{ecosystem}") as span: - try: - critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) - vulnerable_functions_set = set(vulnerable_functions) - - if ecosystem == "go": - candidate_packages, vulnerable_functions = await _enrich_go_candidates( - workflow_state.cve_intel, - workflow_state.original_input.input.image.source_info, - critical_context, - candidate_packages, - vulnerable_functions_set, - ) - - selected_package = None - app_package = None - if len(candidate_packages) > 1: - image_input = workflow_state.original_input.input.image - image_name = image_input.name - source_repos = image_input.source_info - image_repo = source_repos[0].git_repo if source_repos else None - filter_prompt = build_package_filter_prompt( - ecosystem, candidate_packages, - image_name=image_name, image_repo=image_repo, - critical_context=critical_context, - ) - selection: PackageSelection = await package_filter_llm.ainvoke([HumanMessage(content=filter_prompt)]) - selected_package = selection.selected_package - app_package = selected_package - logger.info("Package filter selected '%s' from %d candidates (reason: %s)", - selected_package, len(candidate_packages), selection.reason) - critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) - elif len(candidate_packages) == 1: - selected_package = candidate_packages[0].get("name") - app_package = selected_package - logger.info("Single candidate package after validation: '%s'", selected_package) - critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) - - critical_context.append( - "TASK: Investigate usage and reachability of the vulnerable function/module in the container. " - "Use the vulnerable module name from GHSA as primary investigation target." - ) - - question = state.get("input") or "" - context_block = "\n".join(critical_context) - classification_prompt = build_classification_prompt(context_block, question) - classification_result: Classification = await reachability_llm.ainvoke([HumanMessage(content=classification_prompt)]) - span.set_output({ - "critical_context": critical_context, - "candidate_packages": candidate_packages, - "selected_package": selected_package, - "app_package": app_package if selected_package else None, - "reachability_question": classification_result.is_reachability, - }) - - is_reachability = classification_result.is_reachability - - if is_reachability == "yes": - tool_guidance_local, descriptions_local = _build_tool_guidance_for_ecosystem(ecosystem, tools) - go_instructions = {"instructions": AGENT_THOUGHT_INSTRUCTIONS_GO} if ecosystem == "go" else {} - runtime_prompt = build_system_prompt(descriptions_local, tool_guidance_local, **go_instructions) - active_tool_names = [t.name for t in tools] - else: - reachability_tool_names = { ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER} - non_reach_tools = [t for t in tools if t.name not in reachability_tool_names] - tool_guidance_local, descriptions_local = _build_tool_guidance_for_ecosystem(ecosystem, non_reach_tools, is_reachability="no") - runtime_prompt = build_system_prompt( - descriptions_local, tool_guidance_local, - instructions=AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY, - sys_prompt=AGENT_SYS_PROMPT_NON_REACHABILITY, - ) - active_tool_names = [t.name for t in non_reach_tools] - logger.info("Non-reachability question detected; removed reachability tools from prompt") - rules_tracker = state.get("rules_tracker") - app_package = app_package if selected_package else None - rules_tracker.set_target_package(app_package) - rules_tracker.set_allowed_tools(active_tool_names) - if is_reachability == "yes": - rules_tracker.set_target_functions(vulnerable_functions) - return { - "ecosystem": ecosystem, - "runtime_prompt": runtime_prompt, - "is_reachability": is_reachability, - "observation": Observation(memory=critical_context, results=[]), - "critical_context": critical_context, - "app_package": app_package if selected_package else None, - } - except Exception as e: - logger.exception("pre_process_node failed") - span.set_output({"error": str(e), "exception_type": type(e).__name__}) - raise - - - async def thought_node(state: AgentState) -> AgentState: - step_num = state.get("step", 0) - with AGENT_TRACER.push_active_function("thought node", input_data=f"step:{step_num}") as span: - try: - active_prompt = state.get("runtime_prompt") or default_system_prompt - messages = [SystemMessage(content=active_prompt)] + state["messages"] - obs = state.get("observation", None) - if obs is not None: - memory_list = obs.memory if obs.memory else ["No prior knowledge."] - recent_findings = obs.results if obs.results else ["No recent findings."] - memory_context = "\n".join(f"- {m}" for m in memory_list) - findings_context = "\n".join(f"- {f}" for f in recent_findings) - context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" - messages.append(SystemMessage(content=context_block)) - response: Thought = await thought_llm.ainvoke(messages) - - final_answer = "waiting for the agent to respond" - if response.mode == "finish": - ai_message = AIMessage(content=response.final_answer) - final_answer = response.final_answer - elif response.actions is None: - logger.warning("LLM returned mode='act' but actions is None, forcing finish") - ai_message = AIMessage(content=response.thought or "No actions provided, finishing.") - response = Thought( - thought=response.thought or "No actions provided", - mode="finish", - actions=None, - final_answer=response.thought or "Insufficient evidence to provide a definitive answer." - ) - final_answer = response.final_answer - else: - tool_name = response.actions.tool - arguments = _build_tool_arguments(response.actions) - tool_call_id = str(uuid.uuid4()) - ai_message = AIMessage( - content=response.thought, - tool_calls=[{ - "name": tool_name, - "args": arguments, - "id": tool_call_id - }] - ) - - span.set_output({"mode": response.mode, "step": step_num + 1}) - return { - "messages": [ai_message], - "thought": response, - "step": step_num + 1, - "max_steps": config.max_iterations, - "output": final_answer - } - except Exception as e: - logger.exception("thought_node failed at step %d", step_num) - span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) - raise - - async def should_continue(state: AgentState) -> str: - thought = state.get("thought", None) - if thought is not None and thought.mode == "finish": - return END - if state.get("step", 0) >= state.get("max_steps", config.max_iterations): - return FORCED_FINISH_NODE - return TOOL_NODE - - async def forced_finish_node(state: AgentState) -> AgentState: - step_num = state.get("step", 0) - with AGENT_TRACER.push_active_function("forced_finish node", input_data=f"step:{step_num}") as span: - try: - active_prompt = state.get("runtime_prompt") or default_system_prompt - messages = [SystemMessage(content=active_prompt)] + state["messages"] - messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) - obs = state.get("observation", None) - if obs is not None and obs.memory: - memory_context = "\n".join(f"- {m}" for m in obs.memory) - messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) - response: Thought = await thought_llm.ainvoke(messages) - if response.mode == "finish" and response.final_answer: - ai_message = AIMessage(content=response.final_answer) - final_answer = response.final_answer - else: - final_answer = "Failed to generate a final answer within the maximum allowed steps." - ai_message = AIMessage(content=final_answer) - response = Thought( - thought=response.thought or "Max steps exceeded", - mode="finish", - actions=None, - final_answer=final_answer - ) - span.set_output({"final_answer_length": len(final_answer), "step": step_num}) - return { - "messages": [ai_message], - "thought": response, - "step": step_num, - "max_steps": state.get("max_steps", config.max_iterations), - "observation": state.get("observation", None), - "output": final_answer - } - except Exception as e: - logger.exception("forced_finish_node failed at step %d", step_num) - span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) - raise - - async def observation_node(state: AgentState) -> AgentState: - tool_message = state["messages"][-1] - last_thought_text = state["thought"].thought if state.get("thought") else "No previous thought." - tool_used = state["thought"].actions.tool if state.get("thought") and state["thought"].actions else "Unknown" - tool_input_detail = "" - if state.get("thought") and state["thought"].actions: - actions = state["thought"].actions - if actions.package_name and actions.function_name: - tool_input_detail = f"{actions.package_name},{actions.function_name}" - elif actions.query: - tool_input_detail = actions.query - elif actions.tool_input: - tool_input_detail = actions.tool_input - previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] - rules_tracker = state.get("rules_tracker") - with AGENT_TRACER.push_active_function("observation node", input_data=f"tool used:{tool_used}") as span: - try: - tool_output_for_llm = tool_message.content - result, error_message = rules_tracker.check_thought_behavior(tool_used, tool_input_detail, tool_output_for_llm) - if result: - span.set_output({"rule_error": error_message}) - return {"messages": [HumanMessage(content=error_message)]} - - if state.get("ecosystem", "").lower() == "java": - truncated_output = _truncate_tool_output(tool_output_for_llm, tool_used) - else: - truncated_output = tool_output_for_llm - - # Step 1: Comprehension -- reads raw tool output, produces compact findings - ctx_lines = state.get("critical_context", []) - critical_context_text = "\n".join(ctx_lines) if ctx_lines else "N/A" - comp_prompt = COMPREHENSION_PROMPT.format( - goal=state.get('input'), - selected_package=state.get('app_package') or "N/A", - critical_context=critical_context_text, - tool_used=tool_used, - tool_input_detail=tool_input_detail, - last_thought_text=last_thought_text, - tool_output=truncated_output, - ) - code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) - - findings_text = "\n".join(f"- {f}" for f in code_findings.findings) - - # Step 2: Memory update -- merges compressed findings into cumulative memory - mem_prompt = MEMORY_UPDATE_PROMPT.format( - goal=state.get('input'), - selected_package=state.get('app_package') or "N/A", - previous_memory=previous_memory, - findings=findings_text, - tool_outcome=code_findings.tool_outcome, - ) - new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) - - messages = state["messages"] - active_prompt = state.get("runtime_prompt") or default_system_prompt - estimated = _estimate_tokens(active_prompt, messages, new_observation) - prune_messages = [] - orig_estimated = estimated - - span_trace_dict = {"comprehension_findings": code_findings.findings, "tool_outcome": code_findings.tool_outcome} - - if estimated > config.context_window_token_limit and len(messages) > 3: - prunable = messages[1:-2] - for msg in prunable: - prune_messages.append(RemoveMessage(id=msg.id)) - estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 - if estimated <= config.context_window_token_limit: - break - logger.info( - "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", - len(prune_messages), estimated, config.context_window_token_limit, - ) - span_trace_dict["orig_estimated"] = orig_estimated - span_trace_dict["estimated"] = estimated - span.set_output(span_trace_dict) - cca_results = list(state.get("cca_results", [])) - if tool_used == ToolNames.CALL_CHAIN_ANALYZER: - stripped = tool_output_for_llm.strip().lstrip("([") - first_token = stripped.split(",", 1)[0].strip().lower() - if first_token == "true": - cca_results.append(True) - elif first_token == "false": - cca_results.append(False) - package_validated = state.get("package_validated") - if tool_used == ToolNames.FUNCTION_LOCATOR and state.get("is_reachability") == "yes": - input_pkg = package_name_from_locator_query(tool_input_detail) - target_pkg = (state.get("app_package") or "").strip().lower() - if target_pkg and input_pkg == target_pkg: - if "Package is valid" in tool_output_for_llm: - package_validated = True - elif "Package is not valid" in tool_output_for_llm and package_validated is None: - package_validated = False - return { - "messages": prune_messages, - "observation": new_observation, - "step": state.get("step", 0), - "cca_results": cca_results, - "package_validated": package_validated, - } - except Exception as e: - logger.exception("observation_node failed") - span.set_output({"error": str(e), "exception_type": type(e).__name__}) - raise - - async def create_graph(): - flow = StateGraph(AgentState) - flow.add_node(THOUGHT_NODE, thought_node) - flow.add_node(TOOL_NODE, tool_node) - flow.add_node(FORCED_FINISH_NODE, forced_finish_node) - flow.add_node(PRE_PROCESS_NODE, pre_process_node) - flow.add_node(OBSERVATION_NODE, observation_node) - flow.add_edge(START, PRE_PROCESS_NODE) - flow.add_edge(PRE_PROCESS_NODE, THOUGHT_NODE) - flow.add_conditional_edges( - THOUGHT_NODE, - should_continue, - {END: END, TOOL_NODE: TOOL_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE} - ) - flow.add_edge(TOOL_NODE, OBSERVATION_NODE) - flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) - flow.add_edge(FORCED_FINISH_NODE, END) - - app = flow.compile() - if config.verbose: - app.get_graph().draw_mermaid_png(output_file_path="flow.png") - return app - return await create_graph() -async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, - state: AgentMorpheusEngineState) -> AgentExecutor: - - tools, tool_descriptions,_ = await common_build_tools(config, builder, state) - - llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - tool_guidance = "\n".join(tool_descriptions) - - # Get prompt template - prompt_template_str = get_agent_prompt(config.prompt, config.prompt_examples) - - # Create prompt with tool_selection_strategy as partial variable - prompt = PromptTemplate.from_template( - prompt_template_str, - partial_variables={ - 'tool_selection_strategy': tool_guidance if tool_guidance else "Use available tools as appropriate." - } - ) - # using langchain create_react_agent to create the agent - agent = create_react_agent(llm=llm, - tools=tools, - prompt=prompt, - output_parser=MRKLOutputParser(), - stop_sequence=["\nObservation:", "\n\tObservation:"]) - - agent_executor = AgentExecutor( - agent=agent, - tools=tools, - early_stopping_method="force", - handle_parsing_errors="Check your output and make sure it conforms, use the Action/Action Input syntax", - max_iterations=config.max_iterations, - return_intermediate_steps=config.return_intermediate_steps, - verbose=config.verbose) - - # Disable streaming for accurate token counts - if isinstance(agent_executor.agent, RunnableAgent): - agent_executor.agent.stream_runnable = False - - return agent_executor - -async def _process_steps(agent, steps, semaphore, max_iterations: int = 10): - +async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterations: int = 10): + workflow_state = ctx_state.get() + critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + context_block = "\n".join(critical_context) + precomputed_intel = (critical_context, candidate_packages, vulnerable_functions) async def _process_step(step): - async def call_agent(initial_state,config=None): - if config: - return await agent.ainvoke(initial_state,config=config) - else: - return await agent.ainvoke(initial_state) + routing = await dispatch_question(routing_llm, step, context_block) + agent_type = routing.agent_type + + actual_type = agent_type if agent_type in agents else "reachability" + compiled_graph = agents[actual_type] + tracker = get_agent_class(actual_type).create_rules_tracker() + + initial_state = { + "input": step, + "messages": [HumanMessage(content=step)], + "step": 0, + "max_steps": max_iterations, + "thought": None, + "observation": None, + "output": "waiting for the agent to respond", + "rules_tracker": tracker, + "precomputed_intel": precomputed_intel, + } + graph_config = {"recursion_limit": 50} - initial_state = {"input": step} - config = None - if not isinstance(agent, AgentExecutor): - initial_state = { - "input": step, - "messages": [HumanMessage(content=step)], - "step": 0, - "max_steps": max_iterations, - "thought": None, - "observation": None, - "output": "waiting for the agent to respond", - "rules_tracker": SystemRulesTracker(), - } - config = { - "recursion_limit": 50 - } - with AGENT_TRACER.push_active_function("checklist_question", input_data=step[:80]): + with AGENT_TRACER.push_active_function( + "checklist_question", + input_data=f"[{actual_type}] {step[:80]}", + ): if semaphore: async with semaphore: - return await call_agent(initial_state, config) + return await compiled_graph.ainvoke(initial_state, config=graph_config) else: - return await call_agent(initial_state, config) + return await compiled_graph.ainvoke(initial_state, config=graph_config) return await asyncio.gather(*(_process_step(step) for step in steps), return_exceptions=True) -def _parse_intermediate_step(step: tuple[typing.Any, typing.Any]) -> dict[str, typing.Any]: - """ - Parse an agent intermediate step into an AgentIntermediateStep object. Return the dictionary representation for - compatibility with cudf. - """ - if len(step) != 2: - raise ValueError(f"Expected 2 values in each intermediate step but got {len(step)}.") - - action, output = step - - return {"tool_name": action.tool, "action_log": action.log, "tool_input": action.tool_input, "tool_output": output} - - def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, replace_exceptions_value: str | None, - checklist_questions: list[list]) -> tuple[list[list[str]], list[list[list]]]: - """ - Post-process results into lists of outputs and intermediate steps. Replace exceptions with placholder values if - config.replace_exceptions = True. - :param values: + checklist_questions: list[list]) -> list[list[dict]]: + """Post-process graph agent results into a uniform list of output dicts. + + Replaces exceptions with placeholder values if replace_exceptions is True. """ outputs = [[] for _ in range(len(results))] for i, answer_list in enumerate(results): for j, answer in enumerate(answer_list): - # Handle exceptions returned by the agent - # OutputParserException is not a subclass of Exception, so we need to check for it separately if isinstance(answer, (ToolRaisedException, OutputParserException, Exception)): if replace_exceptions: - # If the agent encounters a parsing error or a server error after retries, replace the error - # with default values to prevent the pipeline from crashing outputs[i].append({"input": checklist_questions[i][j], "output": replace_exceptions_value, "intermediate_steps": None, "cca_results": [], "package_validated": None}) if isinstance(answer, ToolRaisedException): - tool_raised_exception: ToolRaisedException = answer - logger.warning(f"An exception encountered during tool execution, in result [{i}][{j}]. for " - f"question : {checklist_questions[i][j]}" - f".tool raised exception details=> {tool_raised_exception}" - f", replacing with default output -> {replace_exceptions_value}") + logger.warning( + "Tool execution exception in result[%d][%d], replacing with default output: %s", + i, j, type(answer).__name__) else: logger.warning( - "General Exception encountered in result[%d][%d]: %s, for question -> %s, " - "Replacing with default output: \"%s\" and intermediate_steps: None", - i, - j, - str(answer), - checklist_questions[i][j], - replace_exceptions_value) + "General exception in result[%d][%d]: %s, replacing with default output", + i, j, type(answer).__name__) - # For successful agent responses, extract the output, and intermediate steps if available else: - # intermediate_steps availability depends on config.return_intermediate_steps - if "intermediate_steps" in answer: - results[i][j]["intermediate_steps"] = [ - _parse_intermediate_step(step) for step in answer["intermediate_steps"] - ] - else: - results[i][j]["intermediate_steps"] = None - outputs[i].append({"input": answer["input"], "output": answer["output"], - "intermediate_steps": results[i][j]["intermediate_steps"], + "intermediate_steps": None, "cca_results": answer.get("cca_results", []), "package_validated": answer.get("package_validated")}) return outputs @@ -768,13 +157,38 @@ async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: ctx_state.set(state) checklist_plans = state.checklist_plans - - agent = await _create_graph_agent(config, builder, state) - results = await asyncio.gather(*(_process_steps(agent, steps, semaphore, config.max_iterations) - for steps in checklist_plans.values()), return_exceptions=True) - results = _postprocess_results(results, config.replace_exceptions, config.replace_exceptions_value, - list(checklist_plans.values())) + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + agents = {} + for agent_type in get_all_agent_types(): + agent_cls = get_agent_class(agent_type) + tools = agent_cls.get_tools(builder, config, state) + if tools: + agent_instance = agent_cls(tools=tools, llm=llm, config=config) + agents[agent_type] = await agent_instance.build_graph() + + routing_llm = llm.with_structured_output(QuestionRouting) + + results = await asyncio.gather( + *( + _process_steps( + agents, + routing_llm, + steps, + semaphore, + config.max_iterations, + ) + for steps in checklist_plans.values() + ), + return_exceptions=True, + ) + results = _postprocess_results( + results, + config.replace_exceptions, + config.replace_exceptions_value, + list(checklist_plans.values()), + ) state.checklist_results = dict(zip(checklist_plans.keys(), results)) with AGENT_TRACER.push_active_function("agent_finish", input_data={ diff --git a/src/vuln_analysis/functions/dispatcher.py b/src/vuln_analysis/functions/dispatcher.py new file mode 100644 index 000000000..0ab1240b1 --- /dev/null +++ b/src/vuln_analysis/functions/dispatcher.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Literal + +from langchain_core.messages import HumanMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +class QuestionRouting(BaseModel): + """Structured output for LLM-based question routing classification. + + The dispatcher classifies each checklist question into one of two agent types: + - reachability: call chain tracing (is vulnerable code reachable from app code?) + - code_understanding: configuration/version/presence/usage investigation + """ + agent_type: Literal["reachability", "code_understanding"] = Field( + description="Route to 'reachability' if the question asks about call chains, " + "code paths, whether vulnerable code is used/called/reachable, " + "or whether untrusted data can reach a function. " + "Route to 'code_understanding' for configuration, version, " + "or application-level settings questions." + ) + reason: str = Field( + description="One-sentence justification for the routing decision." + ) + + +ROUTING_PROMPT_TEMPLATE = """You are classifying a CVE investigation question to route it to the correct sub-agent. + +Two sub-agents are available: +- **reachability**: Traces call chains and code paths. Use when the question asks whether vulnerable code is CALLED, USED, or REACHABLE from application code, or whether untrusted data can reach a specific function. IMPORTANT: If the question asks whether a class/function/module from the VULNERABLE PACKAGE (listed in the context below) is used, imported, or called — route to reachability. The reachability agent has Call Chain Analyzer and Function Locator to verify actual code-level usage and call paths, which is needed to distinguish "imported but never called on vulnerable path" from "actively used." +- **code_understanding**: Investigates configuration, version, presence, and general application behavior. Use when the question asks about application-level configuration (e.g., XML parsing settings, TLS options), environment setup, or properties — things that do NOT require tracing whether a specific vulnerable function is called. + +Context (CVE / vulnerable packages): +{context_block} + +Question: {question} + +Examples: +- "Is the vulnerable function XStream.fromXML() called from application code?" → reachability +- "Is the application configured to use the affected XML parser?" → code_understanding +- "Can untrusted data reach BeanUtils.populate() through the call chain?" → reachability +- "Is the vulnerable version of commons-beanutils installed?" → code_understanding +- "Is the function parseXML() reachable from any HTTP handler?" → reachability +- "Does the application enable external entity processing in its XML configuration?" → code_understanding +- "Is SslHandler used in the application?" → reachability (SslHandler is from the vulnerable package) +- "Is HttpPostStandardRequestDecoder.offer() called by application code?" → reachability +- "Is the vulnerable newTransformer() method invoked anywhere in the application's XSLT processing?" → reachability +- "Does application code call deserialize() or readObject() on the affected library?" → reachability +- "Does the code sanitize input against path traversal characters and command injection metacharacters?" → code_understanding +- "Are there input validation mechanisms to prevent malicious Markdown input from reaching the paragraph function?" → code_understanding +- "Can a single request force the application server to load unbounded data into memory?" → code_understanding +- "Can malformed input cause an unhandled exception that crashes or restarts the process?" → code_understanding + +Classify the question above.""" + + +def build_routing_prompt(context_block: str, question: str) -> str: + """Format the routing prompt template with CVE context and question text.""" + return ROUTING_PROMPT_TEMPLATE.format( + context_block=context_block, + question=question, + ) + + +async def dispatch_question( + routing_llm, + question: str, + context_block: str, +) -> QuestionRouting: + """Classify a checklist question and return routing decision. + + Uses the routing LLM (with structured output) to decide whether the question + should go to the reachability agent (call chain tracing) or the code understanding + agent (configuration/presence/usage investigation). + """ + prompt = build_routing_prompt(context_block, question) + with AGENT_TRACER.push_active_function("dispatch_question", input_data=f"question:{question[:80]}") as span: + logger.debug("Dispatching question to routing LLM: %.80s", question) + result: QuestionRouting = await routing_llm.ainvoke([HumanMessage(content=prompt)]) + logger.info( + "Question routed to '%s' (reason: %s): %.80s", + result.agent_type, + result.reason, + question, + ) + span.set_output({"agent_type": result.agent_type, "reason": result.reason}) + return result \ No newline at end of file diff --git a/src/vuln_analysis/functions/reachability_agent.py b/src/vuln_analysis/functions/reachability_agent.py new file mode 100644 index 000000000..1801e82d3 --- /dev/null +++ b/src/vuln_analysis/functions/reachability_agent.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from langchain_core.messages import HumanMessage, SystemMessage, AIMessage + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.functions.agent_registry import register_agent +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent, _is_tool_available +from vuln_analysis.functions.react_internals import ( + AgentState, + Classification, + Observation, + Thought, + ReachabilityRulesTracker, + build_reachability_system_prompt, + build_classification_prompt, + FORCED_FINISH_PROMPT, + REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO, + REACHABILITY_AGENT_NON_REACH_SYS_PROMPT, + REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.tools.transitive_code_search import package_name_from_locator_query +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_from_osv +from vuln_analysis.utils.prompting import build_tool_descriptions +from vuln_analysis.utils.prompt_factory import ( + TOOL_SELECTION_STRATEGY, + TOOL_SELECTION_STRATEGY_NON_REACHABILITY, + FEW_SHOT_EXAMPLES, +) +from pathlib import Path +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +def _validate_go_vendor_packages(source_info, candidate_packages): + code_si = next((si for si in source_info if si.type == "code"), None) + if code_si is None: + return candidate_packages, [] + repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) + vendor_path = repo_path / "vendor" + if not vendor_path.is_dir(): + return candidate_packages, [] + validated = [] + removed = [] + for pkg in candidate_packages: + pkg_name = pkg.get("name", "") + if (vendor_path / pkg_name).is_dir(): + validated.append(pkg) + else: + removed.append(pkg_name) + if validated: + return validated, removed + return candidate_packages, [] + + +async def _enrich_go_candidates(cve_intel, source_info, critical_context, candidate_packages, vulnerable_functions_set): + ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) + if not ghsa_has_packages or not vulnerable_functions_set: + intel = cve_intel[0] if cve_intel else None + if intel: + await enrich_go_from_osv(intel, critical_context, candidate_packages, vulnerable_functions_set) + if candidate_packages: + candidate_packages, removed_pkgs = _validate_go_vendor_packages(source_info, candidate_packages) + if removed_pkgs: + logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) + return candidate_packages, sorted(vulnerable_functions_set) + + +@register_agent("reachability") +class ReachabilityAgent(BaseGraphAgent): + + def __init__(self, tools, llm, config): + super().__init__(tools, llm, config) + self._classification_llm = llm.with_structured_output(Classification) + + @property + def agent_type(self) -> str: + return "reachability" + + _REACHABILITY_TOOLS = frozenset({ + ToolNames.FUNCTION_LOCATOR, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_LIBRARY_VERSION_FINDER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + }) + + @staticmethod + def get_tools(builder, config, state) -> list: + all_tools = BaseGraphAgent._load_all_tools(builder, config) + return [ + t for t in all_tools + if t.name in ReachabilityAgent._REACHABILITY_TOOLS + and _is_tool_available(t.name, config, state) + ] + + @staticmethod + def create_rules_tracker() -> ReachabilityRulesTracker: + return ReachabilityRulesTracker() + + def should_truncate_tool_output(self, state: AgentState, tool_used: str) -> bool: + return state.get("ecosystem", "").lower() == "java" + + def _build_tool_guidance_for_ecosystem(self, ecosystem: str, available_tools: list, + is_reachability: str = "yes") -> tuple[str, str]: + filtered_tools = [ + t for t in available_tools + if (t.name != ToolNames.FUNCTION_CALLER_FINDER or ecosystem == "go") and + (t.name != ToolNames.FUNCTION_LIBRARY_VERSION_FINDER or ecosystem == "java") + ] + list_of_tool_names = [t.name for t in filtered_tools] + list_of_tool_descriptions = [t.name + ": " + t.description for t in filtered_tools] + + strategy = TOOL_SELECTION_STRATEGY if is_reachability == "yes" else TOOL_SELECTION_STRATEGY_NON_REACHABILITY + lang = ecosystem.lower() if ecosystem else "" + if lang in strategy: + tool_guidance_local = strategy[lang] + if lang == "java": + tool_guidance_local += ( + " Use Function Library Version Finder to verify the installed version of a library " + "before concluding exploitability (e.g., input 'commons-beanutils')." + ) + if is_reachability == "yes": + hint = FEW_SHOT_EXAMPLES.get(lang, "") + if hint: + tool_guidance_local += f"\nHint: {hint}" + else: + tool_guidance_list_local = build_tool_descriptions(list_of_tool_names) + tool_guidance_local = "\n".join(tool_guidance_list_local) + + descriptions_local = "\n".join(list_of_tool_descriptions) + return tool_guidance_local, descriptions_local + + async def pre_process_node(self, state: AgentState) -> AgentState: + workflow_state = ctx_state.get() + ecosystem = workflow_state.original_input.input.image.ecosystem.value if workflow_state.original_input.input.image.ecosystem else "" + with AGENT_TRACER.push_active_function("pre_process node", input_data=f"ecosystem:{ecosystem}") as span: + try: + precomputed = state.get("precomputed_intel") + if precomputed is not None: + critical_context = list(precomputed[0]) + candidate_packages = [dict(p) for p in precomputed[1]] + vulnerable_functions = list(precomputed[2]) + else: + critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + vulnerable_functions_set = set(vulnerable_functions) + + if ecosystem == "go": + candidate_packages, vulnerable_functions = await _enrich_go_candidates( + workflow_state.cve_intel, + workflow_state.original_input.input.image.source_info, + critical_context, + candidate_packages, + vulnerable_functions_set, + ) + + critical_context, selected_package = await self._select_package( + ecosystem, candidate_packages, critical_context, workflow_state, + ) + app_package = selected_package + + critical_context.append( + "TASK: Investigate usage and reachability of the vulnerable function/module in the container. " + "Use the vulnerable module name from GHSA as primary investigation target." + ) + + question = state.get("input") or "" + context_block = "\n".join(critical_context) + classification_prompt = build_classification_prompt(context_block, question) + classification_result: Classification = await self._classification_llm.ainvoke([HumanMessage(content=classification_prompt)]) + span.set_output({ + "critical_context": critical_context, + "candidate_packages": candidate_packages, + "selected_package": selected_package, + "app_package": app_package if selected_package else None, + "reachability_question": classification_result.is_reachability, + }) + + is_reachability = classification_result.is_reachability + + if is_reachability == "yes": + tool_guidance_local, descriptions_local = self._build_tool_guidance_for_ecosystem(ecosystem, self.tools) + go_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO} if ecosystem == "go" else {} + runtime_prompt = build_reachability_system_prompt(descriptions_local, tool_guidance_local, **go_instructions) + active_tool_names = [t.name for t in self.tools] + else: + reachability_tool_names = {ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER} + non_reach_tools = [t for t in self.tools if t.name not in reachability_tool_names] + tool_guidance_local, descriptions_local = self._build_tool_guidance_for_ecosystem(ecosystem, non_reach_tools, is_reachability="no") + runtime_prompt = build_reachability_system_prompt( + descriptions_local, tool_guidance_local, + instructions=REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS, + sys_prompt=REACHABILITY_AGENT_NON_REACH_SYS_PROMPT, + ) + active_tool_names = [t.name for t in non_reach_tools] + logger.info("Non-reachability question detected; removed reachability tools from prompt") + + rules_tracker = state.get("rules_tracker") + app_package = app_package if selected_package else None + rules_tracker.set_ecosystem(ecosystem) + rules_tracker.set_target_package(app_package) + rules_tracker.set_allowed_tools(active_tool_names) + if is_reachability == "yes": + rules_tracker.set_target_functions(vulnerable_functions) + return { + "ecosystem": ecosystem, + "runtime_prompt": runtime_prompt, + "is_reachability": is_reachability, + "observation": Observation(memory=critical_context, results=[]), + "critical_context": critical_context, + "app_package": app_package, + } + except Exception as e: + logger.exception("pre_process_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + def check_finish_allowed(self, state: AgentState) -> tuple[bool, str]: + if state.get("is_reachability") != "yes": + return True, "" + rules_tracker = state.get("rules_tracker") + cca_results = state.get("cca_results", []) + return rules_tracker.check_finish_allowed(cca_results) + + async def forced_finish_node(self, state: AgentState) -> AgentState: + """Override forced_finish to inject a no-CCA warning for reachability questions. + + When the reachability agent exhausts all iterations without ever calling CCA, + the LLM has no reachability evidence. Without this override the LLM can + hallucinate exploitability based on library presence alone. The injected + prompt steers it toward 'insufficient evidence / not exploitable'. + """ + cca_results = state.get("cca_results", []) + is_reachability = state.get("is_reachability") + if is_reachability == "yes" and not cca_results: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function( + f"{self.agent_type}_forced_finish", input_data=f"step:{step_num}" + ) as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + context_block = self._build_observation_context(state.get("observation", None)) + if context_block: + messages.append(SystemMessage(content=context_block)) + question = state.get("input", "") + no_cca_prompt = ( + (f"QUESTION: {question}\n\n" if question else "") + + FORCED_FINISH_PROMPT + "\n\n" + "CRITICAL: Call Chain Analyzer was NEVER called during this investigation. " + "Without Call Chain Analyzer verification, there is NO evidence that the " + "vulnerable function is reachable from application code. Library presence " + "in dependencies alone does NOT constitute exploitability. " + "You MUST conclude that there is insufficient evidence to confirm " + "exploitability — the function was NOT confirmed reachable." + ) + messages.append(HumanMessage(content=no_cca_prompt)) + keep_tail = 2 if context_block else 1 + self._prune_messages_to_fit(messages, keep_tail=keep_tail, step_num=step_num, caller="reachability_forced_finish") + response: Thought = await self.thought_llm.ainvoke(messages) + if response.mode == "finish" and response.final_answer: + final_answer = response.final_answer + else: + final_answer = ( + "Insufficient evidence: Call Chain Analyzer was never invoked. " + "Cannot confirm the vulnerable function is reachable from application code." + ) + response = Thought( + thought="Max steps exceeded without CCA verification", + mode="finish", + actions=None, + final_answer=final_answer, + ) + ai_message = AIMessage(content=final_answer) + logger.info("Reachability forced_finish: no CCA calls, injected no-CCA warning") + span.set_output({"final_answer_length": len(final_answer), "step": step_num, "no_cca_warning": True}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", self.config.max_iterations), + "observation": state.get("observation", None), + "output": final_answer, + } + except Exception as e: + logger.exception("%s forced_finish_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + return await super().forced_finish_node(state) + + def post_observation(self, state: AgentState, tool_used: str, + tool_output: str, tool_input_detail: str) -> dict: + extra = {} + cca_results = list(state.get("cca_results", [])) + if tool_used == ToolNames.CALL_CHAIN_ANALYZER: + stripped = tool_output.strip().lstrip("([") + first_token = stripped.split(",", 1)[0].strip().lower() + if first_token == "true": + cca_results.append(True) + elif first_token == "false": + cca_results.append(False) + extra["cca_results"] = cca_results + + package_validated = state.get("package_validated") + rules_tracker = state.get("rules_tracker") + if tool_used == ToolNames.FUNCTION_LOCATOR and state.get("is_reachability") == "yes": + input_pkg = package_name_from_locator_query(tool_input_detail) + target_pkg = (state.get("app_package") or "").strip().lower() + if input_pkg and "Package is valid" in tool_output: + rules_tracker.add_validated_package(input_pkg) + if target_pkg and input_pkg == target_pkg: + package_validated = True + elif input_pkg and "Package is not valid" in tool_output: + if target_pkg and input_pkg == target_pkg and package_validated is None: + package_validated = False + extra["package_validated"] = package_validated + return extra diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 9561c349c..803400675 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -17,11 +17,16 @@ from typing import Any from typing import Literal from langgraph.graph import MessagesState -#---- REACT Schemas ----# + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) + +# ---- Pydantic Schemas ---- # class ToolCall(BaseModel): tool: str = Field(description="Exact tool name from AVAILABLE_TOOLS") - #tool_input: str = Field(description="The input for the tool. Example: Code Keyword Search: PQescapeLiteral") package_name: str | None = Field( default=None, description="Package/module name. REQUIRED when using Function Locator, Function Caller Finder, or Call Chain Analyzer. E.g. libpq, urllib, github.com/org/pkg" @@ -30,12 +35,12 @@ class ToolCall(BaseModel): default=None, description="Function or method name with optional args. REQUIRED with package_name for code path tools. E.g. PQescapeLiteral(), parse(), errors.New(\"x\")" ) - # For search tools (Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search) + # For search tools: Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search query: str | None = Field( default=None, description="Search query. Use for search tools when package_name/function_name don't apply" ) - # Fallback: if LLM uses tool_input for simple query-only tools + # Fallback for when LLM uses tool_input for simple query-only tools tool_input: str | None = Field( default=None, description="Legacy/fallback input. Prefer package_name+function_name or query." @@ -94,18 +99,18 @@ class PackageSelection(BaseModel): description="One-sentence justification for why this package is the best investigation target." ) -class SystemRulesTracker: +class BaseRulesTracker: + """Shared behavioral rule infrastructure for all sub-agents.""" def __init__(self): self.action_history = {} self.target_package = None self.allowed_tools = [] - self.target_functions: dict[str, bool] = {} + def set_allowed_tools(self, allowed_tools: list[str]): self.allowed_tools = allowed_tools + def set_target_package(self, target_package: str): self.target_package = target_package - def set_target_functions(self, functions: list[str]): - self.target_functions = {f: False for f in functions} @staticmethod def _is_empty_result(output) -> bool: @@ -127,6 +132,11 @@ def add_action(self, action: str, action_input: str, output): else: self.action_history[action].append(entry) + def _rule_duplicate_call(self, action: str, action_input: str) -> bool: + if action not in self.action_history: + return False + return any(prev["input"] == action_input for prev in self.action_history[action]) + def _rule_number_7(self, action: str, action_input: str, output) -> bool: if action != "Code Keyword Search": return False @@ -141,10 +151,73 @@ def _rule_number_7(self, action: str, action_input: str, output) -> bool: return True return False + def _rule_use_allowed_tools(self, action: str) -> bool: + if action not in self.allowed_tools: + return True + return False + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + if self._rule_duplicate_call(action, action_input): + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) + if self._rule_number_7(action, action_input, output): + return True, ("You are NOT following Rule 7. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules.") + if self._rule_use_allowed_tools(action): + return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") + self.add_action(action, action_input, output) + return False, "" + + +class ReachabilityRulesTracker(BaseRulesTracker): + """Behavioral rules for the reachability sub-agent.""" + def __init__(self): + super().__init__() + self.target_functions: dict[str, bool] = {} + self.ecosystem: str = "" + self.validated_packages: set[str] = set() + + def set_ecosystem(self, ecosystem: str): + self.ecosystem = ecosystem.lower() if ecosystem else "" + + def check_finish_allowed(self, cca_results: list[bool]) -> tuple[bool, str]: + """Block finish when mandatory reachability tools were not used. + + Two rules enforced: + 1. CCA must be called before finishing any reachability question. + 2. (Java only) FLVF must be called when CCA found reachability. + """ + if not cca_results and ToolNames.CALL_CHAIN_ANALYZER not in self.action_history: + return False, ( + "You MUST use Function Locator and Call Chain Analyzer before concluding. " + "Code Keyword Search alone is NOT sufficient to determine reachability. " + "Call Function Locator to validate the package, then Call Chain Analyzer " + "to check if the vulnerable function is reachable from application code." + ) + if self.ecosystem == "java" and any(cca_results): + if ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in self.action_history: + return False, ( + "MANDATORY VERSION CHECK: Call Chain Analyzer found the vulnerable function is reachable, " + "but you have NOT verified the installed library version. " + "You MUST call Function Library Version Finder (e.g., input 'commons-beanutils') " + "to check whether the installed version is within the vulnerable range before concluding." + ) + return True, "" + + def set_target_functions(self, functions: list[str]): + self.target_functions = {f: False for f in functions} + @staticmethod def _normalize_package_name(name: str) -> str: return name.strip().lower().replace("-", "_") + def add_validated_package(self, package_name: str): + """Register a package that FL confirmed as valid (e.g., uber-jar alternative).""" + self.validated_packages.add(self._normalize_package_name(package_name)) + def _rule_number_8(self, action: str, action_input: str, output) -> bool: if self.target_package is None: return False @@ -155,11 +228,12 @@ def _rule_number_8(self, action: str, action_input: str, output) -> bool: target_pkg = self._normalize_package_name(self.target_package) # Allow Java GAV with version suffix: input "group:artifact:version" # should match target "group:artifact" - if input_pkg != target_pkg and not input_pkg.startswith(target_pkg + ":"): - return True - return False - def _rule_use_allowed_tools(self, action: str) -> bool: - if action not in self.allowed_tools: + if input_pkg == target_pkg or input_pkg.startswith(target_pkg + ":"): + return False + # Allow packages that FL already validated (handles uber-jars like + # netty-all containing netty-codec-http classes) + if any(input_pkg == vp or input_pkg.startswith(vp + ":") for vp in self.validated_packages): + return False return True return False @@ -186,24 +260,35 @@ def _rule_number_9(self, action: str, action_input: str) -> tuple[bool, str]: return False, "" def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + if self._rule_duplicate_call(action, action_input): + logger.debug("Duplicate call rule triggered: '%s' with same input", action) + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) if self._rule_number_7(action, action_input, output): + logger.debug("Reachability Rule 7 triggered: dotted query with empty results for tool '%s'", action) return True, ("You are NOT following Rule 7. Your query contains dots and returned " "no results. You MUST retry with just the final component. Follow the rules.") if self._rule_number_8(action, action_input, output): + logger.debug("Reachability Rule 8 triggered: wrong package for tool '%s', expected '%s'", action, self.target_package) return True, (f"You are NOT following Rule 8. You are using the wrong package name. You MUST use the target package name {self.target_package} see KNOWLEDGE as the package_name before trying alternative packages. Follow the rules.") if self._rule_use_allowed_tools(action): + logger.debug("Reachability allowed-tools rule triggered: '%s' not in %s", action, self.allowed_tools) return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") rule9, msg9 = self._rule_number_9(action, action_input) if rule9: + logger.debug("Reachability Rule 9 triggered: must investigate target functions first for tool '%s'", action) return True, msg9 self.add_action(action, action_input, output) return False, "" - + + class AgentState(MessagesState): input: str = "" step: int = 0 max_steps: int = 10 - #memory: str | None = None thought: Thought | None = None observation: Observation | None = None output: str = "" @@ -211,14 +296,19 @@ class AgentState(MessagesState): runtime_prompt: str | None = None app_package: str | None = None is_reachability: str = "yes" - rules_tracker: SystemRulesTracker = SystemRulesTracker() + rules_tracker: BaseRulesTracker = ReachabilityRulesTracker() critical_context: list[str] = [] cca_results: list[bool] = [] package_validated: bool | None = None - -### --- End of REACT Schemas ----# -#---- REACT Prompt Templates ----# -AGENT_SYS_PROMPT = ( + precomputed_intel: tuple | None = None + + +# ---- Reachability Sub-Agent Prompts ---- # +# The following prompts (REACHABILITY_AGENT_SYS_PROMPT, REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS, +# REACHABILITY_REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO) are used exclusively by the reachability sub-agent, +# which traces call chains to determine if vulnerable code is reachable from application code. + +REACHABILITY_AGENT_SYS_PROMPT = ( "You are a security analyst investigating CVE exploitability in container images.\n" "MANDATORY STEPS (follow in order, do NOT skip any):\n" "1. IDENTIFY the vulnerable component/function from the CVE description.\n" @@ -249,7 +339,6 @@ class AgentState(MessagesState): "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." ) -# Update LANGGRAPH_SYSTEM_PROMPT_TEMPLATE in react_internals.py LANGGRAPH_SYSTEM_PROMPT_TEMPLATE = """{sys_prompt} @@ -265,7 +354,7 @@ class AgentState(MessagesState): RESPONSE: {{""" -AGENT_THOUGHT_INSTRUCTIONS = """ +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS = """ 1. Output valid JSON only. thought < 100 words. final_answer < 150 words. 2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. 3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. @@ -286,7 +375,7 @@ class AgentState(MessagesState): {{"thought": "Function Locator confirmed the package. Now trace reachability with Call Chain Analyzer", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} """ -AGENT_THOUGHT_INSTRUCTIONS_GO = """ +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO = """ 1. Output valid JSON only. thought < 100 words. final_answer < 150 words. 2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. 3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. @@ -312,7 +401,7 @@ class AgentState(MessagesState): {{"thought": "Function Caller Finder returned no callers, but this does not prove unreachable. MUST call Call Chain Analyzer to confirm reachability", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} """ -AGENT_SYS_PROMPT_NON_REACHABILITY = ( +REACHABILITY_AGENT_NON_REACH_SYS_PROMPT = ( "You are a security analyst investigating CVE exploitability in container images.\n" "This is NOT a reachability question -- do NOT trace call chains.\n" "MANDATORY STEPS (follow in order, do NOT skip any):\n" @@ -345,7 +434,7 @@ class AgentState(MessagesState): "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." ) -AGENT_THOUGHT_INSTRUCTIONS_NON_REACHABILITY = """ +REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS = """ 1. Output valid JSON only. thought < 100 words. final_answer < 150 words. 2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. 3. Function Locator: MUST set package_name AND function_name. Do NOT use query. @@ -404,6 +493,12 @@ class AgentState(MessagesState): NEW OUTPUT: {tool_output} +CRITICAL — CALL CHAIN ANALYZER REACHABILITY: +When TOOL USED is "Call Chain Analyzer": +- If the result is POSITIVE (reachable): your first finding MUST be "REACHABLE via [package] - sufficient evidence." Do NOT hedge, qualify, or say "further investigation required." +- If the result is NEGATIVE (not reachable): your first finding MUST be "NOT reachable via [package]." +These override all other rules for Call Chain Analyzer results. + CODE COMPREHENSION RULES: 1. READ the actual code snippets in NEW OUTPUT. Do NOT just check whether something was "found" or "not found." 2. For each code snippet or function returned, determine: @@ -452,17 +547,13 @@ class AgentState(MessagesState): RESPONSE: {{""" -# Legacy prompt kept for backwards compatibility with older traces -OBSERVATION_NODE_PROMPT = COMPREHENSION_PROMPT - -### --- End of REACT Prompt Templates ----# -def build_system_prompt( +def build_reachability_system_prompt( tool_descriptions: str, tool_guidance: str, - instructions: str = AGENT_THOUGHT_INSTRUCTIONS, + instructions: str = REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS, sys_prompt: str | None = None, ) -> str: - sys_prompt = sys_prompt or AGENT_SYS_PROMPT + sys_prompt = sys_prompt or REACHABILITY_AGENT_SYS_PROMPT return LANGGRAPH_SYSTEM_PROMPT_TEMPLATE.format( sys_prompt=sys_prompt, tools=tool_descriptions, @@ -545,6 +636,8 @@ def _build_tool_arguments(actions: ToolCall)->dict[str, Any]: return {"query": actions.query} if actions.tool_input: return {"query": actions.tool_input} # fallback + logger.warning("Tool '%s' called without required arguments (package_name=%s, function_name=%s, query=%s)", + actions.tool, actions.package_name, actions.function_name, actions.query) raise ValueError(f"Tool {actions.tool} requires package_name+function_name or query/tool_input") diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index dc847a87f..fe3b4b755 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -47,6 +47,8 @@ from vuln_analysis.tools import container_image_analysis_data from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp +from vuln_analysis.tools import configuration_scanner +from vuln_analysis.tools import import_usage_analyzer from vuln_analysis.utils.error_handling_decorator import catch_pipeline_errors_async # pylint: enable=unused-import from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input @@ -255,7 +257,7 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph = graph_builder.compile() def convert_str_to_agent_morpheus_input(input: str) -> AgentMorpheusInput: - logger.debug("Converting input to AgentMorpheusInput: %s", input) + logger.debug("Converting JSON string input to AgentMorpheusInput (length: %d)", len(input)) try: return AgentMorpheusInput.model_validate_json(input) except Exception as e: @@ -263,7 +265,7 @@ def convert_str_to_agent_morpheus_input(input: str) -> AgentMorpheusInput: raise e def convert_textio_to_agent_morpheus_input(input: TextIOWrapper) -> AgentMorpheusInput: - logger.debug("Converting input to AgentMorpheusInput: %s", input) + logger.debug("Converting TextIOWrapper input to AgentMorpheusInput") try: data = input.read() return AgentMorpheusInput.model_validate_json(data) @@ -273,7 +275,7 @@ def convert_textio_to_agent_morpheus_input(input: TextIOWrapper) -> AgentMorpheu raise e def convert_agent_morpheus_output_to_str(output: AgentMorpheusOutput) -> str: - logger.debug("Converting AgentMorpheusOutput to str: %s", output) + logger.debug("Converting AgentMorpheusOutput to JSON string") try: return output.model_dump_json() except Exception as e: diff --git a/src/vuln_analysis/runtime_context.py b/src/vuln_analysis/runtime_context.py index fceb67aad..adca2360e 100644 --- a/src/vuln_analysis/runtime_context.py +++ b/src/vuln_analysis/runtime_context.py @@ -19,4 +19,9 @@ import contextvars # Holds the current AgentMorpheusEngineState for the active task -ctx_state = contextvars.ContextVar("ctx_state", default="default_value") \ No newline at end of file +ctx_state = contextvars.ContextVar("ctx_state", default="default_value") + +# Source scope for CU agent tools (Docs Semantic Search, Code Keyword Search). +# List of path substrings; tools keep results whose path matches any entry. +# None means no scoping (search all sources). +cu_source_scope = contextvars.ContextVar("cu_source_scope", default=None) \ No newline at end of file diff --git a/src/vuln_analysis/tools/configuration_scanner.py b/src/vuln_analysis/tools/configuration_scanner.py new file mode 100644 index 000000000..0ed1ada34 --- /dev/null +++ b/src/vuln_analysis/tools/configuration_scanner.py @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +import re +from collections import OrderedDict +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope, format_app_dep_output + +CONFIGURATION_SCANNER = "configuration_scanner" + +logger = LoggingFactory.get_agent_logger(__name__) + + +def format_context_snippet(lines: list[str], match_line: int, context_lines: int) -> str: + """Format source lines around a match, marking the match line with '>'.""" + start = max(0, match_line - context_lines) + end = min(len(lines), match_line + context_lines + 1) + return "\n".join( + f"{'>' if i == match_line else ' '} {i+1}: {lines[i]}" + for i in range(start, end) + ) + +# File patterns and directory names considered as configuration files +CONFIG_FILE_PATTERNS = [ + # Named config files — matched anywhere in the repo + "application.yml", "application.yaml", "application.properties", + "config.yaml", "config.yml", "config.xml", + "settings.toml", "settings.yaml", "settings.yml", + "web.xml", "beans.xml", + "Dockerfile", "Dockerfile.*", "docker-compose*.yml", + # Config-specific extensions — safe to match anywhere + "*.properties", "*.env", "*.conf", "*.ini", +] + +# Directory names that typically contain configuration files +CONFIG_DIR_PATTERNS = ["config", "conf", "conf.d", "etc", "resources"] + +# Avoids per-file regex compilation +_CONFIG_EXTENSIONS = [] +_CONFIG_EXACT_NAMES = [] +_CONFIG_WILDCARD_PATTERNS = [] + +for _pat in CONFIG_FILE_PATTERNS: + if _pat.startswith("*."): + _CONFIG_EXTENSIONS.append(_pat[1:]) + elif "*" in _pat: + _CONFIG_WILDCARD_PATTERNS.append( + re.compile(_pat.replace(".", r"\.").replace("*", ".*"), re.IGNORECASE) + ) + else: + _CONFIG_EXACT_NAMES.append(_pat.lower()) + + +class ConfigurationScannerToolConfig(FunctionBaseConfig, name=CONFIGURATION_SCANNER): + """Scans configuration files for vulnerability-relevant patterns.""" + max_results: int = Field(default=15, description="Maximum config entries to return") + context_lines: int = Field(default=5, description="Lines of context around matches") + + +def _is_config_file(file_path: str) -> bool: + """Check if a file path matches any known configuration file pattern.""" + name = os.path.basename(file_path) + lower_name = name.lower() + + # Check in order of speed: extension → exact name → wildcard regex + if any(lower_name.endswith(ext) for ext in _CONFIG_EXTENSIONS): + return True + if lower_name in _CONFIG_EXACT_NAMES: + return True + if any(p.match(lower_name) for p in _CONFIG_WILDCARD_PATTERNS): + return True + + return False + + +def _is_in_config_dir(file_path: str) -> bool: + """Check if a file resides under a known configuration directory.""" + parts = Path(file_path).parts + return any(p.lower() in CONFIG_DIR_PATTERNS for p in parts) + + +def _collect_config_files(repo_path: str) -> list[tuple[str, str]]: + """Walk repo and collect config file paths and contents. + + Skips .git, __pycache__, node_modules, .tox directories. + Skips files larger than 500KB to avoid memory issues with large generated configs. + """ + config_files = [] + for root, dirs, files in os.walk(repo_path): + dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", "node_modules", ".tox")] + for fname in files: + full_path = os.path.join(root, fname) + rel_path = os.path.relpath(full_path, repo_path) + if _is_config_file(rel_path) or _is_in_config_dir(rel_path): + try: + with open(full_path, "r", errors="ignore") as f: + content = f.read() + if len(content) < 500_000: + config_files.append((rel_path, content)) + else: + logger.warning("Skipping config file %s: size %d exceeds 500KB limit", rel_path, len(content)) + except Exception: + continue + logger.debug("Collected %d config files from %s", len(config_files), repo_path) + return config_files + + +def _count_config_matches(config_files: list[tuple[str, str]], keywords: list[str]) -> int: + """Count total keyword-matching lines across config files (no formatting, no cap).""" + count = 0 + for _, content in config_files: + for line in content.split("\n"): + if any(kw in line.lower() for kw in keywords): + count += 1 + return count + + +def search_config_content( + config_files: list[tuple[str, str]], + keywords: list[str], + max_results: int = 15, + context_lines: int = 5, + source_label: str = "unknown", +) -> list[str]: + """Match keywords against config file contents, returning formatted snippets.""" + matches = [] + for rel_path, content in config_files: + lines = content.split("\n") + for line_num, line in enumerate(lines): + line_lower = line.lower() + if any(kw in line_lower for kw in keywords): + snippet = format_context_snippet(lines, line_num, context_lines) + matches.append(f"--- {rel_path} (source: {source_label}) line {line_num+1} ---\n{snippet}") + if len(matches) >= max_results: + break + if len(matches) >= max_results: + break + return matches + + +@register_function(config_type=ConfigurationScannerToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def configuration_scanner(config: ConfigurationScannerToolConfig, builder: Builder): + from vuln_analysis.runtime_context import ctx_state, cu_source_scope + + _CONFIG_CACHE_MAX_SIZE = 20 + _config_files_cache: OrderedDict[tuple, list[tuple[str, str]]] = OrderedDict() + _repo_locks: dict[tuple, asyncio.Lock] = {} + _repo_locks_guard = asyncio.Lock() + + @catch_tool_errors(CONFIGURATION_SCANNER) + async def _arun(query: str) -> str: + workflow_state = ctx_state.get() + source_infos = workflow_state.original_input.input.image.source_info + source_scope = cu_source_scope.get() + logger.debug("Configuration scanner: searching for '%s' across %d source(s), scope=%s", + query, len(source_infos), source_scope) + + keywords = [w.strip().lower() for w in re.split(r"[,\s]+", query) if len(w.strip()) >= 2] + all_app_configs = [] + all_dep_configs = [] + + for si in source_infos: + if not hasattr(si, "git_repo"): + continue + repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(si.git_repo) + if not repo_path.is_dir(): + continue + + repo_key = (si.git_repo, si.ref) + if repo_key in _config_files_cache: + async with _repo_locks_guard: + _config_files_cache.move_to_end(repo_key) + else: + async with _repo_locks_guard: + if repo_key not in _repo_locks: + _repo_locks[repo_key] = asyncio.Lock() + repo_lock = _repo_locks[repo_key] + async with repo_lock: + if repo_key not in _config_files_cache: + _config_files_cache[repo_key] = _collect_config_files(str(repo_path)) + if len(_config_files_cache) > _CONFIG_CACHE_MAX_SIZE: + _config_files_cache.popitem(last=False) + + for cfg in _config_files_cache[repo_key]: + if is_dependency_path(cfg[0]): + all_dep_configs.append(cfg) + else: + all_app_configs.append(cfg) + + all_dep_configs = filter_by_source_scope(all_dep_configs, source_scope, lambda x: x[0]) + + source_label = source_infos[0].git_repo if source_infos and hasattr(source_infos[0], "git_repo") else "unknown" + app_matches = search_config_content( + all_app_configs, keywords, + max_results=config.max_results, + context_lines=config.context_lines, + source_label=source_label, + ) + remaining = config.max_results - len(app_matches) + dep_matches = search_config_content( + all_dep_configs, keywords, + max_results=max(remaining, 0), + context_lines=config.context_lines, + source_label=source_label, + ) if remaining > 0 else [] + + no_results_msg = f"No configuration entries found matching: {query}" + total_app = _count_config_matches(all_app_configs, keywords) + total_dep = _count_config_matches(all_dep_configs, keywords) + logger.debug("Configuration scanner: %d app + %d dep match(es) for '%s'", + total_app, total_dep, query) + return format_app_dep_output(app_matches, dep_matches, total_app, total_dep, no_results_msg) + + yield FunctionInfo.from_fn( + _arun, + description=( + "Scans configuration files (YAML, XML, properties, build files, Dockerfiles) " + "for patterns related to the vulnerability. Input: keywords describing what " + "configuration to look for (e.g. 'XML external entity processing', " + "'deserialization enabled', 'xstream allowTypes'). " + "Searches all indexed sources including framework dependencies." + ), + ) \ No newline at end of file diff --git a/src/vuln_analysis/tools/import_usage_analyzer.py b/src/vuln_analysis/tools/import_usage_analyzer.py new file mode 100644 index 000000000..755c82f2a --- /dev/null +++ b/src/vuln_analysis/tools/import_usage_analyzer.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope, format_app_dep_output + +IMPORT_USAGE_ANALYZER = "import_usage_analyzer" + +logger = LoggingFactory.get_agent_logger(__name__) + + +def _find_usage_in_file(content: str, imported_names: list[str], max_usages: int = 5) -> list[str]: + """Find usage sites of imported names, excluding import lines themselves.""" + usages = [] + lines = content.split("\n") + for line_num, line in enumerate(lines): + for name in imported_names: + short_name = name.rsplit(".", 1)[-1] if "." in name else name + if re.search(rf'\b{re.escape(short_name)}\b', line) and not line.strip().startswith(("import ", "from ", "#include")): + usages.append(f" L{line_num+1}: {line.strip()}") + if len(usages) >= max_usages: + return usages + return usages + + +class ImportUsageAnalyzerToolConfig(FunctionBaseConfig, name=IMPORT_USAGE_ANALYZER): + """Analyzes imports and usage patterns of a package across indexed sources.""" + max_files: int = Field(default=20, description="Maximum files to report") + + +def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: str, + max_files: int = 20, source_scope: list[str] | None = None, + ecosystem_label: str = "") -> str: + """Scan a Tantivy searcher for import patterns, with app/dep source awareness. + + Results are separated into application code and dependency code sections, + with dependency results filtered by source_scope when provided. + """ + num_docs = searcher.num_docs + app_results = [] + dep_results = [] + + for doc_id in range(num_docs): + try: + raw = searcher.doc(doc_id) + file_path = raw["file_path"][0] + content = raw["content"][0] + except Exception: + continue + + found_imports = [] + for pattern in import_patterns: + for match in pattern.finditer(content): + found_imports.append(match.group(0)) + + if not found_imports: + continue + + imported_names = [] + for imp in found_imports: + parts = imp.split() + for p in parts: + cleaned = p.strip(";'\",<>(){}").strip() + if package_name.lower() in cleaned.lower() and len(cleaned) > 2: + imported_names.append(cleaned) + + usages = _find_usage_in_file(content, imported_names) + + entry = f"--- {file_path} ---\n" + entry += f" Imports: {'; '.join(found_imports[:3])}\n" + if usages: + entry += f" Usages ({len(usages)}):\n" + "\n".join(usages) + else: + entry += " No usage sites found beyond import." + + if is_dependency_path(file_path): + dep_results.append((file_path, entry)) + else: + app_results.append(entry) + + dep_results = filter_by_source_scope(dep_results, source_scope, lambda x: x[0]) + dep_entries = [entry for _, entry in dep_results] + + no_results_msg = f"No imports of '{package_name}' found in indexed sources (ecosystem: {ecosystem_label})." + total_app = len(app_results) + total_dep = len(dep_entries) + + trimmed_app = app_results[:max_files] + remaining = max_files - len(trimmed_app) + trimmed_dep = dep_entries[:max(remaining, 0)] + + return format_app_dep_output(trimmed_app, trimmed_dep, total_app, total_dep, no_results_msg) + + +@register_function(config_type=ImportUsageAnalyzerToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def import_usage_analyzer(config: ImportUsageAnalyzerToolConfig, builder: Builder): + from vuln_analysis.runtime_context import ctx_state, cu_source_scope + from vuln_analysis.utils.full_text_search import FullTextSearch + + @catch_tool_errors(IMPORT_USAGE_ANALYZER) + async def _arun(query: str) -> str: + workflow_state = ctx_state.get() + code_index_path = workflow_state.code_index_path + ecosystem = workflow_state.original_input.input.image.ecosystem + source_scope = cu_source_scope.get() + + fts = FullTextSearch.get_instance(cache_path=code_index_path) + if fts.is_empty(): + logger.debug("Import usage analyzer: no source code indexed at %s", code_index_path) + return "No source code indexed." + + package_name = query.strip() + parser = get_language_function_parser(ecosystem, tree=None) if ecosystem else None + import_patterns = parser.get_import_search_patterns(package_name) if parser else [re.compile(re.escape(package_name), re.IGNORECASE)] + ecosystem_label = ecosystem.value if ecosystem else "" + logger.debug("Import usage analyzer: searching for '%s' (ecosystem: %s), scope=%s", + package_name, ecosystem_label, source_scope) + + result = analyze_imports( + fts.index.searcher(), import_patterns, package_name, + max_files=config.max_files, source_scope=source_scope, + ecosystem_label=ecosystem_label, + ) + logger.debug("Import usage analyzer: %s", result.split("\n", 1)[0]) + return result + + yield FunctionInfo.from_fn( + _arun, + description=( + "Finds all imports and usage patterns of a specific package/module across indexed sources. " + "Input: package or module name (e.g. 'encoding/xml', 'com.thoughtworks.xstream', " + "'urllib.parse'). Reports which files import it, how many, and how the package is used. " + "Searches all sources including framework dependencies." + ), + ) \ No newline at end of file diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 0b24fcc1e..8117e990d 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -37,19 +37,25 @@ class LexicalSearchToolConfig(FunctionBaseConfig, name=LEXICAL_CODE_SEARCH): @register_function(config_type=LexicalSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def lexical_search(config: LexicalSearchToolConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.runtime_context import ctx_state + from vuln_analysis.runtime_context import ctx_state, cu_source_scope from vuln_analysis.utils.full_text_search import FullTextSearch @catch_tool_errors(LEXICAL_CODE_SEARCH) async def _arun(query: str) -> str: workflow_state = ctx_state.get() code_index_path = workflow_state.code_index_path - full_text_search = FullTextSearch(cache_path=code_index_path) + full_text_search = FullTextSearch.get_instance(cache_path=code_index_path) if full_text_search.is_empty(): + logger.debug("Lexical search: index is empty at %s", code_index_path) raise ValueError(f"Invalid code index at: {code_index_path}, index is empty") - result = full_text_search.search_index(query, config.top_k) + # Pass source_scope from ContextVar to restrict dependency results + # to the current question's target package (set by CU pre_process_node) + source_scope = cu_source_scope.get() + result = full_text_search.search_index(query, config.top_k, source_scope=source_scope) + logger.debug("Lexical search: query='%.80s', top_k=%d, source_scope=%s", + query, config.top_k, source_scope) return result diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 7549032fa..98b701d64 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -47,15 +47,22 @@ async def load_vectordb_asretriever(config: LocalVDBRetrieverToolConfig, builder from langchain_community.vectorstores import FAISS from langchain_core.prompts import PromptTemplate - from vuln_analysis.runtime_context import ctx_state + from vuln_analysis.runtime_context import ctx_state, cu_source_scope embedder = await builder.get_embedder(embedder_name=config.embedder_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + qa_prompt = PromptTemplate(template=("Use the following pieces of context to answer the question at the end. " + "If you don't know the answer, just say that you don't know, " + "don't try to make up an answer.\n\n{context}\n\n" + "Question: {question}\nHelpful Answer:"), + input_variables=['context', 'question']) + # FAISS deserialization is expensive — cache per db_source path + _retrieval_cache: dict[str, RetrievalQA] = {} + @catch_tool_errors(LOCAL_VDB_RETRIEVER) async def _arun(query: str) -> str | dict: - # workaround since the agent executor only accepts strings. workflow_state = ctx_state.get() if config.vdb_type == VdbType.CODE: @@ -65,21 +72,32 @@ async def _arun(query: str) -> str | dict: else: raise ValueError(f"Invalid VDB type: {config.vdb_type}. Must be one of {VdbType.CODE} or {VdbType.DOC}.") - qa_prompt = PromptTemplate(template=("Use the following pieces of context to answer the question at the end. " - "If you don't know the answer, just say that you don't know, " - "don't try to make up an answer.\n\n{context}\n\n" - "Question: {question}\nHelpful Answer:"), - input_variables=['context', 'question']) - - vector_db = FAISS.load_local(db_source, embedder, allow_dangerous_deserialization=True) - retrieval_qa_tool = RetrievalQA.from_chain_type(llm=llm, - chain_type="stuff", - chain_type_kwargs={"prompt": qa_prompt}, - retriever=vector_db.as_retriever(), - return_source_documents=config.return_source_documents) + if db_source not in _retrieval_cache: + logger.info("Loading FAISS index from %s (type: %s)", db_source, config.vdb_type) + vector_db = FAISS.load_local(db_source, embedder, allow_dangerous_deserialization=True) + _retrieval_cache[db_source] = RetrievalQA.from_chain_type( + llm=llm, + chain_type="stuff", + chain_type_kwargs={"prompt": qa_prompt}, + retriever=vector_db.as_retriever(), + return_source_documents=config.return_source_documents, + ) + retrieval_qa_tool = _retrieval_cache[db_source] output_dict = await retrieval_qa_tool.ainvoke(query) + if config.return_source_documents and "source_documents" in output_dict: + source_scope = cu_source_scope.get() + if source_scope: + pre_filter = len(output_dict["source_documents"]) + output_dict["source_documents"] = [ + doc for doc in output_dict["source_documents"] + if any(scope in doc.metadata.get("source", "") for scope in source_scope) + ] + if pre_filter != len(output_dict["source_documents"]): + logger.debug("Source scope %s filtered doc results from %d to %d", + source_scope, pre_filter, len(output_dict["source_documents"])) + # If returning source documents, include the result and source_documents keys in the output if config.return_source_documents: return {k: v for k, v in output_dict.items() if k in ["result", "source_documents"]} @@ -96,7 +114,9 @@ async def _arun(query: str) -> str | dict: elif config.vdb_type == VdbType.DOC: description = ( "Searches container documentation using semantic search. " - "Answers questions about application purpose, architecture, and features." + "Answers questions about application purpose, architecture, and features. " + "Queries must be specific — include a concrete term (library name, config property, protocol). " + "Vague queries like 'configuration' or 'security settings' return unhelpful results." ) else: raise ValueError(f"Invalid VDB type: {config.vdb_type}. Must be one of {VdbType.CODE} or {VdbType.DOC}.") diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index b432fc257..65037fe5f 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -1075,3 +1075,27 @@ def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): """Test that query cleaning handles both ASCII and Unicode smart quotes.""" cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" + + +@pytest.mark.parametrize("raw_query, expected_cleaned", [ + # Standard ASCII quotes + ("'commons-beanutils:commons-beanutils:1.9.4'", "commons-beanutils:commons-beanutils:1.9.4"), + ('"commons-beanutils:commons-beanutils:1.9.4"', "commons-beanutils:commons-beanutils:1.9.4"), + # Unicode smart quotes (left/right single) + ("\u2018commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), + # Unicode smart quotes (left/right double) + ("\u201ccommons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), + # Mixed: ASCII left, unicode right + ("'commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), + ("\"commons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), + # No quotes + ("commons-beanutils:commons-beanutils:1.9.4", "commons-beanutils:commons-beanutils:1.9.4"), + # Whitespace + quotes + (" 'commons-beanutils:commons-beanutils:1.9.4' ", "commons-beanutils:commons-beanutils:1.9.4"), + # Trailing newline junk from LLM + ("'commons-beanutils:commons-beanutils:1.9.4'\nPlease wait...", "commons-beanutils:commons-beanutils:1.9.4"), +]) +def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): + """Test that query cleaning handles both ASCII and Unicode smart quotes.""" + cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() + assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" diff --git a/src/vuln_analysis/tools/tool_names.py b/src/vuln_analysis/tools/tool_names.py index fa1ddf72c..f07d40d50 100644 --- a/src/vuln_analysis/tools/tool_names.py +++ b/src/vuln_analysis/tools/tool_names.py @@ -22,45 +22,51 @@ class ToolNames: - """ - Constants for agent tool names matching YAML configuration keys. - + """Constants for agent tool names matching YAML configuration keys. + These names correspond to the function keys in the YAML config files - (e.g., config.yml, config-http-openai-local.yml). + (e.g., config.yml, config-http-nim.yml). """ - + # Code analysis tools CODE_SEMANTIC_SEARCH = "Code Semantic Search" - """Searches container source code using semantic vector embeddings""" - + """Searches container source code using semantic vector embeddings.""" + DOCS_SEMANTIC_SEARCH = "Docs Semantic Search" - """Searches container documentation using semantic vector embeddings""" - + """Searches container documentation using semantic vector embeddings.""" + CODE_KEYWORD_SEARCH = "Code Keyword Search" - """Searches container source code for exact keyword matches""" - - # Code path analysis + """Searches container source code for exact keyword matches.""" + + # Code path analysis tools FUNCTION_LOCATOR = "Function Locator" - """Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, provides ecosystem type.""" + """Mandatory first step for code path analysis. Validates package names and locates functions using fuzzy matching.""" CALL_CHAIN_ANALYZER = "Call Chain Analyzer" - """Checks if a function is reachable from application code""" - + """Checks if a function is reachable from application code.""" + FUNCTION_CALLER_FINDER = "Function Caller Finder" - """Golang only. Finds functions calling specific standard library methods.""" - + """Go only. Finds functions calling specific standard library methods.""" + # External and cached data sources CVE_WEB_SEARCH = "CVE Web Search" - """Searches the web for CVE and vulnerability information""" - + """Searches the web for CVE and vulnerability information.""" + CONTAINER_ANALYSIS_DATA = "Container Analysis Data" - """Retrieves pre-analyzed data from earlier container scan steps""" + """Retrieves pre-analyzed data from earlier container scan steps.""" FUNCTION_LIBRARY_VERSION_FINDER = "Function Library Version Finder" - """Checks in which library version the function is used""" + """Java only. Checks in which library version the function is used.""" + + # Code Understanding agent tools + CONFIGURATION_SCANNER = "Configuration Scanner" + """Scans configuration files (YAML, XML, properties, build files) for vulnerability-relevant patterns.""" + + IMPORT_USAGE_ANALYZER = "Import Usage Analyzer" + """Finds all imports and usage patterns of a specific package across indexed sources.""" -# Export as module-level constants +# Module-level constants for convenience imports CODE_SEMANTIC_SEARCH = ToolNames.CODE_SEMANTIC_SEARCH DOCS_SEMANTIC_SEARCH = ToolNames.DOCS_SEMANTIC_SEARCH CODE_KEYWORD_SEARCH = ToolNames.CODE_KEYWORD_SEARCH @@ -70,6 +76,8 @@ class ToolNames: CVE_WEB_SEARCH = ToolNames.CVE_WEB_SEARCH CONTAINER_ANALYSIS_DATA = ToolNames.CONTAINER_ANALYSIS_DATA FUNCTION_LIBRARY_VERSION_FINDER = ToolNames.FUNCTION_LIBRARY_VERSION_FINDER +CONFIGURATION_SCANNER = ToolNames.CONFIGURATION_SCANNER +IMPORT_USAGE_ANALYZER = ToolNames.IMPORT_USAGE_ANALYZER @@ -82,6 +90,8 @@ class ToolNames: 'FUNCTION_CALLER_FINDER', 'CVE_WEB_SEARCH', 'CONTAINER_ANALYSIS_DATA', - 'FUNCTION_LOCATOR', + 'PACKAGE_FUNCTION_LOCATOR', 'FUNCTION_LIBRARY_VERSION_FINDER', -] \ No newline at end of file + 'CONFIGURATION_SCANNER', + 'IMPORT_USAGE_ANALYZER', +] diff --git a/src/vuln_analysis/utils/code_understanding_prompt_factory.py b/src/vuln_analysis/utils/code_understanding_prompt_factory.py new file mode 100644 index 000000000..8063b1832 --- /dev/null +++ b/src/vuln_analysis/utils/code_understanding_prompt_factory.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-ecosystem tool strategies and generic tool descriptions for the Code Understanding agent.""" + +from vuln_analysis.tools.tool_names import ToolNames + +_DOCS_SEARCH_STEP = ( + "4. Use Docs Semantic Search for broader architectural questions about the application " + "— queries must include a specific term (library name, config property, protocol)." +) + +# Generic tool descriptions used when no ecosystem-specific strategy is available +CU_TOOL_GENERAL_DESCRIPTIONS: dict[str, str] = { + ToolNames.DOCS_SEMANTIC_SEARCH: ( + "Searches container documentation using semantic search. " + "Answers questions about application purpose, architecture, features, and dependencies. " + "IMPORTANT: Queries must be specific — include at least one concrete term " + "(class name, config property, library name, or protocol). " + "GOOD queries: 'deserialization whitelist configuration', " + "'SSL/TLS certificate validation settings', 'input sanitization before parsing'. " + "BAD queries: 'security settings', 'configuration', 'error handling'. " + "Vague queries return unhelpful results." + ), + ToolNames.CODE_KEYWORD_SEARCH: ( + "Performs keyword search on container source code for exact text matches. " + "Input should be a function name, class name, import statement, or code pattern." + ), + ToolNames.CONFIGURATION_SCANNER: ( + "Scans configuration files (YAML, XML, properties, build files, Dockerfiles) " + "for feature flags, enabled components, and security settings. " + "Searches all sources including framework dependencies." + ), + ToolNames.IMPORT_USAGE_ANALYZER: ( + "Finds all imports and usage patterns of a package/module across sources. " + "Reports which files import it, usage count, and how it's used. " + "Searches all sources including framework dependencies." + ), +} + +# Per-ecosystem tool selection strategies guiding the agent's investigation order +CU_TOOL_SELECTION_STRATEGY: dict[str, str] = { + "python": ( + "1. Start with Configuration Scanner to check if affected features are enabled or configured. " + "2. Use Code Keyword Search for exact import/function lookups (e.g. 'import xml.etree'). " + "3. Use Import Usage Analyzer to understand how the affected package is imported and used. " + + _DOCS_SEARCH_STEP + ), + "go": ( + "1. Start with Configuration Scanner to check configs for feature flags and security settings. " + "2. Use Code Keyword Search for import paths (e.g. 'encoding/xml'). " + "3. Use Import Usage Analyzer to trace imports of the affected package. " + + _DOCS_SEARCH_STEP + ), + "java": ( + "1. Start with Configuration Scanner to check configs, pom.xml, and build.gradle for feature flags and security settings. " + "2. Use Code Keyword Search for import statements (e.g. 'import javax.xml'). " + "3. Use Import Usage Analyzer to trace how the affected library is imported across sources. " + + _DOCS_SEARCH_STEP + ), + "javascript": ( + "1. Start with Configuration Scanner to check package.json, .env, and config files. " + "2. Use Code Keyword Search for require/import patterns (e.g. 'require(\"xml2js\")'). " + "3. Use Import Usage Analyzer to find all imports of the affected package. " + + _DOCS_SEARCH_STEP + ), + "c": ( + "1. Start with Configuration Scanner to check Makefiles, CMakeLists.txt, and config headers. " + "2. Use Code Keyword Search for function names and #include patterns. " + "3. Use Import Usage Analyzer to trace #include usage of the affected library. " + + _DOCS_SEARCH_STEP + ), +} diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 0a02ab991..6fa0e8e55 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -16,6 +16,8 @@ import json import os import re +import threading +from collections import OrderedDict from pathlib import Path from typing import Iterable @@ -35,9 +37,7 @@ def tokenize_code(code: str) -> str: - """ - Tokenize code into simple readable document format - """ + """Normalize source code identifiers for search: split camelCase/snake_case, lowercase, filter noise.""" matches = re.finditer(r"\b\w{2,}\b", code) tokens = [] for m in matches: @@ -47,6 +47,7 @@ def tokenize_code(code: str) -> str: for part in variable_pattern.findall(section): if len(part) < 2: continue + # Exclude hex-like strings and repetitive patterns if (sum(1 for c in part if "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9") > len(part) // 2 and len(part) / len(set(part)) < 4): tokens.append(part.lower()) @@ -55,8 +56,9 @@ def tokenize_code(code: str) -> str: def clean_query(input_query) -> str: - """ - Parse a query string with OR/AND operations and fix quotes issues using regex. + """Parse a query string with OR/AND operations and fix quote issues. + + Sanitizes LLM-generated queries for Tantivy — unmatched quotes crash its parser. """ # Remove anything after newline characters if '\n' in input_query: @@ -85,16 +87,21 @@ def replace_quoted(match): return input_query -ECOSYSTEM_DEP_DIRS = { - "c": "rpm_libs/", - "go": "vendor/", - "javascript": "node_modules/", - "python": "transitive_env/", - "java": "dependencies-sources/", -} +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope class FullTextSearch: + """Tantivy-based full text search index for source code and documentation. + + Wraps a Tantivy index with document management, query sanitization, and + result separation into app code vs. dependency code. Instances are cached + via get_instance() with LRU eviction to avoid holding too many open indexes. + """ INDEX_TYPE = "tantivy" + # LRU cache of FullTextSearch instances keyed by cache_path. + # Prevents unbounded growth of open Tantivy index handles. + _instances: "OrderedDict[str, FullTextSearch]" = None + _INSTANCE_CACHE_MAX = 10 + _instances_lock = threading.Lock() def __init__(self, cache_path: str = None, tokenizer=False): if cache_path: @@ -104,13 +111,37 @@ def __init__(self, cache_path: str = None, tokenizer=False): self.index.reload() self.tokenizer = tokenizer + @classmethod + def get_instance(cls, cache_path: str, tokenizer=False) -> "FullTextSearch": + """Return a cached instance for the given path, with LRU eviction. + + On cache hit, moves the entry to the end (most recently used). + On cache miss, creates a new instance and evicts the least recently used + if the cache exceeds _INSTANCE_CACHE_MAX. + """ + with cls._instances_lock: + if cls._instances is None: + cls._instances = OrderedDict() + if cache_path in cls._instances: + cls._instances.move_to_end(cache_path) + return cls._instances[cache_path] + instance = cls(cache_path=cache_path, tokenizer=tokenizer) + with cls._instances_lock: + if cache_path in cls._instances: + return cls._instances[cache_path] + cls._instances[cache_path] = instance + if len(cls._instances) > cls._INSTANCE_CACHE_MAX: + evicted = cls._instances.popitem(last=False) + logger.debug("Evicted LRU FullTextSearch instance: %s", evicted[0]) + return instance + @classmethod def get_index_directory(cls, base_path: str, hash_value: str) -> Path: """Returns the path where the index should be stored""" return Path(base_path) / cls.INDEX_TYPE / hash_value def _build_schema(self): - """Build schema for the code index""" + """Build Tantivy schema: file_path + content (both stored for retrieval) + doc_id.""" schema_builder = SchemaBuilder() schema_builder.add_text_field("file_path", stored=True) schema_builder.add_text_field("content", stored=True) @@ -119,17 +150,28 @@ def _build_schema(self): return schema def add_documents(self, documents: Iterable): - + """Index an iterable of (file_path, content) tuples into Tantivy.""" writer = self.index.writer() for doc_id, (title, text) in enumerate(documents): writer.add_document(TantivyDocument(file_path=title, content=text, doc_id=doc_id)) writer.commit() def is_empty(self): + """Check if the index contains any documents.""" return self.index.searcher().num_docs == 0 - def search_index(self, query: str, top_k: int = 10) -> str: + def search_index(self, query: str, top_k: int = 10, source_scope: list[str] | None = None) -> str: + """Search the index using Tantivy's ranked query engine. + + Returns results separated into application code and dependency code sections. + When source_scope is provided, dependency results are filtered to only + include files matching the scope (e.g. specific package names). + Args: + query: Search query string (sanitized internally) + top_k: Maximum total results to return (app results prioritized) + source_scope: Optional list of path substrings to filter dependency results + """ self.index.reload() try: if self.tokenizer: @@ -141,16 +183,20 @@ def search_index(self, query: str, top_k: int = 10) -> str: app_docs = [] dep_docs = [] - - vendors_list = list(ECOSYSTEM_DEP_DIRS.values()) for _, doc_id in results: raw = searcher.doc(doc_id) doc = {"source": raw["file_path"][0], "content": raw["content"][0]} - if any(doc["source"].startswith(vendor) for vendor in vendors_list): + if is_dependency_path(doc["source"]): dep_docs.append(doc) else: app_docs.append(doc) + pre_filter_count = len(dep_docs) + dep_docs = filter_by_source_scope(dep_docs, source_scope, lambda d: d["source"]) + if pre_filter_count != len(dep_docs): + logger.debug("Source scope %s filtered dep results from %d to %d", + source_scope, pre_filter_count, len(dep_docs)) + total_app = len(app_docs) total_dep = len(dep_docs) @@ -174,11 +220,11 @@ def search_index(self, query: str, top_k: int = 10) -> str: return "\n".join(parts) except Exception as e: - logger.exception(e) + logger.exception("Search failed for query") return "There was an error searching for the query." def add_documents_from_langchain_chunks(self, documents: list[Document]): - """Create an index from langchain chunked documents""" + """Index pre-chunked LangChain documents (used by document_embedding.py for VDB build).""" try: documents = [(doc.metadata['source'], doc.page_content) for doc in documents] @@ -188,14 +234,18 @@ def add_documents_from_langchain_chunks(self, documents: list[Document]): self.add_documents(tqdm(documents, total=len(documents), desc="Indexing")) except Exception as e: - logger.warning(f"Unable to add documents to the index {e}") + logger.warning("Unable to add documents to the index: %s", e) def add_documents_from_code_path(self, code_path: str, include_extensions: list[str], use_langparser=True, splitter=True): - """Create an index from raw files.""" + """Index raw source files from a directory, optionally using LangChain's LanguageParser. + + When use_langparser=True, files are parsed with language-aware chunking. + When False, files are read as raw text (optionally tokenized for search). + """ doc_content = [] @@ -227,6 +277,6 @@ def add_documents_from_code_path(self, content = tokenize_code(content) doc_content.append((file_path, content)) except Exception as e: - logger.warning(f"Error reading {file_path}: {e}") + logger.warning("Error reading %s: %s", file_path, e) self.add_documents(doc_content) diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 32e6942f2..d8dee99cd 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -26,6 +26,8 @@ logger = LoggingFactory.get_agent_logger(__name__) +_MAX_RHSA_CANDIDATES = 20 + def update_version(incoming_version, current_version, compare): """ @@ -200,12 +202,19 @@ def build_critical_context(cve_intel_list) -> tuple[list[str], list[dict], list[ critical_context.append(f"Search keywords: {', '.join(short_names)}") for f in vf: vulnerable_functions.add(f.rsplit('.', 1)[-1]) + ver_range = vuln.get('vulnerable_version_range', '') if isinstance(vuln, dict) else getattr(v, 'vulnerable_version_range', '') + patched = vuln.get('first_patched_version', '') if isinstance(vuln, dict) else getattr(v, 'first_patched_version', '') if pkg: if isinstance(pkg, dict): pkg_name = pkg.get("name", "") pkg_eco = pkg.get("ecosystem", "") if pkg_name: critical_context.append(f"Vulnerable module ({pkg_eco}): {pkg_name}") + if ver_range: + ver_ctx = f"Vulnerable version range ({pkg_name}): {ver_range}" + if patched: + ver_ctx += f" | First patched version: {patched}" + critical_context.append(ver_ctx) if pkg_name not in seen_packages: seen_packages.add(pkg_name) candidate_packages.append({"name": pkg_name, "source": "ghsa", "ecosystem": pkg_eco}) @@ -227,10 +236,14 @@ def build_critical_context(cve_intel_list) -> tuple[list[str], list[dict], list[ critical_context.append(f"KNOWN MITIGATIONS: {mit_text[:500]}") if cve_intel.rhsa.package_state: pkgs = list(set(p.package_name for p in cve_intel.rhsa.package_state if p.package_name)) + rhsa_added = 0 for p in pkgs: if p not in seen_packages: seen_packages.add(p) candidate_packages.append({"name": p, "source": "rhsa"}) + rhsa_added += 1 + if rhsa_added >= _MAX_RHSA_CANDIDATES: + break if len(pkgs) > 5: critical_context.append( f"Affected across {len(pkgs)} Red Hat products (sample: {', '.join(pkgs[:5])}). " diff --git a/src/vuln_analysis/utils/source_classification.py b/src/vuln_analysis/utils/source_classification.py new file mode 100644 index 000000000..6a74fd12c --- /dev/null +++ b/src/vuln_analysis/utils/source_classification.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, TypeVar + +from exploit_iq_commons.utils.dep_tree import ECOSYSTEM_DEP_DIRS + +_VENDORS = list(ECOSYSTEM_DEP_DIRS.values()) + +T = TypeVar("T") + + +def is_dependency_path(file_path: str) -> bool: + """Return True if file_path starts with any ecosystem dependency directory prefix.""" + return any(file_path.startswith(vendor) for vendor in _VENDORS) + + +def filter_by_source_scope(dep_items: list[T], source_scope: list[str] | None, + path_fn: Callable[[T], str]) -> list[T]: + """Filter dependency items to those whose path contains any source_scope substring. + + Args: + dep_items: Dependency result items to filter. + source_scope: Path substrings to match. None means no filtering. + path_fn: Extracts the file path string from an item. + """ + if not source_scope: + return dep_items + return [item for item in dep_items if any(scope in path_fn(item) for scope in source_scope)] + + +def format_app_dep_output(app_items: list[str], dep_items: list[str], + total_app: int, total_dep: int, + no_results_msg: str) -> str: + """Format results with app/dep section headers matching Code Keyword Search output. + + Output format: + Main application (N of M results) + + Application library dependencies (N of M results) + + """ + if total_app == 0 and total_dep == 0: + return no_results_msg + + app_header = f"Main application ({len(app_items)} of {total_app} results)" + dep_header = f"Application library dependencies ({len(dep_items)} of {total_dep} results)" + + parts = [app_header] + if app_items: + parts.append("\n\n".join(app_items)) + parts.append(dep_header) + if dep_items: + parts.append("\n\n".join(dep_items)) + return "\n".join(parts) diff --git a/src/vuln_analysis/utils/token_utils.py b/src/vuln_analysis/utils/token_utils.py new file mode 100644 index 000000000..c3e435892 --- /dev/null +++ b/src/vuln_analysis/utils/token_utils.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Token counting and truncation utilities for agent context management.""" + +import tiktoken + +from vuln_analysis.tools.tool_names import ToolNames + +_tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + +def count_tokens(text: str) -> int: + """Count the number of tokens in a text string using the cl100k_base encoding. + + Falls back to a rough character-based estimate if encoding fails. + """ + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + +def estimate_tokens(runtime_prompt: str, messages: list, observation) -> int: + """Estimate total token count of the current agent context (prompt + messages + observation).""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return count_tokens("\n".join(parts)) + + +def truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400) -> str: + """Truncate tool output to fit within a token budget. + + Uses tool-specific strategies: Call Chain Analyzer keeps the header, + Code Keyword Search preserves section headers, and the default strategy + keeps head (70%) and tail (30%) of the output. + """ + token_count = count_tokens(tool_output) + if token_count <= max_tokens: + return tool_output + + lines = tool_output.split('\n') + + if tool_name == ToolNames.CALL_CHAIN_ANALYZER: + head = '\n'.join(lines[:3]) + remaining = token_count - count_tokens(head) + return f"{head}\n[... truncated {remaining} tokens ...]" + + if tool_name == ToolNames.CODE_KEYWORD_SEARCH: + kept_lines = [] + kept_tokens = 0 + for line in lines: + is_header = line.startswith("---") or "Main application" in line or "library dependencies" in line or line.strip() == "" + line_tokens = count_tokens(line) + if is_header or kept_tokens + line_tokens <= max_tokens: + kept_lines.append(line) + kept_tokens += line_tokens + if kept_tokens >= max_tokens: + break + kept_lines.append(f"[... truncated {token_count - kept_tokens} tokens ...]") + return '\n'.join(kept_lines) + + head_budget = int(max_tokens * 0.7) + tail_budget = max_tokens - head_budget + head_lines, head_tokens = [], 0 + for line in lines: + lt = count_tokens(line) + if head_tokens + lt > head_budget: + break + head_lines.append(line) + head_tokens += lt + tail_lines, tail_tokens = [], 0 + for line in reversed(lines): + lt = count_tokens(line) + if tail_tokens + lt > tail_budget: + break + tail_lines.insert(0, line) + tail_tokens += lt + truncated = token_count - head_tokens - tail_tokens + return '\n'.join(head_lines) + f"\n[... truncated {truncated} tokens ...]\n" + '\n'.join(tail_lines) diff --git a/tests/agent_test_helpers.py b/tests/agent_test_helpers.py new file mode 100644 index 000000000..2d60aed2b --- /dev/null +++ b/tests/agent_test_helpers.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock + +from vuln_analysis.tools.tool_names import ToolNames + + +class MockTool: + def __init__(self, name: str): + self.name = name + + +ALL_TOOLS = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.CONTAINER_ANALYSIS_DATA), + MockTool(ToolNames.FUNCTION_LIBRARY_VERSION_FINDER), + MockTool(ToolNames.CONFIGURATION_SCANNER), + MockTool(ToolNames.IMPORT_USAGE_ANALYZER), +] + + +def make_builder(tools=None): + builder = MagicMock() + builder.get_tools = MagicMock(return_value=list(tools if tools is not None else ALL_TOOLS)) + return builder + + +def make_config(**overrides): + config = MagicMock() + config.tool_names = overrides.get("tool_names", []) + config.transitive_search_tool_enabled = overrides.get("transitive_search_tool_enabled", True) + config.cve_web_search_enabled = overrides.get("cve_web_search_enabled", True) + config.max_iterations = 10 + return config + + +def make_state(code_vdb_path="/path", doc_vdb_path="/path", code_index_path="/path"): + state = MagicMock() + state.code_vdb_path = code_vdb_path + state.doc_vdb_path = doc_vdb_path + state.code_index_path = code_index_path + return state diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py new file mode 100644 index 000000000..6260495df --- /dev/null +++ b/tests/test_agent_registry.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for agent_registry: register_agent decorator, get_agent_class, get_all_agent_types. +""" + +import pytest + +from vuln_analysis.functions.agent_registry import ( + _AGENT_REGISTRY, + register_agent, + get_agent_class, + get_all_agent_types, +) + + +class TestRealAgentRegistration: + """Verify that importing the agent modules registers both agents.""" + + def test_reachability_registered(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + assert "reachability" in _AGENT_REGISTRY + + def test_code_understanding_registered(self): + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + assert "code_understanding" in _AGENT_REGISTRY + + def test_get_all_agent_types_contains_both(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + types = get_all_agent_types() + assert "reachability" in types + assert "code_understanding" in types + + def test_get_agent_class_reachability(self): + from vuln_analysis.functions.reachability_agent import ReachabilityAgent + assert get_agent_class("reachability") is ReachabilityAgent + + def test_get_agent_class_code_understanding(self): + from vuln_analysis.functions.code_understanding_agent import CodeUnderstandingAgent + assert get_agent_class("code_understanding") is CodeUnderstandingAgent + + +class TestGetAgentClassErrors: + """Test error cases for get_agent_class.""" + + def test_unknown_type_raises_key_error(self): + with pytest.raises(KeyError, match="Unknown agent type 'nonexistent'"): + get_agent_class("nonexistent") + + def test_error_message_lists_registered_types(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + with pytest.raises(KeyError) as exc_info: + get_agent_class("bogus") + msg = str(exc_info.value) + assert "reachability" in msg + assert "code_understanding" in msg + + +class TestRegisterAgentDecorator: + """Test the register_agent decorator mechanics using a dummy class.""" + + def setup_method(self): + self._saved = dict(_AGENT_REGISTRY) + + def teardown_method(self): + _AGENT_REGISTRY.clear() + _AGENT_REGISTRY.update(self._saved) + + def test_decorator_registers_class(self): + @register_agent("test_dummy") + class DummyAgent: + pass + + assert get_agent_class("test_dummy") is DummyAgent + + def test_decorator_returns_original_class(self): + @register_agent("test_dummy2") + class DummyAgent: + pass + + assert DummyAgent.__name__ == "DummyAgent" + + def test_re_registration_replaces_class(self): + @register_agent("test_replace") + class First: + pass + + @register_agent("test_replace") + class Second: + pass + + assert get_agent_class("test_replace") is Second + + def test_registered_type_appears_in_all_types(self): + @register_agent("test_listed") + class Listed: + pass + + assert "test_listed" in get_all_agent_types() diff --git a/tests/test_base_graph_agent.py b/tests/test_base_graph_agent.py new file mode 100644 index 000000000..834c84cf7 --- /dev/null +++ b/tests/test_base_graph_agent.py @@ -0,0 +1,848 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for BaseGraphAgent: should_continue routing, default hooks, agent_type property, +thought_node context pruning. +""" + +import pytest +from unittest.mock import MagicMock, AsyncMock, patch +from langchain_core.messages import SystemMessage, AIMessage, HumanMessage, ToolMessage + +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent +from vuln_analysis.functions.react_internals import Thought, ToolCall, Observation + + +class _ConcreteAgent(BaseGraphAgent): + """Minimal concrete subclass for testing base class behavior.""" + + async def pre_process_node(self, state): + return state + + @staticmethod + def get_tools(builder, config, state): + return [] + + @staticmethod + def create_rules_tracker(): + return MagicMock() + + +def _make_agent(): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return _ConcreteAgent(tools=[], llm=mock_llm, config=config) + + +class TestShouldContinue: + """Test should_continue routing logic.""" + + @pytest.mark.asyncio + async def test_returns_end_on_finish_mode(self): + agent = _make_agent() + thought = Thought(thought="done", mode="finish", actions=None, final_answer="answer") + state = {"thought": thought, "step": 3, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "__end__" + + @pytest.mark.asyncio + async def test_returns_forced_finish_at_max_steps(self): + agent = _make_agent() + thought = Thought( + thought="still working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 10, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_returns_forced_finish_beyond_max_steps(self): + agent = _make_agent() + thought = Thought( + thought="still working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 15, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_returns_tool_node_when_continuing(self): + agent = _make_agent() + thought = Thought( + thought="need more info", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 3, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "tool_node" + + @pytest.mark.asyncio + async def test_returns_thought_node_when_thought_is_none(self): + agent = _make_agent() + state = {"thought": None, "step": 0, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "thought_node" + + @pytest.mark.asyncio + async def test_forced_finish_when_thought_none_at_max_steps(self): + """Step limit must be enforced even when thought is None (e.g. after + check_finish_allowed repeatedly blocks). Without this, the agent + self-loops thought_node→thought_node until GraphRecursionError.""" + agent = _make_agent() + state = {"thought": None, "step": 10, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_forced_finish_when_thought_none_beyond_max_steps(self): + agent = _make_agent() + state = {"thought": None, "step": 15, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_uses_config_max_iterations_as_fallback(self): + agent = _make_agent() + thought = Thought( + thought="working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + +class TestDefaultHooks: + """Test default hook implementations on BaseGraphAgent.""" + + def test_post_observation_returns_empty_dict(self): + agent = _make_agent() + result = agent.post_observation(state={}, tool_used="X", tool_output="Y", tool_input_detail="Z") + assert result == {} + + def test_should_truncate_returns_false(self): + agent = _make_agent() + result = agent.should_truncate_tool_output(state={}, tool_used="X") + assert result is False + + def test_agent_type_property(self): + agent = _make_agent() + assert agent.agent_type == "base" + + def test_build_comprehension_context_returns_full_context(self): + agent = _make_agent() + state = {"critical_context": ["CVE Description: test vuln", "Vulnerable module: xstream"]} + result = agent.build_comprehension_context(state) + assert "CVE Description: test vuln" in result + assert "Vulnerable module: xstream" in result + + def test_build_comprehension_context_empty_state(self): + agent = _make_agent() + assert agent.build_comprehension_context({}) == "N/A" + + def test_build_comprehension_context_empty_list(self): + agent = _make_agent() + assert agent.build_comprehension_context({"critical_context": []}) == "N/A" + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_wrong_cve(self, mock_ctx): + intel = MagicMock() + intel.vuln_id = "CVE-2021-43859" + ws = MagicMock() + ws.cve_intel = [intel] + mock_ctx.get.return_value = ws + + agent = _make_agent() + findings = ["Found CVE-2020-26217 in code", "Package present"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2020-26217" not in result[0] + assert "the investigated vulnerability" in result[0] + assert result[1] == "Package present" + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_keeps_correct_cve(self, mock_ctx): + intel = MagicMock() + intel.vuln_id = "CVE-2021-43859" + ws = MagicMock() + ws.cve_intel = [intel] + mock_ctx.get.return_value = ws + + agent = _make_agent() + findings = ["Affects CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert result == ["Affects CVE-2021-43859"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_empty_list(self, mock_ctx): + ws = MagicMock() + ws.cve_intel = [] + mock_ctx.get.return_value = ws + + agent = _make_agent() + assert agent.sanitize_findings([], {}) == [] + + +class TestInit: + """Test BaseGraphAgent constructor wires up LLM wrappers.""" + + def test_creates_four_structured_output_llms(self): + mock_llm = MagicMock() + config = MagicMock() + config.max_iterations = 10 + agent = _ConcreteAgent(tools=["t1", "t2"], llm=mock_llm, config=config) + + assert mock_llm.with_structured_output.call_count == 4 + assert agent.tools == ["t1", "t2"] + assert agent.config is config + + +def _make_thought_response(mode="finish", final_answer="done"): + return Thought(thought="thinking", mode=mode, actions=None, final_answer=final_answer) + + +def _long_content(n_words=500): + return " ".join(["word"] * n_words) + + +class TestThoughtNodePruning: + """Test that thought_node prunes messages when tokens exceed the limit.""" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_prunes_middle_messages_when_over_limit(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 100 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(200) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content=long, tool_call_id="tc1"), + AIMessage(content=long), + ToolMessage(content="recent tool output", tool_call_id="tc2"), + HumanMessage(content="recent question"), + ], + "observation": None, + "step": 2, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + num_original = 1 + 6 # system prompt + 6 state messages + assert len(invoked_messages) < num_original + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_no_pruning_when_under_limit(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + state = { + "runtime_prompt": "short prompt", + "messages": [ + HumanMessage(content="hello"), + AIMessage(content="response"), + ], + "observation": None, + "step": 1, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert "hello" in contents + assert "response" in contents + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_preserves_system_prompt_and_last_message(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 50 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(200) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content=long, tool_call_id="tc1"), + HumanMessage(content="latest question"), + ], + "observation": None, + "step": 3, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert "system prompt" in contents + assert "latest question" in contents + assert long not in contents + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_pruning_includes_observation_context_in_count(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 200 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(100) + obs = Observation( + memory=[_long_content(50)], + results=[_long_content(50)], + ) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content="tool out", tool_call_id="tc1"), + HumanMessage(content="question"), + ], + "observation": obs, + "step": 2, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + assert any("KNOWLEDGE" in m.content for m in invoked_messages if hasattr(m, "content") and isinstance(m.content, str)) + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert long not in contents + + +class TestThoughtNodeBadToolArguments: + """Test that thought_node recovers from bad tool arguments instead of crashing. + + Mirrors the old AgentExecutor's handle_parsing_errors behavior: when the LLM + produces a ToolCall with missing required fields, the agent should get an error + message and retry rather than killing the entire graph. + """ + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovers_from_missing_arguments(self, mock_tracer): + """When all ToolCall fields are None, thought_node returns an error + HumanMessage with thought=None so should_continue loops back.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Function Library Version Finder", + package_name=None, + function_name=None, + query=None, + tool_input=None, + reason="check version", + ) + bad_response = Thought( + thought="check the version", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="Is SslHandler used?")], + "observation": None, + "step": 2, + } + + result = await agent.thought_node(state) + + assert result["thought"] is None + assert result["step"] == 3 + assert result["output"] == "waiting for the agent to respond" + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "check the version" in ai_msg.content + error_msg = result["messages"][1] + assert isinstance(error_msg, HumanMessage) + assert "ERROR" in error_msg.content + assert "Function Library Version Finder" in error_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovery_routes_back_to_thought_node_bad_args(self, mock_tracer): + """After a bad-arguments recovery, should_continue returns 'thought_node' + because thought is None — the agent gets another chance.""" + agent = _make_agent() + + state = {"thought": None, "step": 3, "max_steps": 10} + route = await agent.should_continue(state) + assert route == "thought_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovery_still_counts_toward_max_steps(self, mock_tracer): + """A bad-arguments iteration increments step, so the agent hits + forced_finish_node when step reaches max_steps.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Some Tool", + reason="testing", + ) + bad_response = Thought( + thought="trying something", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 9, + "max_steps": 10, + } + + result = await agent.thought_node(state) + + assert result["step"] == 10 + assert result["thought"] is None + + route = await agent.should_continue(result) + assert route == "forced_finish_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_valid_tool_call_still_works(self, mock_tracer): + """Verify that valid tool calls are not affected by the ValueError handling.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + good_actions = ToolCall( + tool="Configuration Scanner", + query="netty SSL settings", + reason="check config", + ) + good_response = Thought( + thought="scan for config", + mode="act", + actions=good_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=good_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 1, + } + + result = await agent.thought_node(state) + + assert result["thought"] is good_response + assert result["step"] == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert ai_msg.tool_calls[0]["name"] == "Configuration Scanner" + assert ai_msg.tool_calls[0]["args"] == {"query": "netty SSL settings"} + + +class TestCheckFinishAllowedBlocking: + """Test that blocked finishes include AIMessage and respect step limits.""" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_includes_ai_message(self, mock_tracer): + """When check_finish_allowed blocks, the LLM's finish attempt must be + recorded as an AIMessage so the chat model sees its own response and + the rejection in proper Human/AI alternation.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "You MUST use Function Locator first.") + ) + + finish_response = Thought( + thought="I have enough info", + mode="finish", + actions=None, + final_answer="The function is not reachable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="Is the function reachable?")], + "observation": None, + "step": 2, + } + + result = await agent.thought_node(state) + + assert result["thought"] is None + assert result["step"] == 3 + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "The function is not reachable." in ai_msg.content + human_msg = result["messages"][1] + assert isinstance(human_msg, HumanMessage) + assert "Function Locator" in human_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_ai_message_falls_back_to_thought(self, mock_tracer): + """When final_answer is None, the AIMessage should use the thought text.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "Call CCA first.") + ) + + finish_response = Thought( + thought="seems done", + mode="finish", + actions=None, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 0, + } + + result = await agent.thought_node(state) + + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "seems done" in ai_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_at_max_steps_routes_to_forced_finish(self, mock_tracer): + """If check_finish_allowed blocks at step 9 (incrementing to 10), + should_continue must route to forced_finish_node, not self-loop.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "Call FL and CCA first.") + ) + + finish_response = Thought( + thought="done", + mode="finish", + actions=None, + final_answer="answer", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 9, + "max_steps": 10, + } + + result = await agent.thought_node(state) + assert result["step"] == 10 + assert result["thought"] is None + + route = await agent.should_continue(result) + assert route == "forced_finish_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_bad_args_includes_ai_message(self, mock_tracer): + """Bad tool arguments error must include an AIMessage with the LLM's + original thought for proper chat alternation.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Function Locator", + reason="locate function", + ) + bad_response = Thought( + thought="Let me find the function", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 3, + } + + result = await agent.thought_node(state) + + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "Let me find the function" in ai_msg.content + error_msg = result["messages"][1] + assert isinstance(error_msg, HumanMessage) + assert "ERROR" in error_msg.content + + +class TestSelectPackage: + """Tests for _select_package image-match fast path and LLM fallback.""" + + def _make_workflow_state(self, image_name="registry.redhat.io/openshift4/ose-docker-builder", + git_repo="https://github.com/openshift/builder"): + si = MagicMock() + si.git_repo = git_repo + image = MagicMock() + image.name = image_name + image.source_info = [si] + ws = MagicMock() + ws.original_input.input.image = image + return ws + + @pytest.mark.asyncio + async def test_image_match_skips_llm(self): + """When a candidate name matches the image/repo, LLM is not called.""" + agent = _make_agent() + candidates = [ + {"name": "builder", "source": "rhsa"}, + {"name": "kernel", "source": "rhsa"}, + {"name": "glibc", "source": "rhsa"}, + ] + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "go", candidates, ["CVE desc"], ws, + ) + + assert selected == "builder" + agent.package_filter_llm.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_no_match_calls_llm(self): + """When no candidate matches the image, LLM is called.""" + agent = _make_agent() + mock_selection = MagicMock() + mock_selection.selected_package = "xstream" + mock_selection.reason = "ecosystem match" + agent.package_filter_llm.ainvoke = AsyncMock(return_value=mock_selection) + + candidates = [ + {"name": "xstream", "source": "ghsa", "ecosystem": "Maven"}, + {"name": "kernel", "source": "rhsa"}, + ] + ws = self._make_workflow_state(image_name="registry.redhat.io/infinispan/server", + git_repo="https://github.com/infinispan/infinispan") + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "java", candidates, ["CVE desc"], ws, + ) + + assert selected == "xstream" + agent.package_filter_llm.ainvoke.assert_called_once() + + @pytest.mark.asyncio + async def test_image_match_with_many_candidates(self): + """1000+ candidates with image match -> LLM skipped, no overflow.""" + agent = _make_agent() + candidates = [{"name": f"rhsa-product-{i}", "source": "rhsa"} for i in range(1200)] + candidates.append({"name": "builder", "source": "rhsa"}) + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "go", candidates, ["CVE desc"], ws, + ) + + assert selected == "builder" + agent.package_filter_llm.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_single_candidate_no_llm(self): + """Single candidate is used directly without LLM.""" + agent = _make_agent() + candidates = [{"name": "jinja2", "source": "ghsa"}] + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "python", candidates, ["CVE desc"], ws, + ) + + assert selected == "jinja2" + agent.package_filter_llm.ainvoke.assert_not_called() + + +class TestForcedFinishNode: + """Tests for forced_finish_node: includes conversation history with selective pruning.""" + + @pytest.mark.asyncio + async def test_includes_history_and_observation(self): + """forced_finish_node should include conversation history AND observation memory.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="summarizing", mode="finish", actions=None, + final_answer="Based on evidence, the function is not reachable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + obs = Observation( + results=["CCA returned (False, [])"], + memory=["Package validated: commons-beanutils:1.9.4", + "FL found PropertyUtilsBean.getProperty", + "CCA: function not reachable from app code"], + ) + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [ + AIMessage(content="I will check the function"), + HumanMessage(content="CCA result: (False, [])"), + ], + "observation": obs, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "I will check the function" in contents, "Conversation history should be in the prompt" + assert "CCA result: (False, [])" in contents, "Tool output should be in the prompt" + knowledge_msgs = [m for m in call_args if hasattr(m, "content") and "KNOWLEDGE" in m.content] + assert len(knowledge_msgs) == 1, "Observation memory should be in the prompt" + assert "LATEST FINDINGS" in knowledge_msgs[0].content + assert "CCA returned (False, [])" in knowledge_msgs[0].content + assert result["output"] == "Based on evidence, the function is not reachable." + + @pytest.mark.asyncio + async def test_prunes_history_when_over_token_limit(self): + """forced_finish_node should prune oldest messages when over the token limit, + while preserving the system prompt, observation, and finish prompt.""" + agent = _make_agent() + agent.config.context_window_token_limit = 100 + + mock_response = Thought( + thought="done", mode="finish", actions=None, + final_answer="Not exploitable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + obs = Observation( + results=["CCA: not reachable"], + memory=["FL found the function"], + ) + long = _long_content(200) + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + HumanMessage(content=long), + AIMessage(content="recent reasoning"), + ], + "observation": obs, + "input": "Is the function reachable?", + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "system prompt" in contents, "System prompt must survive pruning" + assert any("KNOWLEDGE" in c for c in contents), "Observation must survive pruning" + assert any("FORCED" in c or "Is the function reachable?" in c for c in contents), \ + "Finish prompt must survive pruning" + assert long not in contents, "Old long messages should be pruned" + assert result["output"] == "Not exploitable." + + @pytest.mark.asyncio + async def test_works_without_observation(self): + """forced_finish_node should work even when no observations exist.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="no evidence", mode="finish", actions=None, + final_answer="No evidence found.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [HumanMessage(content="some message")], + "observation": None, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "some message" in contents, "History should be included when no pruning needed" + assert result["output"] == "No evidence found." + + @pytest.mark.asyncio + async def test_fallback_on_non_finish_response(self): + """forced_finish_node returns default message when LLM doesn't finish.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="I want to call another tool", mode="act", + actions=ToolCall(tool="Function Locator", package_name="pkg", function_name="fn", reason="test"), + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [], + "observation": None, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + assert "Failed to generate a final answer" in result["output"] diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py index 37929060c..f457f59ff 100644 --- a/tests/test_base_tool_descriptions.py +++ b/tests/test_base_tool_descriptions.py @@ -13,10 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Tests for the base build_tool_descriptions() function. +"""Tests for the base build_tool_descriptions() function. -Tests that the consolidated base function provides simple tool descriptions +Verifies that the consolidated base function provides simple tool descriptions that can be formatted by specialized functions for different contexts. """ @@ -76,7 +75,7 @@ def test_base_all_tools(): result = build_tool_descriptions(tool_names) - # Should have 7 descriptions + # Should have 7 descriptions (CONTAINER_ANALYSIS_DATA has no description in the base function) assert len(result) == 7 # Verify all tools are present @@ -155,36 +154,6 @@ def test_mod_few_shot_structure(): print("✓ MOD_FEW_SHOT structure validated") -def test_cve_web_search_description_warns_about_versions(): - """Test that CVE Web Search description includes version warning.""" - # Without Version Finder: generic version warning - tool_names = [ToolNames.CVE_WEB_SEARCH] - result = build_tool_descriptions(tool_names) - assert len(result) == 1 - desc = result[0] - assert "MULTIPLE versions" in desc - assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in desc - - # With Version Finder (Java): references the tool - tool_names = [ToolNames.CVE_WEB_SEARCH, ToolNames.FUNCTION_LIBRARY_VERSION_FINDER] - result = build_tool_descriptions(tool_names) - descs = " ".join(result) - assert "MULTIPLE versions" in descs - assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in descs - - print("✓ CVE Web Search description includes version warning") - - -def test_agent_prompt_contains_version_awareness_instructions(): - """Test that agent prompt template contains version awareness instructions.""" - from vuln_analysis.utils.prompting import get_agent_prompt - - prompt = get_agent_prompt() - assert "VERSION" in prompt - - print("✓ Agent prompt contains version awareness instructions") - - if __name__ == "__main__": print("Running Base Tool Descriptions tests...\n") @@ -194,7 +163,5 @@ def test_agent_prompt_contains_version_awareness_instructions(): test_base_empty_list() test_checklist_formats_descriptions() test_mod_few_shot_structure() - test_cve_web_search_description_warns_about_versions() - test_agent_prompt_contains_version_awareness_instructions() print("\n✅ All base tool descriptions tests passed!") diff --git a/tests/test_build_code_understanding_tools.py b/tests/test_build_code_understanding_tools.py new file mode 100644 index 000000000..1eb880385 --- /dev/null +++ b/tests/test_build_code_understanding_tools.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for CodeUnderstandingAgent: tool selection, availability, and comprehension hooks.""" + +from unittest.mock import MagicMock, patch + +from agent_test_helpers import MockTool, make_builder, make_config, make_state +from vuln_analysis.functions.code_understanding_agent import CodeUnderstandingAgent +from vuln_analysis.functions.code_understanding_internals import CodeUnderstandingRulesTracker +from vuln_analysis.tools.tool_names import ToolNames + + +def _get_tools(builder=None, config=None, state=None): + return CodeUnderstandingAgent.get_tools( + builder or make_builder(), + config or make_config(), + state or make_state(), + ) + + +class TestGetTools: + """Test CodeUnderstandingAgent.get_tools selection and availability logic.""" + + def test_filters_to_exactly_4_cu_tools(self): + result = _get_tools() + assert len(result) == 4 + + def test_output_tool_names(self): + result = _get_tools() + result_names = {t.name for t in result} + expected_names = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + } + assert result_names == expected_names + + def test_excludes_reachability_tools(self): + tools = [ + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LOCATOR not in result_names + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert len(result) == 2 + + def test_excludes_web_and_container_tools(self): + tools = [ + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.CONTAINER_ANALYSIS_DATA), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.CVE_WEB_SEARCH not in result_names + assert ToolNames.CONTAINER_ANALYSIS_DATA not in result_names + assert len(result) == 2 + + def test_excludes_version_finder(self): + tools = [ + MockTool(ToolNames.FUNCTION_LIBRARY_VERSION_FINDER), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in result_names + assert len(result) == 2 + + def test_empty_builder_returns_empty(self): + result = _get_tools(builder=make_builder(tools=[])) + assert result == [] + + def test_no_matching_tools_returns_empty(self): + tools = [ + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + assert result == [] + + def test_preserves_tool_object_identity(self): + docs_tool = MockTool(ToolNames.DOCS_SEMANTIC_SEARCH) + keyword_tool = MockTool(ToolNames.CODE_KEYWORD_SEARCH) + locator_tool = MockTool(ToolNames.FUNCTION_LOCATOR) + builder = make_builder(tools=[docs_tool, keyword_tool, locator_tool]) + result = _get_tools(builder=builder) + assert len(result) == 2 + assert docs_tool in result + assert keyword_tool in result + assert locator_tool not in result + + def test_partial_overlap(self): + tools = [ + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + expected_names = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + } + assert len(result) == 2 + assert result_names == expected_names + + +class TestGetToolsAvailability: + """get_tools filters out tools whose infrastructure prerequisites are not met.""" + + def test_filters_docs_semantic_search_when_no_vdb(self): + state = make_state(doc_vdb_path=None) + result = _get_tools(state=state) + assert ToolNames.DOCS_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_code_keyword_search_when_no_index(self): + state = make_state(code_index_path=None) + result = _get_tools(state=state) + assert ToolNames.CODE_KEYWORD_SEARCH not in {t.name for t in result} + + def test_cu_only_tools_always_kept(self): + state = make_state(code_vdb_path=None, doc_vdb_path=None, code_index_path=None) + result = _get_tools(state=state) + result_names = {t.name for t in result} + assert ToolNames.CONFIGURATION_SCANNER in result_names + + def test_filters_import_usage_analyzer_when_no_index(self): + state = make_state(code_index_path=None) + result = _get_tools(state=state) + assert ToolNames.IMPORT_USAGE_ANALYZER not in {t.name for t in result} + + def test_import_usage_analyzer_available_with_index(self): + state = make_state(code_index_path="/some/path") + result = _get_tools(state=state) + assert ToolNames.IMPORT_USAGE_ANALYZER in {t.name for t in result} + + +class TestCodeUnderstandingAgentMeta: + """Test create_rules_tracker and agent_type for CodeUnderstandingAgent.""" + + def test_create_rules_tracker_returns_cu_tracker(self): + tracker = CodeUnderstandingAgent.create_rules_tracker() + assert isinstance(tracker, CodeUnderstandingRulesTracker) + + def test_create_rules_tracker_returns_fresh_instance(self): + t1 = CodeUnderstandingAgent.create_rules_tracker() + t2 = CodeUnderstandingAgent.create_rules_tracker() + assert t1 is not t2 + + def test_agent_type_is_cu(self): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + agent = CodeUnderstandingAgent(tools=[], llm=mock_llm, config=config) + assert agent.agent_type == "cu" + + +def _make_cu_agent(): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return CodeUnderstandingAgent(tools=[], llm=mock_llm, config=config) + + +def _mock_ctx_state(*vuln_ids): + """Return a mock workflow state with cve_intel entries for the given vuln IDs.""" + intel_list = [] + for vid in vuln_ids: + intel = MagicMock() + intel.vuln_id = vid + intel_list.append(intel) + ws = MagicMock() + ws.cve_intel = intel_list + return ws + + +class TestCUComprehensionHooks: + """Test CU agent comprehension context reduction and CVE sanitization.""" + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_contains_vuln_id_and_package(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + state = {"app_package": "com.thoughtworks.xstream:xstream"} + result = agent.build_comprehension_context(state) + assert "CVE-2021-43859" in result + assert "com.thoughtworks.xstream:xstream" in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_includes_grounding_instruction(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + result = agent.build_comprehension_context({"app_package": "pkg"}) + assert "Only extract facts explicitly stated in the tool output" in result + assert "Do not add CVE IDs" in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_excludes_critical_context(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + state = { + "app_package": "xstream", + "critical_context": ["GHSA description: XStream can cause DoS", "NVD: high severity"], + } + result = agent.build_comprehension_context(state) + assert "GHSA description" not in result + assert "NVD" not in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_unknown_fallbacks(self, mock_ctx): + ws = MagicMock() + ws.cve_intel = [] + mock_ctx.get.return_value = ws + agent = _make_cu_agent() + result = agent.build_comprehension_context({}) + assert "unknown" in result + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_wrong_cve(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["XStream 1.4.18 is vulnerable to CVE-2020-26217"] + result = agent.sanitize_findings(findings, {}) + assert result == ["XStream 1.4.18 is vulnerable to the investigated vulnerability"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_keeps_correct_cve(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["Affects CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert result == ["Affects CVE-2021-43859"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_multiple_wrong_cves(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["CVE-2020-26217 and CVE-2019-10086 affect this, also CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2020-26217" not in result[0] + assert "CVE-2019-10086" not in result[0] + assert "CVE-2021-43859" in result[0] + assert result[0].count("the investigated vulnerability") == 2 + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_no_cve_ids_unchanged(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["XStream 1.4.18 found in dependencies", "Package is present"] + result = agent.sanitize_findings(findings, {}) + assert result == findings + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_multi_cve_intel(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859", "CVE-2021-39144") + agent = _make_cu_agent() + findings = ["CVE-2021-43859 and CVE-2021-39144 and CVE-2020-26217"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2021-43859" in result[0] + assert "CVE-2021-39144" in result[0] + assert "CVE-2020-26217" not in result[0] diff --git a/tests/test_code_understanding_internals.py b/tests/test_code_understanding_internals.py new file mode 100644 index 000000000..ad52a8c5f --- /dev/null +++ b/tests/test_code_understanding_internals.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.functions.code_understanding_internals import ( + CodeUnderstandingRulesTracker, + CU_AGENT_SYS_PROMPT, + CU_AGENT_THOUGHT_INSTRUCTIONS, +) + + +class TestCodeUnderstandingRulesTracker: + def test_iua_allowed_without_survey(self): + """IUA can be called at any time — no gating.""" + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Import Usage Analyzer"]) + + violated, msg = tracker.check_thought_behavior( + "Import Usage Analyzer", + "com.example.package", + ["imports found"] + ) + + assert violated is False + assert msg == "" + + def test_allowed_tool_passes(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "some query", + ["results"] + ) + + assert violated is False + + def test_docs_semantic_search_passes(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Docs Semantic Search"]) + + violated, msg = tracker.check_thought_behavior( + "Docs Semantic Search", + "query", + ["docs"] + ) + + assert violated is False + + def test_check_rule7_fires(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + tracker.check_thought_behavior( + "Code Keyword Search", + "com.example.Class", + [] + ) + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "com.example.Another", + [] + ) + + assert violated is True + assert "Rule 7" in msg + assert "dots" in msg + + def test_check_allowed_tools_rejects_unknown(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + violated, msg = tracker.check_thought_behavior( + "Unknown Tool", + "query", + ["results"] + ) + + assert violated is True + assert "AVAILABLE_TOOLS" in msg + assert "Code Keyword Search" in msg + + def test_check_passes_and_adds_action(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + assert "Code Keyword Search" not in tracker.action_history + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "import xstream", + ["result1", "result2"] + ) + + assert violated is False + assert msg == "" + assert "Code Keyword Search" in tracker.action_history + assert len(tracker.action_history["Code Keyword Search"]) == 1 + assert tracker.action_history["Code Keyword Search"][0]["input"] == "import xstream" + + def test_duplicate_config_scanner_blocked(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + violated, msg = tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + assert violated is True + assert "already called" in msg + + def test_config_scanner_different_query_allowed(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + violated, msg = tracker.check_thought_behavior( + "Configuration Scanner", "security allowlist", ["other config"] + ) + assert violated is False + assert msg == "" + + def test_check_failing_does_not_add_action(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + + violated, msg = tracker.check_thought_behavior( + "Unknown Tool", + "query", + ["results"] + ) + + assert violated is True + assert "Unknown Tool" not in tracker.action_history + + +class TestCUConstants: + def test_sys_prompt_is_nonempty_string(self): + assert isinstance(CU_AGENT_SYS_PROMPT, str) + assert len(CU_AGENT_SYS_PROMPT) > 0 + assert len(CU_AGENT_SYS_PROMPT.strip()) > 0 + + def test_sys_prompt_mentions_code_understanding(self): + assert "code understanding" in CU_AGENT_SYS_PROMPT.lower() + + def test_sys_prompt_does_not_mention_call_chain_analyzer(self): + assert "Call Chain Analyzer" not in CU_AGENT_SYS_PROMPT + assert "call chain analyzer" not in CU_AGENT_SYS_PROMPT.lower() + + def test_thought_instructions_have_rules_tags(self): + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + + def test_thought_instructions_have_three_examples(self): + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + + def test_thought_instructions_rules_numbered_1_to_9(self): + for i in range(1, 10): + assert f"{i}." in CU_AGENT_THOUGHT_INSTRUCTIONS diff --git a/tests/test_code_understanding_prompt_factory.py b/tests/test_code_understanding_prompt_factory.py new file mode 100644 index 000000000..e05484b3b --- /dev/null +++ b/tests/test_code_understanding_prompt_factory.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.utils.code_understanding_prompt_factory import ( + CU_TOOL_GENERAL_DESCRIPTIONS, + CU_TOOL_SELECTION_STRATEGY, +) + + +class TestCUToolGeneralDescriptions: + def test_has_4_entries(self): + assert len(CU_TOOL_GENERAL_DESCRIPTIONS) == 4 + + def test_keys_match_cu_tool_names(self): + expected_keys = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + } + assert set(CU_TOOL_GENERAL_DESCRIPTIONS.keys()) == expected_keys + + def test_values_non_empty(self): + for key, value in CU_TOOL_GENERAL_DESCRIPTIONS.items(): + assert isinstance(value, str), f"Value for '{key}' is not a string" + assert len(value.strip()) > 0, f"Value for '{key}' is empty" + + +class TestCUToolSelectionStrategy: + def test_has_5_ecosystems(self): + expected_ecosystems = {"python", "go", "java", "javascript", "c"} + assert set(CU_TOOL_SELECTION_STRATEGY.keys()) == expected_ecosystems + + def test_strategies_non_empty(self): + for ecosystem, strategy in CU_TOOL_SELECTION_STRATEGY.items(): + assert isinstance(strategy, str), f"Strategy for '{ecosystem}' is not a string" + assert len(strategy.strip()) > 0, f"Strategy for '{ecosystem}' is empty" + + def test_each_mentions_at_least_3_tool_names(self): + cu_tool_names = [ + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + ] + + for ecosystem, strategy in CU_TOOL_SELECTION_STRATEGY.items(): + mentioned_tools = sum(1 for tool_name in cu_tool_names if tool_name in strategy) + assert mentioned_tools >= 3, f"Strategy for '{ecosystem}' mentions only {mentioned_tools} tool(s)" diff --git a/tests/test_configuration_scanner.py b/tests/test_configuration_scanner.py new file mode 100644 index 000000000..ca9a462c8 --- /dev/null +++ b/tests/test_configuration_scanner.py @@ -0,0 +1,420 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from vuln_analysis.tools.configuration_scanner import ( + _is_config_file, + _is_in_config_dir, + _collect_config_files, + search_config_content, +) + + +class TestIsConfigFile: + @pytest.mark.parametrize( + "file_path,expected", + [ + # Named config files — matched anywhere + ("config.yaml", True), + ("config.yml", True), + ("config.xml", True), + ("application.properties", True), + ("application.yml", True), + ("application.yaml", True), + ("settings.toml", True), + ("settings.yaml", True), + ("settings.yml", True), + ("beans.xml", True), + ("web.xml", True), + ("Dockerfile", True), + ("Dockerfile.prod", True), + ("Dockerfile.dev", True), + ("docker-compose.yml", True), + ("docker-compose.prod.yml", True), + # Config-specific extensions — matched anywhere + ("app.env", True), + (".env", True), + ("nginx.conf", True), + ("config.ini", True), + ("app.properties", True), + # Removed: build/dependency files (handled by other tools) + ("pom.xml", False), + ("build.gradle", False), + ("build.gradle.kts", False), + ("go.mod", False), + ("go.sum", False), + ("package.json", False), + ("requirements.txt", False), + ("setup.py", False), + ("setup.cfg", False), + ("pyproject.toml", False), + # Removed: catch-all extensions (only matched inside config dirs) + ("app.yml", False), + ("app.cfg", False), + ("data.yaml", False), + ("docker-compose-dev.yaml", False), + # Non-config files + ("main.py", False), + ("utils.go", False), + ("README.md", False), + ("app.js", False), + ("test.txt", False), + ("data.json", False), + ], + ) + def test_config_file_patterns(self, file_path, expected): + assert _is_config_file(file_path) == expected + + @pytest.mark.parametrize( + "file_path,expected", + [ + ("DOCKERFILE", True), + ("CONFIG.YAML", True), + ("APPLICATION.PROPERTIES", True), + ("DOCKER-COMPOSE.YML", True), + ("BEANS.XML", True), + ("SETTINGS.TOML", True), + ], + ) + def test_case_insensitive(self, file_path, expected): + assert _is_config_file(file_path) == expected + + def test_file_in_subdirectory(self): + assert _is_config_file("path/to/config.yaml") is True + assert _is_config_file("deep/nested/path/application.properties") is True + + +class TestIsInConfigDir: + @pytest.mark.parametrize( + "file_path,expected", + [ + ("config/app.txt", True), + ("conf/settings.txt", True), + ("conf.d/default.conf", True), + ("etc/nginx/nginx.conf", True), + ("src/main/resources/app.yml", True), + ("app/config/database.yml", True), + ("project/conf/server.xml", True), + ("src/main/java/App.java", False), + ("lib/utils.py", False), + ("tests/test_app.py", False), + ("data/sample.csv", False), + ], + ) + def test_config_dir_patterns(self, file_path, expected): + assert _is_in_config_dir(file_path) == expected + + @pytest.mark.parametrize( + "file_path,expected", + [ + ("Config/app.txt", True), + ("RESOURCES/app.yml", True), + ("ETC/nginx.conf", True), + ("CONF.D/default.conf", True), + ], + ) + def test_case_insensitive_dir(self, file_path, expected): + assert _is_in_config_dir(file_path) == expected + + +class TestCollectConfigFiles: + def test_finds_config_files(self, tmp_path): + (tmp_path / "config.yaml").write_text("key: value") + (tmp_path / "application.properties").write_text("app.name=test") + (tmp_path / "nginx.conf").write_text("server {}") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 3 + paths = {path for path, _ in result} + assert "config.yaml" in paths + assert "application.properties" in paths + assert "nginx.conf" in paths + + for path, content in result: + assert len(content) > 0 + + def test_finds_files_in_config_dir(self, tmp_path): + config_dir = tmp_path / "config" + config_dir.mkdir() + (config_dir / "database.txt").write_text("db_config") + (config_dir / "server.txt").write_text("server_config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 2 + paths = {path for path, _ in result} + assert "config/database.txt" in paths + assert "config/server.txt" in paths + + def test_excludes_git_dir(self, tmp_path): + git_dir = tmp_path / ".git" + git_dir.mkdir() + (git_dir / "config").write_text("git config") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert ".git/config" not in paths + + def test_excludes_pycache(self, tmp_path): + pycache_dir = tmp_path / "__pycache__" + pycache_dir.mkdir() + (pycache_dir / "config.yaml").write_text("cached") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert "__pycache__/config.yaml" not in paths + + def test_excludes_node_modules(self, tmp_path): + node_modules_dir = tmp_path / "node_modules" + node_modules_dir.mkdir() + (node_modules_dir / "package.json").write_text("{}") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert "node_modules/package.json" not in paths + + def test_excludes_tox(self, tmp_path): + tox_dir = tmp_path / ".tox" + tox_dir.mkdir() + (tox_dir / "config.ini").write_text("[tox]") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert ".tox/config.ini" not in paths + + def test_skips_large_files(self, tmp_path): + large_content = "x" * 500_001 + (tmp_path / "large.properties").write_text(large_content) + (tmp_path / "small.properties").write_text("key=value") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "small.properties" in paths + assert "large.properties" not in paths + + def test_empty_repo_returns_empty(self, tmp_path): + result = _collect_config_files(str(tmp_path)) + assert result == [] + + def test_nested_directory_structure(self, tmp_path): + src_dir = tmp_path / "src" / "main" / "resources" + src_dir.mkdir(parents=True) + (src_dir / "application.yml").write_text("app: test") + + conf_dir = tmp_path / "conf" + conf_dir.mkdir() + (conf_dir / "server.txt").write_text("server config") + + (tmp_path / "config.yaml").write_text("key: value") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 3 + paths = {path for path, _ in result} + assert "src/main/resources/application.yml" in paths + assert "conf/server.txt" in paths + assert "config.yaml" in paths + + def test_handles_read_errors_gracefully(self, tmp_path): + (tmp_path / "good.properties").write_text("key=value") + bad_file = tmp_path / "bad.properties" + bad_file.write_bytes(b"\xff\xfe invalid utf-8") + bad_file.chmod(0o000) + + result = _collect_config_files(str(tmp_path)) + + paths = {path for path, _ in result} + assert "good.properties" in paths + + bad_file.chmod(0o644) + + def test_returns_relative_paths(self, tmp_path): + subdir = tmp_path / "config" + subdir.mkdir() + (subdir / "app.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + path, _ = result[0] + assert path == "config/app.yml" + assert not path.startswith("/") + assert str(tmp_path) not in path + + def test_returns_file_contents(self, tmp_path): + expected_content = "database:\n host: localhost\n port: 5432" + (tmp_path / "config.yaml").write_text(expected_content) + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + path, content = result[0] + assert path == "config.yaml" + assert content == expected_content + + +class TestSearchConfigContent: + def test_empty_config_files(self): + result = search_config_content([], ["keyword"]) + assert result == [] + + def test_no_matches(self): + config_files = [("app.yml", "database:\n host: localhost")] + result = search_config_content(config_files, ["xstream"]) + assert result == [] + + def test_single_match(self): + config_files = [("app.yml", "line1\nxstream_enabled: true\nline3")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 1 + assert "app.yml" in result[0] + assert "xstream_enabled: true" in result[0] + + def test_source_label(self): + config_files = [("app.yml", "match_keyword")] + result = search_config_content(config_files, ["match"], source_label="git://repo", context_lines=0) + assert "source: git://repo" in result[0] + + def test_multiple_files(self): + config_files = [ + ("a.yml", "xstream: yes"), + ("b.yml", "no match here"), + ("c.xml", "xstream config"), + ] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 2 + assert "a.yml" in result[0] + assert "c.xml" in result[1] + + def test_max_results_cap(self): + config_files = [("big.yml", "\n".join(f"keyword_{i}" for i in range(50)))] + result = search_config_content(config_files, ["keyword"], max_results=5, context_lines=0) + assert len(result) == 5 + + def test_context_lines(self): + content = "line1\nline2\nMATCH_LINE\nline4\nline5" + config_files = [("app.yml", content)] + result = search_config_content(config_files, ["match_line"], context_lines=1) + assert len(result) == 1 + assert "line2" in result[0] + assert "> 3: MATCH_LINE" in result[0] + assert "line4" in result[0] + assert "line1" not in result[0] + assert "line5" not in result[0] + + def test_case_insensitive_keywords(self): + config_files = [("app.yml", "XStream_Config: enabled")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 1 + + def test_multiple_keywords_match_any(self): + config_files = [ + ("a.yml", "xstream: true"), + ("b.yml", "deserialization: false"), + ("c.yml", "no relevant content"), + ] + result = search_config_content(config_files, ["xstream", "deserialization"], context_lines=0) + assert len(result) == 2 + + def test_line_number_in_output(self): + config_files = [("app.yml", "line1\nline2\nxstream: true")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert "line 3" in result[0] + + def test_default_source_label(self): + config_files = [("app.yml", "match_keyword")] + result = search_config_content(config_files, ["match"], context_lines=0) + assert "source: unknown" in result[0] + + +class TestConfigScannerSourceAwareness: + """Tests for app/dep classification and source scoping in Configuration Scanner.""" + + def _make_configs(self): + return [ + ("config.yaml", "xstream: app-level"), + ("src/main/resources/application.yml", "xstream: app-config"), + ("dependencies-sources/xstream-1.4/config.properties", "xstream: dep-xstream"), + ("dependencies-sources/commons-io-2.6/config.properties", "xstream: dep-commons"), + ] + + def test_app_config_prioritized_over_dependency(self): + from vuln_analysis.utils.source_classification import ( + is_dependency_path, format_app_dep_output, + ) + configs = self._make_configs() + app_configs = [c for c in configs if not is_dependency_path(c[0])] + dep_configs = [c for c in configs if is_dependency_path(c[0])] + + app_matches = search_config_content(app_configs, ["xstream"], max_results=10, context_lines=0) + dep_matches = search_config_content(dep_configs, ["xstream"], max_results=10, context_lines=0) + + result = format_app_dep_output(app_matches, dep_matches, len(app_matches), len(dep_matches), + "No matches") + assert "Main application (2 of 2 results)" in result + assert "Application library dependencies (2 of 2 results)" in result + app_pos = result.index("config.yaml") + dep_pos = result.index("Application library dependencies") + assert app_pos < dep_pos + + def test_source_scope_filters_dependency_configs(self): + from vuln_analysis.utils.source_classification import ( + is_dependency_path, filter_by_source_scope, + ) + configs = self._make_configs() + dep_configs = [c for c in configs if is_dependency_path(c[0])] + + filtered = filter_by_source_scope(dep_configs, ["xstream"], lambda x: x[0]) + assert len(filtered) == 1 + assert "xstream-1.4" in filtered[0][0] + + def test_app_configs_unaffected_by_source_scope(self): + from vuln_analysis.utils.source_classification import is_dependency_path + configs = self._make_configs() + app_configs = [c for c in configs if not is_dependency_path(c[0])] + + app_matches = search_config_content(app_configs, ["xstream"], max_results=10, context_lines=0) + assert len(app_matches) == 2 + + def test_app_dep_headers_in_output(self): + from vuln_analysis.utils.source_classification import format_app_dep_output + result = format_app_dep_output(["match1"], ["match2"], 1, 1, "No matches") + assert "Main application" in result + assert "Application library dependencies" in result + + def test_no_matches_preserves_message(self): + from vuln_analysis.utils.source_classification import format_app_dep_output + result = format_app_dep_output([], [], 0, 0, "No configuration entries found matching: xstream") + assert result == "No configuration entries found matching: xstream" diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py new file mode 100644 index 000000000..bdbeef344 --- /dev/null +++ b/tests/test_dispatcher.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from unittest.mock import AsyncMock +from pydantic import ValidationError +from langchain_core.messages import HumanMessage + +from vuln_analysis.functions.dispatcher import ( + QuestionRouting, + build_routing_prompt, + dispatch_question, +) + + +class TestQuestionRouting: + def test_valid_reachability_type(self): + routing = QuestionRouting(agent_type="reachability", reason="test reason") + assert routing.agent_type == "reachability" + assert routing.reason == "test reason" + + def test_valid_code_understanding_type(self): + routing = QuestionRouting(agent_type="code_understanding", reason="test reason") + assert routing.agent_type == "code_understanding" + assert routing.reason == "test reason" + + def test_invalid_type_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(agent_type="invalid", reason="test") + assert "agent_type" in str(exc_info.value) + + def test_missing_agent_type_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(reason="test") + assert "agent_type" in str(exc_info.value) + + def test_missing_reason_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(agent_type="reachability") + assert "reason" in str(exc_info.value) + + +class TestBuildRoutingPrompt: + def test_context_and_question_inserted(self): + context = "CVE-2024-1234, package: foo:bar" + question = "Is the function vulnerable()?" + result = build_routing_prompt(context, question) + + assert context in result + assert question in result + + def test_all_few_shot_examples_present(self): + result = build_routing_prompt("context", "question") + + expected_examples = [ + "XStream.fromXML()", + "XML parser", + "BeanUtils.populate()", + "commons-beanutils", + "parseXML()", + "external entity processing", + "newTransformer()", + "deserialize()", + ] + + for example in expected_examples: + assert example in result, f"Missing example: {example}" + + def test_both_agent_types_mentioned(self): + result = build_routing_prompt("context", "question") + + assert "reachability" in result + assert "code_understanding" in result + + def test_empty_inputs(self): + result = build_routing_prompt("", "") + assert isinstance(result, str) + assert len(result) > 0 + + def test_special_chars_in_inputs(self): + context = "CVE {test} with % and {{nested}}" + question = "Is {function}() reachable? % complete" + + result = build_routing_prompt(context, question) + + assert context in result + assert question in result + + +class TestDispatchQuestion: + @pytest.mark.asyncio + async def test_dispatch_returns_routing_result(self): + expected_result = QuestionRouting( + agent_type="reachability", + reason="Test reason" + ) + + mock_llm = AsyncMock() + mock_llm.ainvoke.return_value = expected_result + + result = await dispatch_question( + routing_llm=mock_llm, + question="Test question?", + context_block="Test context", + ) + + assert result == expected_result + assert result.agent_type == "reachability" + assert result.reason == "Test reason" + + @pytest.mark.asyncio + async def test_dispatch_passes_human_message(self): + mock_result = QuestionRouting( + agent_type="code_understanding", + reason="Config check" + ) + + mock_llm = AsyncMock() + mock_llm.ainvoke.return_value = mock_result + + question = "Is the XML parser configured?" + context = "CVE-2024-5678" + + await dispatch_question( + routing_llm=mock_llm, + question=question, + context_block=context, + ) + + mock_llm.ainvoke.assert_called_once() + + call_args = mock_llm.ainvoke.call_args[0][0] + assert len(call_args) == 1 + assert isinstance(call_args[0], HumanMessage) + assert question in call_args[0].content + assert context in call_args[0].content + + @pytest.mark.asyncio + async def test_dispatch_propagates_exception(self): + mock_llm = AsyncMock() + mock_llm.ainvoke.side_effect = ValueError("LLM error") + + with pytest.raises(ValueError, match="LLM error"): + await dispatch_question( + routing_llm=mock_llm, + question="Test question", + context_block="Test context", + ) diff --git a/tests/test_import_usage_analyzer.py b/tests/test_import_usage_analyzer.py new file mode 100644 index 000000000..98d770013 --- /dev/null +++ b/tests/test_import_usage_analyzer.py @@ -0,0 +1,278 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser +from vuln_analysis.tools.import_usage_analyzer import ( + _find_usage_in_file, + analyze_imports, +) + + +def _get_patterns(ecosystem: Ecosystem, package_name: str) -> list[re.Pattern]: + parser = get_language_function_parser(ecosystem, tree=None) + return parser.get_import_search_patterns(package_name) + + +class TestGetImportSearchPatterns: + def test_python_import_pattern(self): + patterns = _get_patterns(Ecosystem.PYTHON, "xml.etree") + + assert any(p.search("import xml.etree.ElementTree") for p in patterns) + assert any(p.search("from xml.etree import ElementTree") for p in patterns) + + def test_java_import_pattern(self): + patterns = _get_patterns(Ecosystem.JAVA, "com.thoughtworks.xstream") + + assert any(p.search("import com.thoughtworks.xstream.XStream;") for p in patterns) + assert any(p.search("import static com.thoughtworks.xstream.XStream;") for p in patterns) + + def test_go_import_pattern(self): + patterns = _get_patterns(Ecosystem.GO, "encoding/xml") + + assert any(p.search('import "encoding/xml"') for p in patterns) + assert any(p.search('import (\n "encoding/xml"\n)') for p in patterns) + + def test_javascript_require_pattern(self): + patterns = _get_patterns(Ecosystem.JAVASCRIPT, "xml2js") + + assert any(p.search("const xml = require('xml2js')") for p in patterns) + assert any(p.search('const xml = require("xml2js")') for p in patterns) + + def test_javascript_import_from_pattern(self): + patterns = _get_patterns(Ecosystem.JAVASCRIPT, "xml2js") + + assert any(p.search("import xml from 'xml2js'") for p in patterns) + assert any(p.search('import { parseString } from "xml2js"') for p in patterns) + + def test_c_include_pattern(self): + patterns = _get_patterns(Ecosystem.C_CPP, "libxml") + + assert any(p.search('#include ') for p in patterns) + assert any(p.search('#include "libxml/tree.h"') for p in patterns) + + def test_special_chars_escaped(self): + patterns = _get_patterns(Ecosystem.PYTHON, "foo.bar[baz]") + + assert len(patterns) > 0 + for p in patterns: + assert isinstance(p, re.Pattern) + + def test_case_insensitive(self): + patterns = _get_patterns(Ecosystem.PYTHON, "XmlParser") + + assert any(p.search("import xmlparser") for p in patterns) + assert any(p.search("import XMLPARSER") for p in patterns) + assert any(p.search("import XmlParser") for p in patterns) + + +class TestFindUsageInFile: + def test_finds_usage_sites(self): + content = """import xml.etree.ElementTree +tree = ElementTree.parse('data.xml') +root = tree.getroot() +for child in root: + print(child.tag) +""" + usages = _find_usage_in_file(content, ["xml.etree.ElementTree"]) + + assert len(usages) > 0 + assert any("ElementTree.parse" in u for u in usages) + + def test_skips_import_lines(self): + content = """import xml.etree.ElementTree +from xml.etree import ElementTree +tree = ElementTree.parse('data.xml') +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert len(usages) == 1 + assert "L3:" in usages[0] + assert "parse" in usages[0] + + def test_skips_from_lines(self): + content = """from xml.etree import ElementTree +tree = ElementTree.parse('data.xml') +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + + def test_skips_include_lines(self): + content = """#include +xmlDocPtr doc = xmlParseFile("data.xml"); +""" + usages = _find_usage_in_file(content, ["xmlParseFile"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + + def test_max_usages_cap(self): + content = "\n".join([f"var x{i} = XStream();" for i in range(20)]) + + usages = _find_usage_in_file(content, ["XStream"], max_usages=5) + + assert len(usages) == 5 + + def test_word_boundary_matching(self): + content = """class MyXStreamHelper: + pass +xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert len(usages) == 1 + assert any("L3:" in u and "XStream()" in u for u in usages) + + def test_dotted_name_uses_short_component(self): + content = """import com.thoughtworks.xstream.XStream; +XStream xs = new XStream(); +""" + usages = _find_usage_in_file(content, ["com.thoughtworks.xstream.XStream"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + assert "XStream xs" in usages[0] + + def test_no_usages_returns_empty(self): + content = """import xml.etree.ElementTree +from xml.etree import ElementTree +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert usages == [] + + def test_empty_content(self): + usages = _find_usage_in_file("", ["XStream"]) + + assert usages == [] + + def test_empty_names(self): + content = "xs = XStream();" + usages = _find_usage_in_file(content, []) + + assert usages == [] + + def test_multiple_names_same_line(self): + content = """import foo, bar +result = foo.process(bar.data) +""" + usages = _find_usage_in_file(content, ["foo", "bar"]) + + assert len(usages) == 2 + assert all("L2:" in u for u in usages) + assert any("foo.process" in u for u in usages) + assert any("bar.data" in u for u in usages) + + def test_line_number_format(self): + content = """import XStream +xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert usages[0].startswith(" L2:") + + def test_strips_line_content(self): + content = """import XStream + xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert "xs = XStream()" in usages[0] + assert usages[0].count(" " * 8) == 0 + + +class _MockSearcher: + def __init__(self, docs): + self.docs = docs + self.num_docs = len(docs) + + def doc(self, doc_id): + if doc_id >= len(self.docs): + raise IndexError(f"doc_id {doc_id} out of range") + return self.docs[doc_id] + + +def _import_searcher(*docs): + return _MockSearcher(list(docs)) + + +class TestAnalyzeImportsSourceAwareness: + + def _java_patterns(self, package_name): + return _get_patterns(Ecosystem.JAVA, package_name) + + def test_app_imports_prioritized(self): + docs = [ + {"file_path": ["dependencies-sources/xstream-1.4/XStreamConverter.java"], + "content": ["import com.thoughtworks.xstream.XStream;\nXStream xs = new XStream();"]}, + {"file_path": ["src/main/java/App.java"], + "content": ["import com.thoughtworks.xstream.XStream;\nXStream converter = new XStream();"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application" in result + assert "Application library dependencies" in result + app_pos = result.index("Main application") + dep_pos = result.index("Application library dependencies") + assert app_pos < dep_pos + assert "src/main/java/App.java" in result + + def test_source_scope_filters_dep_imports(self): + docs = [ + {"file_path": ["dependencies-sources/xstream-1.4/Converter.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + {"file_path": ["dependencies-sources/commons-io/IOUtils.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream", + source_scope=["xstream"]) + assert "xstream-1.4" in result + assert "commons-io" not in result + + def test_no_imports_unchanged(self): + docs = [ + {"file_path": ["src/App.java"], "content": ["public class App {}"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream", + ecosystem_label="java") + assert "No imports of 'com.thoughtworks.xstream' found" in result + + def test_app_dep_headers_in_output(self): + docs = [ + {"file_path": ["src/App.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + {"file_path": ["vendor/lib/Helper.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application" in result + assert "Application library dependencies" in result + + def test_only_dep_imports(self): + docs = [ + {"file_path": ["dependencies-sources/lib/Foo.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application (0 of 0 results)" in result + assert "Application library dependencies (1 of 1 results)" in result diff --git a/tests/test_intel_utils.py b/tests/test_intel_utils.py new file mode 100644 index 000000000..831cea727 --- /dev/null +++ b/tests/test_intel_utils.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for intel_utils: build_critical_context RHSA candidate capping.""" + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelRhsa +from vuln_analysis.utils.intel_utils import build_critical_context, _MAX_RHSA_CANDIDATES + + +class TestBuildCriticalContextRhsaCap: + """Tests for RHSA candidate package capping in build_critical_context.""" + + def _make_cve_intel_with_rhsa_packages(self, count): + """Create a CveIntel with `count` RHSA package_state entries.""" + package_states = [ + CveIntelRhsa.PackageState(package_name=f"rhsa-product-{i}") + for i in range(count) + ] + rhsa = CveIntelRhsa(package_state=package_states) + return CveIntel(vuln_id="CVE-2026-99999", rhsa=rhsa) + + def test_rhsa_cap_limits_candidates(self): + """RHSA with 100+ packages should only add _MAX_RHSA_CANDIDATES to candidates.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(100) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_rhsa_below_cap_all_included(self): + """RHSA with fewer than _MAX_RHSA_CANDIDATES packages includes all.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(5) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == 5 + + def test_rhsa_cap_with_1000_packages(self): + """Reproduces production scenario: 1000+ RHSA packages capped at limit.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(1234) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_rhsa_cap_does_not_affect_ghsa(self): + """GHSA candidates are not affected by the RHSA cap.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(100) + # Manually add GHSA data + from exploit_iq_commons.data_models.cve_intel import CveIntelGhsa + cve_intel.ghsa = CveIntelGhsa( + ghsa_id="GHSA-test-0001", + vulnerabilities=[{"package": {"name": "xstream", "ecosystem": "Maven"}}], + ) + _, candidates, _ = build_critical_context([cve_intel]) + + ghsa_candidates = [c for c in candidates if c["source"] == "ghsa"] + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(ghsa_candidates) == 1 + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_context_note_still_shows_total_count(self): + """The critical_context note should still mention the full package count.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(50) + context, _, _ = build_critical_context([cve_intel]) + + affected_notes = [c for c in context if "Affected across" in c] + assert len(affected_notes) == 1 + assert "50" in affected_notes[0] \ No newline at end of file diff --git a/tests/test_process_steps.py b/tests/test_process_steps.py new file mode 100644 index 000000000..381d44922 --- /dev/null +++ b/tests/test_process_steps.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for _process_steps in cve_agent.py. + +Focus: tracker/graph alignment on fallback, initial_state construction, +semaphore usage, concurrent step processing. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from langchain_core.messages import HumanMessage + +from vuln_analysis.functions.cve_agent import _process_steps +from vuln_analysis.functions.dispatcher import QuestionRouting +from vuln_analysis.functions.react_internals import ReachabilityRulesTracker +from vuln_analysis.functions.code_understanding_internals import CodeUnderstandingRulesTracker + + +@pytest.fixture +def mock_workflow_state(): + state = MagicMock() + state.cve_intel = [] + return state + + +@pytest.fixture +def mock_graph(): + graph = AsyncMock() + graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "answer"}) + return graph + + +@pytest.fixture +def patch_externals(mock_workflow_state): + """Patch ctx_state, build_critical_context, dispatch_question, and AGENT_TRACER.""" + with ( + patch("vuln_analysis.functions.cve_agent.ctx_state") as mock_ctx, + patch("vuln_analysis.functions.cve_agent.build_critical_context") as mock_bcc, + patch("vuln_analysis.functions.cve_agent.dispatch_question") as mock_dispatch, + patch("vuln_analysis.functions.cve_agent.AGENT_TRACER") as mock_tracer, + ): + mock_ctx.get.return_value = mock_workflow_state + mock_bcc.return_value = (["ctx_line"], [{"name": "pkg"}], ["vuln_func"]) + mock_tracer.push_active_function.return_value.__enter__ = MagicMock() + mock_tracer.push_active_function.return_value.__exit__ = MagicMock(return_value=False) + yield { + "ctx_state": mock_ctx, + "build_critical_context": mock_bcc, + "dispatch_question": mock_dispatch, + "tracer": mock_tracer, + } + + +class TestFallbackAlignment: + """When routed agent_type is not in agents dict, both graph AND tracker + must fall back to reachability.""" + + @pytest.mark.asyncio + async def test_cu_fallback_uses_reachability_graph(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + mock_graph.ainvoke.assert_called_once() + + @pytest.mark.asyncio + async def test_cu_fallback_uses_reachability_tracker(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + call_args = mock_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, ReachabilityRulesTracker) + assert not isinstance(tracker, CodeUnderstandingRulesTracker) + + @pytest.mark.asyncio + async def test_direct_route_uses_correct_tracker(self, patch_externals): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + reach_graph = AsyncMock() + agents = {"reachability": reach_graph, "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + call_args = cu_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, CodeUnderstandingRulesTracker) + reach_graph.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_reachability_route_uses_reachability_tracker(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="function call check" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is func reachable?"], None) + + call_args = mock_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, ReachabilityRulesTracker) + + +class TestInitialState: + """Verify initial_state dict has correct keys and values.""" + + @pytest.mark.asyncio + async def test_initial_state_keys(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["test question"], None) + + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["input"] == "test question" + assert call_args["step"] == 0 + assert call_args["max_steps"] == 10 + assert call_args["thought"] is None + assert call_args["observation"] is None + assert call_args["output"] == "waiting for the agent to respond" + assert isinstance(call_args["messages"][0], HumanMessage) + assert call_args["messages"][0].content == "test question" + + @pytest.mark.asyncio + async def test_precomputed_intel_passed(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None) + + call_args = mock_graph.ainvoke.call_args[0][0] + intel = call_args["precomputed_intel"] + assert intel == (["ctx_line"], [{"name": "pkg"}], ["vuln_func"]) + + @pytest.mark.asyncio + async def test_custom_max_iterations(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, max_iterations=25) + + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["max_steps"] == 25 + + @pytest.mark.asyncio + async def test_graph_config_recursion_limit(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None) + + config_arg = mock_graph.ainvoke.call_args[1]["config"] + assert config_arg == {"recursion_limit": 50} + + +class TestConcurrency: + """Test concurrent step processing and semaphore behavior.""" + + @pytest.mark.asyncio + async def test_multiple_steps_all_processed(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + steps = ["q1", "q2", "q3"] + + results = await _process_steps(agents, MagicMock(), steps, None) + + assert len(results) == 3 + assert mock_graph.ainvoke.call_count == 3 + + @pytest.mark.asyncio + async def test_semaphore_limits_concurrency(self, patch_externals): + max_concurrent = 0 + current_concurrent = 0 + + async def slow_invoke(state, config=None): + nonlocal max_concurrent, current_concurrent + current_concurrent += 1 + max_concurrent = max(max_concurrent, current_concurrent) + await asyncio.sleep(0.01) + current_concurrent -= 1 + return {"input": state["input"], "output": "done"} + + mock_graph = AsyncMock() + mock_graph.ainvoke = slow_invoke + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + semaphore = asyncio.Semaphore(2) + + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], semaphore) + + assert max_concurrent <= 2 + + @pytest.mark.asyncio + async def test_no_semaphore_allows_full_concurrency(self, patch_externals): + max_concurrent = 0 + current_concurrent = 0 + + async def slow_invoke(state, config=None): + nonlocal max_concurrent, current_concurrent + current_concurrent += 1 + max_concurrent = max(max_concurrent, current_concurrent) + await asyncio.sleep(0.01) + current_concurrent -= 1 + return {"input": state["input"], "output": "done"} + + mock_graph = AsyncMock() + mock_graph.ainvoke = slow_invoke + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], None) + + assert max_concurrent == 4 + + @pytest.mark.asyncio + async def test_exception_in_step_returned_not_raised(self, patch_externals): + mock_graph = AsyncMock() + mock_graph.ainvoke = AsyncMock(side_effect=RuntimeError("graph failed")) + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + results = await _process_steps(agents, MagicMock(), ["q1"], None) + + assert len(results) == 1 + assert isinstance(results[0], RuntimeError) + + @pytest.mark.asyncio + async def test_empty_steps_returns_empty(self, patch_externals, mock_graph): + agents = {"reachability": mock_graph} + results = await _process_steps(agents, MagicMock(), [], None) + assert results == [] + + +class TestTracingSpan: + """Test that the tracing span receives actual_type, not routed type.""" + + @pytest.mark.asyncio + async def test_span_logs_actual_type_on_fallback(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None) + + call_args = patch_externals["tracer"].push_active_function.call_args + assert "[reachability]" in call_args[1]["input_data"] + + @pytest.mark.asyncio + async def test_span_logs_routed_type_when_available(self, patch_externals): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + agents = {"reachability": MagicMock(), "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["q"], None) + + call_args = patch_externals["tracer"].push_active_function.call_args + assert "[code_understanding]" in call_args[1]["input_data"] diff --git a/tests/test_python_segmenter.py b/tests/test_python_segmenter.py index c909d0c04..d2158cafa 100644 --- a/tests/test_python_segmenter.py +++ b/tests/test_python_segmenter.py @@ -89,7 +89,7 @@ ("import os", False, "py3 import statement"), ], ) -def test_is_python2_code(self, code: str, expected: bool, description: str): +def test_is_python2_code(code: str, expected: bool, description: str): """Test that Python 2/3 patterns are correctly detected.""" result = is_python2_code(code) assert result is expected, f"Expected {expected} for {description}, got {result}" \ No newline at end of file diff --git a/tests/test_reachability_agent.py b/tests/test_reachability_agent.py new file mode 100644 index 000000000..590a80426 --- /dev/null +++ b/tests/test_reachability_agent.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ReachabilityAgent: get_tools, create_rules_tracker, +agent_type, should_truncate_tool_output.""" + +import pytest +from unittest.mock import MagicMock + +from agent_test_helpers import MockTool, ALL_TOOLS, make_builder, make_config, make_state +from vuln_analysis.functions.reachability_agent import ReachabilityAgent +from vuln_analysis.functions.react_internals import ReachabilityRulesTracker +from vuln_analysis.tools.tool_names import ToolNames + + +def _make_reachability_agent(tools=None): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return ReachabilityAgent(tools=tools or [], llm=mock_llm, config=config) + + +class TestGetTools: + """ReachabilityAgent.get_tools selects reachability tools and filters by availability.""" + + def test_keeps_reachability_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LOCATOR in result_names + assert ToolNames.CALL_CHAIN_ANALYZER in result_names + assert ToolNames.CODE_KEYWORD_SEARCH in result_names + assert ToolNames.CVE_WEB_SEARCH in result_names + + def test_excludes_cu_only_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CONFIGURATION_SCANNER not in result_names + assert ToolNames.IMPORT_USAGE_ANALYZER not in result_names + + def test_excludes_container_analysis_data(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CONTAINER_ANALYSIS_DATA not in result_names + + def test_keeps_all_8_reachability_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert len(result) == 8 + + def test_empty_builder_returns_empty(self): + builder = make_builder(tools=[]) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert result == [] + + def test_preserves_tool_order(self): + ordered_tools = [ + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + ] + builder = make_builder(tools=ordered_tools) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert [t.name for t in result] == [ + ToolNames.CVE_WEB_SEARCH, + ToolNames.FUNCTION_LOCATOR, + ToolNames.CALL_CHAIN_ANALYZER, + ] + + def test_unknown_tools_excluded(self): + tools = [MockTool(ToolNames.FUNCTION_LOCATOR), MockTool("Some Future Tool")] + builder = make_builder(tools=tools) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert len(result) == 1 + assert result[0].name == ToolNames.FUNCTION_LOCATOR + + +class TestGetToolsAvailability: + """get_tools filters out tools whose infrastructure prerequisites are not met.""" + + def test_filters_code_semantic_search_when_no_vdb(self): + builder = make_builder() + config = make_config() + state = make_state(code_vdb_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CODE_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_docs_semantic_search_when_no_vdb(self): + builder = make_builder() + config = make_config() + state = make_state(doc_vdb_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.DOCS_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_code_keyword_search_when_no_index(self): + builder = make_builder() + config = make_config() + state = make_state(code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CODE_KEYWORD_SEARCH not in {t.name for t in result} + + def test_filters_transitive_tools_when_no_index(self): + builder = make_builder() + config = make_config() + state = make_state(code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert ToolNames.FUNCTION_LOCATOR not in result_names + + def test_filters_cve_web_search_when_disabled(self): + builder = make_builder() + config = make_config(cve_web_search_enabled=False) + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CVE_WEB_SEARCH not in {t.name for t in result} + + def test_filters_transitive_tools_when_disabled(self): + builder = make_builder() + config = make_config(transitive_search_tool_enabled=False) + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert ToolNames.FUNCTION_LOCATOR not in result_names + + def test_version_finder_always_kept(self): + builder = make_builder() + config = make_config() + state = make_state(code_vdb_path=None, doc_vdb_path=None, code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in {t.name for t in result} + + +class TestReachabilityDuplicateCall: + + def test_duplicate_call_blocked(self): + tracker = ReachabilityRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.set_target_package("commons-beanutils") + tracker.check_thought_behavior("Function Locator", "commons-beanutils,getProperty", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "commons-beanutils,getProperty", ["result"]) + assert violated is True + assert "already called" in msg + + +class TestCreateRulesTracker: + + def test_returns_reachability_rules_tracker(self): + tracker = ReachabilityAgent.create_rules_tracker() + assert isinstance(tracker, ReachabilityRulesTracker) + + def test_returns_fresh_instance_each_call(self): + t1 = ReachabilityAgent.create_rules_tracker() + t2 = ReachabilityAgent.create_rules_tracker() + assert t1 is not t2 + + +class TestAgentType: + + def test_agent_type_is_reachability(self): + agent = _make_reachability_agent() + assert agent.agent_type == "reachability" + + +class TestShouldTruncateToolOutput: + + def test_true_for_java(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "java"}, "any_tool") is True + + def test_true_for_java_uppercase(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "Java"}, "any_tool") is True + + def test_false_for_go(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "go"}, "any_tool") is False + + def test_false_for_python(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "python"}, "any_tool") is False + + def test_false_for_empty_ecosystem(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": ""}, "any_tool") is False + + def test_false_when_ecosystem_missing(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({}, "any_tool") is False + + +class TestInit: + + def test_creates_fifth_classification_llm(self): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + agent = ReachabilityAgent(tools=[], llm=mock_llm, config=config) + assert mock_llm.with_structured_output.call_count == 5 + assert hasattr(agent, "_classification_llm") diff --git a/tests/test_react_internals_rules.py b/tests/test_react_internals_rules.py new file mode 100644 index 000000000..2440aecd5 --- /dev/null +++ b/tests/test_react_internals_rules.py @@ -0,0 +1,418 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.functions.react_internals import ( + BaseRulesTracker, + ReachabilityRulesTracker, + AgentState, + _find_image_matching_candidate, +) + + +class TestBaseRulesTracker: + def test_init_defaults(self): + tracker = BaseRulesTracker() + assert tracker.action_history == {} + assert tracker.target_package is None + assert tracker.allowed_tools == [] + + def test_set_allowed_tools(self): + tracker = BaseRulesTracker() + tools = ["Function Locator", "Code Keyword Search"] + tracker.set_allowed_tools(tools) + assert tracker.allowed_tools == tools + + def test_set_target_package(self): + tracker = BaseRulesTracker() + tracker.set_target_package("commons-beanutils") + assert tracker.target_package == "commons-beanutils" + + def test_is_empty_result_empty_list(self): + assert BaseRulesTracker._is_empty_result([]) is True + + def test_is_empty_result_bracket_string(self): + assert BaseRulesTracker._is_empty_result("[]") is True + + def test_is_empty_result_empty_string(self): + assert BaseRulesTracker._is_empty_result("") is True + + def test_is_empty_result_whitespace(self): + assert BaseRulesTracker._is_empty_result(" ") is True + + def test_is_empty_result_none(self): + assert BaseRulesTracker._is_empty_result(None) is False + + def test_is_empty_result_int(self): + assert BaseRulesTracker._is_empty_result(0) is False + + def test_is_empty_result_nonempty_list(self): + assert BaseRulesTracker._is_empty_result(["item"]) is False + + def test_add_action_accumulates(self): + tracker = BaseRulesTracker() + tracker.add_action("Function Locator", "pkg,fn1", ["result1"]) + tracker.add_action("Function Locator", "pkg,fn2", ["result2"]) + assert len(tracker.action_history["Function Locator"]) == 2 + assert tracker.action_history["Function Locator"][0]["input"] == "pkg,fn1" + assert tracker.action_history["Function Locator"][1]["input"] == "pkg,fn2" + + +class TestDuplicateCallRule: + def test_duplicate_call_blocked(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is True + assert "already called" in msg + + def test_different_input_allowed(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn1", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn2", ["result"]) + assert violated is False + assert msg == "" + + def test_different_tool_same_input_allowed(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + + def test_first_call_always_passes(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + + +class TestBaseRule7: + def test_rule7_non_cks_tool_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Function Locator", "org.Class", []) + assert result is False + + def test_rule7_no_dot_in_query_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "ClassName", []) + result = tracker._rule_number_7("Code Keyword Search", "ClassName", []) + assert result is False + + def test_rule7_nonempty_output_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", ["match"]) + assert result is False + + def test_rule7_no_prior_history_ignored(self): + tracker = BaseRulesTracker() + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_prior_non_dotted_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "ClassName", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_prior_had_results_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", ["match"]) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_consecutive_dotted_empty_fires(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Another", []) + assert result is True + + +class TestBaseRuleAllowedTools: + def test_allowed_tools_in_list_passes(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + result = tracker._rule_use_allowed_tools("Function Locator") + assert result is False + + def test_allowed_tools_not_in_list_fails(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + result = tracker._rule_use_allowed_tools("CVE Web Search") + assert result is True + + def test_allowed_tools_empty_list_rejects_all(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools([]) + result = tracker._rule_use_allowed_tools("Function Locator") + assert result is True + + +class TestBaseCheckThoughtBehavior: + def test_happy_path_returns_false_and_adds_to_history(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + assert "Function Locator" in tracker.action_history + assert tracker.action_history["Function Locator"][0]["input"] == "pkg,fn" + + def test_duplicate_priority_over_rule7(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Class", []) + assert violated is True + assert "already called" in msg + assert "Rule 7" not in msg + + def test_rule7_priority_over_allowed_tools(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Other Tool"]) + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) + assert violated is True + assert "Rule 7" in msg + assert "AVAILABLE_TOOLS" not in msg + + def test_allowed_tools_error_message_format(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + violated, msg = tracker.check_thought_behavior("CVE Web Search", "query", []) + assert violated is True + assert "AVAILABLE_TOOLS" in msg + assert "['Function Locator', 'Code Keyword Search']" in msg + + +class TestReachabilityRulesTracker: + def test_extends_base(self): + tracker = ReachabilityRulesTracker() + assert isinstance(tracker, BaseRulesTracker) + + def test_set_target_functions_creates_dict(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["fn1", "fn2"]) + assert tracker.target_functions == {"fn1": False, "fn2": False} + + def test_rule8_target_package_enforcement(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils") + result = tracker._rule_number_8("Function Locator", "wrong-package,fn", []) + assert result is True + + def test_rule8_prefix_matching_for_java_gav(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils:commons-beanutils") + result = tracker._rule_number_8("Function Locator", "commons-beanutils:commons-beanutils:1.9.4,fn", []) + assert result is False + + def test_rule8_skipped_after_first_call(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils") + tracker.set_allowed_tools(["Function Locator"]) + tracker.add_action("Function Locator", "commons-beanutils,fn", ["result"]) + result = tracker._rule_number_8("Function Locator", "wrong-package,fn", []) + assert result is False + + def test_rule8_no_target_package_skipped(self): + tracker = ReachabilityRulesTracker() + result = tracker._rule_number_8("Function Locator", "any-package,fn", []) + assert result is False + + def test_rule9_vulnerable_functions_first(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["getProperty", "setProperty"]) + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,someOtherFunction") + assert violated is True + assert "Rule 9" in msg + assert "getProperty" in msg or "setProperty" in msg + + def test_rule9_passes_after_checking_vulnerable(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["getProperty"]) + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,PropertyUtilsBean.getProperty") + assert violated is False + assert msg == "" + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,someOtherFunction") + assert violated is False + assert msg == "" + + def test_reachability_check_order(self): + tracker = ReachabilityRulesTracker() + tracker.set_allowed_tools(["Other Tool"]) + tracker.set_target_package("commons-beanutils") + tracker.set_target_functions(["getProperty"]) + + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) + assert violated is True + assert "Rule 7" in msg + + tracker2 = ReachabilityRulesTracker() + tracker2.set_allowed_tools(["Call Chain Analyzer"]) + tracker2.set_target_package("commons-beanutils") + violated, msg = tracker2.check_thought_behavior("Call Chain Analyzer", "wrong-package,fn", []) + assert violated is True + assert "Rule 8" in msg + + tracker3 = ReachabilityRulesTracker() + tracker3.set_allowed_tools(["Other Tool"]) + violated, msg = tracker3.check_thought_behavior("Call Chain Analyzer", "pkg,fn", []) + assert violated is True + assert "AVAILABLE_TOOLS" in msg + + tracker4 = ReachabilityRulesTracker() + tracker4.set_allowed_tools(["Call Chain Analyzer"]) + tracker4.set_target_package("pkg") + tracker4.set_target_functions(["getProperty"]) + violated, msg = tracker4.check_thought_behavior("Call Chain Analyzer", "pkg,otherFunction", []) + assert violated is True + assert "Rule 9" in msg + + +class TestCheckFinishAllowed: + """Tests for ReachabilityRulesTracker.check_finish_allowed.""" + + def test_blocks_finish_when_cca_never_called(self): + """Reachability agent must call CCA before finishing.""" + tracker = ReachabilityRulesTracker() + allowed, msg = tracker.check_finish_allowed(cca_results=[]) + assert allowed is False + assert "Function Locator" in msg + assert "Call Chain Analyzer" in msg + + def test_allows_finish_when_cca_returned_false(self): + """CCA was called and returned not-reachable — finish is allowed.""" + tracker = ReachabilityRulesTracker() + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(False, [])") + allowed, msg = tracker.check_finish_allowed(cca_results=[False]) + assert allowed is True + + def test_allows_finish_when_cca_returned_true_non_java(self): + """Non-Java: CCA found reachable — finish allowed (no FLVF requirement).""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("go") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is True + + def test_blocks_finish_java_cca_true_no_flvf(self): + """Java: CCA found reachable but FLVF not called — block finish.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is False + assert "VERSION CHECK" in msg + assert "Function Library Version Finder" in msg + + def test_allows_finish_java_cca_true_with_flvf(self): + """Java: CCA found reachable and FLVF was called — finish allowed.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + tracker.add_action("Function Library Version Finder", "pkg", "1.9.4") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is True + + def test_allows_finish_java_cca_false(self): + """Java: CCA returned not-reachable — no FLVF requirement.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(False, [])") + allowed, msg = tracker.check_finish_allowed(cca_results=[False]) + assert allowed is True + + def test_allows_finish_when_cca_in_history_but_results_empty(self): + """Edge case: CCA was called (in action_history) but cca_results is + empty — e.g., output parsing failed. Don't block since CCA was attempted.""" + tracker = ReachabilityRulesTracker() + tracker.add_action("Call Chain Analyzer", "pkg,fn", "some unparseable output") + allowed, msg = tracker.check_finish_allowed(cca_results=[]) + assert allowed is True + + +class TestAgentState: + def test_default_tracker_type_annotation(self): + assert "rules_tracker" in AgentState.__annotations__ + + def test_default_is_reachability_field(self): + assert "is_reachability" in AgentState.__annotations__ + + +class TestFindImageMatchingCandidate: + """Tests for _find_image_matching_candidate used by package filter fast path.""" + + def test_match_in_image_name(self): + candidates = [ + {"name": "builder", "source": "rhsa"}, + {"name": "kernel", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", None, + ) + assert result == "builder" + + def test_match_in_repo(self): + candidates = [ + {"name": "kernel", "source": "rhsa"}, + {"name": "infinispan", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/some-image", "https://github.com/infinispan/infinispan", + ) + assert result == "infinispan" + + def test_no_match(self): + candidates = [ + {"name": "kernel", "source": "rhsa"}, + {"name": "glibc", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", + "https://github.com/openshift/builder", + ) + assert result is None + + def test_short_name_skipped(self): + """Candidate names shorter than 3 chars are not matched.""" + candidates = [{"name": "go", "source": "rhsa"}] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/golang-builder", None, + ) + assert result is None + + def test_no_image_no_repo(self): + candidates = [{"name": "builder", "source": "rhsa"}] + result = _find_image_matching_candidate(candidates, None, None) + assert result is None + + def test_first_match_wins(self): + """When multiple candidates could match, the first one wins.""" + candidates = [ + {"name": "openshift", "source": "rhsa"}, + {"name": "builder", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", + "https://github.com/openshift/builder", + ) + assert result == "openshift" diff --git a/tests/test_source_classification.py b/tests/test_source_classification.py new file mode 100644 index 000000000..53f072125 --- /dev/null +++ b/tests/test_source_classification.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from vuln_analysis.utils.source_classification import ( + is_dependency_path, + filter_by_source_scope, + format_app_dep_output, +) + + +class TestIsDependencyPath: + @pytest.mark.parametrize("path", [ + "dependencies-sources/commons-beanutils/PropertyUtilsBean.java", + "vendor/github.com/foo/bar/bar.go", + "transitive_env/lib/requests/models.py", + "node_modules/express/index.js", + "rpm_libs/libxml2/parser.c", + ]) + def test_dependency_paths(self, path): + assert is_dependency_path(path) is True + + @pytest.mark.parametrize("path", [ + "src/main/java/com/example/App.java", + "cmd/main.go", + "app.py", + "pom.xml", + "src/main/resources/application.yml", + ]) + def test_app_paths(self, path): + assert is_dependency_path(path) is False + + def test_empty_path(self): + assert is_dependency_path("") is False + + +class TestFilterBySourceScope: + def test_filters_by_scope(self): + items = [ + ("dependencies-sources/xstream/XStream.java", "entry1"), + ("dependencies-sources/commons-io/IOUtils.java", "entry2"), + ("dependencies-sources/xstream/Converter.java", "entry3"), + ] + result = filter_by_source_scope(items, ["xstream"], lambda x: x[0]) + assert len(result) == 2 + assert result[0][1] == "entry1" + assert result[1][1] == "entry3" + + def test_none_scope_returns_all(self): + items = [("a", "x"), ("b", "y")] + result = filter_by_source_scope(items, None, lambda x: x[0]) + assert result == items + + def test_empty_scope_returns_all(self): + items = [("a", "x"), ("b", "y")] + result = filter_by_source_scope(items, [], lambda x: x[0]) + assert result == items + + def test_multiple_scope_terms(self): + items = [ + ("vendor/foo/main.go", "e1"), + ("vendor/bar/main.go", "e2"), + ("vendor/baz/main.go", "e3"), + ] + result = filter_by_source_scope(items, ["foo", "baz"], lambda x: x[0]) + assert len(result) == 2 + assert result[0][1] == "e1" + assert result[1][1] == "e3" + + def test_no_matches_returns_empty(self): + items = [("vendor/foo/main.go", "e1")] + result = filter_by_source_scope(items, ["nonexistent"], lambda x: x[0]) + assert result == [] + + +class TestFormatAppDepOutput: + def test_both_sections(self): + result = format_app_dep_output( + ["app_match_1", "app_match_2"], + ["dep_match_1"], + total_app=2, total_dep=1, + no_results_msg="No results", + ) + assert "Main application (2 of 2 results)" in result + assert "Application library dependencies (1 of 1 results)" in result + assert "app_match_1" in result + assert "dep_match_1" in result + + def test_empty_returns_no_results_msg(self): + result = format_app_dep_output([], [], 0, 0, "Nothing found") + assert result == "Nothing found" + + def test_app_only(self): + result = format_app_dep_output( + ["app1"], [], total_app=1, total_dep=0, + no_results_msg="No results", + ) + assert "Main application (1 of 1 results)" in result + assert "Application library dependencies (0 of 0 results)" in result + assert "app1" in result + + def test_dep_only(self): + result = format_app_dep_output( + [], ["dep1"], total_app=0, total_dep=1, + no_results_msg="No results", + ) + assert "Main application (0 of 0 results)" in result + assert "Application library dependencies (1 of 1 results)" in result + assert "dep1" in result + + def test_trimmed_counts(self): + result = format_app_dep_output( + ["a1", "a2"], ["d1"], + total_app=5, total_dep=3, + no_results_msg="No results", + ) + assert "Main application (2 of 5 results)" in result + assert "Application library dependencies (1 of 3 results)" in result + + def test_app_before_dep(self): + result = format_app_dep_output( + ["APP_SECTION"], ["DEP_SECTION"], + total_app=1, total_dep=1, + no_results_msg="No results", + ) + app_pos = result.index("APP_SECTION") + dep_pos = result.index("DEP_SECTION") + assert app_pos < dep_pos diff --git a/tests/test_transitive_detection.py b/tests/test_transitive_detection.py index 5a5fa5a0e..cd65acb9f 100644 --- a/tests/test_transitive_detection.py +++ b/tests/test_transitive_detection.py @@ -1,28 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for detect_ecosystem in dep_tree.py.""" + import pytest -from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher - -C_CPP_DETECTION_SCENARIOS = [ - # Positive cases - (["src/main.c"], True), - (["src/utils.cpp"], True), - (["include/lib.h"], True), - (["src/deep/nested/logic.cxx"], True), - # Negative cases - (["Makefile", "app.py"], False), - (["README.md", "LICENSE"], False), - (["main.cs"], False), - (["style.css"], False), - ([], False), - # Ignored directories - ([".git/objects/obj.c"], False), - ([".venv/lib/site/test.c"], False), - ([".config/settings.h"], False), +from exploit_iq_commons.utils.dep_tree import Ecosystem, detect_ecosystem + + +# --- C/C++ detection: requires manifest + source files --- + +C_CPP_POSITIVE_SCENARIOS = [ + # manifest + source file combinations + ("CMakeLists.txt", ["src/main.c"]), + ("CMakeLists.txt", ["src/utils.cpp"]), + ("Makefile", ["include/lib.h"]), + ("meson.build", ["src/deep/nested/logic.cxx"]), + ("CMakeLists.txt", ["lib.cc"]), + ("CMakeLists.txt", ["api.hpp"]), ] -@pytest.mark.parametrize("file_paths, expected", C_CPP_DETECTION_SCENARIOS) -def test_has_c_cpp_sources_scenarios(tmp_path, file_paths, expected): + +C_CPP_NEGATIVE_SCENARIOS = [ + # manifest present but no C/C++ source files + ("Makefile", ["app.py"]), + ("CMakeLists.txt", ["README.md", "LICENSE"]), + ("Makefile", ["main.cs"]), + ("CMakeLists.txt", ["style.css"]), + ("Makefile", []), + # C/C++ sources but NO manifest — should not detect + (None, ["src/main.c"]), + (None, ["src/utils.cpp", "include/lib.h"]), + # manifest + source files in ignored dotfile directories + ("CMakeLists.txt", [".git/objects/obj.c"]), + ("CMakeLists.txt", [".venv/lib/site/test.c"]), + ("CMakeLists.txt", [".config/settings.h"]), +] + + +@pytest.mark.parametrize("manifest, file_paths", C_CPP_POSITIVE_SCENARIOS) +def test_c_cpp_detected(tmp_path, manifest, file_paths): + (tmp_path / manifest).touch() for path in file_paths: full_path = tmp_path / path full_path.parent.mkdir(parents=True, exist_ok=True) full_path.touch() - result = TransitiveCodeSearcher._has_c_cpp_sources(tmp_path) - assert result is expected, f"Failed for scenario: {file_paths}" \ No newline at end of file + assert detect_ecosystem(tmp_path) == Ecosystem.C_CPP + + +@pytest.mark.parametrize("manifest, file_paths", C_CPP_NEGATIVE_SCENARIOS) +def test_c_cpp_not_detected(tmp_path, manifest, file_paths): + if manifest: + (tmp_path / manifest).touch() + for path in file_paths: + full_path = tmp_path / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.touch() + assert detect_ecosystem(tmp_path) != Ecosystem.C_CPP + + +# --- Other ecosystem detection --- + +OTHER_ECOSYSTEM_SCENARIOS = [ + ("go.mod", Ecosystem.GO), + ("requirements.txt", Ecosystem.PYTHON), + ("pyproject.toml", Ecosystem.PYTHON), + ("setup.py", Ecosystem.PYTHON), + ("package.json", Ecosystem.JAVASCRIPT), + ("pom.xml", Ecosystem.JAVA), +] + + +@pytest.mark.parametrize("manifest, expected", OTHER_ECOSYSTEM_SCENARIOS) +def test_ecosystem_detected_by_manifest(tmp_path, manifest, expected): + (tmp_path / manifest).touch() + assert detect_ecosystem(tmp_path) == expected + + +def test_no_manifest_returns_none(tmp_path): + (tmp_path / "README.md").touch() + assert detect_ecosystem(tmp_path) is None + + +def test_empty_dir_returns_none(tmp_path): + assert detect_ecosystem(tmp_path) is None + + +# --- Priority order: Go > Python > JS > Java > C/C++ --- + +def test_go_takes_priority_over_python(tmp_path): + (tmp_path / "go.mod").touch() + (tmp_path / "requirements.txt").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.GO + + +def test_python_takes_priority_over_java(tmp_path): + (tmp_path / "requirements.txt").touch() + (tmp_path / "pom.xml").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.PYTHON + + +def test_java_takes_priority_over_c_cpp(tmp_path): + (tmp_path / "pom.xml").touch() + (tmp_path / "CMakeLists.txt").touch() + (tmp_path / "main.c").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.JAVA + + +def test_configure_in_auto_subdir(tmp_path): + """detect_ecosystem checks for auto/configure as a C/C++ manifest.""" + (tmp_path / "auto").mkdir() + (tmp_path / "auto" / "configure").touch() + (tmp_path / "src").mkdir() + (tmp_path / "src" / "main.c").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.C_CPP From 2700093f3be87fd413e1bce3600df14718a31080 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Wed, 20 May 2026 16:48:37 +0300 Subject: [PATCH 237/286] Fix false CCA chains from ancestor interface casts (#234) * Fix false CCA chains from ancestor interface casts Signed-off-by: Theodor Mihalache --- .../java_functions_parsers.py | 59 ++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index 3a2a1d2c9..971d50508 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -1044,6 +1044,26 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: declaring_fqcn_dot = declaring_fqcn.replace('$', '.') declaring_simple = declaring_fqcn_dot.rsplit('.', 1)[-1] + # Early exit: if the caller's source file cannot reference the declaring + # class, skip all pattern matching and type resolution work. + # Checks three ways the declaring class could be visible: + # 1. Simple class name appears in the source text (direct usage or import) + # 2. Wildcard import of the declaring class's package (import pkg.*) + # 3. Caller is in the same package as the declaring class + # In large repos, ~90%+ of source files fail all three checks, avoiding + # expensive regex/type resolution for each one. + if declaring_fqcn: + _caller_src = caller_function.metadata.get('source') + _caller_full_doc = code_documents.get(_caller_src) + if _caller_full_doc: + _full_text = _caller_full_doc.page_content + if declaring_simple not in _full_text: + _declaring_pkg = declaring_fqcn_dot.rsplit('.', 1)[0] + if f"import {_declaring_pkg}.*" not in _full_text: + _pkg_m = re.search(r'^\s*package\s+([\w.]+)\s*;', _full_text, re.MULTILINE) + if not _pkg_m or _pkg_m.group(1) != _declaring_pkg: + return False + # CHANGED: support ctor targeting by simple name, no-package inner name, or fqcn. def _strip_package_prefix(fqcn_dot: str) -> str: parts = fqcn_dot.split('.') @@ -1511,6 +1531,26 @@ def _match_paren_reverse(text: str, close_idx: int) -> int: else: _explicit_imports[imp.rsplit('.', 1)[-1]] = imp + def _is_concrete_or_paired(candidate_fqcn: str) -> bool: + """Guard against false call chains from ancestor interface/abstract casts. + + When a caller file imports a broad interface (e.g. Map, Transformer) + that appears in the callee's type hierarchy (target_class_names), a + naive match would treat the caller as invoking the callee — even + though the caller never references the callee's declaring class. + + This function accepts a candidate only if it is: + - "concrete": the candidate FQCN IS the callee's declaring class, OR + - "paired": the callee's declaring class is explicitly imported in + the same file, meaning the caller is aware of the concrete type. + """ + cand_dot = candidate_fqcn.replace('$', '.') + decl_dot = callee_declaring_fqcn.replace('$', '.') + if cand_dot == decl_dot: + return True + callee_simple = decl_dot.rsplit('.', 1)[-1] + return callee_simple in _explicit_imports + def _strip_type_syntax(token: str) -> str: """Strip generics, arrays, and wildcard bounds from a type-ish token (CHANGED).""" t = (token or "").strip() @@ -1573,17 +1613,23 @@ def _iter_fqcn_candidates(raw_type_token: str): # Explicit import is the definitive binding for this simple name. # Yield only if it resolves to a target type; either way, block # the _target_by_simple fallthrough. + # _is_concrete_or_paired rejects ancestor interfaces (e.g. Transformer) + # that match target_class_names but aren't the callee's declaring class. imp = _explicit_imports.get(t) if imp: if imp in target_class_names: - yield imp + if not callee_declaring_fqcn or _is_concrete_or_paired(imp): + yield imp return - # Target allow-list by simple name (only reached when no explicit import) + # Target allow-list by simple name (only reached when no explicit import). + # Same _is_concrete_or_paired guard: a simple-name match on a broad + # interface (Map, Collection) must not create a false edge to the callee. cands = _target_by_simple.get(t) if cands: for fq in cands: - yield fq + if not callee_declaring_fqcn or _is_concrete_or_paired(fq): + yield fq # Wildcard imports: only usable if we can cheaply construct candidates # (we only build candidates that are already in target_class_names to avoid work) @@ -3516,7 +3562,7 @@ def __lookup_package( caller_package: str = "", ) -> bool: if not struct_initializer_expression and resolved_type not in JAVA_METHOD_PRIM_TYPES: - if resolved_type and resolved_type not in JAVA_METHOD_PRIM_TYPES: + if resolved_type: if self._has_matching_type_in_package( callee_package, resolved_type, type_documents, target_class_names, caller_explicit_imports=caller_explicit_imports, @@ -3590,10 +3636,7 @@ def _find_method_return_type_in_type_docs( m = sig.search(src) if not m: - matches = list(sig.finditer(src)) - if not matches: - continue - m = matches[0] + continue ret = m.group("ret").strip() ret = re.sub(r"<[^<>]*>", "", ret) From 1a34bbe83d45ca1347908918c16facfe45780a67 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Wed, 20 May 2026 20:39:51 +0300 Subject: [PATCH 238/286] Fix shallow clone tag checkout failure in source_code_git_loader() * Fix shallow clone tag checkout failure in source_code_git_loader Signed-off-by: Theodor Mihalache --- .../utils/source_code_git_loader.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 13379cc7f..f6cca7386 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -253,22 +253,39 @@ def _fetch_authenticated(self, repo: Repo, credential_id: str) -> None: del cred def _do_fetch(self, repo: Repo) -> None: - """Core fetch logic: fetch ref and tags.""" + """Core fetch logic: fetch ref (branch, tag, or commit SHA). + + Tries the tag-specific refspec first because it creates a local + tag ref that ``git checkout`` can resolve. A generic fetch is + tried second (handles branches and commit SHAs). The two fetches + must NOT both run on a shallow clone — consecutive shallow fetches + trigger ``fatal: shallow file has changed since we read it``. + """ + # Try as a tag first — creates a local ref under refs/tags/ + try: + repo.git.fetch("origin", f"refs/tags/{self.ref}:refs/tags/{self.ref}", "--depth=1", "--force") + logger.info("Fetched ref '%s' as tag", self.ref) + return + except GitCommandError as e: + logger.debug("Tag fetch failed for '%s': %s", self.ref, e) + + # Not a tag — try as a branch or commit SHA try: repo.git.fetch("origin", self.ref, "--depth=1", "--force") + logger.info("Fetched ref '%s' as branch/commit", self.ref) + return except GitCommandError as e: - # If fetch fails, check if the ref already exists locally - try: - repo.commit(self.ref) - logger.info("Ref %s already exists locally", self.ref) - return - except GitCommandError: - # Ref doesn't exist locally either, re-raise original fetch error - raise e from None - - # Try to fetch tags, but don't fail if they don't exist - with contextlib.suppress(GitCommandError): - repo.git.fetch("origin", f"refs/tags/{self.ref}:refs/tags/{self.ref}", "--depth=1", "--force") + logger.debug("Branch/commit fetch failed for '%s': %s", self.ref, e) + + # Neither fetch worked — check if ref already exists locally + try: + repo.commit(self.ref) + logger.info("Ref '%s' already exists locally", self.ref) + except GitCommandError: + raise GitCommandError( + ["git", "fetch"], 128, + f"Could not fetch ref '{self.ref}' from origin (tried as tag, branch, and commit)" + ) from None def _do_fetch_with_pat(self, repo: Repo, pat: str, username: str | None) -> None: """Fetch with PAT: temporarily update URL, fetch, restore URL.""" From 75bc6bb4aeee844b6242d31618ac07d49a273932 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Mon, 25 May 2026 17:27:39 +0300 Subject: [PATCH 239/286] Remove unreachable code across Java analysis files and fix count_of_dots bug in function_name_locator (#238) * Remove unreachable code across Java analysis files and fix count_of_dots bug in function_name_locator Signed-off-by: Theodor Mihalache --- src/exploit_iq_commons/utils/dep_tree.py | 21 +++++------------- .../utils/java_chain_of_calls_retriever.py | 17 -------------- .../utils/java_segmenters_with_methods.py | 22 ++++--------------- src/exploit_iq_commons/utils/java_utils.py | 1 - .../tools/transitive_code_search.py | 4 +--- .../utils/function_name_locator.py | 5 +---- 6 files changed, 12 insertions(+), 58 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index f32d8bb12..e4ba1d55d 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -326,13 +326,10 @@ def module_from_path(self, path: str) -> str: rel_path = Path(os.path.relpath(os.path.dirname(path), self.root_dir)) parts = rel_path.parts if self.RPM_LIBS_DIR in parts: - try: - idx = parts.index(self.RPM_LIBS_DIR) - if idx + 1 < len(parts): - module = parts[idx + 1] - return module - except ValueError: - pass + idx = parts.index(self.RPM_LIBS_DIR) + if idx + 1 < len(parts): + module = parts[idx + 1] + return module module = self.prj_name or "ROOT" return module @@ -482,13 +479,8 @@ def _batch_resolve_header(self, header: str, including_module: str, mod = self.module_from_path(candidate) if mod != self.C_STANDARD_LIB: return mod - # Fallback to glibc - for candidate in candidates: - mod = self.module_from_path(candidate) - if mod == self.C_STANDARD_LIB: - return self.C_STANDARD_LIB - # Fallback to first - return self.module_from_path(candidates[0]) + # All candidates are glibc + return self.C_STANDARD_LIB # Build a sort of "upside down" tree - a dict containing mapping of each # package to a list of all consuming packages @@ -788,7 +780,6 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: root_package_name = self._get_parent(lines[0]) tree = dict() for line in lines: - line.split(" ") parent = self.extract_package_name(self._get_parent(line)) son = self.extract_package_name(self._get_son(line)) if tree.get(son, None) is None: diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index a27662a69..8e305a4d3 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -756,20 +756,6 @@ def extract_from_query(self, query: str) -> tuple[str, str, str]: return class_name, method_name, package_name - def __get_parents(self, importing_docs: list[Document], prefix_of_3rd_parties_libs: str) -> set[str]: - pkg_names = { - names[0] - for doc in importing_docs - if doc.metadata['source'].startswith(prefix_of_3rd_parties_libs) - for names in [self.language_parser.get_package_names(doc)] - if names - } - - # 2) Keep only k's where the package name appears in them (substring logic preserved) - parents = {k for k in self.tree_dict if any(pkg in k for pkg in pkg_names)} - - return parents - def __determine_doc_package_name(self, target_function_doc, ctx: _JavaSearchCtx): """ Determine the logical package/dependency identifier for a given document. @@ -871,12 +857,9 @@ def _looks_like_type_name(s: str) -> bool: if ( target_simple not in src and (not target_qual or target_qual not in src) - and f"new {target_simple}" not in src - and f"::{target_simple}" not in src and not ( "::new" in src and ( (_looks_like_type_name(target_simple)) or - (target_qual and target_qual in src) or (target_qual and target_qual.split(".")[-1] in src) ) ) diff --git a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py index c8dce5deb..c0dbf2d95 100644 --- a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py +++ b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py @@ -351,8 +351,7 @@ def _source_is_test_source(self, src: str) -> bool: if ident == "extends": saw_extends = True elif saw_extends: - # Allow qualified form: `extends junit.framework.TestCase` - if ident.endswith("TestCase") and ident.rsplit(".", 1)[-1] == "TestCase": + if ident == "TestCase": return True saw_extends = False @@ -1050,7 +1049,7 @@ def _match_balanced_angles(source: str, open_angle_index: int) -> int: continue # Skip over literals so '<' or '>' inside them don't affect nesting. - if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + if source.startswith('"""', i) or source[i] in ('"', "'"): i = _skip_java_string_like(source, i) continue @@ -1117,7 +1116,7 @@ def _match_balanced_parens(source: str, open_paren_index: int) -> int: continue # Skip over literals so any '(' or ')' inside them don't affect nesting. - if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + if source.startswith('"""', i) or source[i] in ('"', "'"): i = _skip_java_string_like(source, i) continue @@ -1189,7 +1188,7 @@ def _match_balanced_braces(source: str, open_brace_index: int) -> int: continue # Skip over literals so braces inside them don't affect nesting. - if i < length and (source.startswith('"""', i) or source[i] in ('"', "'")): + if source.startswith('"""', i) or source[i] in ('"', "'"): i = _skip_java_string_like(source, i) continue @@ -1355,15 +1354,6 @@ def _skip_throws_clause(src: str, i: int) -> int: i += 1 return _skip_ws_comments(src, i) -# ----------------------- type discovery for constructor filter ---------------- - -class _TypeRegion: - __slots__ = ("name", "start", "end") # '{' index .. matching '}' index - def __init__(self, name: str, start: int, end: int) -> None: - self.name = name - self.start = start - self.end = end - # --------------------------------- lambdas ------------------------------------ def _capture_lambda_at_arrow(src: str, arrow: int) -> Optional[Tuple[int, int]]: @@ -1886,8 +1876,6 @@ def _line_is_annotation_only(s: str, line_start: int, line_end: int) -> bool: break i = _skip_ws_and_comments(s, i, line_end) - while i < line_end and s[i].isspace(): - i += 1 return i >= line_end def _skip_annotation_only_lines(s: str, i: int, lim: int) -> int: @@ -2961,8 +2949,6 @@ def _line_is_annotation_only(line_start: int, line_end: int) -> bool: break i = _skip_ws_comments(s, i) - while i < line_end and s[i].isspace(): - i += 1 return i >= line_end def _consume_annotations(i: int) -> int: diff --git a/src/exploit_iq_commons/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py index 395b4b88a..1a1b37786 100644 --- a/src/exploit_iq_commons/utils/java_utils.py +++ b/src/exploit_iq_commons/utils/java_utils.py @@ -338,7 +338,6 @@ def keep_type(typ: str) -> bool: simple = base.split(".")[-1] if simple.lower() in JAVA_METHOD_PRIM_TYPES: return False if simple in WRAPS: return False - if base.startswith("java.lang.") and simple in WRAPS: return False return True def parse_field_header(header: str) -> List[Tuple[str, str]]: diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index fa50e6fd1..3aea7d6df 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -492,9 +492,7 @@ async def _arun(query: str) -> dict: # Java GAV format: "groupId:artifactId:version" package_lower = package.lower() parts = package_lower.split(":") - if (search_term in package_lower or - any(search_term == part for part in parts) or - any(search_term in part for part in parts)): + if search_term in package_lower: matching_packages.append(package) if not matching_packages: diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index 047113396..ccdd35798 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -145,10 +145,7 @@ def python_flow_control(self, input_function: str, package_docs) -> list[str]: List of function names in format 'package,function_name' that match the search term, or error message with package suggestions if package is not found """ - count_of_dots = 0 - splitters = [splitter for splitter in ['.'] if splitter in input_function] - if splitters: - count_of_dots = len(splitters) + count_of_dots = min(input_function.count('.'), 2) list_of_matching_combinations = set() for doc in package_docs: From bb95dff0474dbe289e730453f33d61a89d62d763 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Sun, 31 May 2026 20:30:23 +0300 Subject: [PATCH 240/286] Fix Go method regex regression and add patch-based function enrichment (#240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix Go method regex regression and add patch-based function enrichment Go segmenter regex fixes: - Fix parse_all_methods regex that failed to match Go methods with pointer parameters (e.g. *wire.AckFrame) — old param pattern [,a-zA-Z0-9\s\[\].]* excluded '*', new pattern [^)]* matches all param types - Fix receiver pattern to support bracket types [a-zA-Z0-9\s\*.\[\]]+ and underscores in method names [a-zA-Z_][a-zA-Z0-9_]* - Fix return type group to use (\(?[^{]*\)?)? instead of the (\(?(?:[^{}]|\{[^}]*\})*\)?)? variant which greedily consumed function bodies through the \{[^}]*\} alternation, causing short functions (e.g. GetLowestPacketNotConfirmedAcked) to swallow the next method's signature - Result: 31 methods extracted from sent_packet_handler.go (was 26/27), including detectAndRemoveAckedPackets which is critical for CVE-2025-29785 Patch-based vulnerable function enrichment (intel_utils.py): - Add enrich_vulnerable_functions_from_patch() — when GHSA/OSV have no vulnerable_functions, extract them from GitHub fix commit diffs - Parse function names from unified diff hunk headers and added lines using ecosystem-aware regex patterns (Go, Python, Java, JS, C) - Prioritize GHSA references tagged as type=FIX - Authenticate GitHub API calls via existing GHSA_API_KEY env var - Limit to 3 commits max to avoid excessive API calls Go enrichment pipeline refactoring (cve_agent.py): - Move enrich_go_candidates and validate_go_vendor_packages from reachability_agent.py to intel_utils.py for reuse across agents - Call Go enrichment and patch enrichment in _process_steps before dispatching checklist questions - Batch-dispatch all questions to routing LLM via asyncio.gather instead of sequential per-step dispatch - Inject synthetic reachability question when no checklist question targets reachability but candidate packages exist - Handle routing failures gracefully (log warning, skip failed steps) - Fix _postprocess_results index-out-of-bounds for synthetic questions CCA debug logging (chain_of_calls_retriever.py): - Add structured logging throughout DFS traversal: query parsing, tree lookup, __find_initial_function (doc counts, found/not-found), each DFS iteration (function, package, path length, caller found), backtracking, and final result - Fix potential IndexError: check parent_parents is non-empty before accessing [0] in __find_caller_function_dfs - Fix CCA query parsing for Go sub-packages containing '/' in the function portion (e.g. internal/ackhandler.func) Go parser debug logging (golang_functions_parsers.py): - Add logging to search_for_called_function, __check_identifier, and __trace_down_package for tracing Go method resolution - Fix __check_identifier to pass receiver chain (parts[:-1]) instead of full identifier expression to __trace_down_package - Fix struct field type extraction to handle fields without trailing space (find(" ") returning -1) Other changes: - Add FL debug log showing package doc count before fuzzy matching - Expand full_text_search to widen to 500 results when top-50 returns only dependency docs and no application docs - Relax Go transitive search test assertion from exact path length to len > 1 with root package check * Removed debug logging and fixed tests * Removed unused logger * Fix IUA tantivy DocAddress bug and add synthetic reachability safety net - Fix Import Usage Analyzer: use all_query() to get proper DocAddress objects instead of broken range(num_docs) iteration (tool was non-functional) - Relax synthetic reachability question condition to fire when candidate_packages exists, even without vulnerable_functions - Consolidate Go import regex into single pattern handling aliased/grouped imports - Fix _find_usage_in_file to extract short names from Go slash-separated paths - Always run patch-based function enrichment, not only when vuln functions empty * Fix patch enrichment accuracy regression with test file filtering - Skip test files at file level in enrich_vulnerable_functions_from_patch (_TEST_FILE_RE matches _test.go, *Test.java, test_*.py, *.test.js, *_test.c, src/test/ across all ecosystems) - Restore vulnerable_functions.add() — patch-extracted functions from non-test files are safe for Rule 9 enforcement - Restore conditional enrichment pattern in _process_steps so enriched functions reach precomputed_intel * Changed redpanda pull policy to IfNotPresent * CCA Go sub-package disambiguation - Same-package shortcut returned True without verifying the caller was in the callee package (time.Parse matching strvals.Parse) - Function name regex matched suffixes (MustParse matching Parse) — added negative lookbehind for word boundary - Type resolution matched types globally without checking the caller imports the callee package (pattern.Parser matching jwt.Parser) * Fix Go CCA false-positive chains, demote OSV/patch enrichment to hints - Demote OSV-enriched data to critical_context hints instead of candidate_packages/vulnerable_functions - Demote patch-extracted functions to critical_context hints instead of vulnerable_functions - Fix reachability_agent already_enriched check to match new hint format - Handle missing requirements.txt in Python dep tree builder (fallback to pyproject.toml/setup.py) - Set UV_CACHE_DIR fallback for container permission issues - Add PythonDependencyTreeBuilder manifest detection and fallback tests - Update intel_utils tests to assert hint-only enrichment * Changes following code review -Small param name fix -Reverted some python dependencies changes since there are handled better in https://github.com/RHEcosystemAppEng/vulnerability-analysis/pull/242 -Show actual synthetic question text instead of constant "(synthetic)" placeholder * Changes following code review -Removed incorrect tests * Changes following code review Fall back to reachability agent on routing failure instead of dropping questions - Upgrade routing failure log level from warning to error - Replace failed routings with default reachability QuestionRouting instead of removing them from routed_steps Signed-off-by: Theodor Mihalache --- .tekton/on-cm-runner.yaml | 1 + .tekton/on-pull-request.yaml | 1 + .../utils/chain_of_calls_retriever.py | 15 +- .../golang_functions_parsers.py | 62 ++-- .../utils/go_segmenters_with_methods.py | 2 +- src/vuln_analysis/functions/cve_agent.py | 78 ++++- .../functions/reachability_agent.py | 45 +-- .../tools/import_usage_analyzer.py | 78 ++++- .../tests/test_transitive_code_search.py | 114 +++++-- src/vuln_analysis/utils/full_text_search.py | 16 + src/vuln_analysis/utils/intel_utils.py | 278 ++++++++++++++++-- tests/test_import_usage_analyzer.py | 137 ++++++++- tests/test_intel_utils.py | 249 +++++++++++++++- tests/test_process_steps.py | 102 ++++++- 14 files changed, 1029 insertions(+), 149 deletions(-) diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 6630f8f27..0a47b652e 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -261,6 +261,7 @@ spec: periodSeconds: 5 - name: redpanda image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + imagePullPolicy: IfNotPresent script: | #!/bin/bash set -e diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index c96501987..30c99ba4d 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -405,6 +405,7 @@ spec: periodSeconds: 5 - name: redpanda image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + imagePullPolicy: IfNotPresent script: | #!/bin/bash set -e diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index e47b68e9e..198dfe8b6 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -200,9 +200,6 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa parents = self._get_parents(package_name, ctx) if parents: direct_parents.extend(parents) - # Add same package itself to search path. - # direct_parents.extend([function_package]) - # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) if function_name_to_search == self.language_parser.get_constructor_method_name(): function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) @@ -215,7 +212,8 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa # Search for caller functions only at parents according to dependency tree. for package in direct_parents[last_visited_package_index:]: sources_location_packages = True - if self._get_parents(package, ctx)[0] == ROOT_LEVEL_SENTINEL: + package_parents = self._get_parents(package, ctx) + if package_parents and package_parents[0] == ROOT_LEVEL_SENTINEL: sources_location_packages = False possible_docs = self.get_possible_docs(function_name_to_search, package, @@ -223,7 +221,6 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa sources_location_packages, frozenset(), dict()) - # Collect all potential caller functions for doc in self.get_functions_for_package(package_name=package, documents=possible_docs, @@ -481,19 +478,15 @@ def _depth_first_search(self, matching_documents: List[Document], target_functio # extract package name from function document current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) else: - # end loop because didn't find a caller for initial function if len(matching_documents) == 1: end_loop = True # Backtrack - means that we're going back because current function has no callers anywhere, so we # need to remove it, and continue with other possible potential called functions from its caller else: dead_end_node = matching_documents.pop() - # Excludes dead end function node from future searches, as it led to nowhere. ctx.exclusions[current_package_name].append(dead_end_node) target_function_doc = matching_documents[-1] current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) - # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was - # found or not. return matching_documents, ctx.found_path # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, @@ -509,6 +502,9 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: splitters = [splitter for splitter in ['.'] if splitter in function] if splitters: class_name, function = function.split(splitters[0]) + if class_name and '/' in class_name: + package_name = f"{package_name}/{class_name}" + class_name = None found_package = False matching_documents = [] standard_libs_cache = StandardLibraryCache.get_instance() @@ -611,7 +607,6 @@ def __find_initial_function(self, function_name: str, package_name: str, documen self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=False), ): if function_name.lower() == self.language_parser.get_function_name(document).lower(): - # if language_parser.search_for_called_function(document, callee_function=function_name): package_exclusions.append(document) return document diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index ed7e94703..54d1edcaf 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -321,13 +321,19 @@ def parse_one_type(self, the_type, types_mapping): else: parts = current_line.split(sep=" ", maxsplit=1) if len(parts) == 2: + type_str = parts[1].lstrip() + space_pos = type_str.find(" ") + field_type = type_str[:space_pos] if space_pos != -1 else type_str fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), - parts[1].lstrip()[:parts[1].lstrip().find(" ")])) + field_type)) elif len(parts) == 1: fields_list.append((EMBEDDED_TYPE, parts[0])) elif len(parts) > 2: + type_str_raw = parts[1] + space_pos_raw = type_str_raw.find(" ") + field_type_raw = type_str_raw[:space_pos_raw].strip() if space_pos_raw != -1 else type_str_raw.strip() fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), - parts[1][:parts[1].find(" ")].strip())) + field_type_raw)) else: pass @@ -409,9 +415,11 @@ def search_for_called_function(self, caller_function: Document, callee_function_ caller_function_body = str( caller_function.page_content[index_of_function_opening + 1: index_of_function_closing]) re.search("", caller_function_body) - regex = fr'[a-zA-Z0-9_\[\]\(\).]*.?{callee_function_name}\(' + escaped_name = re.escape(callee_function_name) + regex = fr'(? 0 or re.search(regex_arguments, function_header): - # match_variable = matches[-1] - # split = match_variable.split(":=") - # if split[0] == match_variable: - # split = match_variable.split("=") - # if len(split) > 1: - return self.__trace_down_package(expression=identifier.strip(), code_documents=code_documents, + receiver_parts = list(parts[:-1]) + if receiver_parts and receiver_parts[0].startswith("return"): + receiver_parts[0] = receiver_parts[0].replace("return", "", 1).strip() + receiver_chain = ".".join(p for p in receiver_parts if p) + return self.__trace_down_package(expression=receiver_chain.strip(), code_documents=code_documents, type_documents=type_documents, callee_package=callee_package, fields_of_types=fields_of_types, functions_local_variables_index=functions_local_variables_index, @@ -603,6 +628,5 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: escaped = re.escape(package_name) return [ - re.compile(rf'import\s+"({escaped}[^"]*)"', re.IGNORECASE | re.MULTILINE), - re.compile(rf'import\s+\(\s*[^)]*"({escaped}[^"]*)"', re.IGNORECASE | re.MULTILINE), + re.compile(rf'^\s*(?:import\s+)?(?:[\w.]+\s+)?"({escaped}[^"]*)"', re.MULTILINE), ] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/go_segmenters_with_methods.py b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py index fe8c23f84..364bdfbe2 100644 --- a/src/exploit_iq_commons/utils/go_segmenters_with_methods.py +++ b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py @@ -26,7 +26,7 @@ def parse_all_methods(code: str) -> list[str]: :return: List of all golang methods in the source, including signatures + bodies ( implementation) """ # regex = r"func\s*\([a-zA-Z0-9\s]*\) [a-zA-X]+\([a-zA-Z0-9\s]*\)([a-zA-Z0-9\s]*){" - regex = r"func\s*\([a-zA-Z0-9\s\*.]+\) [a-zA-Z]+\([,a-zA-Z0-9\s\[\].]*\)\s*(\(?[a-zA-Z0-9\s.,*]+\)?)?\s*{" + regex = r"func\s*\([a-zA-Z0-9\s\*.\[\]]+\) [a-zA-Z_][a-zA-Z0-9_]*\([^)]*\)\s*(\(?[^{]*\)?)?\s*{" methods = get_all_functions(code, regex) return methods diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index d1fa7d4d7..19f6cb9b4 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -27,7 +27,7 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.error_handling_decorator import ToolRaisedException -from vuln_analysis.utils.intel_utils import build_critical_context +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_candidates, enrich_vulnerable_functions_from_patch from vuln_analysis.runtime_context import ctx_state from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id from nat.builder.context import Context @@ -79,13 +79,62 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterations: int = 10): workflow_state = ctx_state.get() critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + + ecosystem = (workflow_state.original_input.input.image.ecosystem.value + if workflow_state.original_input.input.image.ecosystem else "") + if ecosystem == "go": + candidate_packages, vulnerable_functions = await enrich_go_candidates( + workflow_state.cve_intel, + workflow_state.original_input.input.image.source_info, + critical_context, candidate_packages, set(vulnerable_functions), + ) + + if not vulnerable_functions: + vf_set: set[str] = set() + await enrich_vulnerable_functions_from_patch( + workflow_state.cve_intel, critical_context, vf_set, ecosystem, + ) + if vf_set: + vulnerable_functions = sorted(vf_set) + context_block = "\n".join(critical_context) precomputed_intel = (critical_context, candidate_packages, vulnerable_functions) - async def _process_step(step): - routing = await dispatch_question(routing_llm, step, context_block) - agent_type = routing.agent_type + routings = await asyncio.gather( + *(dispatch_question(routing_llm, step, context_block) for step in steps), + return_exceptions=True, + ) + default_routing = QuestionRouting( + agent_type="reachability", + reason="Fallback: routing failed, defaulting to reachability agent.", + ) + for i, r in enumerate(routings): + if isinstance(r, Exception): + logger.error("dispatch_question failed for step %d, falling back to reachability: %s", i, r) + routings[i] = default_routing + routed_steps = list(zip(steps, routings)) + + has_reachability = any(r.agent_type == "reachability" for r in routings) + if not has_reachability and "reachability" in agents and candidate_packages: + pkg_name = candidate_packages[0].get("name", "") + if pkg_name: + synthetic_q = ( + f"Is the code from the vulnerable package '{pkg_name}' actually " + f"imported, called, or reachable from the application's own source code?" + ) + synthetic_routing = QuestionRouting( + agent_type="reachability", + reason="Injected: no checklist question targeted reachability — " + "verifying whether vulnerable code is actually used.", + ) + routed_steps.append((synthetic_q, synthetic_routing)) + logger.info( + "No reachability questions in checklist; injected synthetic question for '%s'", + pkg_name, + ) + async def _execute_step(step, routing): + agent_type = routing.agent_type actual_type = agent_type if agent_type in agents else "reachability" compiled_graph = agents[actual_type] tracker = get_agent_class(actual_type).create_rules_tracker() @@ -113,22 +162,35 @@ async def _process_step(step): else: return await compiled_graph.ainvoke(initial_state, config=graph_config) - return await asyncio.gather(*(_process_step(step) for step in steps), return_exceptions=True) + raw_results = await asyncio.gather( + *(_execute_step(step, routing) for step, routing in routed_steps), + return_exceptions=True, + ) + questions = [step for step, _ in routed_steps] + return raw_results, questions -def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, replace_exceptions_value: str | None, +def _postprocess_results(results: list[tuple], replace_exceptions: bool, replace_exceptions_value: str | None, checklist_questions: list[list]) -> list[list[dict]]: """Post-process graph agent results into a uniform list of output dicts. Replaces exceptions with placeholder values if replace_exceptions is True. + Each entry in *results* is a ``(raw_results, questions)`` tuple returned by + ``_process_steps``, where *questions* contains the actual question text + (including any synthetic questions that were injected). """ outputs = [[] for _ in range(len(results))] - for i, answer_list in enumerate(results): + for i, result_entry in enumerate(results): + if isinstance(result_entry, Exception): + logger.warning("_process_steps failed entirely for group %d: %s", i, result_entry) + continue + answer_list, questions = result_entry for j, answer in enumerate(answer_list): if isinstance(answer, (ToolRaisedException, OutputParserException, Exception)): if replace_exceptions: - outputs[i].append({"input": checklist_questions[i][j], "output": replace_exceptions_value, + question_text = questions[j] if j < len(questions) else checklist_questions[i][j] + outputs[i].append({"input": question_text, "output": replace_exceptions_value, "intermediate_steps": None, "cca_results": [], "package_validated": None}) if isinstance(answer, ToolRaisedException): diff --git a/src/vuln_analysis/functions/reachability_agent.py b/src/vuln_analysis/functions/reachability_agent.py index 1801e82d3..6bda95f9f 100644 --- a/src/vuln_analysis/functions/reachability_agent.py +++ b/src/vuln_analysis/functions/reachability_agent.py @@ -35,55 +35,17 @@ from vuln_analysis.runtime_context import ctx_state from vuln_analysis.tools.tool_names import ToolNames from vuln_analysis.tools.transitive_code_search import package_name_from_locator_query -from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_from_osv +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_candidates from vuln_analysis.utils.prompting import build_tool_descriptions from vuln_analysis.utils.prompt_factory import ( TOOL_SELECTION_STRATEGY, TOOL_SELECTION_STRATEGY_NON_REACHABILITY, FEW_SHOT_EXAMPLES, ) -from pathlib import Path -from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path -from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY - logger = LoggingFactory.get_agent_logger(__name__) AGENT_TRACER = Context.get() -def _validate_go_vendor_packages(source_info, candidate_packages): - code_si = next((si for si in source_info if si.type == "code"), None) - if code_si is None: - return candidate_packages, [] - repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) - vendor_path = repo_path / "vendor" - if not vendor_path.is_dir(): - return candidate_packages, [] - validated = [] - removed = [] - for pkg in candidate_packages: - pkg_name = pkg.get("name", "") - if (vendor_path / pkg_name).is_dir(): - validated.append(pkg) - else: - removed.append(pkg_name) - if validated: - return validated, removed - return candidate_packages, [] - - -async def _enrich_go_candidates(cve_intel, source_info, critical_context, candidate_packages, vulnerable_functions_set): - ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) - if not ghsa_has_packages or not vulnerable_functions_set: - intel = cve_intel[0] if cve_intel else None - if intel: - await enrich_go_from_osv(intel, critical_context, candidate_packages, vulnerable_functions_set) - if candidate_packages: - candidate_packages, removed_pkgs = _validate_go_vendor_packages(source_info, candidate_packages) - if removed_pkgs: - logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) - return candidate_packages, sorted(vulnerable_functions_set) - - @register_agent("reachability") class ReachabilityAgent(BaseGraphAgent): @@ -166,8 +128,9 @@ async def pre_process_node(self, state: AgentState) -> AgentState: critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) vulnerable_functions_set = set(vulnerable_functions) - if ecosystem == "go": - candidate_packages, vulnerable_functions = await _enrich_go_candidates( + already_enriched = any("Go vuln DB hint" in c for c in critical_context) + if ecosystem == "go" and not already_enriched: + candidate_packages, vulnerable_functions = await enrich_go_candidates( workflow_state.cve_intel, workflow_state.original_input.input.image.source_info, critical_context, diff --git a/src/vuln_analysis/tools/import_usage_analyzer.py b/src/vuln_analysis/tools/import_usage_analyzer.py index 755c82f2a..f5be30aa0 100644 --- a/src/vuln_analysis/tools/import_usage_analyzer.py +++ b/src/vuln_analysis/tools/import_usage_analyzer.py @@ -15,6 +15,7 @@ import re +import tantivy from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo @@ -38,7 +39,11 @@ def _find_usage_in_file(content: str, imported_names: list[str], max_usages: int lines = content.split("\n") for line_num, line in enumerate(lines): for name in imported_names: - short_name = name.rsplit(".", 1)[-1] if "." in name else name + short_name = name + if "/" in short_name: + short_name = short_name.rsplit("/", 1)[-1] + if "." in short_name: + short_name = short_name.rsplit(".", 1)[-1] if re.search(rf'\b{re.escape(short_name)}\b', line) and not line.strip().startswith(("import ", "from ", "#include")): usages.append(f" L{line_num+1}: {line.strip()}") if len(usages) >= max_usages: @@ -60,15 +65,26 @@ def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: s with dependency results filtered by source_scope when provided. """ num_docs = searcher.num_docs + logger.debug("analyze_imports: scanning %d docs for '%s' (patterns: %d)", num_docs, package_name, len(import_patterns)) app_results = [] dep_results = [] + docs_with_pkg_name = 0 + docs_with_regex_match = 0 + docs_with_fallback_match = 0 + doc_errors = 0 - for doc_id in range(num_docs): + all_docs = searcher.search(tantivy.Query.all_query(), limit=num_docs).hits + + logged_samples = 0 + for _score, doc_address in all_docs: try: - raw = searcher.doc(doc_id) + raw = searcher.doc(doc_address) file_path = raw["file_path"][0] content = raw["content"][0] - except Exception: + except Exception as e: + doc_errors += 1 + if doc_errors <= 3: + logger.debug("analyze_imports: doc_address=%s error: %s %s", doc_address, type(e).__name__, e) continue found_imports = [] @@ -76,8 +92,37 @@ def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: s for match in pattern.finditer(content): found_imports.append(match.group(0)) + pkg_lower = package_name.lower() + content_has_pkg = pkg_lower in content.lower() + + if found_imports: + docs_with_regex_match += 1 + if logged_samples < 5: + logger.debug("analyze_imports: REGEX MATCH file=%s imports=%s", + file_path, found_imports[:3]) + logged_samples += 1 + elif content_has_pkg: + docs_with_pkg_name += 1 + if logged_samples < 5: + import_lines = [line.strip() for line in content.split('\n') + if pkg_lower in line.lower()] + logger.debug("analyze_imports: PKG IN CONTENT but no regex match " + "file=%s lines_with_pkg=%s", + file_path, import_lines[:3]) + logged_samples += 1 + if not found_imports: - continue + # Fallback: regex may miss imports in chunked documents where the + # import keyword is in a different chunk than the package path. + if not content_has_pkg: + continue + for line in content.split('\n'): + stripped = line.strip() + if pkg_lower in stripped.lower() and len(stripped) > len(package_name): + found_imports.append(stripped) + if not found_imports: + continue + docs_with_fallback_match += 1 imported_names = [] for imp in found_imports: @@ -101,8 +146,17 @@ def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: s else: app_results.append(entry) + logger.debug("analyze_imports: docs_scanned=%d, regex_matches=%d, fallback_matches=%d, " + "pkg_name_present=%d, errors=%d, app_results=%d, dep_results=%d", + num_docs, docs_with_regex_match, docs_with_fallback_match, + docs_with_pkg_name, doc_errors, len(app_results), len(dep_results)) + + pre_filter_count = len(dep_results) dep_results = filter_by_source_scope(dep_results, source_scope, lambda x: x[0]) dep_entries = [entry for _, entry in dep_results] + if pre_filter_count != len(dep_entries): + logger.debug("analyze_imports: source_scope filter reduced dep_results from %d to %d (scope=%s)", + pre_filter_count, len(dep_entries), source_scope) no_results_msg = f"No imports of '{package_name}' found in indexed sources (ecosystem: {ecosystem_label})." total_app = len(app_results) @@ -136,11 +190,17 @@ async def _arun(query: str) -> str: parser = get_language_function_parser(ecosystem, tree=None) if ecosystem else None import_patterns = parser.get_import_search_patterns(package_name) if parser else [re.compile(re.escape(package_name), re.IGNORECASE)] ecosystem_label = ecosystem.value if ecosystem else "" - logger.debug("Import usage analyzer: searching for '%s' (ecosystem: %s), scope=%s", - package_name, ecosystem_label, source_scope) - + logger.debug("Import usage analyzer: searching for '%s' (ecosystem: %s), scope=%s, " + "parser=%s, patterns=%s", + package_name, ecosystem_label, source_scope, + type(parser).__name__ if parser else "None", + [p.pattern for p in import_patterns]) + + fts.index.reload() + searcher = fts.index.searcher() + logger.debug("Import usage analyzer: index has %d docs after reload", searcher.num_docs) result = analyze_imports( - fts.index.searcher(), import_patterns, package_name, + searcher, import_patterns, package_name, max_files=config.max_files, source_scope=source_scope, ecosystem_label=ecosystem_label, ) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 65037fe5f..b638d2498 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -31,7 +31,8 @@ async def test_transitive_search_golang_1(): result = await transitive_code_search_runner_coroutine("crypto/x509,ParsePKCS1PrivateKey") (path_found, list_path) = result assert path_found is True - assert len(list_path) == 3 + assert len(list_path) > 1 + assert not list_path[-1].startswith("vendor/") @pytest.mark.asyncio @@ -1077,25 +1078,92 @@ def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" -@pytest.mark.parametrize("raw_query, expected_cleaned", [ - # Standard ASCII quotes - ("'commons-beanutils:commons-beanutils:1.9.4'", "commons-beanutils:commons-beanutils:1.9.4"), - ('"commons-beanutils:commons-beanutils:1.9.4"', "commons-beanutils:commons-beanutils:1.9.4"), - # Unicode smart quotes (left/right single) - ("\u2018commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), - # Unicode smart quotes (left/right double) - ("\u201ccommons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), - # Mixed: ASCII left, unicode right - ("'commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), - ("\"commons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), - # No quotes - ("commons-beanutils:commons-beanutils:1.9.4", "commons-beanutils:commons-beanutils:1.9.4"), - # Whitespace + quotes - (" 'commons-beanutils:commons-beanutils:1.9.4' ", "commons-beanutils:commons-beanutils:1.9.4"), - # Trailing newline junk from LLM - ("'commons-beanutils:commons-beanutils:1.9.4'\nPlease wait...", "commons-beanutils:commons-beanutils:1.9.4"), -]) -def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): - """Test that query cleaning handles both ASCII and Unicode smart quotes.""" - cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() - assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" +# --------------------------------------------------------------------------- +# Go CCA sub-package disambiguation tests +# +# These reproduce false-positive chains where CCA matches a function in +# the wrong Go sub-package (e.g. url.Parse instead of strvals.Parse). +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_go_cca_jwt_parse_not_reachable_in_hypershift(): + """CVE-2025-30204: golang-jwt/jwt ParseWithClaims is NOT reachable + in hypershift — jwt is only used for token signing (NewWithClaims), + never for parsing incoming tokens.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/hypershift", + git_ref="ffe47c7d9ded2a528862acbb8dd0a0f105dd801c", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("github.com/golang-jwt/jwt/v4,ParseWithClaims") + (path_found, list_path) = result + assert path_found is False, ( + f"ParseWithClaims should NOT be reachable in hypershift — " + f"jwt is only used for signing, not parsing. Path: {list_path}" + ) + + +@pytest.mark.asyncio +async def test_go_cca_strvals_parse_not_reachable_in_oc_mirror(): + """CVE-2022-23524: helm.sh/helm/v3/pkg/strvals.Parse is NOT reachable + in oc-mirror. CCA must not follow url.Parse or other same-named + functions from different packages.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/oc-mirror", + git_ref="0a2b069f4859606f9ac85e8d15761fdfb259cf55", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("helm.sh/helm/v3/pkg/strvals,Parse") + (path_found, list_path) = result + assert path_found is False, ( + f"strvals.Parse should NOT be reachable in oc-mirror — " + f"CCA likely followed url.Parse or another same-named function. " + f"Path: {list_path}" + ) + + +@pytest.mark.asyncio +async def test_go_cca_yaml_unmarshal_in_coredns(): + """CVE-2021-4235: gopkg.in/yaml.v2.Unmarshal in coredns. + + coredns imports sigs.k8s.io/yaml which wraps gopkg.in/yaml.v2, but + the CCA dep tree may not connect them. Before the sub-package + disambiguation fix this chain was found via a buggy same-package + shortcut (any "package yaml" file was treated as the callee). + After the fix CCA correctly requires import validation, so the + indirect chain through sigs.k8s.io/yaml may not resolve.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/coredns", + git_ref="e4c9977928b7f36962351b387fafff474b238311", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("gopkg.in/yaml.v2,Unmarshal") + (path_found, list_path) = result + if path_found: + assert len(list_path) > 1 + + +@pytest.mark.asyncio +async def test_go_cca_yaml_unmarshal_not_reachable_in_egress_router(): + """CVE-2022-3064: gopkg.in/yaml.v2.Unmarshal should NOT be reachable + in egress-router-cni — app code never imports yaml.v2, only JSON.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/egress-router-cni", + git_ref="a92e415791b531ca15ec84953550b71bd3534566", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("gopkg.in/yaml.v2,Unmarshal") + (path_found, list_path) = result + if path_found: + last_in_chain = list_path[-1] if list_path else "" + assert not last_in_chain.startswith("vendor/"), ( + f"yaml.Unmarshal chain should reach app code, not stop in vendor/. Path: {list_path}" + ) diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 6fa0e8e55..9ace44b9e 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -191,6 +191,22 @@ def search_index(self, query: str, top_k: int = 10, source_scope: list[str] | No else: app_docs.append(doc) + if not app_docs and dep_docs: + seen_sources = {d["source"] for d in dep_docs} + wider = searcher.search(query, limit=500).hits + for _, doc_id in wider: + raw = searcher.doc(doc_id) + src = raw["file_path"][0] + if src in seen_sources: + continue + doc = {"source": src, "content": raw["content"][0]} + if is_dependency_path(src): + dep_docs.append(doc) + else: + app_docs.append(doc) + if len(app_docs) >= top_k: + break + pre_filter_count = len(dep_docs) dep_docs = filter_by_source_scope(dep_docs, source_scope, lambda d: d["source"]) if pre_filter_count != len(dep_docs): diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index d8dee99cd..f7e6175ca 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re +from pathlib import Path from packaging.version import InvalidVersion from packaging.version import parse as parse_version @@ -21,6 +23,8 @@ from pydpkg.exceptions import DpkgVersionError from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -28,6 +32,49 @@ _MAX_RHSA_CANDIDATES = 20 +_GO_VULN_RE = re.compile(r"pkg\.go\.dev/vuln/(GO-\d{4}-\d+)") +_OSV_API_URL = "https://api.osv.dev/v1/vulns/" +_OSV_TIMEOUT_SECONDS = 5 + +_GITHUB_COMMIT_RE = re.compile(r"github\.com/([^/]+)/([^/]+)/commit/([0-9a-f]{7,40})") +_GITHUB_API_TIMEOUT = 10 +_PATCH_MAX_COMMITS = 3 + +_FUNC_PATTERNS = { + "go": re.compile(r"func\s+(?:\([^)]*\)\s+)?(\w+)\s*\("), + "python": re.compile(r"def\s+(\w+)\s*\("), + "java": re.compile( + r"(?:public|private|protected|static|final|abstract|synchronized|native" + r"|void|int|long|float|double|boolean|char|byte|short|String|Object)" + r"\s+(\w+)\s*\(" + ), + "javascript": re.compile( + r"(?:function\s+(\w+)\s*\(" + r"|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function|\([^)]*\)\s*=>|\w+\s*=>)" + r"|(\w+)\s*\([^)]*\)\s*\{)" + ), + "c": re.compile(r"(?:^|\s)(\w+)\s*\([^)]*\)\s*\{"), +} + +_NOISE_FUNC_NAMES = frozenset({ + "main", "init", "test", "setup", "teardown", + "if", "else", "for", "while", "do", "switch", "return", "sizeof", "typeof", +}) + +_JAVA_TEST_RE = re.compile(r"^test[A-Z]") + +_TEST_FILE_RE = re.compile( + r"(?:_test\.go" + r"|Test(?:s|Case)?\.java" + r"|(?:^|/)test_[^/]*\.py" + r"|_test\.py" + r"|\.(?:test|spec)\.(?:js|ts|jsx|tsx)" + r"|_test\.(?:c|cpp|cc|cxx|h)" + r"|(?:^|/)test(?:s)?/)" +) + +_PACKAGE_TOKEN_CHARS = r"A-Za-z0-9_.\/:@+\-" + def update_version(incoming_version, current_version, compare): """ @@ -266,23 +313,18 @@ def build_critical_context(cve_intel_list) -> tuple[list[str], list[dict], list[ return critical_context, candidate_packages, sorted(vulnerable_functions) -_GO_VULN_RE = re.compile(r"pkg\.go\.dev/vuln/(GO-\d{4}-\d+)") -_OSV_API_URL = "https://api.osv.dev/v1/vulns/" -_OSV_TIMEOUT_SECONDS = 5 - - async def enrich_go_from_osv( cve_intel, critical_context: list[str], - candidate_packages: list[dict], - vulnerable_functions: set[str] | None = None, ) -> None: """Query the OSV API for Go module paths when GHSA has no package data. Looks for a pkg.go.dev/vuln/GO-XXXX-XXXX link in GHSA or NVD - references, fetches the advisory from OSV, and injects module - paths and vulnerable symbols into critical_context and - candidate_packages. Fails silently on any error. + references, fetches the advisory from OSV, and adds module paths + and vulnerable symbols as hints into critical_context only. + OSV data is lower confidence than GHSA advisory data and should + not be added to candidate_packages or vulnerable_functions. + Fails silently on any error. """ refs: list[str] = [] if cve_intel.ghsa is not None: @@ -318,24 +360,17 @@ async def enrich_go_from_osv( path = imp.get("path", "") if path and path not in seen_paths: seen_paths.add(path) - candidate_packages.append( - {"name": path, "source": "osv", "ecosystem": "Go"}, - ) - critical_context.append(f"Vulnerable module (Go): {path}") + critical_context.append(f"Vulnerable module (Go vuln DB hint): {path}") symbols = imp.get("symbols", []) all_symbols.extend(symbols) if all_symbols: critical_context.append( - f"Vulnerable functions (Go vuln DB): {', '.join(all_symbols)}" + f"Vulnerable functions (Go vuln DB hint): {', '.join(all_symbols)}" ) short_names = [s.rsplit(".", 1)[-1] for s in all_symbols if "." in s] unique_keywords = list(dict.fromkeys(all_symbols + short_names)) critical_context.append(f"Search keywords: {', '.join(unique_keywords)}") - if vulnerable_functions is not None: - for s in all_symbols: - vulnerable_functions.add(s.rsplit(".", 1)[-1]) - vulnerable_functions.add(s) if seen_paths: logger.info("OSV enrichment for %s added Go modules: %s", go_id, seen_paths) @@ -343,9 +378,208 @@ async def enrich_go_from_osv( logger.warning("OSV enrichment failed for %s", go_id, exc_info=True) -# Characters treated as part of a package / module / coordinate token for boundary -# detection (avoids stripping short names like "com" from inside "github.com"). -_PACKAGE_TOKEN_CHARS = r"A-Za-z0-9_.\/:@+\-" +def validate_go_vendor_packages(source_info, candidate_packages): + """Filter candidate packages to those present in the vendor/ directory. + + Returns (validated_packages, removed_package_names). If no packages + survive validation the original list is returned unchanged. + """ + code_si = next((si for si in source_info if si.type == "code"), None) + if code_si is None: + return candidate_packages, [] + repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) + vendor_path = repo_path / "vendor" + if not vendor_path.is_dir(): + return candidate_packages, [] + validated = [] + removed = [] + for pkg in candidate_packages: + pkg_name = pkg.get("name", "") + if (vendor_path / pkg_name).is_dir(): + validated.append(pkg) + else: + removed.append(pkg_name) + if validated: + return validated, removed + return candidate_packages, [] + + +async def enrich_go_candidates(cve_intel, source_info, critical_context, candidate_packages, vulnerable_functions_set): + """Enrich Go candidate packages with OSV data and validate against vendor/. + + Calls ``enrich_go_from_osv`` when GHSA lacks package data or vulnerable + function symbols, then prunes candidates not found in vendor/. + + Returns (candidate_packages, sorted_vulnerable_functions). + """ + ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) + if not ghsa_has_packages or not vulnerable_functions_set: + intel = cve_intel[0] if cve_intel else None + if intel: + await enrich_go_from_osv(intel, critical_context) + if candidate_packages: + candidate_packages, removed_pkgs = validate_go_vendor_packages(source_info, candidate_packages) + if removed_pkgs: + logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) + return candidate_packages, sorted(vulnerable_functions_set) + + +def _ref_to_url(ref) -> str: + """Extract a URL string from a reference (GHSA dict or NVD string).""" + if isinstance(ref, str): + return ref + if isinstance(ref, dict): + return ref.get("url", "") + return "" + + +def _is_fix_ref(ref) -> bool: + """Return True if a GHSA reference is tagged as a fix.""" + return isinstance(ref, dict) and ref.get("type", "").upper() == "FIX" + + +def _extract_functions_from_patch(patch_text: str, ecosystem: str) -> set[str]: + """Extract function names from a unified diff patch using hunk headers and modified lines.""" + funcs: set[str] = set() + eco_lower = ecosystem.lower() if ecosystem else "" + + patterns: list[re.Pattern] = [] + if eco_lower in ("go", "golang"): + patterns = [_FUNC_PATTERNS["go"]] + elif eco_lower in ("pypi", "python", "pip"): + patterns = [_FUNC_PATTERNS["python"]] + elif eco_lower in ("npm", "javascript", "node"): + patterns = [_FUNC_PATTERNS["javascript"]] + elif eco_lower in ("maven", "java"): + patterns = [_FUNC_PATTERNS["java"]] + elif eco_lower in ("conan", "c", "cpp", "c++"): + patterns = [_FUNC_PATTERNS["c"]] + else: + patterns = list(_FUNC_PATTERNS.values()) + + for line in patch_text.splitlines(): + context = None + if line.startswith("@@"): + parts = line.split("@@") + if len(parts) >= 3: + context = parts[2].strip() + elif line.startswith("+") and not line.startswith("+++"): + context = line[1:] + + if context is None: + continue + + for pat in patterns: + m = pat.search(context) + if m: + name = next((g for g in m.groups() if g), None) + if (name and len(name) > 1 + and name.lower() not in _NOISE_FUNC_NAMES + and not name.startswith("Test") + and not name.startswith("test_") + and not _JAVA_TEST_RE.match(name)): + funcs.add(name) + break + + return funcs + + +async def enrich_vulnerable_functions_from_patch( + cve_intel_list, + critical_context: list[str], + vulnerable_functions: set[str], + ecosystem: str = "", +) -> None: + """Extract vulnerable function names from remediation patch commits. + + When GHSA/OSV have no vulnerable_functions, looks for GitHub commit URLs + in references, fetches the diff, and extracts function names from + hunk headers and modified function definitions. Test files are + skipped entirely to avoid injecting test-only names. + """ + if vulnerable_functions: + return + + refs: list = [] + fix_refs: list = [] + for cve_intel in cve_intel_list: + if cve_intel.ghsa is not None: + ghsa_refs = getattr(cve_intel.ghsa, "references", None) or [] + for r in ghsa_refs: + if _is_fix_ref(r): + fix_refs.append(r) + else: + refs.append(r) + if cve_intel.nvd is not None: + refs.extend(cve_intel.nvd.references or []) + + all_refs = fix_refs + refs + + commits: list[tuple[str, str, str]] = [] + seen: set[tuple[str, str, str]] = set() + for ref in all_refs: + url = _ref_to_url(ref) + m = _GITHUB_COMMIT_RE.search(url) + if m: + key = (m.group(1), m.group(2), m.group(3)) + if key not in seen: + seen.add(key) + commits.append(key) + if len(commits) >= _PATCH_MAX_COMMITS: + break + + if not commits: + return + + ghsa_api_key = os.environ.get("GHSA_API_KEY") + headers = {"Accept": "application/vnd.github+json"} + if ghsa_api_key: + headers["Authorization"] = f"Bearer {ghsa_api_key}" + else: + logger.warning("GHSA_API_KEY not set — GitHub API calls use unauthenticated rate limit (60 req/hr)") + + try: + import aiohttp + timeout = aiohttp.ClientTimeout(total=_GITHUB_API_TIMEOUT) + async with aiohttp.ClientSession(timeout=timeout) as session: + for owner, repo, sha in commits: + url = f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}" + async with session.get(url, headers=headers) as resp: + if resp.status != 200: + logger.warning( + "GitHub commit fetch for %s/%s/%s returned %d", + owner, repo, sha[:8], resp.status, + ) + continue + commit_data = await resp.json() + + all_funcs: set[str] = set() + for file_info in commit_data.get("files", []): + filename = file_info.get("filename", "") + if _TEST_FILE_RE.search(filename): + continue + patch = file_info.get("patch", "") + if patch: + all_funcs |= _extract_functions_from_patch(patch, ecosystem) + + if not all_funcs: + logger.info("Patch commit %s/%s@%s parsed but no functions extracted", owner, repo, sha[:8]) + + if all_funcs: + critical_context.append( + f"Vulnerable functions (remediation patch hint): {', '.join(sorted(all_funcs))}" + ) + short_names = [f.rsplit(".", 1)[-1] for f in all_funcs if "." in f] + if short_names: + unique = list(dict.fromkeys(list(all_funcs) + short_names)) + critical_context.append(f"Search keywords: {', '.join(unique)}") + logger.info( + "Patch enrichment extracted functions from %s/%s@%s: %s", + owner, repo, sha[:8], sorted(all_funcs), + ) + return + except Exception: + logger.warning("Patch-based function extraction failed", exc_info=True) def _package_token_boundary_pattern(rn: str) -> str: diff --git a/tests/test_import_usage_analyzer.py b/tests/test_import_usage_analyzer.py index 98d770013..247408138 100644 --- a/tests/test_import_usage_analyzer.py +++ b/tests/test_import_usage_analyzer.py @@ -28,6 +28,68 @@ def _get_patterns(ecosystem: Ecosystem, package_name: str) -> list[re.Pattern]: return parser.get_import_search_patterns(package_name) +class TestGoImportSearchPatterns: + """Go-specific pattern tests: single-line, grouped, aliased, dot, blank imports.""" + + def test_single_line_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + assert any(p.search('import "github.com/quic-go/quic-go"') for p in patterns) + + def test_grouped_import_all_matches(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"github.com/quic-go/quic-go"\n\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert "github.com/quic-go/quic-go" in matches + assert "github.com/quic-go/quic-go/http3" in matches + assert "github.com/quic-go/quic-go/qlog" in matches + + def test_sub_packages_only(self): + """File imports only sub-packages, not the root module.""" + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert len(matches) == 2 + assert "github.com/quic-go/quic-go/http3" in matches + assert "github.com/quic-go/quic-go/qlog" in matches + + def test_aliased_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\tquic "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_blank_identifier_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t_ "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_dot_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t. "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_no_false_positive_on_string_literal(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\tfmt.Println("using github.com/quic-go/quic-go")' + assert not any(p.search(content) for p in patterns) + + def test_no_false_positive_on_comment(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t// See github.com/quic-go/quic-go for details' + assert not any(p.search(content) for p in patterns) + + def test_unrelated_import_ignored(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"go.uber.org/zap"\n\t"net/http"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert len(matches) == 0 + + class TestGetImportSearchPatterns: def test_python_import_pattern(self): patterns = _get_patterns(Ecosystem.PYTHON, "xml.etree") @@ -149,6 +211,35 @@ def test_dotted_name_uses_short_component(self): assert "L2:" in usages[0] assert "XStream xs" in usages[0] + def test_go_path_short_name_extraction(self): + """Go import paths use / separator — short name should be last path segment.""" + content = """import "github.com/quic-go/quic-go/http3" +server := http3.Server{} +""" + usages = _find_usage_in_file(content, ["github.com/quic-go/quic-go/http3"]) + + assert len(usages) == 1 + assert "http3.Server" in usages[0] + + def test_go_path_encoding_xml(self): + content = """import "encoding/xml" +data, err := xml.Marshal(v) +""" + usages = _find_usage_in_file(content, ["encoding/xml"]) + + assert len(usages) == 1 + assert "xml.Marshal" in usages[0] + + def test_java_dotted_still_works(self): + """Java dotted package names still extract correct short name.""" + content = """import com.thoughtworks.xstream.XStream; +XStream xs = new XStream(); +""" + usages = _find_usage_in_file(content, ["com.thoughtworks.xstream.XStream"]) + + assert len(usages) == 1 + assert "XStream" in usages[0] + def test_no_usages_returns_empty(self): content = """import xml.etree.ElementTree from xml.etree import ElementTree @@ -197,15 +288,21 @@ def test_strips_line_content(self): assert usages[0].count(" " * 8) == 0 +class _MockSearchResult: + def __init__(self, hits): + self.hits = hits + + class _MockSearcher: def __init__(self, docs): self.docs = docs self.num_docs = len(docs) - def doc(self, doc_id): - if doc_id >= len(self.docs): - raise IndexError(f"doc_id {doc_id} out of range") - return self.docs[doc_id] + def doc(self, doc_address): + return self.docs[doc_address] + + def search(self, query, limit=10, count=True): + return _MockSearchResult([(0.0, i) for i in range(min(limit, len(self.docs)))]) def _import_searcher(*docs): @@ -267,6 +364,38 @@ def test_app_dep_headers_in_output(self): assert "Main application" in result assert "Application library dependencies" in result + def test_content_fallback_on_chunked_doc(self): + """When a chunked doc has the package name but no import keyword, fallback triggers.""" + docs = [ + {"file_path": ["src/listeners.go"], + "content": ['\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)\n\nfunc Listen() {}']}, + ] + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + result = analyze_imports(_import_searcher(*docs), patterns, "github.com/quic-go/quic-go") + assert "listeners.go" in result + assert "No imports" not in result + + def test_content_fallback_skips_unrelated_doc(self): + """Fallback doesn't match docs that don't contain the package name.""" + docs = [ + {"file_path": ["src/main.go"], + "content": ["package main\nfunc main() { fmt.Println() }"]}, + ] + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + result = analyze_imports(_import_searcher(*docs), patterns, "github.com/quic-go/quic-go", + ecosystem_label="go") + assert "No imports" in result + + def test_content_fallback_java_chunked(self): + """Java chunked doc with import line split from keyword.""" + docs = [ + {"file_path": ["src/Converter.java"], + "content": ["com.thoughtworks.xstream.XStream;\nXStream xs = new XStream();"]}, + ] + patterns = _get_patterns(Ecosystem.JAVA, "com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Converter.java" in result + def test_only_dep_imports(self): docs = [ {"file_path": ["dependencies-sources/lib/Foo.java"], diff --git a/tests/test_intel_utils.py b/tests/test_intel_utils.py index 831cea727..a6a3d2f61 100644 --- a/tests/test_intel_utils.py +++ b/tests/test_intel_utils.py @@ -1,10 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for intel_utils: build_critical_context RHSA candidate capping.""" +"""Tests for intel_utils: build_critical_context RHSA candidate capping and patch enrichment.""" + +import pytest from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelRhsa -from vuln_analysis.utils.intel_utils import build_critical_context, _MAX_RHSA_CANDIDATES +from vuln_analysis.utils.intel_utils import ( + build_critical_context, + _MAX_RHSA_CANDIDATES, + _extract_functions_from_patch, + enrich_vulnerable_functions_from_patch, +) class TestBuildCriticalContextRhsaCap: @@ -66,4 +73,240 @@ def test_context_note_still_shows_total_count(self): affected_notes = [c for c in context if "Affected across" in c] assert len(affected_notes) == 1 - assert "50" in affected_notes[0] \ No newline at end of file + assert "50" in affected_notes[0] + + +class TestExtractFunctionsFromPatchNoiseFilter: + """Tests for test function filtering in _extract_functions_from_patch.""" + + JAVA_PATCH_WITH_TEST_METHODS = """\ +@@ -10,6 +10,30 @@ public class BeanUtilsTestCase { ++ public void testAllowAccessToClassPropertyFromBeanUtilsBean() { ++ // test body ++ } ++ public void testSuppressClassPropertyByDefaultFromPropertyUtilsBean() { ++ // test body ++ } ++ public String getName() { ++ return this.name; ++ } ++ public void setName(String name) { ++ this.name = name; ++ } +""" + + def test_java_testFoo_methods_filtered(self): + """Java testFoo naming convention (lowercase test + uppercase letter) must be filtered.""" + funcs = _extract_functions_from_patch(self.JAVA_PATCH_WITH_TEST_METHODS, "java") + assert "testAllowAccessToClassPropertyFromBeanUtilsBean" not in funcs + assert "testSuppressClassPropertyByDefaultFromPropertyUtilsBean" not in funcs + + def test_java_real_methods_kept(self): + """Real methods (getName, setName) must not be filtered.""" + funcs = _extract_functions_from_patch(self.JAVA_PATCH_WITH_TEST_METHODS, "java") + assert "getName" in funcs + assert "setName" in funcs + + def test_go_Test_prefix_filtered(self): + """Go Test prefix (capital T) is filtered.""" + patch = """\ +@@ -5,0 +5,3 @@ ++func TestDoSAttack(t *testing.T) { ++func convert(input string) string { +""" + funcs = _extract_functions_from_patch(patch, "go") + assert "TestDoSAttack" not in funcs + assert "convert" in funcs + + def test_python_test_underscore_filtered(self): + """Python test_ prefix is filtered.""" + patch = """\ +@@ -1,0 +1,4 @@ ++def test_parse_input(): ++def parse_input(data): +""" + funcs = _extract_functions_from_patch(patch, "python") + assert "test_parse_input" not in funcs + assert "parse_input" in funcs + + def test_java_testDoS_variants_filtered(self): + """All testXxx variants are filtered regardless of suffix.""" + patch = """\ +@@ -1,0 +1,6 @@ ++ public void testCannotInjectEventHandler() { ++ public void testCannotUseJaxwsInputStreamToDeleteFile() { ++ public void testDoSAttackWithCollections() { ++ public void unmarshal() { +""" + funcs = _extract_functions_from_patch(patch, "java") + assert "testCannotInjectEventHandler" not in funcs + assert "testCannotUseJaxwsInputStreamToDeleteFile" not in funcs + assert "testDoSAttackWithCollections" not in funcs + assert "unmarshal" in funcs + + def test_noise_words_filtered(self): + """Built-in noise words (main, init, setup, etc.) are filtered.""" + patch = """\ +@@ -1,0 +1,4 @@ ++func main() { ++func init() { ++func Setup() { ++func realFunc() { +""" + funcs = _extract_functions_from_patch(patch, "go") + assert "main" not in funcs + assert "init" not in funcs + assert "realFunc" in funcs + + +class TestEnrichVulnerableFunctionsFromPatch: + """Patch-extracted functions populate vulnerable_functions; test files are skipped.""" + + def _make_mock_intel(self, commit_url): + from unittest.mock import MagicMock + mock_ghsa = MagicMock() + mock_ghsa.references = [{"type": "FIX", "url": commit_url}] + mock_cve_intel = MagicMock() + mock_cve_intel.ghsa = mock_ghsa + mock_cve_intel.nvd = None + return mock_cve_intel + + def _make_mock_session(self, commit_response): + from unittest.mock import AsyncMock, MagicMock + mock_resp = AsyncMock() + mock_resp.status = 200 + mock_resp.json = AsyncMock(return_value=commit_response) + mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) + mock_resp.__aexit__ = AsyncMock(return_value=False) + + mock_session = AsyncMock() + mock_session.get = MagicMock(return_value=mock_resp) + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=False) + return MagicMock(return_value=mock_session) + + @pytest.mark.asyncio + async def test_source_functions_added_to_critical_context_as_hints(self): + """Functions from non-test source files are added to critical_context as hints.""" + from unittest.mock import patch + import aiohttp as real_aiohttp + + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + mock_cve_intel = self._make_mock_intel( + "https://github.com/apache/commons-beanutils/commit/bd20740d" + ) + commit_response = { + "files": [{ + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "patch": """\ +@@ -10,6 +10,10 @@ public class PropertyUtilsBean { ++ public Object getProperty(Object bean, String name) { ++ return null; ++ } +""" + }] + } + mock_client_session = self._make_mock_session(commit_response) + + with patch.object(real_aiohttp, "ClientSession", mock_client_session): + await enrich_vulnerable_functions_from_patch( + [mock_cve_intel], critical_context, vulnerable_functions, "java", + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "getProperty" in patch_context[0] + + @pytest.mark.asyncio + async def test_test_files_skipped(self): + """Files matching test patterns are skipped entirely.""" + from unittest.mock import patch + import aiohttp as real_aiohttp + + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + mock_cve_intel = self._make_mock_intel( + "https://github.com/apache/commons-beanutils/commit/bd20740d" + ) + commit_response = { + "files": [ + { + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "patch": """\ +@@ -10,6 +10,10 @@ public class PropertyUtilsBean { ++ public Object getProperty(Object bean, String name) { ++ return null; ++ } +""" + }, + { + "filename": "src/test/java/org/apache/commons/beanutils/PropertyUtilsBeanTestCase.java", + "patch": """\ +@@ -10,6 +10,10 @@ public class PropertyUtilsBeanTestCase { ++ public void testAllowAccessToClassProperty() { ++ } ++ public void testSuppressClassProperty() { ++ } +""" + }, + ] + } + mock_client_session = self._make_mock_session(commit_response) + + with patch.object(real_aiohttp, "ClientSession", mock_client_session): + await enrich_vulnerable_functions_from_patch( + [mock_cve_intel], critical_context, vulnerable_functions, "java", + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "getProperty" in patch_context[0] + assert "testAllowAccessToClassProperty" not in patch_context[0] + assert "testSuppressClassProperty" not in patch_context[0] + + @pytest.mark.asyncio + async def test_go_test_files_skipped(self): + """Go _test.go files are skipped.""" + from unittest.mock import patch + import aiohttp as real_aiohttp + + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + mock_cve_intel = self._make_mock_intel( + "https://github.com/quic-go/quic-go/commit/abc12345" + ) + commit_response = { + "files": [ + { + "filename": "internal/ackhandler/sent_packet_handler.go", + "patch": """\ +@@ -5,0 +5,3 @@ ++func detectAndRemoveAckedPackets() { +""" + }, + { + "filename": "internal/ackhandler/sent_packet_handler_test.go", + "patch": """\ +@@ -5,0 +5,3 @@ ++func TestDoSAttack(t *testing.T) { ++func BenchmarkAckHandler(b *testing.B) { +""" + }, + ] + } + mock_client_session = self._make_mock_session(commit_response) + + with patch.object(real_aiohttp, "ClientSession", mock_client_session): + await enrich_vulnerable_functions_from_patch( + [mock_cve_intel], critical_context, vulnerable_functions, "go", + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "detectAndRemoveAckedPackets" in patch_context[0] + assert "TestDoSAttack" not in patch_context[0] + assert "BenchmarkAckHandler" not in patch_context[0] \ No newline at end of file diff --git a/tests/test_process_steps.py b/tests/test_process_steps.py index 381d44922..4c0777b3c 100644 --- a/tests/test_process_steps.py +++ b/tests/test_process_steps.py @@ -36,21 +36,23 @@ def mock_graph(): @pytest.fixture def patch_externals(mock_workflow_state): - """Patch ctx_state, build_critical_context, dispatch_question, and AGENT_TRACER.""" + """Patch ctx_state, build_critical_context, dispatch_question, enrich_vulnerable_functions_from_patch, and AGENT_TRACER.""" with ( patch("vuln_analysis.functions.cve_agent.ctx_state") as mock_ctx, patch("vuln_analysis.functions.cve_agent.build_critical_context") as mock_bcc, patch("vuln_analysis.functions.cve_agent.dispatch_question") as mock_dispatch, + patch("vuln_analysis.functions.cve_agent.enrich_vulnerable_functions_from_patch", new_callable=AsyncMock) as mock_enrich, patch("vuln_analysis.functions.cve_agent.AGENT_TRACER") as mock_tracer, ): mock_ctx.get.return_value = mock_workflow_state - mock_bcc.return_value = (["ctx_line"], [{"name": "pkg"}], ["vuln_func"]) + mock_bcc.return_value = (["ctx_line"], [], ["vuln_func"]) mock_tracer.push_active_function.return_value.__enter__ = MagicMock() mock_tracer.push_active_function.return_value.__exit__ = MagicMock(return_value=False) yield { "ctx_state": mock_ctx, "build_critical_context": mock_bcc, "dispatch_question": mock_dispatch, + "enrich_patch": mock_enrich, "tracer": mock_tracer, } @@ -142,6 +144,9 @@ async def test_precomputed_intel_passed(self, patch_externals, mock_graph): patch_externals["dispatch_question"].return_value = QuestionRouting( agent_type="reachability", reason="test" ) + patch_externals["build_critical_context"].return_value = ( + ["ctx_line"], [{"name": "pkg"}], ["vuln_func"] + ) agents = {"reachability": mock_graph} await _process_steps(agents, MagicMock(), ["q"], None) @@ -186,9 +191,9 @@ async def test_multiple_steps_all_processed(self, patch_externals, mock_graph): agents = {"reachability": mock_graph} steps = ["q1", "q2", "q3"] - results = await _process_steps(agents, MagicMock(), steps, None) + raw_results, questions = await _process_steps(agents, MagicMock(), steps, None) - assert len(results) == 3 + assert len(raw_results) == 3 assert mock_graph.ainvoke.call_count == 3 @pytest.mark.asyncio @@ -252,16 +257,95 @@ async def test_exception_in_step_returned_not_raised(self, patch_externals): ) agents = {"reachability": mock_graph} - results = await _process_steps(agents, MagicMock(), ["q1"], None) + raw_results, questions = await _process_steps(agents, MagicMock(), ["q1"], None) - assert len(results) == 1 - assert isinstance(results[0], RuntimeError) + assert len(raw_results) == 1 + assert isinstance(raw_results[0], RuntimeError) @pytest.mark.asyncio async def test_empty_steps_returns_empty(self, patch_externals, mock_graph): agents = {"reachability": mock_graph} - results = await _process_steps(agents, MagicMock(), [], None) - assert results == [] + raw_results, questions = await _process_steps(agents, MagicMock(), [], None) + assert raw_results == [] + assert questions == [] + + +class TestSyntheticReachabilityQuestion: + """Synthetic reachability question is injected whenever candidate_packages is + non-empty and no checklist question routes to reachability.""" + + @pytest.mark.asyncio + async def test_synthetic_question_injected_with_vulnerable_functions(self, patch_externals, mock_graph): + """When vulnerable_functions is non-empty (advisory-sourced), synthetic question is injected.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "xstream"}], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph, "code_understanding": AsyncMock( + ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) + )} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert "reachable" in call_args["input"] + + @pytest.mark.asyncio + async def test_synthetic_question_injected_without_vulnerable_functions(self, patch_externals, mock_graph): + """When vulnerable_functions is empty but candidate_packages exists, synthetic question is still injected.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "moby"}], [] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph, "code_understanding": AsyncMock( + ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) + )} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert "moby" in call_args["input"] + assert "reachable" in call_args["input"] + + @pytest.mark.asyncio + async def test_no_synthetic_question_without_candidate_packages(self, patch_externals, mock_graph): + """When candidate_packages is empty, no synthetic question regardless of vuln functions.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + agents = {"reachability": mock_graph, "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + + mock_graph.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_no_synthetic_when_reachability_already_routed(self, patch_externals, mock_graph): + """When a checklist question already routes to reachability, no synthetic injection.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "xstream"}], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="reachability check" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is convert reachable?"], None) + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["input"] == "Is convert reachable?" class TestTracingSpan: From 24a20513c85aacff2a12016d66825bf79d4a3763 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:45:44 +0300 Subject: [PATCH 241/286] APPENG-4525:Create a tekton task for running Evaluation (#203) --- .tekton/on-cm-runner.yaml | 47 ++++++++++++++++- .tekton/on-pull-request.yaml | 43 +++++++++++++++- .tekton/post-integration-evaluation.yaml | 51 +++++++++++++++++++ .../tools/lexical_full_search.py | 1 - 4 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 .tekton/post-integration-evaluation.yaml diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 0a47b652e..2adfd29ba 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -8,7 +8,7 @@ metadata: pipelinesascode.tekton.dev/on-target-branch: "[rh-aiq-main, main]" pipelinesascode.tekton.dev/on-comment: "/test-heavy" - pipelinesascode.tekton.dev/task: "git-clone" + pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]" pipelinesascode.tekton.dev/max-keep-runs: "3" spec: timeouts: @@ -40,6 +40,34 @@ spec: - name: basic-auth - name: exploit-iq-data tasks: + # Opt-in via comment: /test-heavy eval=on (default: skip post-CM evaluation). + - name: eval-feature-gate + params: + - name: TRIGGER_COMMENT + value: $(params.trigger_comment) + taskSpec: + params: + - name: TRIGGER_COMMENT + type: string + results: + - name: enabled + type: string + steps: + - name: parse-eval-flag + image: registry.redhat.io/ubi9/ubi-micro:latest + env: + - name: TRIGGER_COMMENT + value: $(params.TRIGGER_COMMENT) + script: | + #!/bin/sh + set -eu + enabled=false + case " $TRIGGER_COMMENT " in + *" eval=on "*) enabled=true ;; + esac + echo -n "$enabled" | tee "$(results.enabled.path)" + echo "" + echo "trigger_comment='${TRIGGER_COMMENT}' -> enabled=$(cat "$(results.enabled.path)")" # ------------------------------------------------ # 1. CLONE # ------------------------------------------------ @@ -504,7 +532,7 @@ spec: --target "main" # ------------------------------------------------------- - # STEP D: Cleanup Old Releases (Keep last 5) + # STEP D: Cleanup Old Releases (Keep last 5) # ------------------------------------------------------- echo "--- Running Cleanup (Keeping last 5 releases) ---" @@ -522,6 +550,21 @@ spec: echo "--- Release Process Complete ---" + finally: + - name: post-integration-evaluation + when: + - input: $(tasks.integration-test.status) + operator: in + values: + - Succeeded + - Failed + - input: $(tasks.eval-feature-gate.results.enabled) + operator: in + values: + - "true" + taskRef: + name: post-integration-evaluation + # ------------------------------------------------ # WORKSPACE BINDINGS # ------------------------------------------------ diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 30c99ba4d..a1b085f0d 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -12,7 +12,7 @@ metadata: ("src/**".pathChanged() || "metrics_lib/**".pathChanged() || "pyproject.toml".pathChanged() || "uv.lock".pathChanged() || "Dockerfile".pathChanged() || ".dockerignore".pathChanged()) # Fetch the git-clone task from hub, we are able to reference later on it # with taskRef and it will automatically be embedded into our pipeline. - pipelinesascode.tekton.dev/task: "git-clone" + pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]" # How many runs we want to keep. pipelinesascode.tekton.dev/max-keep-runs: "5" @@ -61,6 +61,32 @@ spec: - name: basic-auth - name: exploit-iq-data tasks: + # Reads evaluation-config POST_INTEGRATION_EVAL_ENABLED (default: enabled if key absent). + # Ops: kubectl edit configmap evaluation-config -> set POST_INTEGRATION_EVAL_ENABLED: "false" + - name: eval-feature-gate + taskSpec: + results: + - name: enabled + type: string + steps: + - name: read-eval-toggle + image: registry.redhat.io/ubi9/ubi-micro:latest + env: + - name: POST_INTEGRATION_EVAL_ENABLED + valueFrom: + configMapKeyRef: + name: evaluation-config + key: POST_INTEGRATION_EVAL_ENABLED + optional: true + script: | + #!/bin/sh + set -eu + if [ "${POST_INTEGRATION_EVAL_ENABLED:-true}" = "true" ]; then + echo -n true | tee "$(results.enabled.path)" + else + echo -n false | tee "$(results.enabled.path)" + fi + echo "POST_INTEGRATION_EVAL_ENABLED=${POST_INTEGRATION_EVAL_ENABLED:-true} -> $(cat "$(results.enabled.path)")" - name: fetch-repository taskRef: resolver: cluster @@ -251,6 +277,7 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME + print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" @@ -481,6 +508,20 @@ spec: echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" + finally: + - name: post-integration-evaluation + when: + - input: $(tasks.integration-test.status) + operator: in + values: + - Succeeded + - Failed + - input: $(tasks.eval-feature-gate.results.enabled) + operator: in + values: + - "true" + taskRef: + name: post-integration-evaluation workspaces: - name: source diff --git a/.tekton/post-integration-evaluation.yaml b/.tekton/post-integration-evaluation.yaml new file mode 100644 index 000000000..e99754f2c --- /dev/null +++ b/.tekton/post-integration-evaluation.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: post-integration-evaluation +spec: + steps: + - name: report + image: quay.io/ecosystem-appeng/cve-evaluation:llama_prompt + env: + - name: EXPLOIT_IQ_API_BASE + valueFrom: + configMapKeyRef: + name: evaluation-config + key: EXPLOIT_IQ_API_BASE + - name: EXPLOIT_IQ_API_TOKEN + valueFrom: + secretKeyRef: + name: evaluation-secret + key: EXPLOIT_IQ_API_TOKEN + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: evaluation-secret + key: NGC_API_KEY + - name: JUDGE_BASE_URL + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_BASE_URL + - name: JUDGE_MODEL + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_MODEL + - name: JUDGE_TEMPERATURE + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_TEMPERATURE + - name: JUDGE_TOP_P + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_TOP_P + args: + - --mode + - api + - --stages + - all + - --submit diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 8117e990d..3fdc0dde6 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -27,7 +27,6 @@ logger = LoggingFactory.get_agent_logger(__name__) - class LexicalSearchToolConfig(FunctionBaseConfig, name=LEXICAL_CODE_SEARCH): """ Lexical search tool used to search source code. From eff8489d259deea217b32cecf56da1448d280e09 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:18:42 +0300 Subject: [PATCH 242/286] APPENG-4467- rpm analyzier first version --- Dockerfile | 7 + README.md | 1 + kustomize/base/exploit-iq-config.yml | 42 + kustomize/base/exploit_iq_service.yaml | 2 + pyproject.toml | 2 + .../data/hardening_kb/__init__.py | 14 + .../data/hardening_kb/hardening_kb.json | 395 + .../data_models/checker_status.py | 269 + src/exploit_iq_commons/data_models/common.py | 21 +- .../data_models/cve_intel.py | 9 +- src/exploit_iq_commons/data_models/info.py | 2 + src/exploit_iq_commons/data_models/input.py | 20 +- .../python_functions_parser.py | 8 +- src/exploit_iq_commons/utils/hardening_kb.py | 180 + .../utils/source_rpm_downloader.py | 38 +- .../configs/brew/external-user-profile.yml | 23 + .../configs/brew/internal-user-profile.yml | 26 + .../configs/config-http-openai.yml | 41 + .../configs/openapi/openapi.json | 7427 ++++++++++------- src/vuln_analysis/data_models/output.py | 1 + .../functions/build_agent_graph_defs.py | 692 ++ .../functions/code_agent_graph_defs.py | 1708 ++++ .../functions/cve_build_agent.py | 784 ++ .../functions/cve_checker_report.py | 1117 +++ .../functions/cve_checker_segmentation.py | 170 + .../functions/cve_http_output.py | 124 +- .../functions/cve_package_code_agent.py | 960 +++ .../functions/cve_source_acquisition.py | 242 + .../functions/react_internals.py | 88 + src/vuln_analysis/register.py | 256 +- src/vuln_analysis/tools/brew_downloader.py | 353 + .../tools/lexical_full_search.py | 17 +- src/vuln_analysis/tools/source_grep.py | 224 + src/vuln_analysis/tools/source_inspector.py | 254 + src/vuln_analysis/tools/tool_names.py | 5 + src/vuln_analysis/utils/clients/nvd_client.py | 19 +- src/vuln_analysis/utils/full_text_search.py | 2 +- src/vuln_analysis/utils/gerrit_client.py | 362 + src/vuln_analysis/utils/intel_utils.py | 81 +- .../utils/osv_patch_retriever.py | 570 ++ src/vuln_analysis/utils/output_formatter.py | 2 +- src/vuln_analysis/utils/package_identifier.py | 495 ++ .../utils/rpm_checker_prompts.py | 1671 ++++ .../test_vulnerability_intel_sanitizer.py | 98 + .../utils/tests/test_web_patch_fetcher.py | 426 + src/vuln_analysis/utils/token_utils.py | 15 + .../utils/vulnerability_intel_sanitizer.py | 68 + src/vuln_analysis/utils/web_patch_fetcher.py | 1073 +++ tests/test_brew_downloader.py | 123 + tests/test_package_identifier.py | 409 + 50 files changed, 17935 insertions(+), 3001 deletions(-) create mode 100644 src/exploit_iq_commons/data/hardening_kb/__init__.py create mode 100644 src/exploit_iq_commons/data/hardening_kb/hardening_kb.json create mode 100644 src/exploit_iq_commons/data_models/checker_status.py create mode 100644 src/exploit_iq_commons/utils/hardening_kb.py create mode 100644 src/vuln_analysis/configs/brew/external-user-profile.yml create mode 100644 src/vuln_analysis/configs/brew/internal-user-profile.yml create mode 100644 src/vuln_analysis/functions/build_agent_graph_defs.py create mode 100644 src/vuln_analysis/functions/code_agent_graph_defs.py create mode 100644 src/vuln_analysis/functions/cve_build_agent.py create mode 100644 src/vuln_analysis/functions/cve_checker_report.py create mode 100644 src/vuln_analysis/functions/cve_checker_segmentation.py create mode 100644 src/vuln_analysis/functions/cve_package_code_agent.py create mode 100644 src/vuln_analysis/functions/cve_source_acquisition.py create mode 100644 src/vuln_analysis/tools/brew_downloader.py create mode 100644 src/vuln_analysis/tools/source_grep.py create mode 100644 src/vuln_analysis/tools/source_inspector.py create mode 100644 src/vuln_analysis/utils/gerrit_client.py create mode 100644 src/vuln_analysis/utils/osv_patch_retriever.py create mode 100644 src/vuln_analysis/utils/package_identifier.py create mode 100644 src/vuln_analysis/utils/rpm_checker_prompts.py create mode 100644 src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py create mode 100644 src/vuln_analysis/utils/tests/test_web_patch_fetcher.py create mode 100644 src/vuln_analysis/utils/vulnerability_intel_sanitizer.py create mode 100644 src/vuln_analysis/utils/web_patch_fetcher.py create mode 100644 tests/test_brew_downloader.py create mode 100644 tests/test_package_identifier.py diff --git a/Dockerfile b/Dockerfile index fa6e6d652..45f492fc9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV AGENT_GIT_COMMIT=${AGENT_GIT_COMMIT} ENV AGENT_GIT_TAG=${AGENT_GIT_TAG} +# System dependencies: +# - build-essential: Required for compiling Python packages with native C extensions +# (e.g., psutil, cryptography, cffi) during `uv sync`. Also needed for libkrb5-dev. +# - libarchive-tools: Provides bsdtar for extracting RPM/SRPM archives in the checker +# - libkrb5-dev: Kerberos headers for gssapi Python package (Brew authentication) RUN apt-get update && apt-get install -y \ + build-essential \ ca-certificates \ curl \ git \ @@ -42,6 +48,7 @@ RUN apt-get update && apt-get install -y \ libarchive-tools \ xz-utils \ libatomic1 \ + libkrb5-dev \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates diff --git a/README.md b/README.md index 659a6a456..f4fab8a9f 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ The current default embedding NIM model is `nv-embedqa-e5-v5`, which was selecte * [git](https://git-scm.com/) * [git-lfs](https://git-lfs.com/) +* bsdtar (from libarchive) - Used for extracting RPM source archives. Install via `apt install libarchive-tools` (Debian/Ubuntu) or `dnf install bsdtar` (Fedora/RHEL). * Since the workflow uses [NVIDIA NeMo Agent Toolkit](https://docs.nvidia.com/aiqtoolkit), the [NeMo Agent toolkit requirements](https://docs.nvidia.com/aiqtoolkit/latest/quick-start/installing.html#prerequisites) also need to be installed. ### Obtain API keys diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index b9eab81d5..abfa7d15b 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -84,6 +84,11 @@ functions: Code Keyword Search: _type: lexical_code_search top_k: 5 + Source Grep: + _type: source_grep + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + max_results: 50 + context_lines: 2 CVE Web Search: _type: serp_wrapper max_retries: 5 @@ -156,6 +161,38 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + cve_source_acquisition: + _type: cve_source_acquisition + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + rpm_user_type: ${RPM_USER_TYPE:-internal} + cve_checker_segmentation: + _type: cve_checker_segmentation + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + cve_package_code_agent: + _type: cve_package_code_agent + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + rpm_user_type: ${RPM_USER_TYPE:-internal} + tool_names: + - Source Grep + - Code Keyword Search + cve_checker_report: + _type: cve_checker_report + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + cve_build_agent: + _type: cve_build_agent + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + max_iterations: 10 + tool_names: + - Source Grep + - Code Keyword Search health_check: _type: health_check @@ -248,6 +285,11 @@ workflow: cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output + cve_source_acquisition_name: cve_source_acquisition + cve_checker_segmentation_name: cve_checker_segmentation + cve_package_code_agent_name: cve_package_code_agent + cve_checker_report_name: cve_checker_report + cve_build_agent_name: cve_build_agent eval: general: diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index 655005ab6..b3ab3677a 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -122,6 +122,8 @@ spec: value: "True" - name: EXPLOIT_IQ_DATA_DIR value: /exploit-iq-data/ + - name: RPM_USER_TYPE + value: "internal" - name: NAMESPACE valueFrom: fieldRef: diff --git a/pyproject.toml b/pyproject.toml index 0d4d36bd5..c3041e8be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dependencies = [ "litellm<=1.75.8", "csaf-tool==0.3.2", "jsonschema>=4.0.0,<5.0.0", + "koji", + "unidiff>=0.7.5", ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" diff --git a/src/exploit_iq_commons/data/hardening_kb/__init__.py b/src/exploit_iq_commons/data/hardening_kb/__init__.py new file mode 100644 index 000000000..cf7c586a5 --- /dev/null +++ b/src/exploit_iq_commons/data/hardening_kb/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json b/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json new file mode 100644 index 000000000..92ab9a8f9 --- /dev/null +++ b/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json @@ -0,0 +1,395 @@ +{ + "kb_version": "1.1", + "last_updated": "2026-05-09", + "flag_type_definitions": { + "warning": "Compile-time warnings only. Does not add runtime protection. Use for 'Best Practices' audits, NOT for mitigation claims.", + "runtime": "Actual runtime protection or detection. Valid for 'Mitigated' status if maps to the specific CWE.", + "optimization": "Changes compiler optimization behavior but does not add runtime detection/prevention. Not valid for mitigation.", + "linker": "Linker-level hardening that affects runtime binary layout/behavior. Valid for mitigation if maps to CWE.", + "architecture": "Platform-specific build flag. Valid for mitigation ONLY if CVE advisory states the architecture is not affected." + }, + "mappings": [ + { + "flag": "-Wall -Wextra", + "flag_type": "warning", + "description": "Enable warnings for constructs often associated with defects.", + "vulnerability_category": "Defensive Coding", + "cwe_ids": [ + "CWE-563", + "CWE-457", + "CWE-480" + ], + "requires": {} + }, + { + "flag": "-Wformat -Wformat=2", + "flag_type": "warning", + "description": "Enable additional format function warnings.", + "vulnerability_category": "Input Validation", + "cwe_ids": [ + "CWE-134" + ], + "requires": {} + }, + { + "flag": "-Wconversion -Wsign-conversion", + "flag_type": "warning", + "description": "Enable implicit conversion warnings.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-681" + ], + "requires": {} + }, + { + "flag": "-Wtrampolines", + "flag_type": "warning", + "description": "Enable warnings about trampolines that require executable stacks.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-Wimplicit-fallthrough", + "flag_type": "warning", + "description": "Warn when a switch case falls through.", + "vulnerability_category": "Defensive Coding", + "cwe_ids": [ + "CWE-484" + ], + "requires": {} + }, + { + "flag": "-Wbidi-chars=any", + "flag_type": "warning", + "description": "Enable warnings for possibly misleading Unicode bidirectional control characters.", + "vulnerability_category": "Code Integrity", + "cwe_ids": [ + "CWE-1301" + ], + "requires": {} + }, + { + "flag": "-Werror", + "flag_type": "warning", + "description": "Treat all or selected compiler warnings as errors.", + "vulnerability_category": "Policy Enforcement", + "cwe_ids": [ + "N/A" + ], + "requires": {} + }, + { + "flag": "-Werror=format-security", + "flag_type": "warning", + "description": "Treat format strings that are not string literals and used without arguments as errors.", + "vulnerability_category": "Input Validation", + "cwe_ids": [ + "CWE-134" + ], + "requires": {} + }, + { + "flag": "-Werror=implicit -Werror=incompatible-pointer-types -Werror=int-conversion", + "flag_type": "warning", + "description": "Treat obsolete C constructs as errors.", + "vulnerability_category": "Type Safety", + "cwe_ids": [ + "CWE-704", + "CWE-843" + ], + "requires": {} + }, + { + "flag": "-D_FORTIFY_SOURCE=3", + "flag_type": "runtime", + "description": "Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-120", + "CWE-121", + "CWE-122" + ], + "requires": {} + }, + { + "flag": "-D_FORTIFY_SOURCE=2", + "flag_type": "runtime", + "description": "Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows (legacy level).", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-120", + "CWE-121", + "CWE-122" + ], + "requires": {} + }, + { + "flag": "-D_GLIBCXX_ASSERTIONS", + "flag_type": "runtime", + "description": "Precondition checks for C++ standard library calls.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-125", + "CWE-787" + ], + "requires": {} + }, + { + "flag": "-fstrict-flex-arrays=3", + "flag_type": "runtime", + "description": "Consider a trailing array in a struct as a flexible array if declared as [].", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-125", + "CWE-787" + ], + "requires": {} + }, + { + "flag": "-fstack-clash-protection", + "flag_type": "runtime", + "description": "Enable run-time checks for variable-size stack allocation validity.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-785" + ], + "requires": {} + }, + { + "flag": "-fstack-protector-strong", + "flag_type": "runtime", + "description": "Enable run-time checks for stack-based buffer overflows.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-121" + ], + "requires": {} + }, + { + "flag": "-fcf-protection=full", + "flag_type": "runtime", + "description": "Enable control-flow protection against return-oriented programming (ROP) and jump-oriented programming (JOP) attacks on x86_64.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-mbranch-protection=standard", + "flag_type": "runtime", + "description": "Enable branch protection against ROP and JOP attacks on AArch64.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-ftrapv", + "flag_type": "runtime", + "description": "Generate traps for signed arithmetic overflow on addition, subtraction, multiplication.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-fsanitize=signed-integer-overflow", + "flag_type": "runtime", + "description": "Enable undefined behavior sanitizer for signed integer overflow detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-fsanitize=unsigned-integer-overflow", + "flag_type": "runtime", + "description": "Enable undefined behavior sanitizer for unsigned integer overflow detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,nodlopen", + "flag_type": "linker", + "description": "Restrict dlopen(3) calls to shared objects.", + "vulnerability_category": "Policy Enforcement", + "cwe_ids": [ + "CWE-269" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,noexecstack", + "flag_type": "linker", + "description": "Enable data execution prevention by marking stack memory as non-executable.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693", + "CWE-94" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,relro -Wl,-z,now", + "flag_type": "linker", + "description": "Mark relocation table entries resolved at load-time as read-only.", + "vulnerability_category": "Code Integrity", + "cwe_ids": [ + "CWE-123" + ], + "requires": {} + }, + { + "flag": "-fPIE -pie", + "flag_type": "linker", + "description": "Build as position-independent executable.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-fPIC -shared", + "flag_type": "linker", + "description": "Build as position-independent code.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-fno-delete-null-pointer-checks", + "flag_type": "optimization", + "description": "Force retention of null pointer checks.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-476" + ], + "requires": {} + }, + { + "flag": "-fno-strict-overflow", + "flag_type": "optimization", + "description": "Do not assume signed integer overflow is undefined behavior. Prevents aggressive optimizations but does NOT add runtime detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190" + ], + "requires": {} + }, + { + "flag": "-fno-strict-aliasing", + "flag_type": "optimization", + "description": "Do not assume strict aliasing.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-416" + ], + "requires": {} + }, + { + "flag": "-ftrivial-auto-var-init", + "flag_type": "runtime", + "description": "Initialize automatic variables that lack explicit initializers.", + "vulnerability_category": "Information Leakage", + "cwe_ids": [ + "CWE-457" + ], + "requires": {} + }, + { + "flag": "-fexceptions", + "flag_type": "runtime", + "description": "Enable exception propagation to harden multi-threaded C code.", + "vulnerability_category": "Error Handling", + "cwe_ids": [ + "CWE-391" + ], + "requires": {} + }, + { + "flag": "-fhardened", + "flag_type": "runtime", + "description": "Enable pre-determined set of hardening options in GCC.", + "vulnerability_category": "Full Hardening", + "cwe_ids": [ + "Multi" + ], + "requires": {} + }, + { + "flag": "-Wl,--as-needed -Wl,--no-copy-dt-needed-entries", + "flag_type": "linker", + "description": "Allow linker to omit libraries specified on the command line to link against if they are not used.", + "vulnerability_category": "Supply Chain Safety", + "cwe_ids": [ + "N/A" + ], + "requires": {} + }, + { + "flag": "-fzero-init-padding-bits=all", + "flag_type": "runtime", + "description": "Guarantee zero initialization of padding bits in all automatic variable initializers.", + "vulnerability_category": "Information Leakage", + "cwe_ids": [ + "CWE-200" + ], + "requires": {} + }, + { + "flag": "-m64", + "flag_type": "architecture", + "description": "Compile for 64-bit x86_64 architecture. Many integer overflow vulnerabilities only affect 32-bit systems.", + "vulnerability_category": "Architecture", + "cwe_ids": [ + "CWE-190", + "CWE-680", + "CWE-681" + ], + "requires": { + "advisory_states": "Mitigation valid ONLY if CVE advisory explicitly states 64-bit systems are not affected." + } + }, + { + "flag": "-m32", + "flag_type": "architecture", + "description": "Compile for 32-bit i686 architecture.", + "vulnerability_category": "Architecture", + "cwe_ids": [], + "requires": { + "advisory_states": "Check CVE advisory for 32-bit specific vulnerabilities." + } + }, + { + "flag": "-march=", + "flag_type": "architecture", + "description": "Target specific CPU architecture. May affect vulnerability applicability.", + "vulnerability_category": "Architecture", + "cwe_ids": [], + "requires": { + "advisory_states": "Check CVE advisory for architecture-specific conditions." + } + } + ] +} diff --git a/src/exploit_iq_commons/data_models/checker_status.py b/src/exploit_iq_commons/data_models/checker_status.py new file mode 100644 index 000000000..029cf8245 --- /dev/null +++ b/src/exploit_iq_commons/data_models/checker_status.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from enum import Enum +from enum import IntEnum +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, Field + + +class PackageCheckerStatus(IntEnum): + """Per-CVE status codes produced by the PackageIdentify phase.""" + OK = 0 + ERROR_PKG_IDENT_NO_INTEL = 1 + PKG_IDENT_NOT_VUL = 2 + ERROR_FAILED_TO_DOWNLOAD_SRPM = 3 + PKG_IDENT_CVE_MISMATCH = 4 + PKG_INTEL_LOW_SCORE = 5 + + +PACKAGE_CHECKER_STATUS_DESCRIPTIONS: dict[PackageCheckerStatus, str] = { + PackageCheckerStatus.OK: + "Package identified and in affected range -- continue investigation", + PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL: + "No Intel found for the package", + PackageCheckerStatus.PKG_IDENT_NOT_VUL: + "Identification state concluded from intel that target package is not vulnerable", + PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM: + "Failed to download the patched SRPM", + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH: + "CVE does not apply to target package - RHSA does not list this package", + PackageCheckerStatus.PKG_INTEL_LOW_SCORE: + "Intel quality score below threshold - insufficient information for reliable analysis", +} + +CHECKER_FAILURE_ERROR_TYPES: dict[PackageCheckerStatus, str] = { + PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL: "no-intel", + PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM: "srpm-download-failed", + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH: "invalid-input", +} + + +class EnumIdentifyResult(str, Enum): + """Result of the PackageIdentify phase for a single CVE.""" + YES = "yes" + NO = "no" + UNKNOWN = "unknown" + +class PackageIdentifyResult(BaseModel): + """Result of the PackageIdentify phase for a single CVE.""" + affected_rpm_list: list[str] = [] + fixed_rpm_list: list[str] = [] + + is_target_package_affected: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN + is_target_package_fixed: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN + + conclusion_reason: str = Field( + default="", + description="Detailed explanation of why the package was determined to be vulnerable or not vulnerable" + ) + + + +class AcquiredArtifacts(BaseModel): + """Resolved file locations populated by source_acquisition, consumed by downstream checker nodes.""" + srpm_path: Path | None = None + source_dir: Path | None = None + build_log_path: Path | None = None + binary_rpm_path: Path | None = None + patch_source_dir: Path | None = None + patch_diff_path: Path | None = None + source_url: str | None = None + + +class VulnerabilityIntel(BaseModel): + """Structured intelligence extracted from CVE advisories and patches. + + Used to provide grep-ready patterns and context for L1 agent source searches. + """ + + affected_files: list[str] = Field( + default_factory=list, + description="Source file paths likely to contain vulnerable code" + ) + vulnerable_functions: list[str] = Field( + default_factory=list, + description="Function names that contain or handle the vulnerability" + ) + vulnerable_variables: list[str] = Field( + default_factory=list, + description="Variable names involved in the vulnerability" + ) + vulnerable_patterns: list[str] = Field( + default_factory=list, + description="Code patterns/snippets indicating vulnerable code (from - lines)" + ) + fix_patterns: list[str] = Field( + default_factory=list, + description="Code patterns/snippets indicating fixed code (from + lines)" + ) + root_cause: str = Field( + default="", + description="Technical explanation of why the code is vulnerable" + ) + vulnerability_type: str = Field( + default="", + description="Category: buffer_overflow, integer_overflow, use_after_free, null_deref, etc." + ) + search_keywords: list[str] = Field( + default_factory=list, + description="Recommended grep patterns ordered by specificity (most specific first)" + ) + affected_bitness: Literal["32-bit", "64-bit", "both"] = Field( + default="both", + description="Which bitness is affected: 32-bit only, 64-bit only, or both (default)" + ) + affected_architectures: list[str] | None = Field( + default=None, + description="CPU families affected (e.g., ['x86', 'arm']). None means all architectures." + ) + is_downstream_patch_available: bool = Field( + default=False, + description="True if a CVE-specific patch file exists in the downstream package" + ) + is_patch_applied_in_build: bool = Field( + default=False, + description="True if the patch was confirmed applied in build logs" + ) + patch_file_name: str = Field( + default="", + description="Name of the CVE-specific patch file (if available)" + ) + known_mitigations: str = Field( + default="", + description="Vendor-provided mitigations from RHSA or other intel sources (e.g., compiler flags, config changes)" + ) + + def format_for_prompt(self) -> str: + """Format VulnerabilityIntel for injection into L1 agent runtime prompt. + + Uses UPPERCASE labels so they can be referenced as anchors in thought prompts. + """ + lines = [] + if self.is_downstream_patch_available: + status = "APPLIED" if self.is_patch_applied_in_build else "AVAILABLE" + lines.append(f"DOWNSTREAM_PATCH_STATUS: {status}") + if self.patch_file_name: + lines.append(f"PATCH_FILE: {self.patch_file_name}") + if self.affected_files: + lines.append(f"AFFECTED_FILES: {', '.join(self.affected_files)}") + if self.vulnerable_functions: + lines.append(f"VULNERABLE_FUNCTIONS: {', '.join(self.vulnerable_functions)}") + if self.vulnerable_variables: + lines.append(f"VULNERABLE_VARIABLES: {', '.join(self.vulnerable_variables)}") + if self.vulnerable_patterns: + lines.append("VULNERABLE_PATTERNS:") + for p in self.vulnerable_patterns: + lines.append(f" - {p}") + if self.fix_patterns: + lines.append("FIX_PATTERNS:") + for p in self.fix_patterns: + lines.append(f" - {p}") + if self.search_keywords: + lines.append(f"SEARCH_KEYWORDS: {', '.join(self.search_keywords)}") + if self.root_cause: + lines.append(f"ROOT_CAUSE: {self.root_cause}") + if self.affected_bitness and self.affected_bitness != "both": + lines.append(f"AFFECTED_BITNESS: {self.affected_bitness}") + if self.affected_architectures: + lines.append(f"AFFECTED_ARCHITECTURES: {', '.join(self.affected_architectures)}") + if self.known_mitigations: + lines.append(f"KNOWN_MITIGATIONS: {self.known_mitigations}") + return "\n".join(lines) + + +class L1InvestigationResult(BaseModel): + """Intermediate result from L1 investigation, input to L2 or report generation.""" + downstream_report: dict[str, Any] | None = Field( + default=None, + description="Serialized DownstreamSearchReport from L1 investigation", + ) + upstream_report: dict[str, Any] | None = Field( + default=None, + description="Serialized UpstreamSearchReport from L1 investigation", + ) + l1_agent_answer: str | None = Field( + default=None, + description="Final answer from the L1 ReAct agent", + ) + vulnerability_intel: VulnerabilityIntel | None = Field( + default=None, + description="Structured vulnerability intelligence extracted from CVE advisories and patches", + ) + preliminary_verdict: Literal["vulnerable", "protected", "not_present", "uncertain"] = Field( + default="uncertain", + description="L1 verdict before L2 refinement", + ) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in the preliminary verdict", + ) + + +class L2BuildResult(BaseModel): + """Result from L2 Build Agent (BuildCompilationCheck + HardeningCheck).""" + compilation_status: Literal["compiled", "not_compiled", "unknown"] = Field( + default="unknown", + description="Whether vulnerable code is compiled into the binary", + ) + compilation_confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in compilation status", + ) + compilation_evidence: str | None = Field( + default=None, + description="Evidence supporting compilation status", + ) + hardening_relevant: bool | None = Field( + default=None, + description="Whether detected hardening flags are relevant to the CVE", + ) + hardening_flags: list[str] = Field( + default_factory=list, + description="Hardening flags detected in build log or binary", + ) + hardening_rationale: str | None = Field( + default=None, + description="Rationale for hardening relevance judgment", + ) + l2_override_verdict: Literal["not_vulnerable", "vulnerable_mitigated", None] = Field( + default=None, + description="L2 verdict override (if any)", + ) + evidence_sources: list[str] = Field( + default_factory=list, + description="Sources used for analysis: 'build_log', 'spec_file', 'build_system_files', 'binary'", + ) + + +class PackageCheckerContext(BaseModel): + """Consolidates all checker-specific state on AgentMorpheusInfo.""" + status: PackageCheckerStatus | None = None + source_key: str | None = None + artifacts: AcquiredArtifacts = Field(default_factory=AcquiredArtifacts) + identify_result: PackageIdentifyResult = Field(default_factory=PackageIdentifyResult) + l1_result: L1InvestigationResult | None = Field( + default=None, + description="Result from L1 Code Agent investigation", + ) + l2_result: L2BuildResult | None = Field( + default=None, + description="Result from L2 Build Agent (optional)", + ) diff --git a/src/exploit_iq_commons/data_models/common.py b/src/exploit_iq_commons/data_models/common.py index 077a98fa9..db848f4ad 100644 --- a/src/exploit_iq_commons/data_models/common.py +++ b/src/exploit_iq_commons/data_models/common.py @@ -28,6 +28,17 @@ class AnalysisType(str, Enum): IMAGE = "image" SOURCE = "source" + +class PipelineMode(str, Enum): + """ + Controls which investigation path the pipeline takes after process_sbom. + Orthogonal to AnalysisType (input format) -- any combination is valid. + """ + FULL_PIPELINE = "full_pipeline" + PACKAGE_CHECKER = "rpm_package_checker" + + + class HashableModel(BaseModel): """ Subclass of a Pydantic BaseModel that is hashable. Use in objects that need to be hashed for caching purposes. @@ -50,7 +61,15 @@ def __ne__(self, other): def __gt__(self, other): return self.__hash__() > other.__hash__() - +class TargetPackage(HashableModel): + """ + A package to investigate. + """ + name: str + version: str | None = None + release: str | None = None # e.g. "1.el8_2.3" (needed for Brew NVR lookup) + arch: str = "x86_64" # e.g. "x86_64", "aarch64", "s390x", "noarch" + class TypedBaseModel(BaseModel, typing.Generic[_LT]): """ Subclass of Pydantic BaseModel that allows for specifying the object type. Use in Pydantic discriminated unions. diff --git a/src/exploit_iq_commons/data_models/cve_intel.py b/src/exploit_iq_commons/data_models/cve_intel.py index 8050ffe26..33f92aecb 100644 --- a/src/exploit_iq_commons/data_models/cve_intel.py +++ b/src/exploit_iq_commons/data_models/cve_intel.py @@ -110,6 +110,7 @@ class Configuration(BaseModel): cvss_vector: str | None = None cvss_base_score: float | None = None cvss_severity: str | None = None + cwe_id: str | None = None cwe_name: str | None = None cwe_description: str | None = None cwe_extended_description: str | None = None @@ -185,8 +186,8 @@ class CVSSV3(BaseModel): class BaseMetricV3(BaseModel): cvssV3: "CVSSV3" - exploitabilityScore: float - impactScore: float + exploitabilityScore: float | None = None + impactScore: float | None = None class Impact(BaseModel): baseMetricV3: "BaseMetricV3" @@ -197,6 +198,10 @@ class Impact(BaseModel): priority: str | None = None ubuntu_description: str | None = None impact: Impact | None = None + patches: dict[str, list[str]] | None = Field( + default=None, + description="Map of package name to patch refs (e.g., 'upstream: https://github.com/.../commit/...')" + ) @property def description_fields(self): diff --git a/src/exploit_iq_commons/data_models/info.py b/src/exploit_iq_commons/data_models/info.py index a01f1dda7..4f7bd1ef1 100644 --- a/src/exploit_iq_commons/data_models/info.py +++ b/src/exploit_iq_commons/data_models/info.py @@ -15,6 +15,7 @@ from pydantic import BaseModel +from exploit_iq_commons.data_models.checker_status import PackageCheckerContext from exploit_iq_commons.data_models.cve_intel import CveIntel from exploit_iq_commons.data_models.dependencies import VulnerableDependencies @@ -62,3 +63,4 @@ class SBOMInfo(BaseModel): intel: list[CveIntel] | None = None sbom: SBOMInfo | None = None vulnerable_dependencies: list[VulnerableDependencies] | None = None + checker_context: PackageCheckerContext | None = None diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index 897c915d1..77a325214 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -25,12 +25,14 @@ from pydantic import Field from pydantic import Tag from pydantic import field_validator +from pydantic import model_validator from exploit_iq_commons.utils.string_utils import is_valid_cve_id from exploit_iq_commons.utils.string_utils import is_valid_ghsa_id from exploit_iq_commons.utils.dep_tree import Ecosystem from exploit_iq_commons.data_models.common import AnalysisType from exploit_iq_commons.data_models.common import HashableModel +from exploit_iq_commons.data_models.common import PipelineMode , TargetPackage from exploit_iq_commons.data_models.common import TypedBaseModel from exploit_iq_commons.data_models.info import AgentMorpheusInfo from exploit_iq_commons.data_models.info import SBOMPackage @@ -168,9 +170,25 @@ class ImageInfoInput(HashableModel): - "source": Analysis of source code and commitId without SBOM data """ - source_info: list[SourceDocumentsInfo] + pipeline_mode: PipelineMode = PipelineMode.FULL_PIPELINE + """ + Controls which investigation path the pipeline takes after process_sbom: + - "full_pipeline": Full transitive analysis (check_vuln_deps -> llm_engine) + - "package_checker": Focused package vulnerability checker (package_checker -> checker_output) + """ + target_package: TargetPackage | None = None + + source_info: list[SourceDocumentsInfo] = [] sbom_info: SBOMInfoInput | None = None + @model_validator(mode="after") + def validate_pipeline_requirements(self) -> "ImageInfoInput": + if self.pipeline_mode == PipelineMode.PACKAGE_CHECKER and self.target_package is None: + raise ValueError("target_package is required when pipeline_mode is PACKAGE_CHECKER") + if self.pipeline_mode == PipelineMode.FULL_PIPELINE and not self.source_info: + raise ValueError("source_info is required and must not be empty when pipeline_mode is FULL_PIPELINE") + return self + @field_validator('source_info', mode='after') @classmethod def check_conflicting_refs(cls, source_info: list[SourceDocumentsInfo]) -> list[SourceDocumentsInfo]: diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index fe7eeec7d..8aa46f681 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -164,8 +164,12 @@ def search_for_called_function(self, caller_function: Document, callee_function_ if len(parts) == 1: identifier = parts[0] identifier = identifier.rstrip('(') - callee_function = code_documents[callee_function_file_name] - callee_function_package = self.get_package_names(callee_function)[0] + try: + callee_function_doc = code_documents[callee_function_file_name] + callee_function_package = self.get_package_names(callee_function_doc)[0] + except KeyError: + # Third-party package without source files - use file name as package name + callee_function_package = callee_function_file_name caller_function_package = self.get_package_names(caller_function)[0] if callee_function_package == caller_function_package: diff --git a/src/exploit_iq_commons/utils/hardening_kb.py b/src/exploit_iq_commons/utils/hardening_kb.py new file mode 100644 index 000000000..22fe6c789 --- /dev/null +++ b/src/exploit_iq_commons/utils/hardening_kb.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Hardening Knowledge Base utilities. + +Loads the hardening_kb.json file containing compiler/linker flags that mitigate +specific CWE vulnerability categories. Provides lookup by CWE ID to retrieve +relevant hardening flags and their descriptions for LLM context. +""" + +from __future__ import annotations + +import json +import threading +from pathlib import Path + +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + + +class HardeningEntry(BaseModel): + """A single hardening flag entry from the knowledge base.""" + + flag: str = Field(description="Compiler/linker flag(s) for hardening") + flag_type: str = Field(description="Type: runtime, linker, warning, optimization, architecture") + description: str = Field(description="Description of what the flag does") + vulnerability_category: str = Field(description="Category of vulnerability this mitigates") + cwe_ids: list[str] = Field(default_factory=list, description="CWE IDs this flag helps mitigate") + + +# Flag types that provide actual runtime mitigation (not just warnings or optimization changes) +MITIGATING_FLAG_TYPES = frozenset({"runtime", "linker"}) + + +class HardeningKB: + """In-memory cache for hardening flags knowledge base. + + Implements singleton pattern to ensure single instance across the application. + Provides lookup by CWE ID to find relevant hardening flags. + """ + + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self) -> None: + if not hasattr(self, '_initialized'): + base_path = Path(__file__).resolve().parents[1] + self.json_path = base_path / "data" / "hardening_kb" / "hardening_kb.json" + + self._entries: list[HardeningEntry] = [] + self._cwe_index: dict[str, list[HardeningEntry]] = {} + self._initialized = True + self._load() + + @classmethod + def get_instance(cls) -> "HardeningKB": + """Get the singleton instance of HardeningKB.""" + return cls() + + def _load(self) -> None: + """Load the hardening KB JSON and build the CWE index.""" + try: + data = json.loads(self.json_path.read_text(encoding="utf-8")) + except FileNotFoundError: + logger.warning("Hardening KB JSON not found at %s", self.json_path) + return + except json.JSONDecodeError as exc: + logger.error("Failed to parse hardening KB JSON: %s", exc) + return + + mappings = data.get("mappings", []) + for mapping in mappings: + try: + entry = HardeningEntry( + flag=mapping.get("flag", "").strip(), + flag_type=mapping.get("flag_type", "unknown"), + description=mapping.get("description", ""), + vulnerability_category=mapping.get("vulnerability_category", ""), + cwe_ids=mapping.get("cwe_ids", []), + ) + self._entries.append(entry) + + for cwe_id in entry.cwe_ids: + normalized = self._normalize_cwe_id(cwe_id) + if normalized: + if normalized not in self._cwe_index: + self._cwe_index[normalized] = [] + self._cwe_index[normalized].append(entry) + + except Exception as exc: + logger.warning("Failed to parse hardening entry: %s - %s", mapping, exc) + + logger.info( + "Loaded hardening KB: %d entries, %d unique CWE mappings", + len(self._entries), + len(self._cwe_index), + ) + + @staticmethod + def _normalize_cwe_id(cwe_id: str) -> str | None: + """Normalize CWE ID to uppercase format (e.g., 'CWE-121'). + + Returns None for special values like 'N/A' or 'Multi'. + """ + if not cwe_id: + return None + cwe_id = cwe_id.strip().upper() + if cwe_id in ("N/A", "MULTI"): + return None + if not cwe_id.startswith("CWE-"): + cwe_id = f"CWE-{cwe_id}" + return cwe_id + + def lookup_by_cwe( + self, + cwe_id: str | None, + include_non_mitigating: bool = False, + ) -> list[HardeningEntry]: + """Return hardening entries that match the given CWE ID. + + By default, only returns flags that provide actual runtime mitigation + (flag_type: runtime, linker). Warning-only and optimization flags are + excluded since they don't mitigate vulnerabilities at runtime. + + Args: + cwe_id: The CWE identifier (e.g., 'CWE-121' or '121') + include_non_mitigating: If True, include warning/optimization flags + that don't provide runtime mitigation (for auditing purposes) + + Returns: + List of HardeningEntry objects that help mitigate this CWE + """ + if not cwe_id: + return [] + + normalized = self._normalize_cwe_id(cwe_id) + if not normalized: + return [] + + entries = self._cwe_index.get(normalized, []) + + if not include_non_mitigating: + entries = [e for e in entries if e.flag_type in MITIGATING_FLAG_TYPES] + + logger.debug( + "HardeningKB lookup for %s: found %d entries (include_non_mitigating=%s)", + normalized, + len(entries), + include_non_mitigating, + ) + return list(entries) + + def get_all_entries(self) -> list[HardeningEntry]: + """Return all hardening entries in the knowledge base.""" + return list(self._entries) + + diff --git a/src/exploit_iq_commons/utils/source_rpm_downloader.py b/src/exploit_iq_commons/utils/source_rpm_downloader.py index 9022b40de..01fa0a240 100644 --- a/src/exploit_iq_commons/utils/source_rpm_downloader.py +++ b/src/exploit_iq_commons/utils/source_rpm_downloader.py @@ -16,7 +16,6 @@ import os from pathlib import Path import gzip -import tarfile import xml.etree.ElementTree as ET import subprocess from concurrent.futures import ThreadPoolExecutor, as_completed @@ -39,17 +38,12 @@ class RepoUrl: def extract_archives_in_folder(folder: str): """ - Extracts all .tar.* files in the given folder to subdirectories. - Supports .tar.gz, .tgz, .tar.xz, .txz, .tar.bz2, .tbz2, and .tar. + Extracts all .tar.* files in the given folder to subdirectories using bsdtar. + Supports .tar.gz, .tgz, .tar.xz, .txz, .tar.bz2, .tbz2, .tar.lzma, .tlz, and .tar. :param folder: Path to the folder to scan. """ - supported_extensions = { - '.tar.gz', '.tgz', - '.tar.xz', '.txz', - '.tar.bz2', '.tbz2', - '.tar' - } + shorthand_extensions = ('.tar', '.tgz', '.txz', '.tbz', '.tbz2', '.tlz') folder_path = Path(folder) @@ -57,25 +51,34 @@ def extract_archives_in_folder(folder: str): if not file.is_file(): continue - suffix = ''.join(file.suffixes[-2:]) if len(file.suffixes) >= 2 else file.suffix - if suffix not in supported_extensions and file.suffix not in supported_extensions: + name_lower = file.name.lower() + is_tarball = '.tar.' in name_lower or name_lower.endswith(shorthand_extensions) + if not is_tarball: continue - # Determine full suffix like .tar.gz or .tgz - full_suffix = suffix if suffix in supported_extensions else file.suffix + suffix = ''.join(file.suffixes[-2:]) if len(file.suffixes) >= 2 else file.suffix + full_suffix = suffix if suffix.startswith('.tar') else file.suffix # Determine output directory name output_dir = file.with_suffix('').with_suffix('').stem if full_suffix.startswith('.tar') else file.stem output_path = folder_path / output_dir logger.debug(f"Extracting: {file.name} → {output_path}") + if output_path.exists(): + logger.debug(f"Output path already exists: {output_path}") + return output_path.mkdir(exist_ok=True) try: - with tarfile.open(file, 'r:*') as tar: - tar.extractall(path=output_path) + result = subprocess.run( + ['bsdtar', '-xf', str(file), '-C', str(output_path)], + capture_output=True, + text=True + ) + if result.returncode != 0: + logger.error(f"Failed to extract {file.name}: {result.stderr.strip()}") except Exception as e: - logger.debug(f"Failed to extract {file.name}: {e}") + logger.error(f"Could not run extraction for {file.name}: {e}") class RPMDependencyManager: """ @@ -476,7 +479,8 @@ def parse_sbom(self): logger.info(f"Found {len(packages)} packages in SBOM, platform: {platform_version}") return packages, platform_version - def extract_src_rpm(self, rpm_path: Path, extract_dir: Path): + @staticmethod + def extract_src_rpm(rpm_path: Path, extract_dir: Path): #logger.info(f" Extracting {rpm_path.name} to {extract_dir} ...") extract_dir.mkdir(parents=True, exist_ok=True) try: diff --git a/src/vuln_analysis/configs/brew/external-user-profile.yml b/src/vuln_analysis/configs/brew/external-user-profile.yml new file mode 100644 index 000000000..3d0dd1984 --- /dev/null +++ b/src/vuln_analysis/configs/brew/external-user-profile.yml @@ -0,0 +1,23 @@ +# External User Profile — public Fedora Koji (no VPN / no auth) +# +# Assumptions: +# - Packages are resolved and downloaded from koji.fedoraproject.org +# - Build logs are not fetched (optional for internal; unavailable or unused for external v1) + +profile: + name: fedora-public + +hosts: + rpm: + brew_hub: https://koji.fedoraproject.org/kojihub + brew_download: https://kojipkgs.fedoraproject.org + +default_arch: x86_64 + +# Dev/local phase: false (matches internal profile). Set to true before cluster deploy. +ssl_verify: true + +build_log: + auto_fetch: false + +download_binary_rpm: false diff --git a/src/vuln_analysis/configs/brew/internal-user-profile.yml b/src/vuln_analysis/configs/brew/internal-user-profile.yml new file mode 100644 index 000000000..35df8e035 --- /dev/null +++ b/src/vuln_analysis/configs/brew/internal-user-profile.yml @@ -0,0 +1,26 @@ +# Internal User Profile — Red Hat VPN-connected environment +# +# Assumptions: +# - User is on the Red Hat VPN (can reach *.redhat.com internal hosts) +# - Build logs are available via Brew task output + +profile: + name: redhat-internal + +hosts: + rpm: + brew_hub: https://brewhub.engineering.redhat.com/brewhub + brew_download: https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot + git: + dist_git: https://pkgs.devel.redhat.com/cgit + +default_arch: x86_64 + +ssl_verify: true +# RH engineering hosts (brewhub, brew CDN) — use git-ca-bundle, not service-ca.crt +verify_path: /app/git-ca-bundle/ca-bundle.crt + +build_log: + auto_fetch: true + +download_binary_rpm: false diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 80c16d7b5..1ee69cfb4 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -80,6 +80,11 @@ functions: Code Keyword Search: _type: lexical_code_search top_k: 5 + Source Grep: + _type: source_grep + base_checker_dir: .cache/am_cache/checker + max_results: 50 + context_lines: 2 CVE Web Search: _type: serp_wrapper max_retries: 5 @@ -157,6 +162,37 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + cve_source_acquisition: + _type: cve_source_acquisition + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + rpm_user_type: ${RPM_USER_TYPE:-internal} + cve_checker_segmentation: + _type: cve_checker_segmentation + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index + cve_package_code_agent: + _type: cve_package_code_agent + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index + rpm_user_type: ${RPM_USER_TYPE:-internal} + tool_names: + - Source Grep + - Code Keyword Search + cve_checker_report: + _type: cve_checker_report + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + cve_build_agent: + _type: cve_build_agent + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + max_iterations: 10 + tool_names: + - Source Grep + - Code Keyword Search health_check: _type: health_check @@ -248,6 +284,11 @@ workflow: cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output + cve_source_acquisition_name: cve_source_acquisition + cve_checker_segmentation_name: cve_checker_segmentation + cve_package_code_agent_name: cve_package_code_agent + cve_checker_report_name: cve_checker_report + cve_build_agent_name: cve_build_agent eval: general: diff --git a/src/vuln_analysis/configs/openapi/openapi.json b/src/vuln_analysis/configs/openapi/openapi.json index 79feca4cc..e0cdb1ad2 100644 --- a/src/vuln_analysis/configs/openapi/openapi.json +++ b/src/vuln_analysis/configs/openapi/openapi.json @@ -1,3043 +1,4602 @@ { "openapi": "3.1.0", "info": { - "title": "FastAPI", - "version": "0.1.0" + "title": "FastAPI", + "version": "0.1.0" }, "paths": { - "/generate": { - "post": { - "summary": "Post Single", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_single_generate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "/generate": { + "post": { + "summary": "Post Single", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_single_generate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } - }, - "/generate/stream": { - "post": { - "summary": "Post Stream", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_stream_generate_stream_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/generate/stream": { + "post": { + "summary": "Post Stream", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_stream_generate_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } - }, - "/generate/full": { - "post": { - "summary": "Post Stream", - "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", - "operationId": "post_stream_generate_full_post", - "parameters": [ - { - "name": "filter_steps", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Filter Steps" - } - } + } + } + }, + "/generate/full": { + "post": { + "summary": "Post Stream", + "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", + "operationId": "post_stream_generate_full_post", + "parameters": [ + { + "name": "filter_steps", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Filter Steps" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" } + } } - }, - "/chat": { - "post": { - "summary": "Post Single", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_single_chat_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatResponse" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/chat/stream": { - "post": { - "summary": "Post Stream", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_stream_chat_stream_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/AIQChatResponseChunk" - }, - { - "$ref": "#/components/schemas/AIQResponseIntermediateStep" - } - ], - "title": "Response Post Stream Chat Stream Post" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/generate/async": { + "post": { + "summary": "Start Async Generation", + "description": "Start an async generate job", + "operationId": "start_async_generation_generate_async_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/AsyncGenerateResponse" + }, + { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + ], + "title": "Response Start Async Generation Generate Async Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/job/last": { - "get": { - "summary": "Get Last Job Status", - "description": "Get the status of the last created evaluation job", - "operationId": "get_last_job_status_evaluate_job_last_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - } - } - } - }, - "404": { - "description": "No jobs found" - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - } + } + } + }, + "/generate/async/job/{job_id}": { + "get": { + "summary": "Get Async Job Status", + "description": "Get the status of an async job", + "operationId": "get_async_job_status_generate_async_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/job/{job_id}": { - "get": { - "summary": "Get Job Status", - "description": "Get the status of an evaluation job", - "operationId": "get_job_status_evaluate_job__job_id__get", - "parameters": [ - { - "name": "job_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Job Id" - } - } + } + } + }, + "/chat": { + "post": { + "summary": "Post Single", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_single_chat_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/chat/stream": { + "post": { + "summary": "Post Stream", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_stream_chat_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatResponseChunk" + }, + { + "$ref": "#/components/schemas/ResponseIntermediateStep" + } + ], + "title": "Response Post Stream Chat Stream Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/chat/completions": { + "post": { + "summary": "Post Openai Api Compatible", + "description": "Executes the default NAT workflow from the loaded configuration (OpenAI Chat Completions API compatible)", + "operationId": "post_openai_api_compatible_v1_chat_completions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatResponse" + }, + { + "$ref": "#/components/schemas/ChatResponseChunk" + } + ], + "title": "Response Post Openai Api Compatible V1 Chat Completions Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/evaluate/job/last": { + "get": { + "summary": "Get Last Job Status", + "description": "Get the status of the last created evaluation job", + "operationId": "get_last_job_status_evaluate_job_last_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + } + } + } + }, + "404": { + "description": "No jobs found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/evaluate/job/{job_id}": { + "get": { + "summary": "Get Job Status", + "description": "Get the status of an evaluation job", + "operationId": "get_job_status_evaluate_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/evaluate/jobs": { + "get": { + "summary": "Get Jobs", + "description": "Get all jobs, optionally filtered by status", + "operationId": "get_jobs_evaluate_jobs_get", + "parameters": [ + { + "name": "status", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - } - } - } - }, - "404": { - "description": "Job not found" - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Status" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + }, + "title": "Response Get Jobs Evaluate Jobs Get" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/jobs": { - "get": { - "summary": "Get Jobs", - "description": "Get all jobs, optionally filtered by status", - "operationId": "get_jobs_evaluate_jobs_get", - "parameters": [ - { - "name": "status", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } - } + } + } + }, + "/evaluate": { + "post": { + "summary": "Start Evaluation", + "description": "Evaluates the performance and accuracy of the workflow on a dataset", + "operationId": "start_evaluation_evaluate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/auth/redirect": { + "get": { + "summary": "Redirect Uri", + "description": "Handles the authorization code and state returned from the Authorization Code Grant Flow.", + "operationId": "redirect_uri_auth_redirect_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + + } + } + } + } + } + } + }, + "/health": { + "get": { + "summary": "Get Single", + "description": "Perform a health check.", + "operationId": "get_single_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthStatusResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/health/stream": { + "get": { + "summary": "Get Stream", + "description": "Perform a health check.", + "operationId": "get_stream_health_stream_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthStatusResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/health/full": { + "get": { + "summary": "Get Stream", + "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", + "operationId": "get_stream_health_full_get", + "parameters": [ + { + "name": "filter_steps", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - }, - "title": "Response Get Jobs Evaluate Jobs Get" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Filter Steps" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate": { - "post": { - "summary": "Start Evaluation", - "description": "Evaluates the performance and accuracy of the workflow on a dataset", - "operationId": "start_evaluation_evaluate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateResponse" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/health/async/job/{job_id}": { + "get": { + "summary": "Get Async Job Status", + "description": "Get the status of an async job", + "operationId": "get_async_job_status_health_async_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } + } } + } }, "components": { - "schemas": { - "AIQChatRequest": { - "properties": { - "messages": { - "items": { - "$ref": "#/components/schemas/Message" - }, - "type": "array", - "title": "Messages" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "temperature": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Temperature" - }, - "max_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Max Tokens" - }, - "top_p": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Top P" - } + "schemas": { + "AcquiredArtifacts": { + "properties": { + "srpm_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "messages" - ], - "title": "AIQChatRequest", - "description": "AIQChatRequest is a data model that represents a request to the AIQ Toolkit chat API." - }, - "AIQChatResponse": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "object": { - "type": "string", - "title": "Object" - }, - "model": { - "type": "string", - "title": "Model", - "default": "" - }, - "created": { - "type": "string", - "format": "date-time", - "title": "Created" - }, - "choices": { - "items": { - "$ref": "#/components/schemas/AIQChoice" - }, - "type": "array", - "title": "Choices" - }, - "usage": { - "anyOf": [ - { - "$ref": "#/components/schemas/AIQUsage" - }, - { - "type": "null" - } - ] - } + { + "type": "null" + } + ], + "title": "Srpm Path" + }, + "source_dir": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "object", - "created", - "choices" - ], - "title": "AIQChatResponse", - "description": "AIQChatResponse is a data model that represents a response from the AIQ Toolkit chat API." - }, - "AIQChatResponseChunk": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "choices": { - "items": { - "$ref": "#/components/schemas/AIQChoice" - }, - "type": "array", - "title": "Choices" - }, - "created": { - "type": "string", - "format": "date-time", - "title": "Created" - }, - "model": { - "type": "string", - "title": "Model", - "default": "" - }, - "object": { - "type": "string", - "title": "Object", - "default": "chat.completion.chunk" - } + { + "type": "null" + } + ], + "title": "Source Dir" + }, + "build_log_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "choices", - "created" - ], - "title": "AIQChatResponseChunk", - "description": "AIQChatResponseChunk is a data model that represents a response chunk from the AIQ Toolkit chat streaming API." + { + "type": "null" + } + ], + "title": "Build Log Path" }, - "AIQChoice": { - "properties": { - "message": { - "$ref": "#/components/schemas/AIQChoiceMessage" - }, - "finish_reason": { - "anyOf": [ - { - "type": "string", - "enum": [ - "stop", - "length", - "tool_calls", - "content_filter", - "function_call" - ] - }, - { - "type": "null" - } - ], - "title": "Finish Reason" - }, - "index": { - "type": "integer", - "title": "Index" - } + "binary_rpm_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "message", - "index" - ], - "title": "AIQChoice" - }, - "AIQChoiceMessage": { - "properties": { - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Content" - }, - "role": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Role" - } + { + "type": "null" + } + ], + "title": "Binary Rpm Path" + }, + "patch_source_dir": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "type": "object", - "title": "AIQChoiceMessage" + { + "type": "null" + } + ], + "title": "Patch Source Dir" }, - "AIQEvaluateRequest": { - "properties": { - "config_file": { - "type": "string", - "title": "Config File", - "description": "Path to the configuration file for evaluation" - }, - "job_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "reps": { - "type": "integer", - "title": "Reps", - "description": "Number of repetitions for the evaluation, defaults to 1", - "default": 1 - }, - "expiry_seconds": { - "type": "integer", - "title": "Expiry Seconds", - "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", - "default": 3600 - } + "patch_diff_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "type": "object", - "required": [ - "config_file" - ], - "title": "AIQEvaluateRequest", - "description": "Request model for the evaluate endpoint." - }, - "AIQEvaluateResponse": { - "properties": { - "job_id": { - "type": "string", - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "status": { - "type": "string", - "title": "Status", - "description": "Current status of the evaluation job" - } + { + "type": "null" + } + ], + "title": "Patch Diff Path" + }, + "source_url": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "job_id", - "status" - ], - "title": "AIQEvaluateResponse", - "description": "Response model for the evaluate endpoint." - }, - "AIQEvaluateStatusResponse": { - "properties": { - "job_id": { - "type": "string", - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "status": { - "type": "string", - "title": "Status", - "description": "Current status of the evaluation job" - }, - "config_file": { - "type": "string", - "title": "Config File", - "description": "Path to the configuration file used for evaluation" - }, - "error": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Error", - "description": "Error message if the job failed" - }, - "output_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Output Path", - "description": "Path to the output file if the job completed successfully" - }, - "created_at": { - "type": "string", - "format": "date-time", - "title": "Created At", - "description": "Timestamp when the job was created" - }, - "updated_at": { - "type": "string", - "format": "date-time", - "title": "Updated At", - "description": "Timestamp when the job was last updated" - }, - "expires_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ], - "title": "Expires At", - "description": "Timestamp when the job will expire" - } + { + "type": "null" + } + ], + "title": "Source Url" + } + }, + "type": "object", + "title": "AcquiredArtifacts", + "description": "Resolved file locations populated by source_acquisition, consumed by downstream checker nodes." + }, + "AgentIntermediateStep": { + "properties": { + "tool_name": { + "type": "string", + "title": "Tool Name" + }, + "action_log": { + "type": "string", + "title": "Action Log" + }, + "tool_input": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "job_id", - "status", - "config_file", - "created_at", - "updated_at" - ], - "title": "AIQEvaluateStatusResponse", - "description": "Response model for the evaluate status endpoint." - }, - "AIQResponseIntermediateStep": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "parent_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Parent Id" - }, - "type": { - "type": "string", - "title": "Type", - "default": "markdown" - }, - "name": { - "type": "string", - "title": "Name" - }, - "payload": { - "type": "string", - "title": "Payload" - } + { + "type": "object" + } + ], + "title": "Tool Input" + }, + "tool_output": { + "title": "Tool Output" + } + }, + "type": "object", + "required": [ + "tool_name", + "action_log", + "tool_input", + "tool_output" + ], + "title": "AgentIntermediateStep", + "description": "Represents info for an intermediate step taken by an agent." + }, + "AgentMorpheusEngineOutput": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "checklist": { + "items": { + "$ref": "#/components/schemas/ChecklistItemOutput" + }, + "type": "array", + "title": "Checklist" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "justification": { + "$ref": "#/components/schemas/JustificationOutput" + }, + "intel_score": { + "type": "integer", + "title": "Intel Score" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSSOutput" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "name", - "payload" - ], - "title": "AIQResponseIntermediateStep", - "description": "AIQResponseSerializedStep is a data model that represents a serialized step in the AIQ Toolkit chat streaming API." - }, - "AIQUsage": { - "properties": { - "prompt_tokens": { - "type": "integer", - "title": "Prompt Tokens" - }, - "completion_tokens": { - "type": "integer", - "title": "Completion Tokens" - }, - "total_tokens": { - "type": "integer", - "title": "Total Tokens" - } + { + "type": "null" + } + ] + } + }, + "type": "object", + "required": [ + "vuln_id", + "checklist", + "summary", + "justification", + "intel_score", + "cvss" + ], + "title": "AgentMorpheusEngineOutput", + "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision.\n- cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability." + }, + "AgentMorpheusInfo": { + "properties": { + "vdb": { + "anyOf": [ + { + "$ref": "#/components/schemas/VdbPaths" }, - "type": "object", - "required": [ - "prompt_tokens", - "completion_tokens", - "total_tokens" - ], - "title": "AIQUsage" + { + "type": "null" + } + ] }, - "AgentIntermediateStep": { - "properties": { - "tool_name": { - "type": "string", - "title": "Tool Name" - }, - "action_log": { - "type": "string", - "title": "Action Log" - }, - "tool_input": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ], - "title": "Tool Input" - }, - "tool_output": { - "title": "Tool Output" - } + "intel": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CveIntel" + }, + "type": "array" }, - "type": "object", - "required": [ - "tool_name", - "action_log", - "tool_input", - "tool_output" - ], - "title": "AgentIntermediateStep", - "description": "Represents info for an intermediate step taken by an agent." - }, - "AgentMorpheusEngineOutput": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "checklist": { - "items": { - "$ref": "#/components/schemas/ChecklistItemOutput" - }, - "type": "array", - "title": "Checklist" - }, - "summary": { - "type": "string", - "title": "Summary" - }, - "justification": { - "$ref": "#/components/schemas/JustificationOutput" - }, - "intel_score": { - "type": "integer", - "title": "Intel Score" - }, - "cvss": { - "anyOf": [ - { - "$ref": "#/components/schemas/CVSSOutput" - }, - { - "type": "null" - } - ] - } + { + "type": "null" + } + ], + "title": "Intel" + }, + "sbom": { + "anyOf": [ + { + "$ref": "#/components/schemas/SBOMInfo" }, - "type": "object", - "required": [ - "vuln_id", - "checklist", - "summary", - "justification", - "intel_score", - "cvss" - ], - "title": "AgentMorpheusEngineOutput", - "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision.\n- intel_score: the intelligence score for the vulnerability.\n- cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability." - }, - "AgentMorpheusInfo": { - "properties": { - "vdb": { - "anyOf": [ - { - "$ref": "#/components/schemas/VdbPaths" - }, - { - "type": "null" - } - ] - }, - "intel": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/CveIntel" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Intel" - }, - "sbom": { - "anyOf": [ - { - "$ref": "#/components/schemas/SBOMInfo" - }, - { - "type": "null" - } - ] - }, - "vulnerable_dependencies": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/VulnerableDependencies" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vulnerable Dependencies" - } + { + "type": "null" + } + ] + }, + "vulnerable_dependencies": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/VulnerableDependencies" + }, + "type": "array" }, - "type": "object", - "title": "AgentMorpheusInfo", - "description": "Information used for decisioning in the Agent Morpheus engine. These information can all be automatically\ngenerated or retrieved by the pipeline from the input information.\n\n- vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability\n is exploitable in the source code.\n- intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability\n databases and APIs.\n- sbom: software bill of materials listing the packages and versions in the container image, used to understand\n whether the vulnerable package exists in the image.\n- vulnerable_dependencies: a list of VulnerableDependencies objects for each vuln_id, representing the SBOM packages\n and transitive dependencies that are vulnerable for the vuln_id." + { + "type": "null" + } + ], + "title": "Vulnerable Dependencies" }, - "AgentMorpheusInput-Input": { - "properties": { - "scan": { - "$ref": "#/components/schemas/ScanInfoInput" - }, - "image": { - "$ref": "#/components/schemas/ImageInfoInput-Input" - } + "checker_context": { + "anyOf": [ + { + "$ref": "#/components/schemas/PackageCheckerContext" }, - "type": "object", - "required": [ - "scan", - "image" - ], - "title": "AgentMorpheusInput", - "description": "Inputs required by the Agent Morpheus pipeline." + { + "type": "null" + } + ] + } + }, + "type": "object", + "title": "AgentMorpheusInfo", + "description": "Information used for decisioning in the Agent Morpheus engine. These information can all be automatically\ngenerated or retrieved by the pipeline from the input information.\n\n- vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability\n is exploitable in the source code.\n- intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability\n databases and APIs.\n- sbom: software bill of materials listing the packages and versions in the container image, used to understand\n whether the vulnerable package exists in the image.\n- vulnerable_dependencies: a list of VulnerableDependencies objects for each vuln_id, representing the SBOM packages\n and transitive dependencies that are vulnerable for the vuln_id." + }, + "AgentMorpheusInput-Input": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" }, - "AgentMorpheusInput-Output": { - "properties": { - "scan": { - "$ref": "#/components/schemas/ScanInfoInput" - }, - "image": { - "$ref": "#/components/schemas/ImageInfoInput-Output" - } + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Input" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "scan", - "image" - ], - "title": "AgentMorpheusInput", - "description": "Inputs required by the Agent Morpheus pipeline." + { + "type": "null" + } + ], + "title": "Credential Id" }, - "AgentMorpheusOutput": { - "properties": { - "input": { - "$ref": "#/components/schemas/AgentMorpheusInput-Output" - }, - "info": { - "$ref": "#/components/schemas/AgentMorpheusInfo" - }, - "output": { - "$ref": "#/components/schemas/OutputPayload" - } + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "input", - "info", - "output" - ], - "title": "AgentMorpheusOutput", - "description": "\"\nThe final output of the Agent Morpheus pipeline.\nContains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability." - }, - "AudioContent": { - "properties": { - "type": { - "type": "string", - "const": "input_audio", - "title": "Type", - "default": "input_audio" - }, - "input_audio": { - "$ref": "#/components/schemas/InputAudio", - "default": { - "data": "default", - "format": "default" - } - } + { + "type": "null" + } + ], + "title": "Code Index Success" + }, + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": false, - "type": "object", - "title": "AudioContent" + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "description": "Inputs required by the Agent Morpheus pipeline." + }, + "AgentMorpheusInput-Output": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" }, - "BaseMetricV3": { - "properties": { - "cvssV3": { - "$ref": "#/components/schemas/CVSSV3" - }, - "exploitabilityScore": { - "type": "number", - "title": "Exploitabilityscore" - }, - "impactScore": { - "type": "number", - "title": "Impactscore" - } + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Output" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "cvssV3", - "exploitabilityScore", - "impactScore" - ], - "title": "BaseMetricV3" - }, - "Bugzilla": { - "properties": { - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Id" - }, - "url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Url" - } + { + "type": "null" + } + ], + "title": "Credential Id" + }, + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "title": "Bugzilla" - }, - "CVSS": { - "properties": { - "score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Score" - }, - "vector_string": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Vector String" - } + { + "type": "null" + } + ], + "title": "Code Index Success" + }, + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "CVSS" - }, - "CVSS3": { - "properties": { - "cvss3_base_score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Cvss3 Base Score" - }, - "cvss3_scoring_vector": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss3 Scoring Vector" - }, - "status": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "description": "Inputs required by the Agent Morpheus pipeline." + }, + "AgentMorpheusOutput": { + "properties": { + "input": { + "$ref": "#/components/schemas/AgentMorpheusInput-Output" + }, + "info": { + "$ref": "#/components/schemas/AgentMorpheusInfo" + }, + "output": { + "$ref": "#/components/schemas/OutputPayload" + } + }, + "type": "object", + "required": [ + "input", + "info", + "output" + ], + "title": "AgentMorpheusOutput", + "description": "\"\nThe final output of the Agent Morpheus pipeline.\nContains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability." + }, + "AnalysisType": { + "type": "string", + "enum": [ + "image", + "source" + ], + "title": "AnalysisType" + }, + "AsyncGenerateRequest": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" + }, + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Input" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "CVSS3" + { + "type": "null" + } + ], + "title": "Credential Id" }, - "CVSSOutput": { - "properties": { - "vector_string": { - "type": "string", - "title": "Vector String" - }, - "score": { - "type": "string", - "title": "Score" - } + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "vector_string", - "score" - ], - "title": "CVSSOutput", - "description": "CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image.\n- vector_string: The CVSS vector string that encodes the metric values used to calculate the score.\n- score: The calculated CVSS base score representing the severity of the vulnerability in the given image." - }, - "CVSSV3": { - "properties": { - "attackComplexity": { - "type": "string", - "title": "Attackcomplexity" - }, - "attackVector": { - "type": "string", - "title": "Attackvector" - }, - "availabilityImpact": { - "type": "string", - "title": "Availabilityimpact" - }, - "baseScore": { - "type": "number", - "title": "Basescore" - }, - "baseSeverity": { - "type": "string", - "title": "Baseseverity" - }, - "confidentialityImpact": { - "type": "string", - "title": "Confidentialityimpact" - }, - "integrityImpact": { - "type": "string", - "title": "Integrityimpact" - }, - "privilegesRequired": { - "type": "string", - "title": "Privilegesrequired" - }, - "scope": { - "type": "string", - "title": "Scope" - }, - "userInteraction": { - "type": "string", - "title": "Userinteraction" - }, - "vectorString": { - "type": "string", - "title": "Vectorstring" - }, - "version": { - "type": "string", - "title": "Version" - } + { + "type": "null" + } + ], + "title": "Code Index Success" + }, + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "attackComplexity", - "attackVector", - "availabilityImpact", - "baseScore", - "baseSeverity", - "confidentialityImpact", - "integrityImpact", - "privilegesRequired", - "scope", - "userInteraction", - "vectorString", - "version" - ], - "title": "CVSSV3" + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" }, - "CWE": { - "properties": { - "cwe_id": { - "type": "string", - "title": "Cwe Id" - }, - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - } + "job_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "cwe_id" - ], - "title": "CWE" + { + "type": "null" + } + ], + "title": "Job Id", + "description": "Unique identifier for the evaluation job" }, - "ChecklistItemOutput": { - "properties": { - "input": { - "type": "string", - "title": "Input" - }, - "response": { - "type": "string", - "title": "Response" - }, - "intermediate_steps": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/AgentIntermediateStep" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Intermediate Steps" - } + "sync_timeout": { + "type": "integer", + "maximum": 300, + "minimum": 0, + "title": "Sync Timeout", + "description": "Attempt to perform the job synchronously up until `sync_timeout` sectonds, if the job hasn't been completed by then a job_id will be returned with a status code of 202.", + "default": 0 + }, + "expiry_seconds": { + "type": "integer", + "maximum": 86400, + "minimum": 600, + "title": "Expiry Seconds", + "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", + "default": 3600 + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AsyncGenerateRequest" + }, + "AsyncGenerateResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the job" + } + }, + "type": "object", + "required": [ + "job_id", + "status" + ], + "title": "AsyncGenerateResponse", + "description": "Response model for the async generation endpoint." + }, + "AsyncGenerationStatusResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the evaluation job" + }, + "error": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "input", - "response" - ], - "title": "ChecklistItemOutput", - "description": "Input, response, and intermediate steps for a single checklist item provided to the LLM agent." - }, - "Configuration": { - "properties": { - "package": { - "type": "string", - "title": "Package" - }, - "system": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "System" - }, - "versionStartExcluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionstartexcluding" - }, - "versionEndExcluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionendexcluding" - }, - "versionStartIncluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionstartincluding" - }, - "versionEndIncluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionendincluding" - } + { + "type": "null" + } + ], + "title": "Error", + "description": "Error message if the job failed" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "Timestamp when the job was created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At", + "description": "Timestamp when the job was last updated" + }, + "expires_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "type": "object", - "required": [ - "package" - ], - "title": "Configuration" + { + "type": "null" + } + ], + "title": "Expires At", + "description": "Timestamp when the job will expire" }, - "CveIntel": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "ghsa": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelGhsa" - }, - { - "type": "null" - } - ] - }, - "nvd": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelNvd" - }, - { - "type": "null" - } - ] - }, - "rhsa": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelRhsa" - }, - { - "type": "null" - } - ] - }, - "ubuntu": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelUbuntu" - }, - { - "type": "null" - } - ] - }, - "epss": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelEpss" - }, - { - "type": "null" - } - ] - }, - "has_sufficient_intel_for_agent": { - "type": "boolean", - "title": "Has Sufficient Intel For Agent", - "description": "Logic to determine if the CVE has sufficient intel and can be passed to the agent.\n\nReturns\n-------\nbool\n True if enough intel has been found for the CVE", - "readOnly": true - } + "output": { + "anyOf": [ + { + "type": "object" }, - "type": "object", - "required": [ - "vuln_id", - "has_sufficient_intel_for_agent" - ], - "title": "CveIntel", - "description": "Information about a CVE (Common Vulnerabilities and Exposures) entry." - }, - "CveIntelEpss": { - "properties": { - "epss": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Epss" - }, - "percentile": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Percentile" - }, - "date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Date" - } + { + "type": "null" + } + ], + "title": "Output", + "description": "Output of the generate request, this is only available if the job completed successfully." + } + }, + "type": "object", + "required": [ + "job_id", + "status", + "created_at", + "updated_at" + ], + "title": "AsyncGenerationStatusResponse" + }, + "AudioContent": { + "properties": { + "type": { + "type": "string", + "const": "input_audio", + "title": "Type", + "default": "input_audio" + }, + "input_audio": { + "$ref": "#/components/schemas/InputAudio", + "default": { + "data": "default", + "format": "default" + } + } + }, + "additionalProperties": false, + "type": "object", + "title": "AudioContent" + }, + "BaseMetricV3": { + "properties": { + "cvssV3": { + "$ref": "#/components/schemas/CVSSV3" + }, + "exploitabilityScore": { + "anyOf": [ + { + "type": "number" }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelEpss", - "description": "Information about an EPSS (Elastic Product Security Service) entry." + { + "type": "null" + } + ], + "title": "Exploitabilityscore" }, - "CveIntelGhsa": { - "properties": { - "ghsa_id": { - "type": "string", - "title": "Ghsa Id" - }, - "cve_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cve Id" - }, - "summary": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Summary" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Severity" - }, - "vulnerabilities": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vulnerabilities" - }, - "cvss": { - "anyOf": [ - { - "$ref": "#/components/schemas/CVSS" - }, - { - "type": "null" - } - ] - }, - "cwes": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/CWE" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Cwes" - }, - "published_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published At" - }, - "updated_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Updated At" - } + "impactScore": { + "anyOf": [ + { + "type": "number" }, - "additionalProperties": true, - "type": "object", - "required": [ - "ghsa_id" - ], - "title": "CveIntelGhsa", - "description": "Information about a GHSA (GitHub Security Advisory) entry." - }, - "CveIntelNvd": { - "properties": { - "cve_id": { - "type": "string", - "title": "Cve Id" - }, - "cve_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cve Description" - }, - "cvss_vector": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss Vector" - }, - "cvss_base_score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Cvss Base Score" - }, - "cvss_severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss Severity" - }, - "cwe_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Name" - }, - "cwe_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Description" - }, - "cwe_extended_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Extended Description" - }, - "configurations": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/Configuration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Configurations" - }, - "vendor_names": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vendor Names" - }, - "references": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "References" - }, - "disputed": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "title": "Disputed" - }, - "published_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published At" - }, - "updated_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Updated At" - } + { + "type": "null" + } + ], + "title": "Impactscore" + } + }, + "type": "object", + "required": [ + "cvssV3" + ], + "title": "BaseMetricV3" + }, + "Bugzilla": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": true, - "type": "object", - "required": [ - "cve_id" - ], - "title": "CveIntelNvd", - "description": "Information about an NVD (National Vulnerability Database) entry." + { + "type": "null" + } + ], + "title": "Description" }, - "CveIntelRhsa": { - "properties": { - "bugzilla": { - "$ref": "#/components/schemas/Bugzilla" - }, - "details": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Details" - }, - "statement": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Statement" - }, - "package_state": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/PackageState" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Package State" - }, - "upstream_fix": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Upstream Fix" - }, - "cvss3": { - "anyOf": [ - { - "$ref": "#/components/schemas/CVSS3" - }, - { - "type": "null" - } - ] - } + "id": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelRhsa", - "description": "Information about a RHSA (Red Hat Security Advisory) entry." - }, - "CveIntelUbuntu": { - "properties": { - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "notes": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/Note" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Notes" - }, - "notices": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Notices" - }, - "priority": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Priority" - }, - "ubuntu_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Ubuntu Description" - }, - "impact": { - "anyOf": [ - { - "$ref": "#/components/schemas/Impact" - }, - { - "type": "null" - } - ] - } - }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelUbuntu", - "description": "Information about a Ubuntu CVE entry." - }, - "DependencyPackage": { - "properties": { - "system": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "System" - }, - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Version" - }, - "relation": { - "anyOf": [ - { - "type": "string", - "enum": [ - "SELF", - "DIRECT", - "INDIRECT" - ] - }, - { - "type": "null" - } - ], - "title": "Relation" - } + { + "type": "null" + } + ], + "title": "Id" + }, + "url": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "DependencyPackage", - "description": "Information about a dependency package as obtained from deps.dev API for a given SBOM package." - }, - "FileSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "file", - "title": "Type", - "description": "The type of the object", - "default": "file" - }, - "file_path": { - "type": "string", - "title": "File Path" - } + { + "type": "null" + } + ], + "title": "Url" + } + }, + "type": "object", + "title": "Bugzilla" + }, + "CVSS": { + "properties": { + "score": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "required": [ - "file_path" - ], - "title": "FileSBOMInfoInput", - "description": "A file path pointing to a Software Bill of Materials file." - }, - "HTTPSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "http", - "title": "Type", - "description": "The type of the object", - "default": "http" - }, - "url": { - "type": "string", - "minLength": 1, - "format": "uri", - "title": "Url" - } + { + "type": "null" + } + ], + "title": "Score" + }, + "vector_string": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "url" - ], - "title": "HTTPSBOMInfoInput", - "description": "A URL pointing to a Software Bill of Materials file." - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "type": "array", - "title": "Detail" - } + { + "type": "null" + } + ], + "title": "Vector String" + } + }, + "type": "object", + "title": "CVSS" + }, + "CVSS3": { + "properties": { + "cvss3_base_score": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "title": "HTTPValidationError" + { + "type": "null" + } + ], + "title": "Cvss3 Base Score" }, - "ImageContent": { - "properties": { - "type": { - "type": "string", - "const": "image_url", - "title": "Type", - "default": "image_url" - }, - "image_url": { - "$ref": "#/components/schemas/ImageUrl", - "default": { - "url": "http://default.com/" - } - } + "cvss3_scoring_vector": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": false, - "type": "object", - "title": "ImageContent" - }, - "ImageInfoInput-Input": { - "properties": { - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "tag": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Tag" - }, - "digest": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Digest" - }, - "platform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Platform" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "analysis_type": { - "type": "string", - "enum": [ - "image", - "source" - ], - "title": "Analysis Type" - }, - "ecosystem": { - "type": "string", - "enum": [ - "go", - "python", - "javascript", - "java", - "c", - ], - "title": "Ecosystem" - }, - "manifest_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Manifest Path" - }, - "source_info": { - "items": { - "$ref": "#/components/schemas/SourceDocumentsInfo" - }, - "type": "array", - "title": "Source Info" - }, - "sbom_info": { - "oneOf": [ - { - "$ref": "#/components/schemas/ManualSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/FileSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/HTTPSBOMInfoInput" - } - ], - "title": "Sbom Info" - } + { + "type": "null" + } + ], + "title": "Cvss3 Scoring Vector" + }, + "status": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "analysis_type", - "source_info", - "sbom_info" - ], - "title": "ImageInfoInput", - "description": "Information about a container image, including the source information and sbom information." - }, - "ImageInfoInput-Output": { - "properties": { - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "tag": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Tag" - }, - "digest": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Digest" - }, - "platform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Platform" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "analysis_type": { - "type": "string", - "enum": [ - "image", - "source" - ], - "title": "Analysis Type" - }, - "ecosystem": { - "type": "string", - "enum": [ - "go", - "python", - "javascript", - "java", - "c", - ], - "title": "Ecosystem" - }, - "manifest_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Manifest Path" - }, - "source_info": { - "items": { - "$ref": "#/components/schemas/SourceDocumentsInfo" - }, - "type": "array", - "title": "Source Info" - }, - "sbom_info": { - "oneOf": [ - { - "$ref": "#/components/schemas/ManualSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/FileSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/HTTPSBOMInfoInput" - } - ], - "title": "Sbom Info" - } + { + "type": "null" + } + ], + "title": "Status" + } + }, + "type": "object", + "title": "CVSS3" + }, + "CVSSOutput": { + "properties": { + "vector_string": { + "type": "string", + "title": "Vector String" + }, + "score": { + "type": "string", + "title": "Score" + } + }, + "type": "object", + "required": [ + "vector_string", + "score" + ], + "title": "CVSSOutput", + "description": "CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image.\n- vector_string: The CVSS vector string that encodes the metric values used to calculate the score.\n- score: The calculated CVSS base score representing the severity of the vulnerability in the given image." + }, + "CVSSV3": { + "properties": { + "attackComplexity": { + "type": "string", + "title": "Attackcomplexity" + }, + "attackVector": { + "type": "string", + "title": "Attackvector" + }, + "availabilityImpact": { + "type": "string", + "title": "Availabilityimpact" + }, + "baseScore": { + "type": "number", + "title": "Basescore" + }, + "baseSeverity": { + "type": "string", + "title": "Baseseverity" + }, + "confidentialityImpact": { + "type": "string", + "title": "Confidentialityimpact" + }, + "integrityImpact": { + "type": "string", + "title": "Integrityimpact" + }, + "privilegesRequired": { + "type": "string", + "title": "Privilegesrequired" + }, + "scope": { + "type": "string", + "title": "Scope" + }, + "userInteraction": { + "type": "string", + "title": "Userinteraction" + }, + "vectorString": { + "type": "string", + "title": "Vectorstring" + }, + "version": { + "type": "string", + "title": "Version" + } + }, + "type": "object", + "required": [ + "attackComplexity", + "attackVector", + "availabilityImpact", + "baseScore", + "baseSeverity", + "confidentialityImpact", + "integrityImpact", + "privilegesRequired", + "scope", + "userInteraction", + "vectorString", + "version" + ], + "title": "CVSSV3" + }, + "CWE": { + "properties": { + "cwe_id": { + "type": "string", + "title": "Cwe Id" + }, + "name": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "analysis_type", - "source_info", - "sbom_info" - ], - "title": "ImageInfoInput", - "description": "Information about a container image, including the source information and sbom information." - }, - "ImageUrl": { - "properties": { - "url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", - "title": "Url", - "default": "http://default.com/" - } + { + "type": "null" + } + ], + "title": "Name" + } + }, + "type": "object", + "required": [ + "cwe_id" + ], + "title": "CWE" + }, + "ChatRequest": { + "properties": { + "messages": { + "items": { + "$ref": "#/components/schemas/Message" + }, + "type": "array", + "title": "Messages" + }, + "model": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "ImageUrl" + { + "type": "null" + } + ], + "title": "Model", + "description": "name of the model to use" }, - "Impact": { - "properties": { - "baseMetricV3": { - "$ref": "#/components/schemas/BaseMetricV3" - } + "frequency_penalty": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "required": [ - "baseMetricV3" - ], - "title": "Impact" - }, - "InputAudio": { - "properties": { - "data": { - "type": "string", - "title": "Data", - "default": "default" - }, - "format": { - "type": "string", - "title": "Format", - "default": "default" - } + { + "type": "null" + } + ], + "title": "Frequency Penalty", + "description": "Penalty for new tokens based on frequency in text", + "default": 0 + }, + "logit_bias": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" }, - "type": "object", - "title": "InputAudio" + { + "type": "null" + } + ], + "title": "Logit Bias", + "description": "Modify likelihood of specified tokens appearing" }, - "JustificationOutput": { - "properties": { - "label": { - "type": "string", - "title": "Label" - }, - "reason": { - "type": "string", - "title": "Reason" - }, - "status": { - "type": "string", - "enum": [ - "TRUE", - "FALSE", - "UNKNOWN" - ], - "title": "Status" - } + "logprobs": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "label", - "reason", - "status" - ], - "title": "JustificationOutput", - "description": "Final justification for the vulnerability.\n\n- label: a categorical justification label classifying the status of an image against a given vulnerability, e.g.\n code_not_present, code_not_reachable, false_positive.\n- reason: a human-readable explanation for why justification label was selected.\n- status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given\n vulnerability. Determined based on a mapping from the justification label." - }, - "ManualSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "manual", - "title": "Type", - "description": "The type of the object", - "default": "manual" - }, - "packages": { - "items": { - "$ref": "#/components/schemas/SBOMPackage" - }, - "type": "array", - "title": "Packages" - } + { + "type": "null" + } + ], + "title": "Logprobs", + "description": "Whether to return log probabilities" + }, + "top_logprobs": { + "anyOf": [ + { + "type": "integer" }, - "type": "object", - "required": [ - "packages" - ], - "title": "ManualSBOMInfoInput", - "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects." - }, - "Message": { - "properties": { - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/TextContent" - }, - { - "$ref": "#/components/schemas/ImageContent" - }, - { - "$ref": "#/components/schemas/AudioContent" - } - ], - "discriminator": { - "propertyName": "type", - "mapping": { - "image_url": "#/components/schemas/ImageContent", - "input_audio": "#/components/schemas/AudioContent", - "text": "#/components/schemas/TextContent" - } - } - }, - "type": "array" - } - ], - "title": "Content" + { + "type": "null" + } + ], + "title": "Top Logprobs", + "description": "Number of most likely tokens to return" + }, + "max_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tokens", + "description": "Maximum number of tokens to generate" + }, + "n": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "N", + "description": "Number of chat completion choices to generate", + "default": 1 + }, + "presence_penalty": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Presence Penalty", + "description": "Penalty for new tokens based on presence in text", + "default": 0 + }, + "response_format": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Response Format", + "description": "Response format specification" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seed", + "description": "Random seed for deterministic sampling" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "auto", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier", + "description": "Service tier for the request" + }, + "stream": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Stream", + "description": "Whether to stream partial message deltas", + "default": false + }, + "stream_options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Stream Options", + "description": "Options for streaming" + }, + "temperature": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Temperature", + "description": "Sampling temperature between 0 and 2", + "default": 1 + }, + "top_p": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Top P", + "description": "Nucleus sampling parameter" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "List of tools the model may call" + }, + "tool_choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Choice", + "description": "Controls which tool is called" + }, + "parallel_tool_calls": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Parallel Tool Calls", + "description": "Whether to enable parallel function calling", + "default": true + }, + "user": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User", + "description": "Unique identifier representing end-user" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "messages" + ], + "title": "ChatRequest", + "description": "ChatRequest is a data model that represents a request to the NAT chat API.\nFully compatible with OpenAI Chat Completions API specification.", + "example": { + "messages": [ + { + "content": "who are you?", + "role": "user" + } + ], + "model": "nvidia/nemotron", + "stream": false, + "temperature": 0.7 + } + }, + "ChatResponse": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "object": { + "type": "string", + "title": "Object", + "default": "chat.completion" + }, + "model": { + "type": "string", + "title": "Model", + "default": "" + }, + "created": { + "type": "integer", + "title": "Created" + }, + "choices": { + "items": { + "$ref": "#/components/schemas/Choice" + }, + "type": "array", + "title": "Choices" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/Usage" + }, + { + "type": "null" + } + ] + }, + "system_fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System Fingerprint" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "scale", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "created", + "choices" + ], + "title": "ChatResponse", + "description": "ChatResponse is a data model that represents a response from the NAT chat API.\nFully compatible with OpenAI Chat Completions API specification." + }, + "ChatResponseChunk": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "choices": { + "items": { + "$ref": "#/components/schemas/Choice" + }, + "type": "array", + "title": "Choices" + }, + "created": { + "type": "integer", + "title": "Created" + }, + "model": { + "type": "string", + "title": "Model", + "default": "" + }, + "object": { + "type": "string", + "title": "Object", + "default": "chat.completion.chunk" + }, + "system_fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System Fingerprint" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "scale", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/Usage" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "choices", + "created" + ], + "title": "ChatResponseChunk", + "description": "ChatResponseChunk is a data model that represents a response chunk from the NAT chat streaming API.\nFully compatible with OpenAI Chat Completions API specification." + }, + "ChecklistItemOutput": { + "properties": { + "input": { + "type": "string", + "title": "Input" + }, + "response": { + "type": "string", + "title": "Response" + }, + "intermediate_steps": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentIntermediateStep" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Intermediate Steps" + } + }, + "type": "object", + "required": [ + "input", + "response" + ], + "title": "ChecklistItemOutput", + "description": "Input, response, and intermediate steps for a single checklist item provided to the LLM agent." + }, + "Choice": { + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChoiceMessage" + }, + { + "type": "null" + } + ] + }, + "delta": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChoiceDelta" + }, + { + "type": "null" + } + ] + }, + "finish_reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call" + ] + }, + { + "type": "null" + } + ], + "title": "Finish Reason" + }, + "index": { + "type": "integer", + "title": "Index" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "index" + ], + "title": "Choice" + }, + "ChoiceDelta": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Role" + } + }, + "type": "object", + "title": "ChoiceDelta", + "description": "Delta object for streaming responses (OpenAI-compatible)" + }, + "ChoiceMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Role" + } + }, + "type": "object", + "title": "ChoiceMessage" + }, + "Configuration": { + "properties": { + "package": { + "type": "string", + "title": "Package" + }, + "vendor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vendor" + }, + "system": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System" + }, + "versionStartExcluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionstartexcluding" + }, + "versionEndExcluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionendexcluding" + }, + "versionStartIncluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionstartincluding" + }, + "versionEndIncluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionendincluding" + } + }, + "type": "object", + "required": [ + "package" + ], + "title": "Configuration" + }, + "CveIntel": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "ghsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelGhsa" + }, + { + "type": "null" + } + ] + }, + "nvd": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelNvd" + }, + { + "type": "null" + } + ] + }, + "rhsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelRhsa" + }, + { + "type": "null" + } + ] + }, + "ubuntu": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelUbuntu" + }, + { + "type": "null" + } + ] + }, + "epss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelEpss" + }, + { + "type": "null" + } + ] + }, + "plugin_data": { + "items": { + "$ref": "#/components/schemas/IntelPluginData" + }, + "type": "array", + "title": "Plugin Data", + "default": [] + }, + "intel_score": { + "type": "integer", + "title": "Intel Score", + "default": 0 + }, + "has_sufficient_intel_for_agent": { + "type": "boolean", + "title": "Has Sufficient Intel For Agent", + "description": "Logic to determine if the CVE has sufficient intel and can be passed to the agent.\n\nReturns\n-------\nbool\n True if enough intel has been found for the CVE", + "readOnly": true + } + }, + "type": "object", + "required": [ + "vuln_id", + "has_sufficient_intel_for_agent" + ], + "title": "CveIntel", + "description": "Information about a CVE (Common Vulnerabilities and Exposures) entry." + }, + "CveIntelEpss": { + "properties": { + "epss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Epss" + }, + "percentile": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Percentile" + }, + "date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Date" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelEpss", + "description": "Information about an EPSS (Elastic Product Security Service) entry." + }, + "CveIntelGhsa": { + "properties": { + "ghsa_id": { + "type": "string", + "title": "Ghsa Id" + }, + "cve_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Id" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Summary" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "vulnerabilities": { + "anyOf": [ + { + "items": { + + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vulnerabilities" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSS" + }, + { + "type": "null" + } + ] + }, + "cwes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CWE" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Cwes" + }, + "published_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ghsa_id" + ], + "title": "CveIntelGhsa", + "description": "Information about a GHSA (GitHub Security Advisory) entry." + }, + "CveIntelNvd": { + "properties": { + "cve_id": { + "type": "string", + "title": "Cve Id" + }, + "cve_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Description" + }, + "cvss_vector": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cvss Vector" + }, + "cvss_base_score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cvss Base Score" + }, + "cvss_severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cvss Severity" + }, + "cwe_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Id" + }, + "cwe_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Name" + }, + "cwe_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Description" + }, + "cwe_extended_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Extended Description" + }, + "configurations": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Configuration" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Configurations" + }, + "vendor_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vendor Names" + }, + "references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "References" + }, + "disputed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Disputed" + }, + "published_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cve_id" + ], + "title": "CveIntelNvd", + "description": "Information about an NVD (National Vulnerability Database) entry." + }, + "CveIntelRhsa": { + "properties": { + "bugzilla": { + "$ref": "#/components/schemas/Bugzilla" + }, + "details": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Details" + }, + "statement": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Statement" + }, + "package_state": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PackageState" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Package State" + }, + "upstream_fix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Fix" + }, + "cvss3": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSS3" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelRhsa", + "description": "Information about a RHSA (Red Hat Security Advisory) entry." + }, + "CveIntelUbuntu": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "notes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Note" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Notes" + }, + "notices": { + "anyOf": [ + { + "items": { + + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Notices" + }, + "priority": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Priority" + }, + "ubuntu_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ubuntu Description" + }, + "impact": { + "anyOf": [ + { + "$ref": "#/components/schemas/Impact" + }, + { + "type": "null" + } + ] + }, + "patches": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Patches", + "description": "Map of package name to patch refs (e.g., 'upstream: https://github.com/.../commit/...')" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelUbuntu", + "description": "Information about a Ubuntu CVE entry." + }, + "DependencyPackage": { + "properties": { + "system": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "relation": { + "anyOf": [ + { + "type": "string", + "enum": [ + "SELF", + "DIRECT", + "INDIRECT" + ] + }, + { + "type": "null" + } + ], + "title": "Relation" + } + }, + "type": "object", + "title": "DependencyPackage", + "description": "Information about a dependency package as obtained from deps.dev API for a given SBOM package." + }, + "Ecosystem": { + "type": "string", + "enum": [ + "go", + "python", + "javascript", + "java", + "c" + ], + "title": "Ecosystem" + }, + "EnumIdentifyResult": { + "type": "string", + "enum": [ + "yes", + "no", + "unknown" + ], + "title": "EnumIdentifyResult", + "description": "Result of the PackageIdentify phase for a single CVE." + }, + "EvaluateRequest": { + "properties": { + "config_file": { + "type": "string", + "title": "Config File", + "description": "Path to the configuration file for evaluation" + }, + "job_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "reps": { + "type": "integer", + "exclusiveMinimum": 0, + "title": "Reps", + "description": "Number of repetitions for the evaluation, defaults to 1", + "default": 1 + }, + "expiry_seconds": { + "type": "integer", + "exclusiveMinimum": 0, + "title": "Expiry Seconds", + "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", + "default": 3600 + } + }, + "type": "object", + "required": [ + "config_file" + ], + "title": "EvaluateRequest", + "description": "Request model for the evaluate endpoint." + }, + "EvaluateResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the job" + } + }, + "type": "object", + "required": [ + "job_id", + "status" + ], + "title": "EvaluateResponse", + "description": "Response model for the evaluate endpoint." + }, + "EvaluateStatusResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the evaluation job" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "Error message if the job failed" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "Timestamp when the job was created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At", + "description": "Timestamp when the job was last updated" + }, + "expires_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "title": "Expires At", + "description": "Timestamp when the job will expire" + }, + "config_file": { + "type": "string", + "title": "Config File", + "description": "Path to the configuration file used for evaluation" + }, + "output_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Path", + "description": "Path to the output file if the job completed successfully" + } + }, + "type": "object", + "required": [ + "job_id", + "status", + "created_at", + "updated_at", + "config_file" + ], + "title": "EvaluateStatusResponse", + "description": "Response model for the evaluate status endpoint." + }, + "FileSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "file", + "title": "Type", + "description": "The type of the object", + "default": "file" + }, + "file_path": { + "type": "string", + "title": "File Path" + } + }, + "type": "object", + "required": [ + "file_path" + ], + "title": "FileSBOMInfoInput", + "description": "A file path pointing to a Software Bill of Materials file." + }, + "HTTPSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "http", + "title": "Type", + "description": "The type of the object", + "default": "http" + }, + "url": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Url" + } + }, + "type": "object", + "required": [ + "url" + ], + "title": "HTTPSBOMInfoInput", + "description": "A URL pointing to a Software Bill of Materials file." + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "HealthStatusResponse": { + "properties": { + "status": { + "type": "string", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "HealthStatusResponse" + }, + "ImageContent": { + "properties": { + "type": { + "type": "string", + "const": "image_url", + "title": "Type", + "default": "image_url" + }, + "image_url": { + "$ref": "#/components/schemas/ImageUrl", + "default": { + "url": "http://default.com/" + } + } + }, + "additionalProperties": false, + "type": "object", + "title": "ImageContent" + }, + "ImageInfoInput-Input": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "ecosystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/Ecosystem" + }, + { + "type": "null" + } + ] + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, + "analysis_type": { + "$ref": "#/components/schemas/AnalysisType" + }, + "pipeline_mode": { + "$ref": "#/components/schemas/PipelineMode", + "default": "full_pipeline" + }, + "target_package": { + "anyOf": [ + { + "$ref": "#/components/schemas/TargetPackage" + }, + { + "type": "null" + } + ] + }, + "source_info": { + "items": { + "$ref": "#/components/schemas/SourceDocumentsInfo" + }, + "type": "array", + "title": "Source Info", + "default": [] + }, + "sbom_info": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/components/schemas/ManualSBOMInfoInput" + }, + { + "$ref": "#/components/schemas/FileSBOMInfoInput" + }, + { + "$ref": "#/components/schemas/HTTPSBOMInfoInput" + } + ] + }, + { + "type": "null" + } + ], + "title": "Sbom Info" + } + }, + "type": "object", + "required": [ + "analysis_type" + ], + "title": "ImageInfoInput", + "description": "Information about a container image, including the source information and sbom information." + }, + "ImageInfoInput-Output": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "ecosystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/Ecosystem" + }, + { + "type": "null" + } + ] + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, + "analysis_type": { + "$ref": "#/components/schemas/AnalysisType" + }, + "pipeline_mode": { + "$ref": "#/components/schemas/PipelineMode", + "default": "full_pipeline" + }, + "target_package": { + "anyOf": [ + { + "$ref": "#/components/schemas/TargetPackage" + }, + { + "type": "null" + } + ] + }, + "source_info": { + "items": { + "$ref": "#/components/schemas/SourceDocumentsInfo" + }, + "type": "array", + "title": "Source Info", + "default": [] + }, + "sbom_info": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/components/schemas/ManualSBOMInfoInput" }, - "role": { - "type": "string", - "title": "Role" + { + "$ref": "#/components/schemas/FileSBOMInfoInput" + }, + { + "$ref": "#/components/schemas/HTTPSBOMInfoInput" } + ] + }, + { + "type": "null" + } + ], + "title": "Sbom Info" + } + }, + "type": "object", + "required": [ + "analysis_type" + ], + "title": "ImageInfoInput", + "description": "Information about a container image, including the source information and sbom information." + }, + "ImageUrl": { + "properties": { + "url": { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri", + "title": "Url", + "default": "http://default.com/" + } + }, + "type": "object", + "title": "ImageUrl" + }, + "Impact": { + "properties": { + "baseMetricV3": { + "$ref": "#/components/schemas/BaseMetricV3" + } + }, + "type": "object", + "required": [ + "baseMetricV3" + ], + "title": "Impact" + }, + "InputAudio": { + "properties": { + "data": { + "type": "string", + "title": "Data", + "default": "default" + }, + "format": { + "type": "string", + "title": "Format", + "default": "default" + } + }, + "type": "object", + "title": "InputAudio" + }, + "IntelPluginData": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "description": { + "type": "string", + "title": "Description" + } + }, + "type": "object", + "required": [ + "label", + "description" + ], + "title": "IntelPluginData" + }, + "JustificationOutput": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "reason": { + "type": "string", + "title": "Reason" + }, + "status": { + "type": "string", + "enum": [ + "TRUE", + "FALSE", + "UNKNOWN" + ], + "title": "Status" + } + }, + "type": "object", + "required": [ + "label", + "reason", + "status" + ], + "title": "JustificationOutput", + "description": "Final justification for the vulnerability.\n\n- label: a categorical justification label classifying the status of an image against a given vulnerability, e.g.\n code_not_present, code_not_reachable, false_positive.\n- reason: a human-readable explanation for why justification label was selected.\n- status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given\n vulnerability. Determined based on a mapping from the justification label." + }, + "L1InvestigationResult": { + "properties": { + "downstream_report": { + "anyOf": [ + { + "type": "object" }, - "type": "object", - "required": [ - "content", - "role" - ], - "title": "Message" - }, - "Note": { - "properties": { - "author": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Author" - }, - "note": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Note" - } + { + "type": "null" + } + ], + "title": "Downstream Report", + "description": "Serialized DownstreamSearchReport from L1 investigation" + }, + "upstream_report": { + "anyOf": [ + { + "type": "object" }, - "type": "object", - "title": "Note" - }, - "OutputPayload": { - "properties": { - "analysis": { - "items": { - "$ref": "#/components/schemas/AgentMorpheusEngineOutput" - }, - "type": "array", - "title": "Analysis" - }, - "vex": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Vex" - } + { + "type": "null" + } + ], + "title": "Upstream Report", + "description": "Serialized UpstreamSearchReport from L1 investigation" + }, + "l1_agent_answer": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "analysis", - "vex" - ], - "title": "OutputPayload", - "description": "Wrapper for final pipeline results.\n- analysis: per-vulnerability analysis results\n- vex: the vulnerability exploitability exchange document JSON" - }, - "PackageState": { - "properties": { - "product_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Product Name" - }, - "fix_state": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Fix State" - }, - "package_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Name" - }, - "cpe": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cpe" - } + { + "type": "null" + } + ], + "title": "L1 Agent Answer", + "description": "Final answer from the L1 ReAct agent" + }, + "vulnerability_intel": { + "anyOf": [ + { + "$ref": "#/components/schemas/VulnerabilityIntel" }, - "type": "object", - "title": "PackageState" - }, - "SBOMInfo": { - "properties": { - "packages": { - "items": { - "$ref": "#/components/schemas/SBOMPackage" - }, - "type": "array", - "title": "Packages" - } + { + "type": "null" + } + ], + "description": "Structured vulnerability intelligence extracted from CVE advisories and patches" + }, + "preliminary_verdict": { + "type": "string", + "enum": [ + "vulnerable", + "protected", + "not_present", + "uncertain" + ], + "title": "Preliminary Verdict", + "description": "L1 verdict before L2 refinement", + "default": "uncertain" + }, + "confidence": { + "type": "number", + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "description": "Confidence in the preliminary verdict", + "default": 0 + } + }, + "type": "object", + "title": "L1InvestigationResult", + "description": "Intermediate result from L1 investigation, input to L2 or report generation." + }, + "L2BuildResult": { + "properties": { + "compilation_status": { + "type": "string", + "enum": [ + "compiled", + "not_compiled", + "unknown" + ], + "title": "Compilation Status", + "description": "Whether vulnerable code is compiled into the binary", + "default": "unknown" + }, + "compilation_confidence": { + "type": "number", + "maximum": 1, + "minimum": 0, + "title": "Compilation Confidence", + "description": "Confidence in compilation status", + "default": 0 + }, + "compilation_evidence": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "packages" - ], - "title": "SBOMInfo", - "description": "List of SBOMPackage objects representing the packages found in the input image." - }, - "SBOMPackage": { - "properties": { - "name": { - "type": "string", - "title": "Name" - }, - "version": { - "type": "string", - "title": "Version" - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Path" - }, - "system": { - "type": "string", - "title": "System" - } + { + "type": "null" + } + ], + "title": "Compilation Evidence", + "description": "Evidence supporting compilation status" + }, + "hardening_relevant": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "name", - "version", - "system" - ], - "title": "SBOMPackage", - "description": "Information about a single package in the container image's Software Bill of Materials (SBOM)." - }, - "ScanInfoInput": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Type" - }, - "started_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Started At" - }, - "completed_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Completed At" - }, - "vulns": { - "items": { - "$ref": "#/components/schemas/VulnInfo" - }, - "type": "array", - "minItems": 1, - "title": "Vulns" - } + { + "type": "null" + } + ], + "title": "Hardening Relevant", + "description": "Whether detected hardening flags are relevant to the CVE" + }, + "hardening_flags": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Hardening Flags", + "description": "Hardening flags detected in build log or binary" + }, + "hardening_rationale": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "vulns" - ], - "title": "ScanInfoInput", - "description": "Information about a unique scan for a container image against a list of vulnerabilies." - }, - "SourceDocumentsInfo": { - "properties": { - "type": { - "type": "string", - "enum": [ - "code", - "doc" - ], - "title": "Type" - }, - "git_repo": { - "type": "string", - "minLength": 1, - "title": "Git Repo" - }, - "ref": { - "type": "string", - "minLength": 1, - "title": "Ref" - }, - "include": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Include", - "default": [ - "*.py", - "*.ipynb" - ] - }, - "exclude": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Exclude", - "default": [] - } + { + "type": "null" + } + ], + "title": "Hardening Rationale", + "description": "Rationale for hardening relevance judgment" + }, + "l2_override_verdict": { + "enum": [ + "not_vulnerable", + "vulnerable_mitigated", + null], + "title": "L2 Override Verdict", + "description": "L2 verdict override (if any)" + } + }, + "type": "object", + "title": "L2BuildResult", + "description": "Result from L2 Build Agent (BuildCompilationCheck + HardeningCheck)." + }, + "ManualSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "manual", + "title": "Type", + "description": "The type of the object", + "default": "manual" + }, + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "ManualSBOMInfoInput", + "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects." + }, + "Message": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "type", - "git_repo", - "ref" - ], - "title": "SourceDocumentsInfo", - "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference, such as tag/branch/commit_id\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents." - }, - "TextContent": { - "properties": { - "type": { - "type": "string", - "const": "text", - "title": "Type", - "default": "text" - }, - "text": { - "type": "string", - "title": "Text", - "default": "default" + { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/TextContent" + }, + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/AudioContent" + } + ], + "discriminator": { + "propertyName": "type", + "mapping": { + "image_url": "#/components/schemas/ImageContent", + "input_audio": "#/components/schemas/AudioContent", + "text": "#/components/schemas/TextContent" + } } + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "type": "string", + "title": "Role" + } + }, + "type": "object", + "required": [ + "content", + "role" + ], + "title": "Message" + }, + "Note": { + "properties": { + "author": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": false, - "type": "object", - "title": "TextContent" - }, - "ValidationError": { - "properties": { - "loc": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "type": "array", - "title": "Location" - }, - "msg": { - "type": "string", - "title": "Message" - }, - "type": { - "type": "string", - "title": "Error Type" - } + { + "type": "null" + } + ], + "title": "Author" + }, + "note": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "loc", - "msg", - "type" - ], - "title": "ValidationError" - }, - "VdbPaths": { - "properties": { - "code_vdb_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Code Vdb Path" - }, - "doc_vdb_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Doc Vdb Path" - }, - "code_index_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Code Index Path" - } + { + "type": "null" + } + ], + "title": "Note" + } + }, + "type": "object", + "title": "Note" + }, + "OutputPayload": { + "properties": { + "analysis": { + "items": { + "$ref": "#/components/schemas/AgentMorpheusEngineOutput" + }, + "type": "array", + "title": "Analysis" + }, + "vex": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vex" + } + }, + "type": "object", + "required": [ + "analysis", + "vex" + ], + "title": "OutputPayload", + "description": "Wrapper for final pipeline results.\n- analysis: per-vulnerability analysis results\n- vex: the vulnerability exploitability exchange document JSON" + }, + "PackageCheckerContext": { + "properties": { + "status": { + "anyOf": [ + { + "$ref": "#/components/schemas/PackageCheckerStatus" }, - "type": "object", - "title": "VdbPaths", - "description": "Paths to where the generated VDBs are stored." + { + "type": "null" + } + ] }, - "VulnInfo": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Score" - }, - "severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Severity" - }, - "published_date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published Date" - }, - "last_modified_date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Last Modified Date" - }, - "url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Url" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "package": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package" - }, - "package_version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Version" - }, - "package_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Name" - }, - "package_type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Type" - } + "source_key": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": true, - "type": "object", - "required": [ - "vuln_id" - ], - "title": "VulnInfo", - "description": "Information about a vulnerability." - }, - "VulnerableDependencies": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "vuln_package_intel_sources": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Vuln Package Intel Sources" - }, - "vulnerable_sbom_packages": { - "items": { - "$ref": "#/components/schemas/VulnerableSBOMPackage" - }, - "type": "array", - "title": "Vulnerable Sbom Packages" - } + { + "type": "null" + } + ], + "title": "Source Key" + }, + "artifacts": { + "$ref": "#/components/schemas/AcquiredArtifacts" + }, + "identify_result": { + "$ref": "#/components/schemas/PackageIdentifyResult" + }, + "l1_result": { + "anyOf": [ + { + "$ref": "#/components/schemas/L1InvestigationResult" }, - "type": "object", - "required": [ - "vuln_id", - "vuln_package_intel_sources", - "vulnerable_sbom_packages" - ], - "title": "VulnerableDependencies", - "description": "Information about the vulnerable SBOM packages associated with the vuln_id.\n\n- vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list.\n- vuln_package_intel_sources: list of sources (e.g. \"ghsa\", \"nvd\", \"ubuntu\", \"rhsa\") that provided\n the vulnerable package/version intel for the vuln_id.\n- vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are\n vulnerable for a given vuln_id." - }, - "VulnerableSBOMPackage": { - "properties": { - "name": { - "type": "string", - "title": "Name" - }, - "version": { - "type": "string", - "title": "Version" - }, - "vulnerable_dependency_package": { - "$ref": "#/components/schemas/DependencyPackage" - } + { + "type": "null" + } + ], + "description": "Result from L1 Code Agent investigation" + }, + "l2_result": { + "anyOf": [ + { + "$ref": "#/components/schemas/L2BuildResult" }, - "type": "object", - "required": [ - "name", - "version", - "vulnerable_dependency_package" - ], - "title": "VulnerableSBOMPackage", - "description": "Information about a vulnerable SBOM package and its related vulnerable dependency package.\n\n- name: SBOM package name\n- version: SBOM package version\n- vulnerable_dependency_package: DependencyPackage object with info about the vulnerable dependency package.\n If an SBOM package itself is vulnerable, the vulnerable_dependency_package.relation will be \"SELF\".\n Otherwise, if it is vulnerable due to its dependency, the vulnerable_dependency_package.relation will be either\n \"DIRECT\" or \"INDIRECT\"." + { + "type": "null" + } + ], + "description": "Result from L2 Build Agent (optional)" + } + }, + "type": "object", + "title": "PackageCheckerContext", + "description": "Consolidates all checker-specific state on AgentMorpheusInfo." + }, + "PackageCheckerStatus": { + "type": "integer", + "enum": [0, 1, 2, 3, 4, 5], + "title": "PackageCheckerStatus", + "description": "Per-CVE status codes produced by the PackageIdentify phase." + }, + "PackageIdentifyResult": { + "properties": { + "affected_rpm_list": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Affected Rpm List", + "default": [] + }, + "fixed_rpm_list": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Fixed Rpm List", + "default": [] + }, + "is_target_package_affected": { + "$ref": "#/components/schemas/EnumIdentifyResult", + "default": "unknown" + }, + "is_target_package_fixed": { + "$ref": "#/components/schemas/EnumIdentifyResult", + "default": "unknown" + }, + "conclusion_reason": { + "type": "string", + "title": "Conclusion Reason", + "description": "Detailed explanation of why the package was determined to be vulnerable or not vulnerable", + "default": "" + } + }, + "type": "object", + "title": "PackageIdentifyResult", + "description": "Result of the PackageIdentify phase for a single CVE." + }, + "PackageState": { + "properties": { + "product_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product Name" + }, + "fix_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Fix State" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Name" + }, + "cpe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cpe" + } + }, + "type": "object", + "title": "PackageState" + }, + "PipelineMode": { + "type": "string", + "enum": [ + "full_pipeline", + "rpm_package_checker" + ], + "title": "PipelineMode", + "description": "Controls which investigation path the pipeline takes after process_sbom.\nOrthogonal to AnalysisType (input format) -- any combination is valid." + }, + "ResponseIntermediateStep": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "parent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Parent Id" + }, + "type": { + "type": "string", + "title": "Type", + "default": "markdown" + }, + "name": { + "type": "string", + "title": "Name" + }, + "payload": { + "type": "string", + "title": "Payload" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name", + "payload" + ], + "title": "ResponseIntermediateStep", + "description": "ResponseSerializedStep is a data model that represents a serialized step in the NAT chat streaming API." + }, + "SBOMInfo": { + "properties": { + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "SBOMInfo", + "description": "List of SBOMPackage objects representing the packages found in the input image." + }, + "SBOMPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "type": "string", + "title": "Version" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + }, + "system": { + "type": "string", + "title": "System" + } + }, + "type": "object", + "required": [ + "name", + "version", + "system" + ], + "title": "SBOMPackage", + "description": "Information about a single package in the container image's Software Bill of Materials (SBOM)." + }, + "ScanInfoInput": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + }, + "started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Started At", + "description": "Scan start time as ISO-8601 with UTC offset (e.g. ...+00:00)." + }, + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Completed At", + "description": "Scan completion time as ISO-8601 with UTC offset (e.g. ...+00:00)." + }, + "vulns": { + "items": { + "$ref": "#/components/schemas/VulnInfo" + }, + "type": "array", + "minItems": 1, + "title": "Vulns" + } + }, + "type": "object", + "required": [ + "vulns" + ], + "title": "ScanInfoInput", + "description": "Information about a unique scan for a container image against a list of vulnerabilies." + }, + "SourceDocumentsInfo": { + "properties": { + "type": { + "type": "string", + "enum": [ + "code", + "doc" + ], + "title": "Type" + }, + "git_repo": { + "type": "string", + "minLength": 1, + "title": "Git Repo" + }, + "ref": { + "type": "string", + "minLength": 1, + "title": "Ref" + }, + "include": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Include", + "default": [ + "*.py", + "*.ipynb" + ] + }, + "exclude": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Exclude", + "default": [] + } + }, + "type": "object", + "required": [ + "type", + "git_repo", + "ref" + ], + "title": "SourceDocumentsInfo", + "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference, such as tag/branch/commit_id\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents." + }, + "TargetPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "release": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Release" + }, + "arch": { + "type": "string", + "title": "Arch", + "default": "x86_64" + } + }, + "type": "object", + "required": [ + "name" + ], + "title": "TargetPackage", + "description": "A package to investigate." + }, + "TextContent": { + "properties": { + "type": { + "type": "string", + "const": "text", + "title": "Type", + "default": "text" + }, + "text": { + "type": "string", + "title": "Text", + "default": "default" + } + }, + "additionalProperties": false, + "type": "object", + "title": "TextContent" + }, + "Usage": { + "properties": { + "prompt_tokens": { + "type": "integer", + "title": "Prompt Tokens" + }, + "completion_tokens": { + "type": "integer", + "title": "Completion Tokens" + }, + "total_tokens": { + "type": "integer", + "title": "Total Tokens" + } + }, + "type": "object", + "required": [ + "prompt_tokens", + "completion_tokens", + "total_tokens" + ], + "title": "Usage" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + }, + "VdbPaths": { + "properties": { + "code_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Vdb Path" + }, + "doc_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Doc Vdb Path" + }, + "code_index_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Index Path" + } + }, + "type": "object", + "title": "VdbPaths", + "description": "Paths to where the generated VDBs are stored." + }, + "VulnInfo": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "published_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Date" + }, + "last_modified_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Modified Date" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package" + }, + "package_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Version" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Name" + }, + "package_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Type" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "vuln_id" + ], + "title": "VulnInfo", + "description": "Information about a vulnerability." + }, + "VulnerabilityIntel": { + "properties": { + "affected_files": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Affected Files", + "description": "Source file paths likely to contain vulnerable code" + }, + "vulnerable_functions": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Functions", + "description": "Function names that contain or handle the vulnerability" + }, + "vulnerable_variables": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Variables", + "description": "Variable names involved in the vulnerability" + }, + "vulnerable_patterns": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Patterns", + "description": "Code patterns/snippets indicating vulnerable code (from - lines)" + }, + "fix_patterns": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Fix Patterns", + "description": "Code patterns/snippets indicating fixed code (from + lines)" + }, + "root_cause": { + "type": "string", + "title": "Root Cause", + "description": "Technical explanation of why the code is vulnerable", + "default": "" + }, + "vulnerability_type": { + "type": "string", + "title": "Vulnerability Type", + "description": "Category: buffer_overflow, integer_overflow, use_after_free, null_deref, etc.", + "default": "" + }, + "search_keywords": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Search Keywords", + "description": "Recommended grep patterns ordered by specificity (most specific first)" + }, + "affected_architectures": { + "type": "string", + "enum": [ + "32-bit", + "64-bit", + "both" + ], + "title": "Affected Architectures", + "description": "Which CPU architectures are affected: 32-bit only, 64-bit only, or both (default)", + "default": "both" + }, + "is_downstream_patch_available": { + "type": "boolean", + "title": "Is Downstream Patch Available", + "description": "True if a CVE-specific patch file exists in the downstream package", + "default": false + }, + "is_patch_applied_in_build": { + "type": "boolean", + "title": "Is Patch Applied In Build", + "description": "True if the patch was confirmed applied in build logs", + "default": false + }, + "patch_file_name": { + "type": "string", + "title": "Patch File Name", + "description": "Name of the CVE-specific patch file (if available)", + "default": "" + } + }, + "type": "object", + "title": "VulnerabilityIntel", + "description": "Structured intelligence extracted from CVE advisories and patches.\n\nUsed to provide grep-ready patterns and context for L1 agent source searches." + }, + "VulnerableDependencies": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "vuln_package_intel_sources": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vuln Package Intel Sources" + }, + "vulnerable_sbom_packages": { + "items": { + "$ref": "#/components/schemas/VulnerableSBOMPackage" + }, + "type": "array", + "title": "Vulnerable Sbom Packages" + } + }, + "type": "object", + "required": [ + "vuln_id", + "vuln_package_intel_sources", + "vulnerable_sbom_packages" + ], + "title": "VulnerableDependencies", + "description": "Information about the vulnerable SBOM packages associated with the vuln_id.\n\n- vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list.\n- vuln_package_intel_sources: list of sources (e.g. \"ghsa\", \"nvd\", \"ubuntu\", \"rhsa\") that provided\n the vulnerable package/version intel for the vuln_id.\n- vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are\n vulnerable for a given vuln_id." + }, + "VulnerableSBOMPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "type": "string", + "title": "Version" + }, + "vulnerable_dependency_package": { + "$ref": "#/components/schemas/DependencyPackage" } + }, + "type": "object", + "required": [ + "name", + "version", + "vulnerable_dependency_package" + ], + "title": "VulnerableSBOMPackage", + "description": "Information about a vulnerable SBOM package and its related vulnerable dependency package.\n\n- name: SBOM package name\n- version: SBOM package version\n- vulnerable_dependency_package: DependencyPackage object with info about the vulnerable dependency package.\n If an SBOM package itself is vulnerable, the vulnerable_dependency_package.relation will be \"SELF\".\n Otherwise, if it is vulnerable due to its dependency, the vulnerable_dependency_package.relation will be either\n \"DIRECT\" or \"INDIRECT\"." } + } } -} \ No newline at end of file + } \ No newline at end of file diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index 2af85daa9..b182c9fa2 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -90,6 +90,7 @@ class AgentMorpheusEngineOutput(BaseModel): justification: JustificationOutput intel_score: int cvss: CVSSOutput | None + details: str | None = None class OutputPayload(BaseModel): diff --git a/src/vuln_analysis/functions/build_agent_graph_defs.py b/src/vuln_analysis/functions/build_agent_graph_defs.py new file mode 100644 index 000000000..698ff1b84 --- /dev/null +++ b/src/vuln_analysis/functions/build_agent_graph_defs.py @@ -0,0 +1,692 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Graph definitions for the L2 Build Agent (BuildCompilationCheck). + +Houses the LangGraph state schema and structured-output schemas for BuildHarvestReport. +Prompt templates are in vuln_analysis.utils.rpm_checker_prompts. +""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Literal, NotRequired + +from langgraph.graph import MessagesState +from pydantic import BaseModel, Field + +from exploit_iq_commons.utils.hardening_kb import HardeningEntry +from vuln_analysis.functions.react_internals import CheckerThought, Observation +from vuln_analysis.utils.rpm_checker_prompts import ( + L2_CONFIG_SYS_PROMPT, + L2_CONFIG_PROMPT_TEMPLATE, + L2_CONFIG_THOUGHT_INSTRUCTIONS, + L2_HARDENING_SYS_PROMPT, + L2_HARDENING_PROMPT_TEMPLATE, + L2_HARDENING_THOUGHT_INSTRUCTIONS, + L2_COMPILATION_VERDICT_PROMPT, + L2_HARDENING_VERDICT_PROMPT, + L2_COMPREHENSION_PROMPT, + L2_MEMORY_UPDATE_PROMPT, + L2_HARDENING_COMPREHENSION_PROMPT, + L2_HARDENING_MEMORY_UPDATE_PROMPT, +) +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + + +class BuildHarvestReport(BaseModel): + """Deterministic data harvested from build artifacts. + + Extracted during the data_harvest_node before the ReAct loop begins. + + Key vulnerability-relevant data: + - Feature disable flags that prevent vulnerable code from being compiled: + - OpenSSL style: no-sm2, no-ssl3, no-md5, no-asm + - Autoconf style: --disable-feature, --without-feature + - CMake style: -DENABLE_FEATURE=OFF + - Feature enable flags that explicitly enable optional features: + - Autoconf style: --enable-feature, --with-feature + - CMake style: -DENABLE_FEATURE=ON + - Architecture flags to understand target platform + - Hardening flags relevant to the CVE's CWE class + + Note: Compiled files are NOT pre-extracted. The LLM searches the build log + for affected files from l1_result.vulnerability_intel.affected_files during the ReAct loop. + """ + + disabled_features: list[str] = Field( + default_factory=list, + description="Feature-disabling flags from build log (e.g., '-DOPENSSL_NO_SM2', '-DNO_GZIP')", + ) + spec_disabled_features: list[str] = Field( + default_factory=list, + description="Feature-disabling flags from spec %build section (e.g., 'no-sm2', '--disable-ssl3', '--without-openssl')", + ) + enabled_features: list[str] = Field( + default_factory=list, + description="Feature-enabling flags from build log (e.g., '-DENABLE_LDAP', '-DLDAP_ENABLED')", + ) + spec_enabled_features: list[str] = Field( + default_factory=list, + description="Feature-enabling flags from spec %build section (e.g., '--enable-ldap', '--with-openssl')", + ) + expected_hardening: list[HardeningEntry] = Field( + default_factory=list, + description="Hardening flags relevant to the CVE's CWE, with descriptions for LLM context", + ) + build_architecture: Literal["32-bit", "64-bit", "unknown"] = Field( + default="unknown", + description="Target architecture from -m64/-m32 flags or build target (x86_64/i686)", + ) + linked_libraries: list[str] = Field( + default_factory=list, + description="Libraries linked at build time (extracted from -l flags in build log)", + ) + built_subpackages: list[str] = Field( + default_factory=list, + description="Subpackages defined in spec %package sections", + ) + excluded_subpackages: list[str] = Field( + default_factory=list, + description="Subpackages excluded via %bcond_without, ExcludeArch, or conditional guards", + ) + kernel_config: dict[str, str] = Field( + default_factory=dict, + description="Kernel CONFIG_* options (y/n/m) - only populated for kernel packages", + ) + is_kernel: bool = Field( + default=False, + description="True if this is a kernel package (detected by Kconfig presence)", + ) + kernel_config_path: str | None = Field( + default=None, + description="Path to kernel config file (e.g., kernel-x86_64-rhel.config) for LLM grep access", + ) + kernel_source_root: str | None = Field( + default=None, + description="Path to kernel source tree root (directory containing Kconfig)", + ) + + +class L2CompileVerdictExtraction(BaseModel): + """LLM-extracted verdict from L2 agent final answer.""" + + compilation_status: str = Field( + description="Whether vulnerable code is compiled into the binary: one of 'compiled', 'not_compiled', or 'unknown'" + ) + confidence: float = Field(description="Confidence in the verdict (0.0 to 1.0)") + reasoning: str = Field(description="Brief explanation of the verdict") + +class L2HardeningVerdictExtraction(BaseModel): + """LLM-extracted verdict from L2 Hardening investigation.""" + hardening_status: str = Field( + description="Whether hardening flags mitigate the vulnerability: one of 'mitigated', 'not_mitigated', 'not_applicable', or 'unknown'" + ) + hardening_flags: list[str] = Field( + default_factory=list, + description="List of specific hardening flags found (e.g., -fstack-protector-strong, -D_FORTIFY_SOURCE=2)", + ) + confidence: float = Field(description="Confidence in the verdict (0.0 to 1.0)") + reasoning: str = Field(description="Brief explanation of the verdict") + + +# --------------------------------------------------------------------------- +# Graph State +# --------------------------------------------------------------------------- + + +class BuildAgentState(MessagesState): + """LangGraph state for the L2 Build Agent.""" + + harvest_report: NotRequired[BuildHarvestReport | None] + vulnerability_intel_str: NotRequired[str | None] + l1_preliminary_verdict: NotRequired[str | None] + runtime_prompt: NotRequired[str | None] + thought: NotRequired[CheckerThought | None] + observation: NotRequired[Observation | None] + step: NotRequired[int] + max_steps: NotRequired[int] + L2CompileVerdict: NotRequired[L2CompileVerdictExtraction | None] + L2HardeningVerdict: NotRequired[L2HardeningVerdictExtraction | None] + evidence_sources: NotRequired[list[str] | None] + +# --------------------------------------------------------------------------- +# Spec File Parsing Helpers +# --------------------------------------------------------------------------- + + +def _extract_spec_build_section(spec_path: Path) -> str: + """Extract the %build section from an RPM spec file. + + The %build section contains configure/Configure commands with feature flags + that determine what code is compiled. + + Args: + spec_path: Path to the RPM spec file + + Returns: + The raw %build section content, or empty string if not found + """ + try: + content = spec_path.read_text(encoding="utf-8", errors="replace") + except OSError as e: + logger.warning("Failed to read spec file %s: %s", spec_path, e) + return "" + + # Find %build section (ends at next RPM section marker or EOF) + # Only match actual section markers, not macros like %configure, %ifarch, %{name} + # Common RPM sections: %prep, %build, %install, %check, %files, %post, %pre, + # %preun, %postun, %changelog, %package, %description + match = re.search( + r"^%build\s*\n(.*?)(?=^%(prep|install|check|files|post|pre|preun|postun|changelog|package|description)\b|\Z)", + content, + re.MULTILINE | re.DOTALL, + ) + return match.group(1).strip() if match else "" + + +def _extract_spec_disabled_features(build_section: str) -> list[str]: + """Extract feature-disable flags from spec %build section. + + Recognizes patterns from common build systems: + - OpenSSL style: no-sm2, no-ssl3, no-asm + - Autoconf style: --disable-feature, --without-feature + - CMake style: -DENABLE_FEATURE=OFF + + Args: + build_section: The raw %build section content + + Returns: + Sorted list of disabled feature names (without prefix) + """ + disabled: set[str] = set() + + # OpenSSL style: no-feature (e.g., no-sm2, no-ssl3, no-asm) + disabled.update(re.findall(r"\bno-(\w+)", build_section)) + + # Autoconf style: --disable-feature (e.g., --disable-static) + disabled.update(re.findall(r"--disable-(\w+)", build_section)) + + # Autoconf style: --without-feature (e.g., --without-openssl) + disabled.update(re.findall(r"--without-(\w+)", build_section)) + + # CMake style: -DENABLE_FEATURE=OFF or =0 or =FALSE + disabled.update( + re.findall(r"-DENABLE_(\w+)=(?:OFF|0|FALSE)", build_section, re.IGNORECASE) + ) + + return sorted(disabled) + + +def _extract_spec_enabled_features(build_section: str) -> list[str]: + """Extract feature-enable flags from spec %build section. + + Recognizes patterns from common build systems: + - Autoconf style: --enable-feature, --with-feature + - CMake style: -DENABLE_FEATURE=ON + + Args: + build_section: The raw %build section content + + Returns: + Sorted list of enabled feature names (without prefix) + """ + enabled: set[str] = set() + + # Autoconf style: --enable-feature (e.g., --enable-ldap) + enabled.update(re.findall(r"--enable-(\w+)", build_section)) + + # Autoconf style: --with-feature (e.g., --with-openssl) + enabled.update(re.findall(r"--with-(\w+)", build_section)) + + # CMake style: -DENABLE_FEATURE=ON or =1 or =TRUE + enabled.update( + re.findall(r"-DENABLE_(\w+)=(?:ON|1|TRUE)", build_section, re.IGNORECASE) + ) + + return sorted(enabled) + + +def _extract_linked_libraries(build_log_content: str) -> list[str]: + """Extract libraries linked at build time from build log. + + Parses -l flags from gcc/g++/ld commands in the build log. + + Args: + build_log_content: Full content of the build log file + + Returns: + Sorted list of unique library names (without 'lib' prefix or path) + """ + libraries: set[str] = set() + + # Match -l flags preceded by whitespace (actual linker flags) + # Library names must start with a letter (filters out false positives like "ib500" from "lib500") + libraries.update(re.findall(r"(?:^|\s)-l([a-zA-Z][a-zA-Z0-9_]*)", build_log_content, re.MULTILINE)) + + return sorted(libraries) + + +def _extract_spec_subpackages(spec_content: str) -> tuple[list[str], list[str]]: + """Extract built and excluded subpackages from spec file. + + Parses %package directives and conditional guards to determine + which subpackages are built vs excluded. + + Args: + spec_content: Full content of the spec file + + Returns: + Tuple of (built_subpackages, excluded_subpackages) + """ + built: set[str] = set() + excluded: set[str] = set() + + # Extract all %package definitions + # Handles: %package -n libfoo, %package devel, %package -n foo-libs + for match in re.finditer(r"^%package\s+(?:-n\s+)?(\S+)", spec_content, re.MULTILINE): + pkg_name = match.group(1) + # Skip macro expansions that we can't resolve + if not pkg_name.startswith("%"): + built.add(pkg_name) + + # Extract %bcond_without directives (features disabled by default) + for match in re.finditer(r"%bcond_without\s+(\w+)", spec_content): + excluded.add(match.group(1)) + + # Extract %bcond_with directives (features that must be explicitly enabled) + # These are also "excluded by default" + for match in re.finditer(r"%bcond_with\s+(\w+)", spec_content): + excluded.add(match.group(1)) + + # Check for ExcludeArch which excludes entire package on certain architectures + if re.search(r"^ExcludeArch:", spec_content, re.MULTILINE): + # Note: We can't determine current arch, but flag that exclusions exist + pass + + return sorted(built), sorted(excluded) + + +def _extract_kernel_config(source_path: Path) -> dict[str, str]: + """Extract kernel CONFIG_* options from .config file. + + Only applicable to kernel packages. Looks for .config file + in the source tree and parses CONFIG_*=y/n/m lines. + + Args: + source_path: Path to the kernel source directory + + Returns: + Dict mapping CONFIG_* names to their values (y/n/m) + """ + config: dict[str, str] = {} + + # Look for .config file in common locations + config_paths = [ + source_path / ".config", + source_path / "configs" / ".config", + ] + + # Also check for arch-specific configs + for arch_config in source_path.glob("configs/kernel-*.config"): + config_paths.append(arch_config) + + for config_path in config_paths: + if config_path.exists() and config_path.is_file(): + try: + content = config_path.read_text(encoding="utf-8", errors="replace") + # Parse CONFIG_*=y/n/m lines + for match in re.finditer(r"^(CONFIG_\w+)=(y|n|m)", content, re.MULTILINE): + config[match.group(1)] = match.group(2) + # Also capture "# CONFIG_X is not set" as CONFIG_X=n + for match in re.finditer(r"^# (CONFIG_\w+) is not set", content, re.MULTILINE): + config[match.group(1)] = "n" + # Found and parsed a config file, no need to continue + if config: + logger.info( + "_extract_kernel_config: parsed %d options from %s", + len(config), config_path + ) + break + except OSError as e: + logger.warning("_extract_kernel_config: failed to read %s: %s", config_path, e) + + return config + + +def _is_kernel_package(source_path: Path) -> bool: + """Detect kernel package by presence of Kconfig in source root or linux-* subdir. + + Kernel source trees contain a top-level Kconfig file that defines the + kernel configuration system. This is a reliable marker for kernel packages. + + Args: + source_path: Path to the source directory + + Returns: + True if this appears to be a kernel package + """ + if not source_path or not source_path.exists(): + return False + + # Check source root for Kconfig + if (source_path / "Kconfig").exists(): + return True + + # Check linux-* subdirectory pattern (RHEL kernel structure) + for subdir in source_path.glob("linux-*"): + if subdir.is_dir(): + # Check nested linux-* (e.g., linux-5.14.0/linux-5.14.0.el9/) + for nested in subdir.glob("linux-*"): + if nested.is_dir() and (nested / "Kconfig").exists(): + return True + # Check direct subdir + if (subdir / "Kconfig").exists(): + return True + + return False + + +def _find_kernel_config_file(source_path: Path, arch: str) -> Path | None: + """Find the kernel config file for a specific architecture. + + RHEL kernel packages store config files in the source root with naming + pattern: kernel-{arch}-rhel.config (base flavor) or + kernel-{arch}-{flavor}-rhel.config (debug, rt, etc.) + + Args: + source_path: Path to the source directory + arch: Target architecture (e.g., 'x86_64', 'aarch64') + + Returns: + Path to the config file, or None if not found + """ + if not source_path or not source_path.exists() or not arch: + return None + + # Try base flavor first: kernel-{arch}-rhel.config + config_path = source_path / f"kernel-{arch}-rhel.config" + if config_path.exists(): + logger.info("_find_kernel_config_file: found config at %s", config_path) + return config_path + + # Fallback: any kernel-{arch}*.config + for config in source_path.glob(f"kernel-{arch}*.config"): + if config.is_file(): + logger.info("_find_kernel_config_file: found fallback config at %s", config) + return config + + logger.warning("_find_kernel_config_file: no config found for arch %s in %s", arch, source_path) + return None + + +def _find_kernel_source_root(source_path: Path) -> Path | None: + """Find the actual kernel source tree root containing Kconfig. + + RHEL kernel packages have a nested structure: + source/ + ├── kernel-x86_64-rhel.config (config files at root) + └── linux-5.14.0/ + └── linux-5.14.0.el9/ (actual kernel source here) + └── Kconfig + + Args: + source_path: Path to the source directory + + Returns: + Path to the kernel source root (directory containing Kconfig), or None + """ + if not source_path or not source_path.exists(): + return None + + # Check if Kconfig is directly in source_path + if (source_path / "Kconfig").exists(): + return source_path + + # Check linux-* subdirectory pattern + for subdir in source_path.glob("linux-*"): + if subdir.is_dir(): + # Check nested linux-* first (RHEL pattern) + for nested in subdir.glob("linux-*"): + if nested.is_dir() and (nested / "Kconfig").exists(): + logger.info("_find_kernel_source_root: found at %s", nested) + return nested + # Check direct subdir + if (subdir / "Kconfig").exists(): + logger.info("_find_kernel_source_root: found at %s", subdir) + return subdir + + logger.warning("_find_kernel_source_root: no kernel source root found in %s", source_path) + return None + + +# --------------------------------------------------------------------------- +# Data Harvesting Functions +# --------------------------------------------------------------------------- + + +async def harvest_build_data( + build_log_path: Path | None, + spec_path: Path | None, + cwe_id: str | None = None, + source_path: Path | None = None, + package_name: str | None = None, + arch: str | None = None, +) -> BuildHarvestReport: + """Extract structured data from build log and spec file. + + Parses: + - Feature-disabling -D defines (e.g., -DOPENSSL_NO_SM2, -DNO_GZIP) + - Feature-enabling -D defines (e.g., -DENABLE_LDAP, -DLDAP_ENABLED) + - Linked libraries from -l flags + - Subpackages from spec %package directives + - Kernel CONFIG_* options (for kernel packages only) + + For kernel packages, uses CONFIG_* based compilation checking instead of + build log analysis (kernel builds use make -s which hides individual compilations). + + Args: + build_log_path: Path to the build log file + spec_path: Path to the RPM spec file + cwe_id: CWE identifier to look up expected hardening flags (e.g., 'CWE-121') + source_path: Path to the source directory (for kernel config extraction) + package_name: Name of the package (to detect kernel packages) + arch: Target architecture (e.g., 'x86_64') for kernel config file selection + + Returns: + BuildHarvestReport with harvested data and expected hardening flags + """ + from exploit_iq_commons.utils.hardening_kb import HardeningKB + from vuln_analysis.tools.source_inspector import SourceInspector + + # Detect kernel package and find kernel-specific paths + is_kernel = _is_kernel_package(source_path) if source_path else False + kernel_config_path: str | None = None + kernel_source_root: str | None = None + + if is_kernel: + logger.info("harvest_build_data: detected kernel package") + # Find kernel config file for the target architecture + if arch: + config_file = _find_kernel_config_file(source_path, arch) + if config_file: + kernel_config_path = str(config_file) + # Find kernel source root (contains Kconfig, Makefiles) + source_root = _find_kernel_source_root(source_path) + if source_root: + kernel_source_root = str(source_root) + + # Handle case where build_log_path is a directory instead of a file + if build_log_path and build_log_path.is_dir(): + log_files = list(build_log_path.glob("*-build.log")) or list(build_log_path.glob("*.log")) + if log_files: + build_log_path = log_files[0] + logger.info("harvest_build_data: resolved build log directory to file: %s", build_log_path) + else: + logger.warning("harvest_build_data: build_log_path is a directory but no .log files found") + build_log_path = None + + # Lookup expected hardening flags from KB based on CWE + expected_hardening = [] + if cwe_id: + kb = HardeningKB.get_instance() + expected_hardening = kb.lookup_by_cwe(cwe_id) + logger.info( + "harvest_build_data: CWE %s maps to %d hardening flags", + cwe_id, + len(expected_hardening), + ) + + # Extract feature-disabling defines from build log + disabled_features: list[str] = [] + if build_log_path: + inspector = SourceInspector(build_log_path.parent) + + # Grep for lines containing -D defines + matches = inspector.grep_content(r"-D\w+", file_path=build_log_path) + + # Extract unique defines from matched lines + all_defines: set[str] = set() + for match in matches: + defines = re.findall(r"-D(\w+)", match.line_content) + all_defines.update(defines) + + # Filter for feature-disabling patterns: + # - NO_* prefix (e.g., NO_GZIP) + # - DISABLE_* prefix (e.g., DISABLE_SSL) + # - WITHOUT_* prefix (e.g., WITHOUT_FEATURE) + # - *_NO_* infix (e.g., OPENSSL_NO_SM2) + # - *_DISABLE_* infix + # - *_DISABLED suffix + disable_pattern = re.compile( + r"^(NO_|DISABLE_|WITHOUT_)|(_NO_|_DISABLE_)|(_DISABLED$)" + ) + disabled_features = sorted( + d for d in all_defines if disable_pattern.search(d) + ) + + if disabled_features: + logger.info( + "harvest_build_data: found %d disabled features in build log", + len(disabled_features), + ) + + # Filter for feature-enabling patterns: + # - ENABLE_* prefix (e.g., ENABLE_LDAP) + # - *_ENABLED suffix (e.g., LDAP_ENABLED) + enable_pattern = re.compile(r"^ENABLE_|_ENABLED$") + enabled_features = sorted( + d for d in all_defines if enable_pattern.search(d) + ) + + if enabled_features: + logger.info( + "harvest_build_data: found %d enabled features in build log", + len(enabled_features), + ) + else: + enabled_features = [] + + # Extract linked libraries from build log (skip for kernel - not meaningful) + # Note: Architecture detection removed - use target_package.arch instead (checked in L1) + build_architecture: Literal["32-bit", "64-bit", "unknown"] = "unknown" + linked_libraries: list[str] = [] + if build_log_path and not is_kernel: + try: + build_log_content = build_log_path.read_text(encoding="utf-8", errors="replace") + linked_libraries = _extract_linked_libraries(build_log_content) + if linked_libraries: + logger.info( + "harvest_build_data: found %d linked libraries", + len(linked_libraries), + ) + except OSError as e: + logger.warning("harvest_build_data: failed to read build log: %s", e) + + # Extract %build section and features from spec file + spec_build_section = "" + spec_disabled_features: list[str] = [] + spec_enabled_features: list[str] = [] + built_subpackages: list[str] = [] + excluded_subpackages: list[str] = [] + if spec_path and spec_path.exists(): + spec_build_section = _extract_spec_build_section(spec_path) + spec_disabled_features = _extract_spec_disabled_features(spec_build_section) + spec_enabled_features = _extract_spec_enabled_features(spec_build_section) + + if spec_disabled_features: + logger.info( + "harvest_build_data: found %d disabled features in spec", + len(spec_disabled_features), + ) + if spec_enabled_features: + logger.info( + "harvest_build_data: found %d enabled features in spec", + len(spec_enabled_features), + ) + + # Extract subpackages from full spec content + try: + spec_content = spec_path.read_text(encoding="utf-8", errors="replace") + built_subpackages, excluded_subpackages = _extract_spec_subpackages(spec_content) + if built_subpackages: + logger.info( + "harvest_build_data: found %d built subpackages", + len(built_subpackages), + ) + if excluded_subpackages: + logger.info( + "harvest_build_data: found %d excluded subpackages", + len(excluded_subpackages), + ) + except OSError as e: + logger.warning("harvest_build_data: failed to read spec for subpackages: %s", e) + + # Extract kernel config if this is a kernel package + # Note: kernel_config dict is kept for backward compatibility but the primary + # mechanism for kernel is now CONFIG lookup via kernel_config_path + kernel_config: dict[str, str] = {} + if is_kernel and source_path and source_path.exists(): + kernel_config = _extract_kernel_config(source_path) + if kernel_config: + logger.info( + "harvest_build_data: extracted %d kernel config options", + len(kernel_config), + ) + + return BuildHarvestReport( + disabled_features=disabled_features, + spec_disabled_features=spec_disabled_features, + enabled_features=enabled_features, + spec_enabled_features=spec_enabled_features, + expected_hardening=expected_hardening, + build_architecture=build_architecture, + linked_libraries=linked_libraries, + built_subpackages=built_subpackages, + excluded_subpackages=excluded_subpackages, + kernel_config=kernel_config, + is_kernel=is_kernel, + kernel_config_path=kernel_config_path, + kernel_source_root=kernel_source_root, + ) + diff --git a/src/vuln_analysis/functions/code_agent_graph_defs.py b/src/vuln_analysis/functions/code_agent_graph_defs.py new file mode 100644 index 000000000..28225f021 --- /dev/null +++ b/src/vuln_analysis/functions/code_agent_graph_defs.py @@ -0,0 +1,1708 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Graph definitions for the L1 Package Code Agent. + +Houses the LangGraph state schema, structured-output schemas for +DownstreamSearchReport/UpstreamSearchReport pipelines, and CodeAgentReport. +Prompt templates are in vuln_analysis.utils.rpm_checker_prompts. +""" + +from __future__ import annotations + +import logging +import re +import shutil +import subprocess +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, NotRequired, TYPE_CHECKING + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + from vuln_analysis.tools.brew_downloader import BrewDownloader + +import aiohttp +import openai +from langchain_core.messages import HumanMessage, SystemMessage +from langgraph.graph import MessagesState +from pydantic import BaseModel, Field +from unidiff import PatchSet + +logger = logging.getLogger(__name__) + +from exploit_iq_commons.data_models.checker_status import L2BuildResult, VulnerabilityIntel +from exploit_iq_commons.data_models.common import TargetPackage +from vuln_analysis.functions.react_internals import CheckerThought, Observation, L1VerdictExtraction +from vuln_analysis.utils.rpm_checker_prompts import ( + L1_VERDICT_EXTRACTION_PROMPT, + VULNERABILITY_INTEL_EXTRACTION_PROMPT, + CODE_AGENT_REPORT_PROMPT, + L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + L1_AGENT_SYS_PROMPT_REBASE_FIX, + L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + L1_AGENT_PROMPT_TEMPLATE, + L1_AGENT_PROMPT_TEMPLATE_NO_PATCH, + L1_AGENT_THOUGHT_INSTRUCTIONS, + L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + L1_COMPREHENSION_PROMPT, + L1_MEMORY_UPDATE_PROMPT, + L1_COMPREHENSION_PROMPT_CVE_DESC, + L1_MEMORY_UPDATE_PROMPT_CVE_DESC, +) + +# --------------------------------------------------------------------------- +# Graph state +# --------------------------------------------------------------------------- + + +class CodeAgentState(MessagesState): + """LangGraph state for the L1 Code Agent (DownstreamSearch -> UpstreamSearch).""" + downstream_report: NotRequired[DownstreamSearchReport | None] + upstream_report: NotRequired[UpstreamSearchReport | None] + runtime_prompt: NotRequired[str | None] + last_thought: NotRequired[CheckerThought | None] + step: NotRequired[int] + max_steps: NotRequired[int] + output: NotRequired[str] + thought: NotRequired[CheckerThought | None] + observation: NotRequired[Observation | None] + vulnerability_intel: NotRequired["VulnerabilityIntel | None"] + arch_mismatch_reason: NotRequired[str | None] + + +# --------------------------------------------------------------------------- +# Patch schemas (must be defined before reports that use them) +# --------------------------------------------------------------------------- + + +class PatchHunk(BaseModel): + """A single hunk from a downstream patch file.""" + source_start: int + source_length: int + target_start: int + target_length: int + context_lines: list[str] = Field(default_factory=list, description="Unchanged lines") + removed_lines: list[str] = Field(default_factory=list, description="Deleted lines (- stripped)") + added_lines: list[str] = Field(default_factory=list, description="Added lines (+ stripped)") + + +class PatchFile(BaseModel): + """Changes to a single file in a downstream patch.""" + source_path: str + target_path: str + hunks: list[PatchHunk] + is_new_file: bool = False + is_deleted_file: bool = False + + +class ParsedPatch(BaseModel): + """Structured representation of a downstream patch file.""" + patch_filename: str + files: list[PatchFile] + + +class OSVPatchResult(BaseModel): + """Result of fetching a patch from OSV/GitHub or intel references.""" + cve_id: str + fixed_commit: str + repo_url: str + patch_url: str + patch_content: str | None = Field(default=None, description="Raw .patch text") + parsed_patch: "ParsedPatch | None" = Field(default=None, description="Structured patch data") + commit_message: str | None = None + commit_author: str | None = None + commit_date: str | None = None + source: str | None = Field(default=None, description="Source provider (ghsa/nvd/rhsa/ubuntu_patches/osv)") + url_type: str | None = Field(default=None, description="URL type (commit/pull)") + platform: str | None = Field(default=None, description="Hosting platform (github/kernel.org)") + + +@dataclass +class SpecPatchMatch: + """Result of finding a CVE patch in a spec file.""" + patch_index: int + patch_filename: str + raw_directive: str + comment_block: str | None + match_source: Literal["filename", "comment"] + + +# --------------------------------------------------------------------------- +# Reflection schemas +# --------------------------------------------------------------------------- +class DownstreamSearchReport(BaseModel): + """Result of a downstream search.""" + is_patch_file_available: bool = Field(default=False, description="True if a patch file is available") + patch_file_name: str = Field(default="", description="The name of the patch file") + is_patch_in_spec_file: bool = Field(default=False, description="True if a patch file is in the spec file") + spec_file_log_change: str = Field( + default="", + description="All lines in the .spec file that match a grep for the CVE id (not changelog-only)", + ) + is_patch_applied_in_build: bool = Field(default=False, description="True if a patch file is applied in the build") + build_log_patch_applied: str = Field(default="", description="The patch applied in the build log") + spec_patch_directives_for_cve: list[str] = Field( + default_factory=list, + description="Raw PatchN: lines from the spec whose patch filename token matches this CVE", + ) + spec_changelog_cve_lines: str = Field( + default="", + description="Lines from the %changelog section of the .spec that mention the CVE", + ) + spec_source0_line: str = Field( + default="", + description="The Source0: line from the spec file (upstream tarball reference)", + ) + spec_version_line: str = Field( + default="", + description="The Version: line from the spec file", + ) + parsed_patch: ParsedPatch | None = Field(default=None, description="The parsed patch file") + + +class UpstreamSearchReport(BaseModel): + """Result of an upstream search.""" + + is_fixed_srpm_is_needed: bool = Field(default=False, description="True if a fixed SRPM is needed downstream style patch files") + fixed_srpm_file_name: str = Field(default="", description="The name of the fixed SRPM file") + fixed_parsed_patch: ParsedPatch | None = Field(default=None, description="The parsed fixed SRPM patch file") + reference_package_nvr: str = Field( + default="", + description="NVR (name-version-release) of the reference fixed package from intel", + ) + reason_cve_code: str = Field( + default="", + description="Does the CVE description match the code which is vulnerable", + ) + is_code_fixed_by_rebase: Literal["yes", "no", "unknown"] = Field( + default="unknown", + description="yes if the code is fixed by rebase", + ) + spec_file_log_change: str = Field( + default="", + description="The log change of patch in the spec file", + ) + spec_fixed_srpm_change: str = Field( + default="", + description="The change of the fixed SRPM in the spec file", + ) + reason_code_fixed_by_rebase: str = Field( + default="", + description="The reason why the code is fixed by rebase", + ) + osv_result: OSVPatchResult | None = Field(default=None, description="The result of the OSV patch retrieval") + + + + + +class ReflectionBase(BaseModel): + """Base schema for phase reports. + + Subclasses add phase-specific fields on top. + """ + instructions: str = Field( + description="Guidance to the generator for the next iteration.") + is_sufficient: bool = Field( + description="True if results are good enough to proceed.") + + +# --------------------------------------------------------------------------- +# Code Agent Report schema +# --------------------------------------------------------------------------- + + +class CodeSnippet(BaseModel): + """A code snippet from the investigation.""" + file_path: str = Field(description="Path to the source file") + line_number: int | None = Field(default=None, description="Starting line number") + code: str = Field(description="The code content") + snippet_type: str = Field( + description="Type of snippet: one of 'vulnerable', 'fix', or 'context'") + source: str = Field( + description="Where this snippet came from: one of 'downstream_patch', 'upstream_patch', or 'source_search'") + + +class CodeAgentReportLLM(BaseModel): + """LLM-facing model for report generation. Excludes code_snippets which are populated programmatically.""" + justification_label: str = Field( + description=( + "Justification category aligned with VEX: one of " + "code_not_present, protected_by_mitigating_control, vulnerable, uncertain" + )) + executive_summary: str = Field( + description=( + "3-4 sentence synthesis. Must include: 1) Final verdict, " + "2) Technical nature of flaw, 3) Why L2 context overrides L1 (if applicable)." + )) + evidence_chain: list[str] = Field( + description="Ordered list of evidence items tracing the vulnerability through phases") + affected_files: list[str] = Field( + description="Source files where vulnerable code was identified") + patch_analysis: str | None = Field( + default=None, + description="Analysis of downstream patches if any were found") + caveats: list[str] = Field( + default_factory=list, + description="Investigation gaps or uncertainties that may need manual review") + + +class CodeAgentReport(CodeAgentReportLLM): + """Final L1 Code Agent investigation report synthesizing all phases.""" + code_snippets: list[CodeSnippet] = Field( + default_factory=list, + description="Populated programmatically from patches after generation.") + + def to_markdown( + self, + vuln_id: str = "", + target_package: str = "", + version: str = "", + release: str = "", + downstream_report: DownstreamSearchReport | None = None, + ) -> str: + """Render the report as a formatted markdown string.""" + lines: list[str] = [] + + # Header with title + lines.append("# L1 Code Agent Investigation Report") + lines.append("") + + # Verdict banner based on justification label + verdict_map = { + "protected_by_mitigating_control": ("NOT VULNERABLE", "Protected by downstream patch"), + "protected_by_compiler": ("NOT VULNERABLE", "Protected by compiler hardening"), + "code_not_present": ("NOT VULNERABLE", "Vulnerable code not present"), + "code_not_reachable": ("NOT VULNERABLE", "Vulnerable code not reachable"), + "requires_environment": ("NOT VULNERABLE", "Requires specific environment"), + "vulnerable": ("VULNERABLE", "Package requires patching"), + "uncertain": ("UNCERTAIN", "Requires manual review"), + } + verdict_status, verdict_desc = verdict_map.get( + self.justification_label, + ("UNKNOWN", "Unknown status") + ) + + lines.append(f"> **Verdict: {verdict_status}** - {verdict_desc}") + lines.append("") + + # Package information table + lines.append("## Package Information") + lines.append("") + lines.append("| Field | Value |") + lines.append("|-------|-------|") + if vuln_id: + lines.append(f"| **CVE ID** | `{vuln_id}` |") + if target_package: + lines.append(f"| **Package** | `{target_package}` |") + if version: + version_str = f"{version}-{release}" if release else version + lines.append(f"| **Version** | `{version_str}` |") + lines.append(f"| **Justification** | `{self.justification_label}` |") + lines.append("") + + # Executive Summary + lines.append("---") + lines.append("") + lines.append("## Executive Summary") + lines.append("") + lines.append(self.executive_summary) + lines.append("") + + # Evidence Chain + lines.append("---") + lines.append("") + lines.append("## Evidence Chain") + lines.append("") + lines.extend(_format_interleaved_evidence( + self.evidence_chain, + downstream_report, + )) + + # Affected Files + if self.affected_files: + lines.append("---") + lines.append("") + lines.append("## Affected Files") + lines.append("") + # Separate source files from test files + source_files = [f for f in self.affected_files if "/test/" not in f and "test_" not in f] + test_files = [f for f in self.affected_files if "/test/" in f or "test_" in f] + + if source_files: + lines.append("**Source files:**") + for f in source_files: + lines.append(f"- `{f}`") + lines.append("") + + if test_files: + lines.append("**Test files:**") + for f in test_files: + lines.append(f"- `{f}`") + lines.append("") + + # Patch Analysis + if self.patch_analysis: + lines.append("---") + lines.append("") + lines.append("## Patch Analysis") + lines.append("") + lines.append(self.patch_analysis) + lines.append("") + + # Code Snippets - separate vulnerable from fix, prioritize main source files + if self.code_snippets: + lines.append("---") + lines.append("") + lines.append("## Code Comparison") + lines.append("") + + # Filter and organize snippets + vuln_snippets = [s for s in self.code_snippets if s.snippet_type == "vulnerable"] + fix_snippets = [s for s in self.code_snippets if s.snippet_type == "fix"] + + # Prioritize main source files (not test/build files) + def is_main_source(path: str) -> bool: + return "/test/" not in path and "test_" not in path and "Makefile" not in path and "CMakeLists" not in path + + main_vuln = [s for s in vuln_snippets if is_main_source(s.file_path)] + main_fix = [s for s in fix_snippets if is_main_source(s.file_path)] + + # Show main vulnerability code + if main_vuln: + lines.append("### Vulnerable Code") + lines.append("") + for snippet in main_vuln[:2]: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**File:** `{file_name}` (Line {snippet.line_number or 'N/A'})") + lines.append("") + lines.append("```c") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + + # Show fix code + if main_fix: + lines.append("### Fix Code") + lines.append("") + for snippet in main_fix[:2]: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**File:** `{file_name}` (Line {snippet.line_number or 'N/A'})") + lines.append("") + lines.append("```c") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + + # Show other snippets (test/build files) in collapsible section if any + other_vuln = [s for s in vuln_snippets if not is_main_source(s.file_path)] + other_fix = [s for s in fix_snippets if not is_main_source(s.file_path)] + + if other_vuln or other_fix: + lines.append("
") + lines.append("Additional Changes (Test/Build Files)") + lines.append("") + for snippet in other_vuln + other_fix: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**{snippet.snippet_type.title()}** - `{file_name}`") + lines.append("") + lines.append("```") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + lines.append("
") + lines.append("") + + # Caveats + if self.caveats: + lines.append("---") + lines.append("") + lines.append("## Caveats") + lines.append("") + for caveat in self.caveats: + lines.append(f"- {caveat}") + lines.append("") + + # Footer + lines.append("---") + lines.append("") + lines.append("*Report generated by L1 Code Agent*") + + return "\n".join(lines) + + +def format_patch_data_for_intel( + parsed_patch: ParsedPatch | None +) -> str: + """Format patch and CVE data for intelligence extraction. + + Parameters + ---------- + parsed_patch: + Parsed patch file structure (may be None if no patch available). + cve_description: + CVE description text from advisories. + + Returns + ------- + str + Formatted string suitable for the VULNERABILITY_INTEL_EXTRACTION_PROMPT. + """ + if not parsed_patch: + return "" + + lines = [f"Patch: {parsed_patch.patch_filename}", ""] + for pf in parsed_patch.files: + lines.append(f"File: {pf.target_path}") + for hunk in pf.hunks: + if hunk.removed_lines: + lines.append(" Removed (vulnerable):") + for line in hunk.removed_lines[:10]: + lines.append(f" - {line}") + if len(hunk.removed_lines) > 10: + lines.append(f" ... (+{len(hunk.removed_lines) - 10} more lines)") + if hunk.added_lines: + lines.append(" Added (fix):") + for line in hunk.added_lines[:10]: + lines.append(f" + {line}") + if len(hunk.added_lines) > 10: + lines.append(f" ... (+{len(hunk.added_lines) - 10} more lines)") + lines.append("") + + return "\n".join(lines) + + +def get_relevant_hunks(parsed_patch: ParsedPatch | None, grep_query: str) -> str: + """Extract unified diff hunks for files matching the grep target. + + Parameters + ---------- + parsed_patch: + Parsed patch file structure (may be None if no patch available). + grep_query: + The grep query string, which may include a file filter (e.g., "pattern,filename.c"). + + Returns + ------- + str + Unified diff format string with relevant hunks, or empty string if no patch/match. + """ + if not parsed_patch: + return "" + + file_pattern = None + if "," in grep_query: + file_pattern = grep_query.split(",")[-1].strip() + + hunks = [] + for pf in parsed_patch.files: + if file_pattern and file_pattern not in pf.target_path: + continue + hunks.append(f"--- a/{pf.target_path}") + hunks.append(f"+++ b/{pf.target_path}") + for hunk in pf.hunks: + for line in hunk.removed_lines: + hunks.append(f"-\t{line}") + for line in hunk.added_lines: + hunks.append(f"+\t{line}") + + return "\n".join(hunks) if hunks else "" + + +# --------------------------------------------------------------------------- +# Report formatting helpers +# --------------------------------------------------------------------------- + +MAX_SNIPPET_CHARS = 500 +L1_EXTRACTED_FACTS_EXCERPT_CHARS = 2000 + + +def _cap_text_excerpt(text: str, max_chars: int) -> tuple[str, bool]: + """Return (possibly truncated) text and whether truncation occurred.""" + t = text.strip() + if len(t) <= max_chars: + return t, False + return t[: max_chars] + "\n[… truncated …]", True + + +def _format_interleaved_evidence( + evidence_chain: list[str], + downstream_report: DownstreamSearchReport | None, + *, + max_excerpt: int = L1_EXTRACTED_FACTS_EXCERPT_CHARS, +) -> list[str]: + """Build audit-ready markdown for the Evidence Chain section. + + Structure follows the 3-pillar model for TARGET package verification: + - Status Summary table for at-a-glance verification + - Target Patch Metadata (the "What") + - Integration Evidence (the "Plan" - spec file directives) + - Execution Evidence (the "Action" - build logs) + - Source Validation (the "Result" - L1 agent findings) + """ + lines: list[str] = [] + + if downstream_report is None: + for ev in evidence_chain: + lines.append(f"- {ev}") + return lines + + d = downstream_report + + # Categorize evidence items by keywords + patch_evidence: list[str] = [] + build_evidence: list[str] = [] + code_evidence: list[str] = [] + other_evidence: list[str] = [] + + patch_keywords = ("patch", "spec", "patchn", "directive", "target", "reference") + build_keywords = ("build", "applied", "log") + code_keywords = ("code", "function", "vulnerable", "fix", "found", "source", "l1", "agent") + + for ev in evidence_chain: + ev_lower = ev.lower() + if any(kw in ev_lower for kw in patch_keywords): + patch_evidence.append(ev) + elif any(kw in ev_lower for kw in build_keywords): + build_evidence.append(ev) + elif any(kw in ev_lower for kw in code_keywords): + code_evidence.append(ev) + else: + other_evidence.append(ev) + + # Status Summary - at-a-glance verification of TARGET package (using bullets for UI compatibility) + lines.append("### Status Summary (Target Package)") + lines.append("") + patch_check = "PASS" if d.is_patch_file_available else "FAIL" + spec_check = "PASS" if d.is_patch_in_spec_file else "FAIL" + build_check = "PASS" if d.is_patch_applied_in_build else "FAIL" + lines.append(f"- **Target patch file exists:** {patch_check}") + lines.append(f"- **Referenced in target spec:** {spec_check}") + lines.append(f"- **Applied in target build:** {build_check}") + lines.append("") + + # Section 1: Target Patch Metadata + if d.patch_file_name or patch_evidence: + lines.append("### 1. Patch Metadata") + lines.append("") + if d.patch_file_name: + lines.append(f"- **Target patch file:** `{d.patch_file_name}`") + for ev in patch_evidence: + lines.append(f"- {ev}") + lines.append("") + + # Section 2: Integration Evidence (Spec File) - the "Plan" + has_integration = d.spec_patch_directives_for_cve or d.spec_changelog_cve_lines.strip() + if has_integration: + lines.append("### 2. Integration Evidence (Spec File)") + lines.append("") + + if d.spec_patch_directives_for_cve: + # Split directives into declaration and application + declarations = [line for line in d.spec_patch_directives_for_cve + if line.strip().startswith("Patch")] + applications = [line for line in d.spec_patch_directives_for_cve + if line.strip().startswith("%patch")] + + if declarations: + lines.append("**Patch declaration:**") + lines.append("") + lines.append("```ini") + lines.append("\n".join(declarations)) + lines.append("```") + lines.append("") + + if applications: + lines.append("**Patch application directive:**") + lines.append("") + lines.append("```ini") + lines.append("\n".join(applications)) + lines.append("```") + lines.append("") + + if d.spec_changelog_cve_lines.strip(): + ex, trunc = _cap_text_excerpt(d.spec_changelog_cve_lines, max_excerpt) + hdr = "**Changelog entry:**" + if trunc: + hdr += " *(truncated)*" + lines.append(hdr) + lines.append("") + lines.append("```ini") + lines.append(ex) + lines.append("```") + lines.append("") + + # Section 3: Execution Evidence (Build Log) - the "Action" + if d.build_log_patch_applied.strip() or build_evidence: + lines.append("### 3. Execution Evidence (Build Log)") + lines.append("") + + for ev in build_evidence: + lines.append(f"- {ev}") + if build_evidence: + lines.append("") + + if d.build_log_patch_applied.strip(): + ex, trunc = _cap_text_excerpt(d.build_log_patch_applied, max_excerpt) + if trunc: + lines.append("**Build output:** *(truncated)*") + else: + lines.append("**Build output:**") + lines.append("") + lines.append("```bash") + lines.append(ex) + lines.append("```") + lines.append("") + + # Section 4: Source Validation - the "Result" + if code_evidence: + lines.append("### 4. Source Validation") + lines.append("") + for ev in code_evidence: + lines.append(f"- {ev}") + lines.append("") + + # Section 5: Tarball Reference + if d.spec_version_line or d.spec_source0_line: + lines.append("### 5. Tarball Reference") + lines.append("") + if d.spec_version_line: + lines.append(f"- `{d.spec_version_line}`") + if d.spec_source0_line: + lines.append(f"- `{d.spec_source0_line}`") + lines.append("") + + # Additional evidence (uncategorized) + if other_evidence: + lines.append("### Additional Evidence") + lines.append("") + for ev in other_evidence: + lines.append(f"- {ev}") + lines.append("") + + return lines + + +def _format_extracted_facts_section( + d: DownstreamSearchReport, + *, + max_excerpt: int = L1_EXTRACTED_FACTS_EXCERPT_CHARS, +) -> list[str]: + """Build markdown lines for the deterministic *Extracted facts* block. + + .. deprecated:: + This function is deprecated. Use `_format_interleaved_evidence()` instead, + which merges Evidence Chain and Extracted facts into a single interleaved + section for better readability. + """ + warnings.warn( + "_format_extracted_facts_section is deprecated. " + "Use _format_interleaved_evidence() instead.", + DeprecationWarning, + stacklevel=2, + ) + lines: list[str] = [ + "## Extracted facts", + "", + "*Verbatim excerpts from spec/build grep and parsers. Narrative sections below are model-generated.*", + "", + ] + lines.append(f"- **Downstream patch file found:** {d.is_patch_file_available}") + if d.patch_file_name: + lines.append(f"- **Patch file name:** `{d.patch_file_name}`") + lines.append(f"- **Patch referenced in spec (CVE grep):** {d.is_patch_in_spec_file}") + lines.append(f"- **Build log shows CVE / patch application:** {d.is_patch_applied_in_build}") + lines.append("") + + if d.spec_patch_directives_for_cve: + lines.append("**Spec `PatchN:` line(s) whose patch filename contains this CVE:**") + block = "\n".join(d.spec_patch_directives_for_cve) + lines.extend(["", "```", block, "```", ""]) + else: + lines.extend(["**Spec `PatchN:` line(s) whose patch filename contains this CVE:** *None found*", ""]) + + if d.spec_changelog_cve_lines.strip(): + ex, trunc = _cap_text_excerpt(d.spec_changelog_cve_lines, max_excerpt) + sub = f" (truncated to ~{max_excerpt} chars)" if trunc else "" + lines.append(f"**%changelog line(s) mentioning this CVE:**{sub}") + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**%changelog line(s) mentioning this CVE:** *No matching lines* ", ""]) + + if d.spec_file_log_change.strip(): + ex, trunc = _cap_text_excerpt(d.spec_file_log_change, max_excerpt) + hdr = "**All spec lines matching CVE grep (may include Patch, changelog, comments):**" + if trunc: + hdr += f" *({max_excerpt} char excerpt)*" + lines.append(hdr) + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**All spec lines matching CVE grep:** *None*", ""]) + + if d.build_log_patch_applied.strip(): + ex, trunc = _cap_text_excerpt(d.build_log_patch_applied, max_excerpt) + hdr = "**Build log line(s) matching CVE grep:**" + if trunc: + hdr += f" *({max_excerpt} char excerpt)*" + lines.append(hdr) + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**Build log line(s) matching CVE grep:** *None or build log not available* ", ""]) + + # Spec tarball reference (Source0/Version) for delivery-model context + if d.spec_version_line or d.spec_source0_line: + lines.append("**Spec tarball reference:**") + if d.spec_version_line: + lines.append(f"- `{d.spec_version_line}`") + if d.spec_source0_line: + lines.append(f"- `{d.spec_source0_line}`") + lines.append("") + + return lines + + +def _format_downstream_for_report(report: DownstreamSearchReport | None) -> str: + """Format target package analysis results for prompt injection. + + This section reports whether the TARGET package (the one being scanned) + contains a CVE-specific patch file. + """ + if report is None: + return "Target package analysis did not produce results." + + lines = [] + lines.append(f"**Target Package Patch Available:** {report.is_patch_file_available}") + + if report.is_patch_file_available: + lines.append(f"**Target Patch File:** `{report.patch_file_name}`") + lines.append(f"**Referenced in Spec:** {report.is_patch_in_spec_file}") + if report.spec_file_log_change: + lines.append(f"**Target Spec Changelog:**\n```\n{report.spec_file_log_change[:500]}\n```") + lines.append(f"**Applied in Build:** {report.is_patch_applied_in_build}") + if report.build_log_patch_applied: + lines.append(f"**Build Log Evidence:**\n```\n{report.build_log_patch_applied[:500]}\n```") + + if report.parsed_patch: + lines.append(f"\n**Parsed Patch ({len(report.parsed_patch.files)} files):**") + for pf in report.parsed_patch.files[:5]: + added = sum(len(h.added_lines) for h in pf.hunks) + removed = sum(len(h.removed_lines) for h in pf.hunks) + lines.append(f"- `{pf.target_path}` (+{added}/-{removed} lines)") + if len(report.parsed_patch.files) > 5: + lines.append(f" (+{len(report.parsed_patch.files) - 5} more files)") + else: + lines.append("No CVE-specific patch file found in target package.") + + return "\n".join(lines) + + +def _format_upstream_for_report(report: UpstreamSearchReport | None) -> str: + """Format reference intel gathering results for prompt injection. + + This section reports TWO distinct pieces of information: + 1. Rebase indicator: Checked TARGET's spec file for CVE mention + 2. Reference package: Downloaded a known-fixed package from intel to extract patch patterns + """ + if report is None: + return "Reference intel gathering did not produce results." + + lines = [] + + # Part 1: Rebase indicator (checked in TARGET's spec file) + rebase_status = report.is_code_fixed_by_rebase + if rebase_status == "unknown": + lines.append("**Target Rebase Indicator:** not found (no CVE mention in target's spec file)") + elif rebase_status == "yes": + lines.append("**Target Rebase Indicator:** found (CVE mentioned in target's spec changelog)") + else: + lines.append(f"**Target Rebase Indicator:** {rebase_status}") + + if report.spec_file_log_change: + lines.append(f"**Target Spec Changelog Match:**\n```\n{report.spec_file_log_change[:500]}\n```") + + # Part 2: Reference package (downloaded from intel for comparison) + if report.is_fixed_srpm_is_needed: + if report.reference_package_nvr: + lines.append(f"**Reference Fixed Package:** `{report.reference_package_nvr}` (from intel)") + else: + lines.append(f"**Reference Fixed Package:** Available (from intel)") + lines.append(f"**Reference Patch File:** `{report.fixed_srpm_file_name}`") + if report.fixed_parsed_patch: + lines.append(f"\n**Reference Patch ({len(report.fixed_parsed_patch.files)} files):**") + for pf in report.fixed_parsed_patch.files[:5]: + added = sum(len(h.added_lines) for h in pf.hunks) + removed = sum(len(h.removed_lines) for h in pf.hunks) + lines.append(f"- `{pf.target_path}` (+{added}/-{removed} lines)") + + + if report.reason_code_fixed_by_rebase: + lines.append(f"\n**Rebase Reasoning:** {report.reason_code_fixed_by_rebase}") + + return "\n".join(lines) + + +def _build_agent_status_line(l2_result: L2BuildResult | None) -> str: + """Explicit build-agent run state for the report LLM (avoids inferring from empty XML).""" + if l2_result is None: + return "Build agent status: not_run" + if l2_result.l2_override_verdict is not None: + return f"Build agent status: ran_with_override ({l2_result.l2_override_verdict})" + return "Build agent status: ran_no_override (code agent verdict stands)" + + +def _format_l2_for_report(l2_result: L2BuildResult | None) -> str: + """Format build agent results for prompt injection. + + Emits context whenever L2 ran, including when there is no override verdict. + """ + if l2_result is None: + return "" + + override = l2_result.l2_override_verdict or "none" + lines = [ + "", + f"**Override verdict:** {override}", + f"**Compilation status:** {l2_result.compilation_status}", + ] + + if l2_result.compilation_evidence: + lines.append(f"**Compilation evidence:** {l2_result.compilation_evidence}") + + if l2_result.hardening_flags: + flags_str = ", ".join(l2_result.hardening_flags[:10]) + if len(l2_result.hardening_flags) > 10: + flags_str += f" (+{len(l2_result.hardening_flags) - 10} more)" + lines.append(f"**Hardening flags:** {flags_str}") + + if l2_result.hardening_rationale: + lines.append(f"**Hardening rationale:** {l2_result.hardening_rationale}") + + if l2_result.hardening_relevant is not None: + lines.append(f"**Hardening relevant to CVE:** {l2_result.hardening_relevant}") + + if l2_result.evidence_sources: + lines.append(f"**Evidence sources:** {', '.join(l2_result.evidence_sources)}") + + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Report generation pipeline +# --------------------------------------------------------------------------- + +MAX_REPORT_CODE_SNIPPETS_VULNERABLE = 3 +MAX_REPORT_CODE_SNIPPETS_FIX = 3 + + +def _normalize_snippet_path(path: str) -> str: + """Stable comparison key for patch vs affected file paths.""" + p = path.strip().replace("\\", "/") + while p.startswith("./"): + p = p[2:] + if p.startswith("ab/"): + p = p[3:] + return p.lower() + + +def _snippet_matches_any_affected_path(snippet_path: str, affected_files: list[str]) -> bool: + if not affected_files: + return False + norm_snip = _normalize_snippet_path(snippet_path) + snip_base = Path(snippet_path).name.lower() + for af in affected_files: + norm_af = _normalize_snippet_path(af) + if norm_snip == norm_af: + return True + if snip_base and snip_base == Path(af).name.lower(): + return True + if norm_snip.endswith(norm_af) or norm_af.endswith(norm_snip): + return True + return False + + +def _rank_patch_snippets_for_relevance( + snippets: list[CodeSnippet], + affected_files: list[str], +) -> list[CodeSnippet]: + """Paths matching affected_files first; preserve original order within each bucket.""" + if not affected_files: + return list(snippets) + indexed = list(enumerate(snippets)) + indexed.sort( + key=lambda pair: ( + 0 if _snippet_matches_any_affected_path(pair[1].file_path, affected_files) else 1, + pair[0], + ), + ) + return [s for _, s in indexed] + + +def _cap_snippets_by_type( + snippets: list[CodeSnippet], + *, + max_vulnerable: int = MAX_REPORT_CODE_SNIPPETS_VULNERABLE, + max_fix: int = MAX_REPORT_CODE_SNIPPETS_FIX, +) -> list[CodeSnippet]: + """Keep insertion order; at most max_vulnerable vulnerable and max_fix fix snippets.""" + n_vuln = n_fix = 0 + out: list[CodeSnippet] = [] + for s in snippets: + if s.snippet_type == "vulnerable": + if n_vuln >= max_vulnerable: + continue + n_vuln += 1 + out.append(s) + elif s.snippet_type == "fix": + if n_fix >= max_fix: + continue + n_fix += 1 + out.append(s) + else: + out.append(s) + return out + + +def _extract_downstream_patch_code_snippets( + downstream_report: DownstreamSearchReport | None, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from the downstream parsed patch only. + + For purely additive patches (no removed lines), shows context lines + as "vulnerable" since they represent the code lacking the fix. + """ + if not downstream_report or not downstream_report.parsed_patch: + return [] + snippets: list[CodeSnippet] = [] + for pf in downstream_report.parsed_patch.files: + for hunk in pf.hunks: + if hunk.removed_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.source_start, + code="\n".join(hunk.removed_lines[:10]), + snippet_type="vulnerable", + source="downstream_patch", + )) + elif hunk.context_lines and hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.source_start, + code="\n".join(hunk.context_lines[:10]), + snippet_type="vulnerable", + source="downstream_patch", + )) + if hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.target_start, + code="\n".join(hunk.added_lines[:10]), + snippet_type="fix", + source="downstream_patch", + )) + return snippets + + +def _extract_code_snippets( + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, +) -> list[CodeSnippet]: + """Extract code snippets from parsed patches. + + For purely additive patches (no removed lines), shows context lines + as "vulnerable" since they represent the code lacking the fix. + """ + snippets: list[CodeSnippet] = _extract_downstream_patch_code_snippets(downstream_report) + + if upstream_report and upstream_report.fixed_parsed_patch: + for pf in upstream_report.fixed_parsed_patch.files: + for hunk in pf.hunks: + if hunk.removed_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.source_start, + code="\n".join(hunk.removed_lines[:10]), + snippet_type="vulnerable", + source="upstream_patch", + )) + elif hunk.context_lines and hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.source_start, + code="\n".join(hunk.context_lines[:10]), + snippet_type="vulnerable", + source="upstream_patch", + )) + if hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.target_path.lstrip("ab/"), + line_number=hunk.target_start, + code="\n".join(hunk.added_lines[:10]), + snippet_type="fix", + source="upstream_patch", + )) + + return snippets + + +async def extract_l1_verdict( + llm, + vuln_id: str, + target_package: str, + final_answer: str, + tracer, +) -> L1VerdictExtraction: + """Use LLM to extract structured verdict from L1 agent's final answer. + + Parameters + ---------- + llm: + LangChain LLM for verdict extraction. + vuln_id: + CVE identifier (e.g. "CVE-2026-5121"). + target_package: + Name of the package being investigated. + final_answer: + The L1 agent's final answer text. + tracer: + Request-scoped tracing context. + + Returns + ------- + L1VerdictExtraction + Structured verdict with confidence and reasoning. + """ + verdict_llm = llm.with_structured_output(L1VerdictExtraction) + prompt = L1_VERDICT_EXTRACTION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package, + final_answer=final_answer, + ) + with tracer.push_active_function("extract_l1_verdict", input_data={"vuln_id": vuln_id}) as span: + result = await verdict_llm.ainvoke([SystemMessage(content=prompt)]) + span.set_output({ + "preliminary_verdict": result.preliminary_verdict, + "confidence": result.confidence, + }) + logger.info( + "extract_l1_verdict: verdict=%s confidence=%.2f", + result.preliminary_verdict, result.confidence, + ) + return result + + +async def generate_code_agent_report( + *, + llm, + vuln_id: str, + target_package: str, + descriptions: list[tuple[str, str]], + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, + l1_agent_answer: str | None, + tracer, + policy_context: str = "", + l2_result: L2BuildResult | None = None, +) -> CodeAgentReport: + """Generate the final L1 Code Agent investigation report. + + Synthesizes results from downstream search, upstream search, L1 agent analysis, + and optionally L2 build analysis into a comprehensive, auditable report with + a clear verdict. + + Parameters + ---------- + llm: + LangChain LLM for report generation. + vuln_id: + CVE identifier (e.g. "CVE-2026-5121"). + target_package: + Name of the package being investigated. + descriptions: + ``(source_name, text)`` pairs from CVE intel. + downstream_report: + Output of downstream search (may be None). + upstream_report: + Output of upstream search (may be None). + l1_agent_answer: + Final answer from the L1 ReAct agent (may be None). + tracer: + Request-scoped tracing context. + policy_context: + Pre-formatted NVR posture and RHSA excerpt context for the LLM prompt. + l2_result: + Output of L2 build analysis (may be None). When present, L2 verdicts + override L1 findings as L2 analyzes actual compiled binaries. + + Returns + ------- + CodeAgentReport + Structured report with verdict, evidence, and recommendations. + """ + from langchain_core.messages import HumanMessage, SystemMessage + + cve_description = "\n".join(f"[{src}] {txt}" for src, txt in descriptions) + + downstream_section = _format_downstream_for_report(downstream_report) + upstream_section = _format_upstream_for_report(upstream_report) + l1_agent_section = l1_agent_answer or "Code agent did not produce a final answer." + l2_context_section = _format_l2_for_report(l2_result) + build_agent_status_section = ( + f"\n{_build_agent_status_line(l2_result)}\n\n" + ) + + # Generate override notice when L2 has an override verdict + if l2_result is not None and l2_result.l2_override_verdict is not None: + override_notice_section = ( + "\n" + f"BUILD AGENT OVERRIDE IN EFFECT: The build agent's verdict ({l2_result.l2_override_verdict}) " + "SUPERSEDES the code agent's source analysis.\n" + "Do NOT use 'vulnerable' as the justification_label. " + "Follow the LABEL SELECTION DECISION TREE in the instructions.\n" + "\n" + ) + else: + override_notice_section = "" + + if policy_context: + policy_context_section = ( + "\n" + + policy_context + + "\n\n" + ) + else: + policy_context_section = "" + + prompt_text = CODE_AGENT_REPORT_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package, + cve_description=cve_description, + policy_context_section=policy_context_section, + downstream_section=downstream_section, + upstream_section=upstream_section, + l1_agent_section=l1_agent_section, + l2_context_section=l2_context_section, + build_agent_status_section=build_agent_status_section, + override_notice_section=override_notice_section, + ) + + report_llm = llm.with_structured_output(CodeAgentReportLLM) + + has_l2_override = l2_result is not None and l2_result.l2_override_verdict is not None + with tracer.push_active_function( + "generate_report", + input_data={ + "vuln_id": vuln_id, + "target_package": target_package, + "has_downstream_patch": downstream_report.is_patch_file_available if downstream_report else False, + "has_upstream_patch": upstream_report.is_fixed_srpm_is_needed if upstream_report else False, + "has_l1_answer": l1_agent_answer is not None, + "has_l2_override": has_l2_override, + }, + ) as span: + messages = [ + SystemMessage(content=prompt_text), + HumanMessage(content="Generate the report."), + ] + try: + llm_report: CodeAgentReportLLM = await report_llm.ainvoke(messages) + except openai.LengthFinishReasonError as e: + logger.error("generate_code_agent_report: generation cut off due to token limits") + raw_partial_content = e.completion.choices[0].message.content + logger.debug("generate_code_agent_report: raw truncated content: %s", raw_partial_content) + logger.debug("generate_code_agent_report: usage details: %s", e.completion.usage) + raise + except openai.OpenAIError as e: + logger.error("generate_code_agent_report: OpenAI error: %s", e) + raise + except Exception as e: + logger.error("generate_code_agent_report: error generating report: %s", e) + raise + + report = CodeAgentReport(**llm_report.model_dump()) + + snippet_source = "unchanged" + downstream_patch_snippet_count_pre_cap = 0 + if downstream_report and downstream_report.parsed_patch: + raw = _extract_downstream_patch_code_snippets(downstream_report) + downstream_patch_snippet_count_pre_cap = len(raw) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "downstream_patch" + elif upstream_report and upstream_report.fixed_parsed_patch: + raw = _extract_code_snippets(downstream_report, upstream_report) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "upstream_patch" + elif not report.code_snippets: + report.code_snippets = _extract_code_snippets(downstream_report, upstream_report) + + span.set_output({ + "justification_label": report.justification_label, + "affected_files_count": len(report.affected_files), + "caveats_count": len(report.caveats), + "code_snippets_count": len(report.code_snippets), + "snippet_source": snippet_source, + "downstream_patch_snippet_count_pre_cap": downstream_patch_snippet_count_pre_cap, + }) + + logger.info( + "generate_code_agent_report: justification=%s", + report.justification_label, + ) + + return report + + +# --------------------------------------------------------------------------- +# Diff and patch helpers +# --------------------------------------------------------------------------- + + +def download_patch_and_gen_diff(fix_info: dict, brew_downloader: BrewDownloader, source_dir: Path, patch_dir: Path) -> Path | None: + """Download the patched SRPM and generate the diff file between the source and the patched SRPM.""" + from exploit_iq_commons.utils.source_rpm_downloader import SourceRPMDownloader + + srpm_path = brew_downloader.download_patched_srpm_by_nevra(fix_info["nevra"]) + if srpm_path is None: + srpm_path = brew_downloader.download_patched_srpm(fix_info["name"], fix_info["version"], fix_info["release"],) + if srpm_path is not None: + patch_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(srpm_path, patch_dir) + SourceRPMDownloader.extract_src_rpm(srpm_path, patch_dir) + + #diff_text = _generate_tree_diff(source_dir, patch_dir) + #diff_output_path = patch_dir.parent / "locate.diff" + #diff_output_path.write_text(diff_text, encoding="utf-8") + #return diff_output_path + return None + + +# --------------------------------------------------------------------------- +# Spec/build log parsing helpers +# --------------------------------------------------------------------------- + +_SPEC_PATCH_RE = re.compile(r"^Patch(\d+)\s*:\s*(.+)$", re.IGNORECASE) + + +def _parse_spec_patch_directives( + inspector, spec_path: Path, +) -> list[tuple[int, str, str]]: + """Return ``[(index, filename, raw_line), ...]`` from ``PatchN:`` lines.""" + matches = inspector.grep_content(_SPEC_PATCH_RE.pattern, spec_path) + results: list[tuple[int, str, str]] = [] + for m in matches: + hit = _SPEC_PATCH_RE.match(m.line_content.strip()) + if hit: + results.append((int(hit.group(1)), hit.group(2).strip(), m.line_content.strip())) + return results + + +def _extract_spec_changelog(inspector, spec_path: Path) -> str | None: + """Return text after the ``%changelog`` directive, or ``None``.""" + content = inspector.read_file(spec_path) + idx = content.find("%changelog") + if idx == -1: + return None + return content[idx + len("%changelog"):] + + +def find_cve_patch_in_spec( + inspector, + cve_id: str, +) -> SpecPatchMatch | None: + """Find a patch file for a CVE by searching filename or spec comments. + + Search order: + 1. Patch files with CVE in filename (fast path) + 2. PatchN: directives with CVE in the contiguous comment block above + + Returns None if no match or patch file doesn't exist. + """ + cve_pattern = re.compile(re.escape(cve_id), re.IGNORECASE) + + spec_files = inspector.find_files("*.spec", recursive=False) + if not spec_files: + return None + spec_path = spec_files[0] + + patch_files = inspector.find_files("*.patch", recursive=False) + patch_filenames = {p.name for p in patch_files} + + for pf in patch_files: + if cve_pattern.search(pf.name): + for idx, fname, raw_line in _parse_spec_patch_directives(inspector, spec_path): + if fname == pf.name: + return SpecPatchMatch( + patch_index=idx, + patch_filename=pf.name, + raw_directive=raw_line, + comment_block=None, + match_source="filename", + ) + return SpecPatchMatch( + patch_index=0, + patch_filename=pf.name, + raw_directive=f"Patch: {pf.name}", + comment_block=None, + match_source="filename", + ) + + content = inspector.read_file(spec_path) + lines = content.splitlines() + + comment_block: list[str] = [] + for line in lines: + stripped = line.strip() + if stripped.startswith("#"): + comment_block.append(stripped) + elif _SPEC_PATCH_RE.match(stripped): + hit = _SPEC_PATCH_RE.match(stripped) + if hit and comment_block: + combined_comments = "\n".join(comment_block) + if cve_pattern.search(combined_comments): + patch_filename = hit.group(2).strip() + if patch_filename in patch_filenames: + return SpecPatchMatch( + patch_index=int(hit.group(1)), + patch_filename=patch_filename, + raw_directive=stripped, + comment_block=combined_comments, + match_source="comment", + ) + comment_block = [] + else: + comment_block = [] + + return None + + +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu','.uue','.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', + '.woff', '.woff2', '.ttf', '.otf', '.eot', + '.mp3', '.mp4', '.wav', '.avi', '.mov', '.mkv', + '.db', '.sqlite', '.sqlite3', +}) + + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def parse_patch_file(patch_path: Path) -> ParsedPatch | None: + """Parse a downstream .patch file into structured data. + + Returns None if the file cannot be parsed. + """ + try: + diff_text = patch_path.read_text(encoding="utf-8", errors="replace") + patch_set = PatchSet.from_string(diff_text) + except Exception: + logger.warning("parse_patch_file: failed to parse %s", patch_path) + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_path.name, files=files) + + +# --------------------------------------------------------------------------- +# Downstream search pipeline +# --------------------------------------------------------------------------- +def _populate_downstream_spec_and_build( + report: DownstreamSearchReport, + inspector, + cve_pattern: str, + build_log_path: Path | None, +) -> None: + """Inspect target spec and build log for CVE fix evidence (runs with or without a patch file).""" + from vuln_analysis.tools.source_inspector import SourceInspector + + spec_files = inspector.find_files("*.spec", recursive=False) + spec_path = spec_files[0] if spec_files else None + + if not spec_path: + report.is_patch_in_spec_file = False + else: + cve_c = re.compile(cve_pattern, re.IGNORECASE) + for _idx, fname, raw_line in _parse_spec_patch_directives(inspector, spec_path): + if cve_c.search(fname): + report.spec_patch_directives_for_cve.append(raw_line) + chlog = _extract_spec_changelog(inspector, spec_path) + if chlog: + cve_in_cl = [ln for ln in chlog.splitlines() if cve_c.search(ln)] + report.spec_changelog_cve_lines = "\n".join(cve_in_cl) + grep_spec_matches = inspector.grep_content(cve_pattern, spec_path) + if grep_spec_matches: + report.is_patch_in_spec_file = True + report.spec_file_log_change = "\n".join(m.line_content for m in grep_spec_matches) + else: + report.is_patch_in_spec_file = False + + source0_matches = inspector.grep_content(r"^Source0:", spec_path) + if source0_matches: + report.spec_source0_line = source0_matches[0].line_content.strip() + version_matches = inspector.grep_content(r"^Version:", spec_path) + if version_matches: + report.spec_version_line = version_matches[0].line_content.strip() + + if build_log_path and build_log_path.exists(): + build_inspector = SourceInspector(build_log_path.parent) + build_log_matches = build_inspector.grep_content(cve_pattern, build_log_path) + if build_log_matches: + report.is_patch_applied_in_build = True + report.build_log_patch_applied = "\n".join(m.line_content for m in build_log_matches) + else: + report.is_patch_applied_in_build = False + else: + report.is_patch_applied_in_build = False + + +async def downstream_search_preprocss( + *, + llm, + vuln_id: str, + descriptions: list[tuple[str, str]], + source_path: Path, + build_log_path: Path | None, + tracer, +) -> DownstreamSearchReport: + """Build the downstream search pipeline.""" + from vuln_analysis.tools.source_inspector import SourceInspector + inspector = SourceInspector(source_path) + + cve_pattern = re.escape(vuln_id) + report = DownstreamSearchReport() + patch_file = None + with tracer.push_active_function("Is_patch_file_available", input_data={"vuln_id": vuln_id}) as span: + spec_match = find_cve_patch_in_spec(inspector, vuln_id) + if spec_match: + report.is_patch_file_available = True + patch_file = inspector.root / spec_match.patch_filename + report.patch_file_name = spec_match.patch_filename + if spec_match.match_source == "comment": + report.spec_patch_directives_for_cve.append(spec_match.raw_directive) + span.set_output({ + "patch_found": True, + "match_source": spec_match.match_source, + "patch_filename": spec_match.patch_filename, + }) + else: + report.is_patch_file_available = False + span.set_output({"patch_found": False}) + + with tracer.push_active_function( + "Is_patch_in_spec_file", + input_data={"patch_file_name": report.patch_file_name or "none"}, + ) as span: + _populate_downstream_spec_and_build(report, inspector, cve_pattern, build_log_path) + + if patch_file: + with tracer.push_active_function( + "Extract_patch_details", input_data={"patch_file_name": patch_file.name} + ) as span: + details = parse_patch_file(patch_file) + report.parsed_patch = details if details else None + + return report + +async def upstream_search_preprocess( + *, + vuln_id: str, + source_path: Path, + fix_info: dict, + brew_downloader: BrewDownloader, + patch_dir: Path, + target_package: TargetPackage, + tracer, + intel: list | None = None, + commit_url_candidates: dict[str, list[str]] | None = None, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, +) -> UpstreamSearchReport: + """Build the upstream search pipeline. + + Args: + intel: Optional list of CveIntel objects for Ubuntu patch lookup. + commit_url_candidates: Optional dict of URLs from intel references for patch fetching. + cve_description: Optional CVE description for Chromium CL disambiguation. + llm: Optional LangChain LLM for Chromium CL selection when multiple MERGED CLs exist. + """ + from vuln_analysis.tools.source_inspector import SourceInspector + inspector = SourceInspector(source_path) + report = UpstreamSearchReport() + cve_pattern = re.escape(vuln_id) + need_to_find_code = True + # Store reference package NVR from fix_info if available + if fix_info and fix_info.get("nevra"): + report.reference_package_nvr = fix_info["nevra"] + + with tracer.push_active_function("Is_upstream_fixed_by_rebase", input_data={"vuln_id": vuln_id}) as span: + spec_files = inspector.find_files("*.spec", recursive=False) + spec_path = spec_files[0] if spec_files else None + + if not spec_path: + report.is_code_fixed_by_rebase = "unknown" + else: + grep_spec_matches = inspector.grep_content(cve_pattern, spec_path) + if grep_spec_matches: + report.is_code_fixed_by_rebase = "yes" + report.spec_file_log_change = "\n".join(m.line_content for m in grep_spec_matches) + else: + report.is_code_fixed_by_rebase = "unknown" + span.set_output({ + "is_code_fixed_by_rebase": report.is_code_fixed_by_rebase, + "spec_file_log_change": report.spec_file_log_change, + }) + + if patch_dir.exists(): + shutil.rmtree(patch_dir, ignore_errors=True) + + if fix_info and brew_downloader is not None: + with tracer.push_active_function( + "download_rpm_patch", input_data={"fix_info": fix_info} + ) as span: + try: + download_patch_and_gen_diff(fix_info, brew_downloader, source_path, patch_dir) + span.set_output({"patch_dir_exists": patch_dir.exists()}) + except Exception as e: + logger.warning("locate: failed to download/extract patched SRPM: %s", e) + span.set_output({"error": str(e), "patch_dir_exists": False}) + + if patch_dir.exists(): + patch_inspector = SourceInspector(patch_dir) + with tracer.push_active_function("is_patch_downsteam_patch_file", input_data={"patch_dir": patch_dir}) as span: + spec_match = find_cve_patch_in_spec(patch_inspector, vuln_id) + if spec_match: + report.is_fixed_srpm_is_needed = True + report.fixed_srpm_file_name = spec_match.patch_filename + report.fixed_parsed_patch = parse_patch_file(patch_inspector.root / spec_match.patch_filename) + span.set_output({ + "is_fixed_srpm_is_needed": True, + "match_source": spec_match.match_source, + "patch_filename": spec_match.patch_filename, + }) + return report + else: + report.is_fixed_srpm_is_needed = False + span.set_output({ + "is_fixed_srpm_is_needed": report.is_fixed_srpm_is_needed}) + + # Try intel references (unified: commit URLs from GHSA/NVD/RHSA/Ubuntu) + if (not patch_dir.exists() or need_to_find_code) and not report.fixed_parsed_patch: + if commit_url_candidates: + from vuln_analysis.utils.web_patch_fetcher import WebPatchFetcher + with tracer.push_active_function( + "fetch_patch_from_intel_refs", + input_data={ + "vuln_id": vuln_id, + "candidates_count": {src: len(urls) for src, urls in commit_url_candidates.items()}, + } + ) as span: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session=session) + result = await fetcher.fetch_from_intel_refs( + commit_url_candidates, + vuln_id, + cve_description=cve_description, + llm=llm, + ) + if result and result.parsed_patch: + report.fixed_parsed_patch = result.parsed_patch + report.fixed_srpm_file_name = result.patch_url + report.is_fixed_srpm_is_needed = True + report.osv_result = result + span.set_output({ + "source_found": result.source, + "url_type": result.url_type, + "platform": result.platform, + "patch_url": result.patch_url, + "commit_message": result.commit_message, + "patch_found": True, + }) + else: + span.set_output({"patch_found": False}) + + # OSV API fallback + if (not patch_dir.exists() or need_to_find_code) and not report.fixed_parsed_patch: + from vuln_analysis.utils.web_patch_fetcher import WebPatchFetcher, OSVClient + with tracer.push_active_function("fetch_patch_from_osv", input_data={"vuln_id": vuln_id}) as span: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session=session) + client = OSVClient(session=session, patch_fetcher=fetcher) + result = await client.get_fix_patch(vuln_id, target_package.version, target_package.name) + if result and result.parsed_patch: + report.fixed_parsed_patch = result.parsed_patch + report.fixed_srpm_file_name = result.patch_url + report.is_fixed_srpm_is_needed = True + report.osv_result = result + span.set_output({ + "source_found": "osv", + "platform": result.platform, + "patch_url": result.patch_url, + "commit_message": result.commit_message, + "patch_found": True, + }) + else: + span.set_output({"patch_found": False}) + return report + diff --git a/src/vuln_analysis/functions/cve_build_agent.py b/src/vuln_analysis/functions/cve_build_agent.py new file mode 100644 index 000000000..a0c4a65d2 --- /dev/null +++ b/src/vuln_analysis/functions/cve_build_agent.py @@ -0,0 +1,784 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Level 2 Build Agent for Package Vulnerability Checker. + +Performs BuildCompilationCheck (Phase 1) and HardeningCheck (Phase 2) to +determine if vulnerable code identified by L1 is actually compiled into +the binary and whether hardening flags provide mitigation. +""" + +from pathlib import Path +from enum import StrEnum + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import L2BuildResult + +from langgraph.graph import StateGraph, START, END +from langgraph.prebuilt import ToolNode +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, RemoveMessage + +from nat.builder.context import Context +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + +from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output + +from vuln_analysis.functions.build_agent_graph_defs import ( + BuildAgentState, + BuildHarvestReport, + harvest_build_data, + L2CompileVerdictExtraction, + L2HardeningVerdictExtraction, +) +from vuln_analysis.utils.rpm_checker_prompts import ( + L2_CONFIG_PROMPT_TEMPLATE, + L2_CONFIG_SYS_PROMPT, + L2_CONFIG_THOUGHT_INSTRUCTIONS, + L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE, + L2_CONFIG_SPEC_ONLY_SYS_PROMPT, + L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS, + L2_COMPREHENSION_PROMPT, + L2_MEMORY_UPDATE_PROMPT, + L2_HARDENING_PROMPT_TEMPLATE, + L2_HARDENING_SYS_PROMPT, + L2_HARDENING_THOUGHT_INSTRUCTIONS, + L2_COMPILATION_VERDICT_PROMPT, + L2_HARDENING_VERDICT_PROMPT, + L2_KERNEL_CONFIG_SYS_PROMPT, + L2_KERNEL_CONFIG_PROMPT_TEMPLATE, + L2_KERNEL_THOUGHT_INSTRUCTIONS, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.utils.token_utils import truncate_tool_output +import uuid +import tiktoken +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVEBuildAgentConfig(FunctionBaseConfig, name="cve_build_agent"): + """ + Level 2 Build Agent. Analyzes build artifacts to determine if vulnerable + code is compiled into the binary and whether hardening flags mitigate. + + Phase 1: BuildCompilationCheck - Is vulnerable code compiled? + Phase 2: HardeningCheck - Do hardening flags mitigate the CVE? + """ + + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + max_iterations: int = Field( + default=5, + description="The maximum number of iterations for the agent.", + ) + llm_name: str = Field(description="The LLM model to use with the L1 code agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to L1 code agent") + context_window_token_limit: int = Field(default=5000, description="Token limit for context window before pruning old messages.") + +def _build_tool_strategy(tool_names: list[str]) -> str: + """Generate tool usage guidance based on available tools.""" + strategies = [] + tool_names_lower = [t.lower().replace("_", " ") for t in tool_names] + + if any("grep" in t for t in tool_names_lower): + strategies.append("- Use Source Grep for exact code patterns from patch (function names, variable names, specific code)") + if any("keyword" in t or "search" in t for t in tool_names_lower): + strategies.append("- Use Code Keyword Search for broader concept searches when grep fails") + if any("read" in t for t in tool_names_lower): + strategies.append("- Use Read File to examine full context around matches") + + return "\n".join(strategies) if strategies else "Use available tools to search for vulnerable and fixed code patterns." + + +class L2InvestigationPhase(StrEnum): + CONFIGURATION = "configuration" + HARDENING = "hardening" + + +async def create_graph_build_agent( + config: CVEBuildAgentConfig, + builder: Builder, + state: AgentMorpheusEngineInput, + tracer, +): + """Build the L2 Build Agent LangGraph. + + Graph structure: + START -> data_harvest_node -> thought_node -+-> END (finish) + | + +-> tool_node -> observation_node -> thought_node + """ + # Node name constants + DATA_HARVEST_NODE = "data_harvest" + THOUGHT_NODE = "thought_node" + TOOL_NODE = "tool_node" + OBSERVATION_NODE = "observation_node" + FORCED_FINISH_NODE = "forced_finish" + INVESTIGATION_PHASE_NODE = "investigation_phase" + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + thought_llm = llm.with_structured_output(CheckerThought) + comprehension_llm = llm.with_structured_output(CodeFindings) + observation_llm = llm.with_structured_output(Observation) + compilation_verdict_llm = llm.with_structured_output(L2CompileVerdictExtraction) + hardening_verdict_llm = llm.with_structured_output(L2HardeningVerdictExtraction) + tools_node = ToolNode(tools, handle_tool_errors=True) if tools else None + enabled_tool_names = [tool.name for tool in tools] + tool_descriptions_list = [t.name + ": " + t.description for t in tools] + tools_str = "\n".join(tool_descriptions_list) + tool_strategy = _build_tool_strategy(enabled_tool_names) + # Extract context from state (guaranteed by early exit checks in _arun) + ctx = state.info.checker_context + assert ctx is not None, "checker_context must exist (checked in _arun)" + assert ctx.l1_result is not None, "l1_result must exist (checked in _arun)" + # build_log_path is now optional - None means spec-only mode + assert ctx.source_key, "source_key must exist when artifacts exist" + + l1_result = ctx.l1_result + artifacts = ctx.artifacts + target_package = state.input.image.target_package + vuln_id = state.input.scan.vulns[0].vuln_id + + # Paths + source_key = ctx.source_key + checker_dir = Path(config.base_checker_dir) / source_key + + build_log_path = Path(artifacts.build_log_path) if artifacts.build_log_path else None + + # L1 results - use full VulnerabilityIntel for richer context + vulnerability_intel = l1_result.vulnerability_intel + vulnerability_intel_str = vulnerability_intel.format_for_prompt() if vulnerability_intel else "No intel available" + l1_preliminary_verdict = l1_result.preliminary_verdict + + # Extract CWE ID from intel (if available) + cwe_id = None + intel_list = state.info.intel + intel = intel_list[0] + if intel.nvd and intel.nvd.cwe_id: + cwe_id = intel.nvd.cwe_id + logger.info("build_agent: CWE ID from intel: %s", cwe_id) + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + investigation_stack: list[L2InvestigationPhase] = [] + investigation_stack.append(L2InvestigationPhase.HARDENING) + investigation_stack.append(L2InvestigationPhase.CONFIGURATION) + # ------------------------------------------------------------------------- + # L2 graph nodes + # ------------------------------------------------------------------------- + def _count_tokens(text: str) -> int: + """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: + """Estimate the token count thought_node will send to the LLM.""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return _count_tokens("\n".join(parts)) + # ------------------------------------------------------------------------- + # Data Harvest Node + # ------------------------------------------------------------------------- + async def build_runtime_prompt(harvest_report: BuildHarvestReport) -> str: + """Generate the runtime prompt for the current investigation phase.""" + current_phase = investigation_stack[-1] + runtime_prompt = "" + if current_phase == L2InvestigationPhase.CONFIGURATION: + # Kernel packages use CONFIG-based checking (not build log grep) + if harvest_report.is_kernel: + logger.info("Using kernel-specific prompt template") + runtime_prompt = L2_KERNEL_CONFIG_PROMPT_TEMPLATE.format( + sys_prompt=L2_KERNEL_CONFIG_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + kernel_config_path=harvest_report.kernel_config_path or "Not available", + kernel_source_root=harvest_report.kernel_source_root or "Not available", + tools=tools_str, + tool_instructions=L2_KERNEL_THOUGHT_INSTRUCTIONS, + ) + elif build_log_path: + # Full mode: build log + spec file (standard packages) + runtime_prompt = L2_CONFIG_PROMPT_TEMPLATE.format( + sys_prompt=L2_CONFIG_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + disabled_features=harvest_report.disabled_features, + spec_disabled_features=harvest_report.spec_disabled_features, + enabled_features=harvest_report.enabled_features, + spec_enabled_features=harvest_report.spec_enabled_features, + linked_libraries=harvest_report.linked_libraries or "None", + built_subpackages=harvest_report.built_subpackages or "None", + excluded_subpackages=harvest_report.excluded_subpackages or "None", + kernel_config=harvest_report.kernel_config or "N/A (not a kernel package)", + tools=tools_str, + tool_instructions=L2_CONFIG_THOUGHT_INSTRUCTIONS, + ) + else: + # Spec-only mode: no build log available + logger.info("Using spec-only prompt template (no build log)") + runtime_prompt = L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE.format( + sys_prompt=L2_CONFIG_SPEC_ONLY_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + spec_disabled_features=harvest_report.spec_disabled_features, + spec_enabled_features=harvest_report.spec_enabled_features, + built_subpackages=harvest_report.built_subpackages or "None", + excluded_subpackages=harvest_report.excluded_subpackages or "None", + kernel_config=harvest_report.kernel_config or "N/A (not a kernel package)", + tools=tools_str, + tool_instructions=L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS, + ) + return runtime_prompt + elif current_phase == L2InvestigationPhase.HARDENING: + runtime_prompt = L2_HARDENING_PROMPT_TEMPLATE.format( + sys_prompt=L2_HARDENING_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + cwe_id=cwe_id, + expected_hardening_table=harvest_report.expected_hardening, + tools=tools_str, + tool_instructions=L2_HARDENING_THOUGHT_INSTRUCTIONS, + ) + return runtime_prompt + else: + raise ValueError(f"Unknown investigation phase: {current_phase}") + + async def data_harvest_node(state: BuildAgentState) -> dict: + """Harvest structured data from build log, spec, and source tree before the ReAct loop.""" + logger.info("data_harvest_node: starting") + + with tracer.push_active_function("data_harvest", input_data={}) as span: + # Find spec file if available + spec_path = None + if checker_dir and checker_dir.exists(): + spec_files = list((checker_dir / "source").glob("*.spec")) + spec_path = spec_files[0] if spec_files else None + + source_path = checker_dir / "source" if checker_dir else None + harvest_report = await harvest_build_data( + build_log_path=build_log_path, + spec_path=spec_path, + cwe_id=cwe_id, + source_path=source_path, + package_name=target_package.name if target_package else None, + arch=target_package.arch if target_package else None, + ) + + runtime_prompt = await build_runtime_prompt(harvest_report) + + affected_files_count = len(vulnerability_intel.affected_files) if vulnerability_intel else 0 + # Compute evidence sources based on available artifacts + evidence_sources: list[str] = [] + if build_log_path: + evidence_sources.append("build_log") + if spec_path: + evidence_sources.append("spec_file") + + span.set_output({ + "disabled_features_count": len(harvest_report.disabled_features), + "spec_disabled_features_count": len(harvest_report.spec_disabled_features), + "enabled_features_count": len(harvest_report.enabled_features), + "spec_enabled_features_count": len(harvest_report.spec_enabled_features), + "expected_hardening_count": len(harvest_report.expected_hardening), + "vulnerability_intel_files_count": affected_files_count, + "linked_libraries_count": len(harvest_report.linked_libraries), + "built_subpackages_count": len(harvest_report.built_subpackages), + "excluded_subpackages_count": len(harvest_report.excluded_subpackages), + "kernel_config_count": len(harvest_report.kernel_config), + "evidence_sources": evidence_sources, + }) + + return { + "harvest_report": harvest_report, + "vulnerability_intel_str": vulnerability_intel_str, + "l1_preliminary_verdict": l1_preliminary_verdict, + "runtime_prompt": runtime_prompt, + "evidence_sources": evidence_sources, + "messages": [AIMessage(content="Build data harvested, beginning analysis.")], + } + + async def thought_node(state: BuildAgentState) -> dict: + """ReAct step: LLM chooses tools or emits a final answer for the current investigation phase.""" + step_num = state.get("step", 0) + logger.info("thought_node: starting step %d", step_num) + + runtime_prompt = state.get("runtime_prompt") + _messages = [SystemMessage(content=runtime_prompt)] + state["messages"] # Reserved for LLM call + + with tracer.push_active_function("thought_node", input_data={}) as span: + obs = state.get("observation", None) + if obs is not None: + memory_list = obs.memory if obs.memory else ["No prior knowledge."] + recent_findings = obs.results if obs.results else ["No recent findings."] + memory_context = "\n".join(f"- {m}" for m in memory_list) + findings_context = "\n".join(f"- {f}" for f in recent_findings) + context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" + _messages.append(SystemMessage(content=context_block)) + + response: CheckerThought = await thought_llm.ainvoke(_messages) + + if response.mode == "finish": + ai_message = AIMessage(content=response.final_answer or "Analysis complete.") + else: + tool_name = response.actions.tool + arguments = response.actions.query + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{"name": tool_name, "args": {"query": arguments}, "id": tool_call_id}] + ) + span.set_output({ + "thought": response.thought, + "mode": response.mode, + "actions": response.actions, + "final_answer": response.final_answer, + }) + + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": config.max_iterations, + } + + async def observation_node(state: BuildAgentState) -> dict: + """Process tool output: comprehension -> memory update for build analysis.""" + logger.info("observation_node: starting") + tool_message = state["messages"][-1] + last_thought = state.get("thought") + if not last_thought: + return { + "messages": [AIMessage(content="No thought found")], + } + last_thought_text = last_thought.thought + tool_used = last_thought.actions.tool + tool_input_detail = last_thought.actions.query + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + + harvest_report = state.get("harvest_report") or BuildHarvestReport() + target_package_name = target_package.name if target_package else "unknown" + + with tracer.push_active_function("observation_node", input_data=f"tool used:{tool_used} + {tool_input_detail}") as span: + tool_output_for_llm = tool_message.content + + # Check for empty/error outputs - bypass LLM if so to prevent hallucination + empty_findings = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) + if empty_findings: + # Build-specific: empty grep for file in logs = NOT_COMPILED evidence + if tool_used == "Source Grep" and "logs:" in tool_input_detail: + file_searched = tool_input_detail.replace("logs:", "") + empty_findings = CodeFindings( + findings=[ + f"Source Grep for '{file_searched}' returned EMPTY - file not in build log", + "This indicates the file was NOT compiled in this build" + ], + tool_outcome=f"CALLED: Source Grep with {tool_input_detail} -> EMPTY (not compiled)" + ) + code_findings = empty_findings + else: + # Step 1: Comprehension - extract findings from tool output + comp_prompt = L2_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=vulnerability_intel_str, + disabled_features=", ".join(harvest_report.disabled_features) if harvest_report.disabled_features else "None", + spec_disabled_features=", ".join(harvest_report.spec_disabled_features) if harvest_report.spec_disabled_features else "None", + enabled_features=", ".join(harvest_report.enabled_features) if harvest_report.enabled_features else "None", + spec_enabled_features=", ".join(harvest_report.spec_enabled_features) if harvest_report.spec_enabled_features else "None", + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), + ) + code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + findings_text = "\n".join(f"- {f}" for f in code_findings.findings) + + # Step 2: Memory update - merge findings into cumulative memory + mem_prompt = L2_MEMORY_UPDATE_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + previous_memory="\n".join(f"- {m}" for m in previous_memory) if isinstance(previous_memory, list) else previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") or "" + estimated = _estimate_tokens(active_prompt, messages, new_observation) + orig_estimated = estimated + prune_messages = [] + if estimated > config.context_window_token_limit: + with tracer.push_active_function("context_pruning", input_data={"estimated_tokens": estimated, "limit": config.context_window_token_limit}) as prune_span: + + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, config.context_window_token_limit, + ) + prune_span.set_output({ + "pruning_triggered": len(prune_messages) > 0, + "messages_pruned": len(prune_messages), + "tokens_before": orig_estimated, + "tokens_after": estimated, + }) + + + span.set_output({ + "last_thought_text": last_thought_text, + "tool_output_for_llm": tool_output_for_llm[:500], + "findings": code_findings.findings, + "tool_outcome": code_findings.tool_outcome, + "new_memory": new_observation.memory, + "amount_of_orig_tokens": orig_estimated, + "amount_of_estimated_tokens": estimated, + }) + return { + "messages": prune_messages, + "observation": new_observation, + } + + async def forced_finish_node(state: BuildAgentState) -> dict: + """Force finish when max iterations reached. + + Invokes the LLM with FORCED_FINISH_PROMPT to generate a final answer + based on evidence gathered so far. + """ + step_num = state.get("step", 0) + with tracer.push_active_function("forced_finish_node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") or "" + messages = [SystemMessage(content=active_prompt)] + state["messages"] + messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) + + obs = state.get("observation") + if obs is not None and obs.memory: + memory_context = "\n".join(f"- {m}" for m in obs.memory) + messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) + + response: CheckerThought = await thought_llm.ainvoke(messages) + + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Unable to determine compilation status within iteration limit." + ai_message = AIMessage(content=final_answer) + response = CheckerThought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer, + ) + + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", config.max_iterations), + "observation": state.get("observation"), + "output": final_answer, + } + except Exception as e: + logger.exception("forced_finish_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def should_continue(state: BuildAgentState) -> str: + """Route based on thought mode.""" + thought = state.get("thought") + if thought is not None and thought.mode == "finish": + return INVESTIGATION_PHASE_NODE + if state.get("step", 0) >= state.get("max_steps", config.max_iterations): + return FORCED_FINISH_NODE + return TOOL_NODE + + + async def is_investigation_finished(state: BuildAgentState) -> str: + """Check if the investigation is finished.""" + if len(investigation_stack) == 0: + return END + return THOUGHT_NODE + + async def investigation_phase_node(state: BuildAgentState) -> dict: + """Determine the next investigation phase.""" + if len(investigation_stack) == 0: + raise ValueError("Investigation stack is empty") + + final_answer = None + thought = state.get("thought") + if thought and thought.mode == "finish": + final_answer = thought.final_answer + + current_phase = investigation_stack[-1] + with tracer.push_active_function("investigation_phase_node", input_data=f"phase :{current_phase}") as span: + investigation_stack.pop() + if current_phase == L2InvestigationPhase.CONFIGURATION: + verdict: L2CompileVerdictExtraction = await compilation_verdict_llm.ainvoke([SystemMessage(content=L2_COMPILATION_VERDICT_PROMPT.format(final_answer=final_answer))]) + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + }) + if verdict.compilation_status == "not_compiled": + return {"L2CompileVerdict": verdict} + + # Check if this is a kernel package - skip hardening phase + # RHEL kernels always have hardening enabled, so checking provides low signal + preprocess_data = state.get("harvest_report") or BuildHarvestReport() + if preprocess_data.is_kernel: + logger.info("investigation_phase_node: kernel package - skipping hardening phase") + # Clear the investigation stack to prevent hardening phase + investigation_stack.clear() + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + "hardening_skipped": True, + "hardening_skip_reason": "kernel_package", + }) + return {"L2CompileVerdict": verdict} + + has_binary = bool( + artifacts + and artifacts.binary_rpm_path + and Path(artifacts.binary_rpm_path).exists() + ) + if not build_log_path and not has_binary: + logger.info( + "investigation_phase_node: no build log or binary RPM - skipping hardening phase", + ) + investigation_stack.clear() + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + "hardening_skipped": True, + "hardening_skip_reason": "no_build_or_binary_evidence", + }) + return {"L2CompileVerdict": verdict} + + else: + # Note: Architecture mismatch check is now handled early in L1 + # (cve_package_code_agent.py) using target_package.arch. The previous + # L2 check using build_architecture from build logs has been removed + # since build_architecture detection is no longer performed. + + # Normal path: proceed to hardening phase (non-kernel packages) + runtime_prompt = await build_runtime_prompt(preprocess_data) + messages = state["messages"] + prune_messages = [] + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + span.set_output({ + "runtime_prompt": runtime_prompt, + }) + return { + "runtime_prompt": runtime_prompt, + "thought": None, + "observation": None, + "step": 0, + "messages": prune_messages, + "L2CompileVerdict": verdict, + } + else: + #state that run was hardening need to extract the hardening verdict + verdict: L2HardeningVerdictExtraction = await hardening_verdict_llm.ainvoke([SystemMessage(content=L2_HARDENING_VERDICT_PROMPT.format(final_answer=final_answer))]) + span.set_output({ + "hardening_status": verdict.hardening_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + }) + return { + "L2HardeningVerdict": verdict, + } + # ------------------------------------------------------------------------- + # Build graph + # ------------------------------------------------------------------------- + + flow = StateGraph(BuildAgentState) + + flow.add_node(DATA_HARVEST_NODE, data_harvest_node) + flow.add_node(THOUGHT_NODE, thought_node) + flow.add_node(FORCED_FINISH_NODE, forced_finish_node) + flow.add_node(OBSERVATION_NODE, observation_node) + flow.add_node(TOOL_NODE, tools_node) + flow.add_node(INVESTIGATION_PHASE_NODE, investigation_phase_node) + + flow.add_edge(START, DATA_HARVEST_NODE) + flow.add_edge(DATA_HARVEST_NODE, THOUGHT_NODE) + edge_map = {INVESTIGATION_PHASE_NODE: INVESTIGATION_PHASE_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE, TOOL_NODE: TOOL_NODE} + flow.add_conditional_edges(THOUGHT_NODE, should_continue, edge_map) + flow.add_edge(TOOL_NODE, OBSERVATION_NODE) + flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) + flow.add_edge(FORCED_FINISH_NODE, INVESTIGATION_PHASE_NODE) + flow.add_conditional_edges(INVESTIGATION_PHASE_NODE, is_investigation_finished, {END: END, THOUGHT_NODE: THOUGHT_NODE}) + + app = flow.compile() + return app + + +@register_function(config_type=CVEBuildAgentConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_build_agent(config: CVEBuildAgentConfig, builder: Builder): + """Level 2 Build Agent entry point.""" + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Run L2 build analysis and populate l2_result on checker_context.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + # Set ctx_state for tools + from types import SimpleNamespace + workflow_state = SimpleNamespace(original_input=message, info=message.info) + ctx_state.set(workflow_state) + + logger.info("build_agent: starting L2 investigation") + + ctx = message.info.checker_context + if not ctx or not ctx.l1_result: + logger.warning("build_agent: no L1 result available, skipping L2") + return message + + if not ctx.artifacts: + logger.warning("build_agent: no artifacts available, skipping L2") + return message + + has_build_log = bool(ctx.artifacts.build_log_path) + if not has_build_log: + logger.info("build_agent: no build log, will use spec/build system files only") + + # Build and run the graph + build_agent_graph = await create_graph_build_agent(config, builder, message, tracer) + initial_state: BuildAgentState = { + "messages": [HumanMessage(content="Begin L2 build analysis")], + "step": 0, + "max_steps": config.max_iterations, + } + + with tracer.push_active_function("l2_build_agent_graph", input_data=initial_state["messages"][0].content): + # Each phase: (max_iterations * 3 react nodes) + data_harvest/forced_finish/investigation_phase + # Two phases (CONFIG + HARDENING) when code is compiled + steps_per_phase = (config.max_iterations * 3) + 4 + recursion_limit = steps_per_phase * 2 + 5 # buffer for edge cases + result = await build_agent_graph.ainvoke( + initial_state, + config={"recursion_limit": recursion_limit}, + ) + + logger.info("build_agent: L2 investigation finished") + + # Extract verdict from result + compile_verdict = result.get("L2CompileVerdict") or None + if compile_verdict is None: + logger.error("build_agent: L2CompileVerdict is none should never happen") + return message + hardening_verdict = result.get("L2HardeningVerdict") or None + harvest_report = result.get("harvest_report") or BuildHarvestReport() + hardening_reason = None + hardening_flags = [] + if compile_verdict.compilation_status == "not_compiled": + hardening_relevant = False + l2_override_verdict = "not_vulnerable" + elif hardening_verdict is None: + # Hardening was skipped - could be kernel package or architecture mismatch + hardening_relevant = False + if harvest_report.is_kernel: + # Kernel package: hardening skipped because RHEL kernels always have it + # Code IS compiled, so defer to L1 verdict (no L2 override) + l2_override_verdict = None + else: + # Architecture mismatch case: compiled but not exploitable on this arch + l2_override_verdict = "not_vulnerable" + else: + hardening_relevant = True + hardening_reason = hardening_verdict.reasoning + hardening_flags = hardening_verdict.hardening_flags or [] + if hardening_verdict.hardening_status == "mitigated": + l2_override_verdict = "vulnerable_mitigated" + else: + l2_override_verdict = None + + # Build L2 result + evidence_sources = result.get("evidence_sources") or [] + l2_result = L2BuildResult( + compilation_status=compile_verdict.compilation_status, + compilation_confidence=compile_verdict.confidence, + compilation_evidence=compile_verdict.reasoning, + hardening_relevant=hardening_relevant, + hardening_flags=hardening_flags, + hardening_rationale=hardening_reason, + l2_override_verdict=l2_override_verdict, + evidence_sources=evidence_sources, + ) + + with tracer.push_active_function( + "l2_agent_finish", + input_data={"compilation_status": l2_result.compilation_status}, + ) as span: + span.set_output({ + "compilation_status": l2_result.compilation_status, + "compilation_confidence": l2_result.compilation_confidence, + "l2_override_verdict": l2_result.l2_override_verdict, + }) + + # Store result on checker_context + if message.info.checker_context is not None: + message.info.checker_context.l2_result = l2_result + else: + logger.warning("build_agent: checker_context is None, cannot store l2_result") + + logger.info( + "build_agent: L2 result - status=%s, confidence=%.2f, override=%s", + l2_result.compilation_status, + l2_result.compilation_confidence, + l2_result.l2_override_verdict, + ) + + return message + + yield FunctionInfo.from_fn( + _arun, + description="Level 2 Build Agent: analyzes build artifacts for compilation status and hardening", + ) diff --git a/src/vuln_analysis/functions/cve_checker_report.py b/src/vuln_analysis/functions/cve_checker_report.py new file mode 100644 index 000000000..4e70b5ae4 --- /dev/null +++ b/src/vuln_analysis/functions/cve_checker_report.py @@ -0,0 +1,1117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +CVE Checker Report Generation Function. + +This module provides the report generation node for the L1/L2 pipeline. +It consumes L1InvestigationResult (and optionally L2BuildResult) from +checker_context and produces the final AgentMorpheusOutput. +""" + +import re +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.checker_status import L1InvestigationResult, L2BuildResult + +from nat.builder.context import Context +from vuln_analysis.data_models.output import ( + AgentMorpheusEngineOutput, + AgentMorpheusOutput, + JustificationOutput, + OutputPayload, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + CodeAgentReport, + CodeSnippet, + DownstreamSearchReport, + ParsedPatch, + UpstreamSearchReport, + generate_code_agent_report, +) + +logger = LoggingFactory.get_agent_logger(__name__) + + +_StatusLiteral = Literal["TRUE", "FALSE", "UNKNOWN"] + +_JUSTIFICATION_LABEL_TO_STATUS: dict[str, _StatusLiteral] = { + "code_not_present": "FALSE", + "code_not_reachable": "FALSE", + "protected_by_mitigating_control": "FALSE", + "protected_by_compiler": "FALSE", + "requires_environment": "FALSE", + "vulnerable": "TRUE", + "uncertain": "UNKNOWN", +} + + +_POLICY_MAX_RPM_LIST_ITEMS = 5 +_POLICY_RHSA_STATEMENT_CAP = 400 +_POLICY_MAX_PACKAGE_STATE_ITEMS = 8 + +_BUILD_LOG_EXCERPT_MAX_LINES = 8 +_EVIDENCE_CHAIN_ITEM_CAP = 8 + + +def _md_section_header(title: str) -> list[str]: + """UI-friendly section break: horizontal rule + level-2 heading (no tables).""" + return ["---", "", f"## {title}", ""] + + +def _md_note(text: str) -> list[str]: + """Callout line for interpretation; renders as blockquote in most markdown UIs.""" + return [f"> **Note:** {text}", ""] + + +def _md_check_result(passed: bool) -> str: + return "**Pass**" if passed else "**Fail**" + + +def _first_nonempty_line(text: str, *, max_len: int = 140) -> str: + for raw in text.splitlines(): + line = raw.strip() + if line: + if len(line) > max_len: + return line[: max_len - 1] + "…" + return line + return "" + + +def _spec_declaration_summary(directives: list[str]) -> str: + for directive in directives: + stripped = directive.strip() + if stripped.startswith("Patch"): + return stripped + return directives[0].strip() if directives else "" + + +def _is_redundant_patch_evidence(evidence: str) -> bool: + """Skip LLM bullets that repeat the patch/spec/build tri-state checks.""" + ev_lower = evidence.lower() + redundant_phrases = ( + "target package patch available", + "target patch file referenced in spec", + "referenced in spec", + "build log evidence", + "patch file exists", + "applied in build", + "patch file on target", + ) + return any(phrase in ev_lower for phrase in redundant_phrases) + + +def _categorize_evidence_chain( + evidence_chain: list[str], +) -> tuple[list[str], list[str], list[str], list[str]]: + patch_evidence: list[str] = [] + build_evidence: list[str] = [] + code_evidence: list[str] = [] + other_evidence: list[str] = [] + + patch_keywords = ("patch", "spec", "patchn", "directive", "target", "reference", "rebase") + build_keywords = ("build", "applied", "log", "compilation") + code_keywords = ("code agent", "code analysis", "function", "vulnerable", "source search") + + for ev in evidence_chain[:_EVIDENCE_CHAIN_ITEM_CAP]: + ev_lower = ev.lower() + if any(kw in ev_lower for kw in code_keywords): + code_evidence.append(ev) + elif any(kw in ev_lower for kw in build_keywords): + if not _is_redundant_patch_evidence(ev): + build_evidence.append(ev) + elif any(kw in ev_lower for kw in patch_keywords): + if _is_redundant_patch_evidence(ev): + continue + patch_evidence.append(ev) + else: + other_evidence.append(ev) + + return patch_evidence, build_evidence, code_evidence, other_evidence + + +def _target_nvr(blocks: "ReportBlocks") -> str: + version_release = ( + f"{blocks.package_version}-{blocks.package_release}" + if blocks.package_release + else blocks.package_version + ) + return f"{blocks.package_name}-{version_release} ({blocks.package_arch})" + + +def _derive_investigation_mode( + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, +) -> str: + if downstream_report and downstream_report.is_patch_file_available: + return "patch_available" + if upstream_report and upstream_report.is_code_fixed_by_rebase == "yes": + return "rebase" + if upstream_report and upstream_report.fixed_parsed_patch: + return "upstream_patch" + return "no_patch" + + +def _derive_arch_gate_reason( + l1_agent_answer: str | None, + affected_bitness: str | None, + package_arch: str, +) -> str | None: + if l1_agent_answer: + answer_lower = l1_agent_answer.lower() + if "bitness mismatch" in answer_lower or ( + "architecture" in answer_lower and "mismatch" in answer_lower + ): + return l1_agent_answer.strip() + if not affected_bitness or affected_bitness == "both": + return None + arch_lower = package_arch.lower() + is_64 = any(token in arch_lower for token in ("x86_64", "amd64", "aarch64", "64")) + is_32 = any(token in arch_lower for token in ("i686", "i386", "32")) + if affected_bitness == "32-bit" and is_64 and not is_32: + return f"CVE affects 32-bit only; target package is {package_arch}." + if affected_bitness == "64-bit" and is_32 and not is_64: + return f"CVE affects 64-bit only; target package is {package_arch}." + return None + + +def _target_fix_search_interpretation(blocks: "ReportBlocks") -> str: + checks = ( + blocks.is_patch_file_available, + blocks.is_patch_in_spec_file, + blocks.is_patch_applied_in_build, + ) + passed = sum(1 for c in checks if c) + if passed == 3: + if blocks.justification_label == "vulnerable": + return ( + "Target fix search (3/3): patch file, spec reference, and build log all match, " + "but vulnerable patterns remain in source—see target source check below." + ) + return ( + "Target fix search (3/3): CVE patch file present, referenced in the spec, " + "and observed in the build log." + ) + if passed == 0: + return ( + "Target fix search (0/3): no CVE-named patch file, spec reference, " + "or build-time application found on the target package." + ) + gaps: list[str] = [] + if not blocks.is_patch_file_available: + gaps.append("no CVE patch file on target") + if not blocks.is_patch_in_spec_file: + gaps.append("not referenced in target spec") + if not blocks.is_patch_applied_in_build: + gaps.append("not observed in target build log") + return ( + f"Target fix search ({passed}/3 checks passed): " + f"{'; '.join(gaps)}." + ) + + +def _summarize_parsed_patch_lines(parsed_patch: ParsedPatch | None) -> list[str]: + if not parsed_patch: + return [] + lines: list[str] = [] + if parsed_patch.patch_filename: + lines.append(f"- **Patch file:** `{parsed_patch.patch_filename}`") + for pf in parsed_patch.files[:2]: + path = pf.target_path.lstrip("ab/") + lines.append(f"- **File in patch:** `{path}`") + for hunk in pf.hunks[:1]: + if hunk.removed_lines: + lines.append(f" - Removed: `{hunk.removed_lines[0].strip()}`") + if hunk.added_lines: + lines.append(f" - Added: `{hunk.added_lines[0].strip()}`") + return lines + + +def _format_target_fix_search_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header(f"CVE fix search on target package ({_target_nvr(blocks)})")) + + patch_detail = f"`{blocks.patch_file_name}`" if blocks.patch_file_name else "_not found_" + spec_detail = ( + f"`{_spec_declaration_summary(blocks.spec_patch_directives)}`" + if blocks.spec_patch_directives + else ("_CVE mentioned in spec_" if blocks.is_patch_in_spec_file else "_not referenced_") + ) + build_detail = _first_nonempty_line(blocks.build_log_evidence) or "_not observed in build log_" + if build_detail and not build_detail.startswith("`") and build_detail != "_not observed in build log_": + build_detail = f"`{build_detail}`" + + lines.append( + f"- **Patch file on target:** {_md_check_result(blocks.is_patch_file_available)} — {patch_detail}" + ) + lines.append( + f"- **Referenced in spec:** {_md_check_result(blocks.is_patch_in_spec_file)} — {spec_detail}" + ) + lines.append( + f"- **Applied in build:** {_md_check_result(blocks.is_patch_applied_in_build)} — {build_detail}" + ) + lines.append("") + lines.extend(_md_note(_target_fix_search_interpretation(blocks))) + + if blocks.spec_patch_directives: + declarations = [d for d in blocks.spec_patch_directives if d.strip().startswith("Patch")] + applications = [d for d in blocks.spec_patch_directives if d.strip().startswith("%patch")] + if declarations: + lines.append("- **Spec patch declaration:**") + for decl in declarations: + lines.append(f" - `{decl.strip()}`") + if applications: + lines.append("- **Spec patch application:**") + for app in applications: + lines.append(f" - `{app.strip()}`") + if blocks.spec_changelog_cve_lines.strip(): + lines.append("- **Spec changelog (CVE):**") + for changelog_line in blocks.spec_changelog_cve_lines.strip().splitlines(): + stripped = changelog_line.strip() + if stripped: + lines.append(f" - `{stripped}`") + if blocks.spec_version_line or blocks.spec_source0_line: + if blocks.spec_version_line: + lines.append(f"- **Spec version line:** `{blocks.spec_version_line.strip()}`") + if blocks.spec_source0_line: + lines.append(f"- **Spec Source0 line:** `{blocks.spec_source0_line.strip()}`") + lines.append( + "> **Note:** Version/Source0 describe the upstream snapshot named in the spec; " + "they may differ from the tree used for source review." + ) + lines.append("") + return "\n".join(lines).strip() + + +def _format_external_fix_clues_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header("Fix clues from advisories and reference builds")) + + if blocks.investigation_mode == "patch_available": + lines.append( + "- **Status:** Not needed — a CVE-named patch file was found on the target package." + ) + lines.append("") + return "\n".join(lines).strip() + + if not blocks.upstream_search_ran: + lines.append( + "- **Status:** Not evaluated — external reference search did not run " + "(target patch workflow did not require it)." + ) + lines.append("") + return "\n".join(lines).strip() + + upstream = blocks.upstream_report + clue_lines: list[str] = [] + + if blocks.fixed_rpm_hints: + shown = blocks.fixed_rpm_hints[:_POLICY_MAX_RPM_LIST_ITEMS] + clue_lines.append("- **Known-fixed packages (intel):**") + for hint in shown: + clue_lines.append(f" - `{hint}`") + + if upstream: + if upstream.reference_package_nvr: + clue_lines.append( + f"- **Reference package NVR:** `{upstream.reference_package_nvr}`" + ) + if upstream.fixed_srpm_file_name: + clue_lines.append( + f"- **Reference patch source:** `{upstream.fixed_srpm_file_name}`" + ) + clue_lines.extend(_summarize_parsed_patch_lines(upstream.fixed_parsed_patch)) + if upstream.osv_result: + osv = upstream.osv_result + if osv.patch_url: + clue_lines.append(f"- **Fetched patch URL:** {osv.patch_url}") + if osv.fixed_commit: + clue_lines.append(f"- **Fixed commit:** `{osv.fixed_commit}`") + if osv.source: + clue_lines.append(f"- **Patch source:** {osv.source}") + if upstream.is_code_fixed_by_rebase == "yes": + clue_lines.append("- **Rebase on target:** Spec/changelog suggests fix via version rebase.") + if upstream.spec_file_log_change.strip(): + excerpt = _first_nonempty_line(upstream.spec_file_log_change, max_len=200) + if excerpt: + clue_lines.append(f" - `{excerpt}`") + if upstream.reason_code_fixed_by_rebase.strip(): + clue_lines.append(f" - {upstream.reason_code_fixed_by_rebase.strip()}") + + if clue_lines: + lines.extend(clue_lines) + else: + lines.append( + "- **Status:** No external fix clue found — advisories and reference builds " + "did not yield a usable patch for comparison." + ) + lines.append("") + return "\n".join(lines).strip() + + +def _format_target_source_check_md(blocks: "ReportBlocks") -> str: + _, _, code_evidence, _ = _categorize_evidence_chain(blocks.evidence_chain) + lines: list[str] = [] + lines.extend(_md_section_header("Target source check")) + + if blocks.l1_agent_answer: + lines.append(f"- **Code agent summary:** {blocks.l1_agent_answer.strip()}") + + if blocks.affected_files: + lines.append("- **Affected source files:**") + for f in blocks.affected_files[:10]: + lines.append(f" - `{f}`") + elif blocks.arch_gate_reason: + lines.append( + f"- **Status:** Source review limited — {blocks.arch_gate_reason}" + ) + else: + lines.append("- **Affected source files:** _not determined_") + + mitigated = blocks.justification_label in ( + "protected_by_mitigating_control", + "protected_by_compiler", + "code_not_present", + "code_not_reachable", + "requires_environment", + ) + vulnerable_snippets = _filter_review_snippets(blocks.vulnerable_snippets) + fix_snippets = _filter_review_snippets(blocks.fix_snippets) + + if vulnerable_snippets: + title = "Pattern before fix (reference or target)" if mitigated else "Vulnerable pattern on target" + lines.append(f"- **{title}:**") + lines.extend(blocks._format_snippet_bullets(vulnerable_snippets)) + elif not blocks.arch_gate_reason: + lines.append("- **Vulnerable pattern on target:** _not shown — no source excerpt collected_") + + if fix_snippets: + lines.append("- **Fix pattern (reference or expected on target):**") + lines.extend(blocks._format_snippet_bullets(fix_snippets)) + elif vulnerable_snippets or blocks.l1_agent_answer: + lines.append("- **Fix pattern:** _not shown — no fix excerpt collected_") + + for ev in code_evidence[:2]: + if not _is_redundant_patch_evidence(ev): + lines.append(f"- {ev}") + + lines.append("") + return "\n".join(lines).strip() + + +def _format_target_build_check_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header("Target build check")) + + l2 = blocks.l2_result + if l2 is None: + lines.append("- **Status:** Not evaluated — build agent did not run for this scan.") + lines.append("") + return "\n".join(lines).strip() + + lines.append(f"- **Compilation:** {l2.compilation_status or 'unknown'}") + if l2.compilation_evidence: + lines.append(f"- **Compilation evidence:** {l2.compilation_evidence.strip()}") + if l2.hardening_flags: + flags_str = ", ".join(l2.hardening_flags[:5]) + lines.append(f"- **Compiler hardening:** `{flags_str}`") + if l2.hardening_rationale: + lines.append(f"- **Hardening rationale:** {l2.hardening_rationale.strip()}") + if l2.l2_override_verdict: + lines.append(f"- **Build agent override:** `{l2.l2_override_verdict}`") + elif l2.hardening_relevant is False: + lines.append("- **Hardening relevant to CVE:** no") + lines.append("") + return "\n".join(lines).strip() + + +def _filter_review_snippets(snippets: list[CodeSnippet]) -> list[CodeSnippet]: + """Prefer primary source files over build-system noise when both are present.""" + primary = [ + s for s in snippets + if s.file_path.endswith((".c", ".h", ".cpp", ".cc", ".cxx")) + and "/test/" not in s.file_path + and "CMakeLists" not in s.file_path + and "Makefile" not in s.file_path + ] + return primary if primary else snippets + + +def _is_reference_tree_path(file_path: str) -> bool: + path_lower = file_path.lower() + return "_patched" in path_lower or "/patched/" in path_lower + + +@dataclass +class ReportBlocks: + """Formatted report blocks - each piece of data formatted once for UI output.""" + + # Package info + package_name: str + package_version: str + package_release: str + package_arch: str + + # CVE info + cve_id: str + cve_description: str + + # Verdict + justification_label: str + executive_summary: str + + # Evidence + evidence_chain: list[str] + affected_files: list[str] + + # Extracted facts from downstream search + patch_file_name: str + spec_patch_directives: list[str] + build_log_evidence: str + spec_changelog_cve_lines: str + spec_version_line: str + spec_source0_line: str + is_patch_file_available: bool + is_patch_in_spec_file: bool + is_patch_applied_in_build: bool + + # Code snippets + vulnerable_snippets: list[CodeSnippet] + fix_snippets: list[CodeSnippet] + + # Investigation context (upstream / L1 / L2) + upstream_report: UpstreamSearchReport | None = None + l1_agent_answer: str | None = None + investigation_mode: str | None = None + arch_gate_reason: str | None = None + l2_result: L2BuildResult | None = None + fixed_rpm_hints: list[str] = field(default_factory=list) + upstream_search_ran: bool = False + + @property + def package_header_md(self) -> str: + """Format package metadata as Markdown header.""" + version_release = f"{self.package_version}-{self.package_release}" if self.package_release else self.package_version + return f"**Package:** `{self.package_name}-{version_release}` ({self.package_arch})" + + @property + def evidence_chain_md(self) -> str: + """Investigation narrative for the details field (four sections, always present).""" + sections = [ + _format_target_fix_search_md(self), + _format_external_fix_clues_md(self), + _format_target_source_check_md(self), + _format_target_build_check_md(self), + ] + return "\n\n".join(s for s in sections if s).strip() + + @property + def affected_files_md(self) -> str: + """Format affected files as Markdown list.""" + if not self.affected_files: + return "" + lines = ["**Affected Files:**"] + for f in self.affected_files[:10]: + lines.append(f"- `{f}`") + return "\n".join(lines) + + @property + def details_md(self) -> str: + """Source snippets are rendered inside evidence_chain_md (target source check).""" + return "" + + def _format_snippet_bullets(self, snippets: list[CodeSnippet]) -> list[str]: + lines: list[str] = [] + for snippet in snippets: + display_path = snippet.file_path + if len(display_path) > 72: + display_path = Path(display_path).name + line_part = f" (line {snippet.line_number})" if snippet.line_number else "" + lines.append(f"### `{display_path}`{line_part}") + if _is_reference_tree_path(snippet.file_path): + lines.append( + "> **Note:** Excerpt path is from a reference/patched tree, not the target SRPM source." + ) + lines.append("") + lang = _infer_language_from_path(snippet.file_path) + code_body = "\n".join(snippet.code.strip().splitlines()[:12]) + if code_body: + lines.append(f"```{lang}") + lines.append(code_body) + lines.append("```") + lines.append("") + return lines + +_NO_PATCH_EVIDENCE_PHRASE = ( + "no CVE fix evidence was found on the target (no patch file, spec reference, " + "or build-time application)" +) + +_REASON_BY_LABEL: dict[str, str] = { + "vulnerable": ( + "Labeled vulnerable because the code agent confirmed vulnerable patterns " + "remain in source." + ), + "protected_by_mitigating_control": ( + "Labeled protected by mitigating control because a downstream patch or fix pattern " + "addresses the vulnerability in this package." + ), + "protected_by_compiler": ( + "Labeled protected by compiler because the build agent found compiler hardening " + "that mitigates exploitation of this flaw." + ), + "code_not_present": ( + "Labeled code not present because the build agent determined the vulnerable code " + "is not compiled into this package." + ), + "code_not_reachable": ( + "Labeled code not reachable because the vulnerable code path is not reachable " + "in this package build." + ), + "requires_environment": ( + "Labeled requires environment because exploitation depends on runtime or architecture " + "conditions not met on this target." + ), + "uncertain": ( + "Labeled uncertain because patch, spec, build, and source evidence are " + "insufficient or conflicting." + ), +} + + +def _all_patch_checks_failed(blocks: ReportBlocks) -> bool: + return not ( + blocks.is_patch_file_available + or blocks.is_patch_in_spec_file + or blocks.is_patch_applied_in_build + ) + + +def _format_justification_reason(blocks: ReportBlocks) -> str: + """One-sentence VEX label rationale (outcome voice); evidence lives in details.""" + label = blocks.justification_label + if label == "vulnerable": + if _all_patch_checks_failed(blocks): + return ( + f"Labeled vulnerable because {_NO_PATCH_EVIDENCE_PHRASE}, and the code agent " + "confirmed vulnerable patterns remain in source." + ) + return _REASON_BY_LABEL["vulnerable"] + if label == "uncertain": + if _all_patch_checks_failed(blocks): + return ( + f"Labeled uncertain because {_NO_PATCH_EVIDENCE_PHRASE}, and source evidence " + "is insufficient or conflicting." + ) + return _REASON_BY_LABEL["uncertain"] + return _REASON_BY_LABEL.get(label, _REASON_BY_LABEL["uncertain"]) + + +_BUILD_AGENT_ABSENCE_PHRASES = ( + "build agent context is not present", + "l2 context is not present", + "l2_build_context is not present", + "based solely on the code agent analysis", + "verdict is based solely on the code agent", + "because the build agent context is not present", + "build agent did not run", + "no build agent context", +) + + +def _strip_build_agent_absence_boilerplate(text: str) -> str: + """Remove LLM sentences that misstate build-agent availability.""" + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + kept = [ + part + for part in parts + if part and not any(phrase in part.lower() for phrase in _BUILD_AGENT_ABSENCE_PHRASES) + ] + return " ".join(kept).strip() + + +def _infer_language_from_path(file_path: str) -> str: + """Infer programming language hint from file extension for syntax highlighting.""" + ext_map = { + ".c": "c", + ".h": "c", + ".cpp": "cpp", + ".hpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".py": "python", + ".go": "go", + ".rs": "rust", + ".java": "java", + ".js": "javascript", + ".ts": "typescript", + ".rb": "ruby", + ".sh": "bash", + } + return ext_map.get(Path(file_path).suffix.lower(), "c") + + +def _build_details_md(blocks: ReportBlocks) -> str | None: + """Audit packet: investigation narrative (four sections, includes source excerpts).""" + investigation = blocks.evidence_chain_md.strip() + return investigation or None + + +def _build_report_blocks( + message: AgentMorpheusEngineInput, + code_agent_report: CodeAgentReport, + cve_description: str, + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None = None, + l1_result: L1InvestigationResult | None = None, + l2_result: L2BuildResult | None = None, + identify_result=None, +) -> ReportBlocks: + """Extract and format all report data into blocks.""" + target_package = message.input.image.target_package + + vulnerable_snippets = [s for s in code_agent_report.code_snippets if s.snippet_type == "vulnerable"] + fix_snippets = [s for s in code_agent_report.code_snippets if s.snippet_type == "fix"] + + patch_file_name = "" + spec_patch_directives: list[str] = [] + build_log_evidence = "" + spec_changelog_cve_lines = "" + spec_version_line = "" + spec_source0_line = "" + is_patch_file_available = False + is_patch_in_spec_file = False + is_patch_applied_in_build = False + + if downstream_report: + patch_file_name = downstream_report.patch_file_name or "" + spec_patch_directives = downstream_report.spec_patch_directives_for_cve or [] + build_log_evidence = downstream_report.build_log_patch_applied or "" + spec_changelog_cve_lines = downstream_report.spec_changelog_cve_lines or "" + spec_version_line = downstream_report.spec_version_line or "" + spec_source0_line = downstream_report.spec_source0_line or "" + is_patch_file_available = downstream_report.is_patch_file_available + is_patch_in_spec_file = downstream_report.is_patch_in_spec_file + is_patch_applied_in_build = downstream_report.is_patch_applied_in_build + + investigation_mode = _derive_investigation_mode(downstream_report, upstream_report) + upstream_search_ran = not ( + downstream_report is not None and downstream_report.is_patch_file_available + ) + + l1_agent_answer = l1_result.l1_agent_answer if l1_result else None + affected_bitness = None + if l1_result and l1_result.vulnerability_intel: + affected_bitness = l1_result.vulnerability_intel.affected_bitness + + package_arch = target_package.arch or "x86_64" if target_package else "x86_64" + arch_gate_reason = _derive_arch_gate_reason(l1_agent_answer, affected_bitness, package_arch) + + fixed_rpm_hints: list[str] = [] + if identify_result and identify_result.fixed_rpm_list: + fixed_rpm_hints = list(identify_result.fixed_rpm_list[:_POLICY_MAX_RPM_LIST_ITEMS]) + + return ReportBlocks( + package_name=target_package.name if target_package else "unknown", + package_version=target_package.version or "" if target_package else "", + package_release=target_package.release or "" if target_package else "", + package_arch=package_arch, + cve_id=message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "", + cve_description=cve_description, + justification_label=code_agent_report.justification_label, + executive_summary=code_agent_report.executive_summary, + evidence_chain=list(code_agent_report.evidence_chain), + affected_files=list(code_agent_report.affected_files), + patch_file_name=patch_file_name, + spec_patch_directives=spec_patch_directives, + build_log_evidence=build_log_evidence, + spec_changelog_cve_lines=spec_changelog_cve_lines, + spec_version_line=spec_version_line, + spec_source0_line=spec_source0_line, + is_patch_file_available=is_patch_file_available, + is_patch_in_spec_file=is_patch_in_spec_file, + is_patch_applied_in_build=is_patch_applied_in_build, + vulnerable_snippets=vulnerable_snippets, + fix_snippets=fix_snippets, + upstream_report=upstream_report, + l1_agent_answer=l1_agent_answer, + investigation_mode=investigation_mode, + arch_gate_reason=arch_gate_reason, + l2_result=l2_result, + fixed_rpm_hints=fixed_rpm_hints, + upstream_search_ran=upstream_search_ran, + ) + + +def _format_policy_context_for_report( + *, + target_nvr: str, + identify_result, + intel, +) -> str: + """Build a context block for the LLM prompt covering NVR posture and RHSA excerpts.""" + lines: list[str] = [] + + if target_nvr: + lines.append(f"**Scanned target NVR:** `{target_nvr}`") + + if identify_result: + affected = identify_result.affected_rpm_list or [] + fixed = identify_result.fixed_rpm_list or [] + + if affected: + shown = affected[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(affected) - len(shown)} more)" if len(affected) > len(shown) else "" + lines.append(f"**Affected NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_affected: `{identify_result.is_target_package_affected.value}`") + + if fixed: + shown = fixed[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(fixed) - len(shown)} more)" if len(fixed) > len(shown) else "" + lines.append(f"**Fixed NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_fixed: `{identify_result.is_target_package_fixed.value}`") + + rhsa = None + if intel and len(intel) > 0: + rhsa = intel[0].rhsa + + if rhsa: + if rhsa.statement: + stmt = rhsa.statement + if len(stmt) > _POLICY_RHSA_STATEMENT_CAP: + stmt = stmt[:_POLICY_RHSA_STATEMENT_CAP] + " …" + lines.append(f"**RHSA statement excerpt:** {stmt}") + + if rhsa.upstream_fix: + lines.append(f"**RHSA upstream_fix:** `{rhsa.upstream_fix}`") + + pkg_states = rhsa.package_state or [] + if pkg_states: + lines.append("**RHSA package_state:**") + for ps in pkg_states[:_POLICY_MAX_PACKAGE_STATE_ITEMS]: + parts = [] + if ps.product_name: + parts.append(ps.product_name) + if ps.package_name: + parts.append(f"pkg={ps.package_name}") + if ps.fix_state: + parts.append(f"fix_state={ps.fix_state}") + if parts: + lines.append(f" - {' | '.join(parts)}") + if len(pkg_states) > _POLICY_MAX_PACKAGE_STATE_ITEMS: + lines.append(f" - (+ {len(pkg_states) - _POLICY_MAX_PACKAGE_STATE_ITEMS} more)") + + return "\n".join(lines) + + +def _apply_l2_verdict( + report: CodeAgentReport, + l2_result: L2BuildResult, +) -> CodeAgentReport: + """Apply L2 Build Agent verdict overrides to the CodeAgentReport. + + .. deprecated:: + This function is deprecated. L2 results are now passed directly to + `generate_code_agent_report()` so the LLM can synthesize L1 and L2 + findings into a cohesive narrative. This function will be removed + in a future release. + """ + warnings.warn( + "_apply_l2_verdict is deprecated. L2 results are now integrated " + "directly into the LLM prompt via generate_code_agent_report().", + DeprecationWarning, + stacklevel=2, + ) + if l2_result.l2_override_verdict is None: + return report + + updated_fields = {} + + if l2_result.l2_override_verdict == "not_vulnerable": + evidence = l2_result.compilation_evidence or "" + if "Architecture mismatch" in evidence: + # Architecture-based not affected - vulnerability cannot occur on this platform + updated_fields["justification_label"] = "requires_environment" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** {evidence} " + f"Vulnerability condition cannot occur on this architecture." + ) + elif l2_result.compilation_status == "not_compiled": + updated_fields["justification_label"] = "code_not_present" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerable code is NOT compiled into the binary. " + f"Evidence: {evidence or 'Build analysis confirmed exclusion.'}" + ) + else: + updated_fields["justification_label"] = "code_not_reachable" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Code determined not vulnerable by L2 analysis." + ) + + elif l2_result.l2_override_verdict == "vulnerable_mitigated": + if l2_result.hardening_relevant and l2_result.hardening_flags: + updated_fields["justification_label"] = "protected_by_compiler" + flags_str = ", ".join(l2_result.hardening_flags[:5]) + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerability mitigated by compiler hardening flags: {flags_str}. " + f"Rationale: {l2_result.hardening_rationale or 'Hardening flags provide protection.'}" + ) + else: + updated_fields["justification_label"] = "protected_by_mitigating_control" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerability mitigated by build-time controls." + ) + + if updated_fields: + evidence = list(report.evidence_chain) + evidence.append(f"L2 Build Agent: {l2_result.l2_override_verdict}") + if l2_result.compilation_evidence: + evidence.append(f"L2 compilation evidence: {l2_result.compilation_evidence}") + if l2_result.hardening_rationale: + evidence.append(f"L2 hardening rationale: {l2_result.hardening_rationale}") + updated_fields["evidence_chain"] = evidence + + return report.model_copy(update=updated_fields) + + return report + + +def _build_analysis( + message: AgentMorpheusEngineInput, + code_agent_report: CodeAgentReport, + intel_score: int, + cve_description: str = "", + downstream_report: DownstreamSearchReport | None = None, + upstream_report: UpstreamSearchReport | None = None, + l1_result: L1InvestigationResult | None = None, + l2_result: L2BuildResult | None = None, +) -> list[AgentMorpheusEngineOutput]: + """Build the final analysis output from the code agent report using ReportBlocks. + + - summary: LLM executive_summary (verdict, reconciliation, technical context) + - justification.reason: one-sentence template from label and patch booleans + - details: investigation narrative markdown (target fix search through build check) + """ + ctx = message.info.checker_context if message.info else None + blocks = _build_report_blocks( + message, + code_agent_report, + cve_description, + downstream_report, + upstream_report=upstream_report, + l1_result=l1_result, + l2_result=l2_result, + identify_result=ctx.identify_result if ctx else None, + ) + + label = blocks.justification_label + status: _StatusLiteral = _JUSTIFICATION_LABEL_TO_STATUS.get(label, "UNKNOWN") + + summary = _strip_build_agent_absence_boilerplate(blocks.executive_summary.strip()) + reason = _format_justification_reason(blocks) + details = _build_details_md(blocks) + + return [ + AgentMorpheusEngineOutput( + vuln_id=intel.vuln_id, + checklist=[], + summary=summary, + justification=JustificationOutput( + label=label, + reason=reason, + status=status, + ), + intel_score=intel_score, + cvss=None, + details=details, + ) + for intel in (message.info.intel if message.info and message.info.intel else []) + ] + + +class CVECheckerReportConfig(FunctionBaseConfig, name="cve_checker_report"): + """Configuration for the CVE Checker Report generation function.""" + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + llm_name: str = Field(description="The LLM model to use for report generation.") + + +@register_function(config_type=CVECheckerReportConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_checker_report(config: CVECheckerReportConfig, builder: Builder): + """Report generation function for the L1/L2 checker pipeline.""" + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Generate the final checker report from L1 (and optionally L2) results.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + logger.info("cve_checker_report: starting report generation") + + ctx = message.info.checker_context + if ctx is None or ctx.l1_result is None: + logger.error("cve_checker_report: no L1 result available") + return AgentMorpheusOutput( + input=message.input, + info=message.info, + output=OutputPayload( + analysis=[ + AgentMorpheusEngineOutput( + vuln_id=intel.vuln_id, + checklist=[], + summary="Rpm scanning investigation did not produce results.", + justification=JustificationOutput( + label="uncertain", + reason="Rpm scanning investigation did not produce results.", + status="UNKNOWN", + ), + intel_score=0, + cvss=None, + details=None, + ) + for intel in (message.info.intel if message.info and message.info.intel else []) + ], + vex=None, + ), + ) + + l1_result = ctx.l1_result + l2_result = ctx.l2_result + + downstream_report: DownstreamSearchReport | None = None + upstream_report: UpstreamSearchReport | None = None + + if l1_result.downstream_report: + downstream_report = DownstreamSearchReport.model_validate(l1_result.downstream_report) + if l1_result.upstream_report: + upstream_report = UpstreamSearchReport.model_validate(l1_result.upstream_report) + + vuln_id = message.input.scan.vulns[0].vuln_id + target_package = message.input.image.target_package + target_package_name = target_package.name if target_package else "unknown" + intel = message.info.intel + + descriptions: list[tuple[str, str]] = [] + if intel: + a_intel = intel[0] + if a_intel.ghsa: + cve_text = a_intel.ghsa.description or a_intel.ghsa.summary or "" + if cve_text: + descriptions.append(("ghsa", cve_text)) + if a_intel.ubuntu and a_intel.ubuntu.description: + descriptions.append(("ubuntu", a_intel.ubuntu.description)) + + version = (target_package.version or "") if target_package else "" + release = (target_package.release or "") if target_package else "" + target_nvr = f"{target_package_name}-{version}-{release}" if target_package_name else "" + + policy_context = _format_policy_context_for_report( + target_nvr=target_nvr, + identify_result=ctx.identify_result, + intel=intel, + ) + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + with tracer.push_active_function("generate_code_agent_report", input_data={"vuln_id": vuln_id}): + code_agent_report: CodeAgentReport = await generate_code_agent_report( + llm=llm, + vuln_id=vuln_id, + target_package=target_package_name, + descriptions=descriptions, + downstream_report=downstream_report, + upstream_report=upstream_report, + l1_agent_answer=l1_result.l1_agent_answer, + tracer=tracer, + policy_context=policy_context, + l2_result=l2_result, + ) + + source_key = ctx.source_key + if source_key: + report_dir = Path(config.base_checker_dir) / source_key / "report" + report_dir.mkdir(parents=True, exist_ok=True) + suffix = f"-{target_package_name}" if target_package_name else "" + if version: + suffix += f"-{version}" + if release: + suffix += f"-{release}" + report_path = report_dir / f"L1_report_{vuln_id}{suffix}.md" + report_path.write_text(code_agent_report.to_markdown( + vuln_id=vuln_id, + target_package=target_package_name, + version=version, + release=release, + downstream_report=downstream_report, + )) + logger.info("cve_checker_report: wrote report to %s", report_path) + + with tracer.push_active_function( + "report_finish", + input_data={ + "justification_label": code_agent_report.justification_label, + "has_l2_override": l2_result is not None and l2_result.l2_override_verdict is not None, + }, + ) as span: + span.set_output({ + "executive_summary": code_agent_report.executive_summary, + "affected_files": code_agent_report.affected_files, + }) + intel_score = intel[0].intel_score + + cve_description = "" + if descriptions: + cve_description = descriptions[0][1] + + return AgentMorpheusOutput( + input=message.input, + info=message.info, + output=OutputPayload( + analysis=_build_analysis( + message, + code_agent_report, + intel_score, + cve_description=cve_description, + downstream_report=downstream_report, + upstream_report=upstream_report, + l1_result=l1_result, + l2_result=l2_result, + ), + vex=None, + ), + ) + + yield FunctionInfo.from_fn( + _arun, + description="Generate final checker report from L1/L2 investigation results", + ) diff --git a/src/vuln_analysis/functions/cve_checker_segmentation.py b/src/vuln_analysis/functions/cve_checker_segmentation.py new file mode 100644 index 000000000..bae7f31ee --- /dev/null +++ b/src/vuln_analysis/functions/cve_checker_segmentation.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from langchain.docstore.document import Document +from exploit_iq_commons.utils.document_embedding import MultiLanguageRecursiveCharacterTextSplitter,ExtendedLanguageParser +from langchain_community.document_loaders.generic import GenericLoader +from langchain.text_splitter import RecursiveCharacterTextSplitter +logger = LoggingFactory.get_agent_logger(__name__) + +_BUILD_FILE_NAMES = {"Makefile", "GNUmakefile", "configure"} +LANG_PARSER_EXTENSIONS = {".c", ".h", ".cpp", ".hpp", ".py", ".go", ".java"} +TEXT_FILE_EXTENSIONS = {".spec", ".conf", ".cfg", ".sh", ".m4",".ac", ".am", ".in", ".txt", ".md", ".rst"} +class RpmDocumentEmbedding: + def __init__(self, source_dir: Path, chunk_size: int = 800, chunk_overlap: int = 160): + self.source_dir = source_dir + self.lang_splitter = MultiLanguageRecursiveCharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap, + ) + self.text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1000, chunk_overlap=200, + ) + + def load_and_chunk_code(self) -> list[Document]: + loader = GenericLoader.from_filesystem( + self.source_dir, + glob="**/*", + suffixes=list(LANG_PARSER_EXTENSIONS), + parser=ExtendedLanguageParser(), + ) + try: + documents = loader.load() + except Exception as e: + logger.warning("LanguageParser failed on %s: %s", self.source_dir, e) + return [] + return self.lang_splitter.split_documents(documents) + + def load_and_chunk_files(self) -> list[Document]: + documents: list[Document] = [] + for root, _, files in os.walk(self.source_dir): + for file in files: + if any(file.endswith(ext) for ext in TEXT_FILE_EXTENSIONS) or file in _BUILD_FILE_NAMES: + file_path = os.path.join(root, file) + try: + with open(file_path, "r") as f: + content = f.read() + documents.append(Document(page_content=content, metadata={"source": file_path})) + except Exception as e: + logger.warning("Error reading %s: %s", file_path, e) + continue + return self.text_splitter.split_documents(documents) + + def load_and_chunk_all(self) -> list[Document]: + documents = self.load_and_chunk_code() + documents.extend(self.load_and_chunk_files()) + return documents + + + + +class CVECheckerSegmentationConfig(FunctionBaseConfig, name="cve_checker_segmentation"): + """ + Builds a scoped Tantivy lexical code index from extracted RPM source files. + Reads source directories populated by source_acquisition, indexes them, + and sets info.vdb.code_index_path for downstream checker nodes. + """ + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) + include_extensions: list[str] = Field( + default=[ + ".c", ".h", ".cpp", ".hpp", ".py", ".go", ".java", + ".spec", ".patch", ".conf", ".cfg", ".sh", ".m4", + ".ac", ".am", ".in", ".txt", ".md", ".rst", + ], + description="File extensions to include when building the code index.", + ) + + +@register_function(config_type=CVECheckerSegmentationConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_checker_segmentation(config: CVECheckerSegmentationConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.full_text_search import FullTextSearch + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + if not message.info.checker_context or not message.info.checker_context.source_key: + logger.info("checker_segmentation: no checker_context.source_keys, skipping indexing") + return message + + source_key = message.info.checker_context.source_key + if not source_key: + logger.info("checker_segmentation: no source_key, skipping indexing") + return message + + index_path = FullTextSearch.get_index_directory(config.base_code_index_dir, source_key) + + if index_path.exists(): + logger.info("checker_segmentation: cache hit on code index: %s", index_path) + else: + start = time.time() + fts = FullTextSearch(cache_path=str(index_path)) + + source_dir = Path(config.base_checker_dir) / source_key / "source" + if not source_dir.is_dir(): + logger.warning("checker_segmentation: source dir missing: %s", source_dir) + return message + + logger.info("checker_segmentation: indexing source dir %s", source_dir) + document_embedding = RpmDocumentEmbedding(source_dir=source_dir) + documents = document_embedding.load_and_chunk_all() + + + fts.add_documents_from_langchain_chunks(documents) + + elapsed = time.time() - start + logger.info("checker_segmentation: indexing completed in %.2fs at %s", elapsed, index_path) + + message.info.vdb = AgentMorpheusInfo.VdbPaths(code_index_path=str(index_path)) + return message + + yield FunctionInfo.from_fn( + _arun, + description="Build scoped Tantivy code index from extracted checker sources", + ) + + +def _index_build_files(fts, source_dir: Path) -> None: + """Walk source_dir for extensionless build files and add them to the index.""" + docs: list[tuple[str, str]] = [] + for root, _, files in os.walk(source_dir): + for fname in files: + if fname in _BUILD_FILE_NAMES: + fpath = os.path.join(root, fname) + try: + with open(fpath, "r", encoding="utf-8", errors="replace") as f: + docs.append((fpath, f.read())) + except Exception as exc: + logger.warning("checker_segmentation: error reading %s: %s", fpath, exc) + if docs: + fts.add_documents(docs) + logger.info("checker_segmentation: indexed %d build files from %s", len(docs), source_dir) diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index b6b475645..ceb2539bf 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import base64 +from dataclasses import dataclass from http import HTTPStatus from datetime import datetime from aiq.builder.builder import Builder @@ -22,13 +23,21 @@ from pydantic import Field from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import ( + CHECKER_FAILURE_ERROR_TYPES, + PACKAGE_CHECKER_STATUS_DESCRIPTIONS, +) +from exploit_iq_commons.data_models.common import PipelineMode, TypedBaseModel +from exploit_iq_commons.data_models.input import SourceDocumentsInfo from vuln_analysis.data_models.job import Job, LocalDateTime -from exploit_iq_commons.data_models.common import TypedBaseModel import typing -from typing import Any +from typing import Any, TYPE_CHECKING import os import re +if TYPE_CHECKING: + from vuln_analysis.data_models.output import AgentMorpheusOutput, FailureReport + logger = LoggingFactory.get_agent_logger(__name__) @@ -91,43 +100,104 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): mlops_config: MLOpsConfig = Field(..., description="MLOps configuration") +@dataclass +class OutputPayload: + """Encapsulates the HTTP output payload details.""" + json: str + url: str + skip_mlops: bool + + +def _build_output_payload( + message: "AgentMorpheusOutput", + config: CVEHttpOutputConfig, + default_json: str, +) -> OutputPayload: + """ + Determine the payload to send - either the full output or a failure report. + + Returns an OutputPayload with the appropriate JSON, URL, and skip_mlops flag. + """ + from vuln_analysis.data_models.output import FailureReport + + default_url = config.url + config.endpoint + failure_url = config.url + config.failure_endpoint + + if message.input.code_index_success is False: + repo_url = message.input.image.source_info[0].git_repo if message.input.image.source_info else "unknown" + report = FailureReport( + scan_id=message.input.scan.id, + error_type="processing-error", + error_message=f"Failed to clone repository {repo_url}--{message.input.failure_reason}", + ) + logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") + return OutputPayload(json=report.model_dump_json(by_alias=True), url=failure_url, skip_mlops=True) + + checker_ctx = message.info.checker_context + if checker_ctx and checker_ctx.status in CHECKER_FAILURE_ERROR_TYPES: + error_type = CHECKER_FAILURE_ERROR_TYPES[checker_ctx.status] + error_msg = PACKAGE_CHECKER_STATUS_DESCRIPTIONS.get( + checker_ctx.status, + f"Checker failed with status {checker_ctx.status}" + ) + cve_id = message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "unknown" + pkg_name = message.input.image.target_package.name if message.input.image.target_package else "unknown" + report = FailureReport( + scan_id=message.input.scan.id, + error_type=error_type, + error_message=f"{error_msg} (CVE: {cve_id}, package: {pkg_name})", + ) + logger.info( + f"Checker early exit for scan {message.input.scan.id} with status {checker_ctx.status}, " + f"sending failure report to {failure_url}" + ) + return OutputPayload(json=report.model_dump_json(by_alias=True), url=failure_url, skip_mlops=True) + + return OutputPayload(json=default_json, url=default_url, skip_mlops=False) + + @register_function(config_type=CVEHttpOutputConfig) async def output_to_http(config: CVEHttpOutputConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.output import AgentMorpheusOutput, FailureReport + from vuln_analysis.data_models.output import AgentMorpheusOutput from vuln_analysis.utils import http_utils async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: trace_id.set(message.input.scan.id) + + model_json = message.model_dump_json(by_alias=True) - url = config.url + config.endpoint + + # Save JSON for debugging - compare with local markdown reports + #from pathlib import Path + #debug_output_dir = Path(".cache/am_cache/checker_json_output") + #debug_output_dir.mkdir(parents=True, exist_ok=True) + #vuln_id = message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "unknown" + #json_file = debug_output_dir / f"{message.input.scan.id}_{vuln_id}.json" + #json_file.write_text(model_json) + #logger.info(f"Saved JSON output to {json_file}") + headers = {'Content-type': 'application/json', 'traceId': trace_id.get()} auth_header = get_auth_header(config) if auth_header is not None: headers['Authorization'] = auth_header - verify = True - if config.verify_path: - verify = config.verify_path + verify = config.verify_path if config.verify_path else True + + payload = _build_output_payload(message, config, model_json) try: - skipped_mlops = False - if message.input.code_index_success is False: - repo_url = message.input.image.source_info[0].git_repo if message.input.image.source_info else "unknown" - failure_report = FailureReport( - scan_id=message.input.scan.id, - error_type="processing-error", - error_message=f"Failed to clone repository {repo_url}--{message.input.failure_reason}", - ) - failure_url = config.url + config.failure_endpoint - logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") - model_json = failure_report.model_dump_json(by_alias=True) - url = failure_url - skipped_mlops = True - logger.info(f"Sending output to {url}") - http_utils.request_with_retry(request_kwargs={ - "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers, "verify": verify - }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) - if config.enable_mlops and not skipped_mlops: + logger.info(f"Sending output to {payload.url}") + http_utils.request_with_retry( + request_kwargs={ + "url": payload.url, + "method": "POST", + "data": payload.json.encode('utf-8'), + "headers": headers, + "verify": verify, + }, + accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED), + ) + if config.enable_mlops and not payload.skip_mlops: mlops_url = None try: job = _extract_job_data(message) @@ -143,9 +213,9 @@ async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: except Exception as mlops_e: logger.error('Unable to send job to MLOps API at %s. Error: %s', mlops_url, mlops_e) except Exception as e: - logger.error('Unable to send output response to %s. Error: %s', url, e) + logger.error('Unable to send output response to %s. Error: %s', payload.url, e) else: - logger.info('Successfully sent output to %s', url) + logger.info('Successfully sent output to %s', payload.url) return message diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py new file mode 100644 index 000000000..0df0b90d0 --- /dev/null +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -0,0 +1,960 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path +from typing import Literal + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import L1InvestigationResult + +from langgraph.graph import StateGraph, START, END +from langgraph.prebuilt import ToolNode +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, RemoveMessage + +from nat.builder.context import Context +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from vuln_analysis.functions.code_agent_graph_defs import ( + CodeAgentState, + DownstreamSearchReport, + UpstreamSearchReport, + ParsedPatch, + downstream_search_preprocss, + upstream_search_preprocess, + extract_l1_verdict, + VulnerabilityIntel, + format_patch_data_for_intel, + get_relevant_hunks, +) +from vuln_analysis.utils.rpm_checker_prompts import ( + L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + L1_AGENT_SYS_PROMPT_REBASE_FIX, + L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + L1_AGENT_PROMPT_TEMPLATE, + L1_AGENT_PROMPT_TEMPLATE_NO_PATCH, + L1_AGENT_THOUGHT_INSTRUCTIONS, + L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + L1_COMPREHENSION_PROMPT, + L1_MEMORY_UPDATE_PROMPT, + VULNERABILITY_INTEL_EXTRACTION_PROMPT, +) +from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile +from vuln_analysis.utils.package_identifier import _extract_dist_tag + +from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output +from vuln_analysis.utils.intel_utils import extract_commit_url_candidates +from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer +from vuln_analysis.utils.token_utils import truncate_tool_output +from vuln_analysis.runtime_context import ctx_state + +logger = LoggingFactory.get_agent_logger(__name__) + +import uuid +import tiktoken + +# NEVRA = Name-Epoch-Version-Release-Architecture, the standard RPM package naming format. +_RPM_NEVRA_RE = re.compile(r"^(.+?)-(?:(\d+):)?(\d\S*?)-(\S+)$") + +# Architecture mappings for early CVE applicability checks +ARCH_TO_BITNESS = { + "x86_64": "64-bit", + "amd64": "64-bit", + "i686": "32-bit", + "i386": "32-bit", + "aarch64": "64-bit", + "arm64": "64-bit", + "ppc64le": "64-bit", + "s390x": "64-bit", +} +ARCH_TO_FAMILY = { + "x86_64": "x86", + "amd64": "x86", + "i686": "x86", + "i386": "x86", + "aarch64": "arm", + "arm64": "arm", + "ppc64le": "ppc", + "s390x": "s390", +} + + +def _parse_fix_info_from_context(ctx, target_name: str, target_release: str | None = None) -> dict: + """Extract {name, version, release} from checker_context.identify_result.fixed_rpm_list. + + Handles both epoch and non-epoch NEVRAs: + - With epoch: libpq-0:13.20-1.el8_6.x86_64 + - Without epoch: libpq-13.20-1.el8_6.x86_64 + + Finds the NEVRA that matches the target package name AND EL dist tag. + Falls back to the first name match if no EL-matching entry is found. + Returns an empty dict if no match is found. + """ + if not ctx or not ctx.identify_result or not ctx.identify_result.fixed_rpm_list: + return {} + + target_dist = _extract_dist_tag(target_release) if target_release else None + fallback: dict = {} + + for nevra in ctx.identify_result.fixed_rpm_list: + m = _RPM_NEVRA_RE.match(nevra) + if not m: + continue + name = m.group(1) + if name.lower() != target_name.lower(): + continue + + version = m.group(3) + release_arch = m.group(4) + release = release_arch.rsplit(".", 1)[0] if "." in release_arch else release_arch + clean_nevra = f"{name}-{version}-{release_arch}" + result = {"nevra": clean_nevra, "name": name, "version": version, "release": release} + + # Store first name match as fallback + if not fallback: + fallback = result + + # Check if EL tag matches - if so, return immediately + candidate_dist = _extract_dist_tag(release_arch) + if target_dist and candidate_dist and target_dist == candidate_dist: + return result + + # No EL-matching entry found, return fallback (first name match) + return fallback + + +def _build_tool_strategy(tool_names: list[str]) -> str: + """Generate tool usage guidance based on available tools.""" + strategies = [] + tool_names_lower = [t.lower().replace("_", " ") for t in tool_names] + + if any("grep" in t for t in tool_names_lower): + strategies.append("- Use Source Grep for exact code patterns from patch (function names, variable names, specific code)") + if any("keyword" in t or "search" in t for t in tool_names_lower): + strategies.append("- Use Code Keyword Search for broader concept searches when grep fails") + if any("read" in t for t in tool_names_lower): + strategies.append("- Use Read File to examine full context around matches") + + return "\n".join(strategies) if strategies else "Use available tools to search for vulnerable and fixed code patterns." + + +# --------------------------------------------------------------------------- +# Policy context formatting for L1 reports (Feedback-2 gap coverage) +# --------------------------------------------------------------------------- + +_POLICY_MAX_RPM_LIST_ITEMS = 5 +_POLICY_RHSA_STATEMENT_CAP = 400 +_POLICY_MAX_PACKAGE_STATE_ITEMS = 8 + + +def _format_policy_context_for_l1_report( + *, + target_nvr: str, + identify_result, + intel, +) -> str: + """Build a context block for the LLM prompt covering NVR posture and RHSA excerpts. + + Returns an empty string if no meaningful context is available. + """ + lines: list[str] = [] + + # 1. Scanned target NVR + if target_nvr: + lines.append(f"**Scanned target NVR:** `{target_nvr}`") + + # 2. PackageIdentifyResult: affected/fixed lists + if identify_result: + affected = identify_result.affected_rpm_list or [] + fixed = identify_result.fixed_rpm_list or [] + + if affected: + shown = affected[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(affected) - len(shown)} more)" if len(affected) > len(shown) else "" + lines.append(f"**Affected NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_affected: `{identify_result.is_target_package_affected.value}`") + + if fixed: + shown = fixed[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(fixed) - len(shown)} more)" if len(fixed) > len(shown) else "" + lines.append(f"**Fixed NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_fixed: `{identify_result.is_target_package_fixed.value}`") + + # 3. RHSA excerpts (if present) + rhsa = None + if intel and len(intel) > 0: + rhsa = intel[0].rhsa + + if rhsa: + # Statement excerpt + if rhsa.statement: + stmt = rhsa.statement + if len(stmt) > _POLICY_RHSA_STATEMENT_CAP: + stmt = stmt[:_POLICY_RHSA_STATEMENT_CAP] + " …" + lines.append(f"**RHSA statement excerpt:** {stmt}") + + # Upstream fix + if rhsa.upstream_fix: + lines.append(f"**RHSA upstream_fix:** `{rhsa.upstream_fix}`") + + # Package state (compact table-like bullets) + pkg_states = rhsa.package_state or [] + if pkg_states: + lines.append("**RHSA package_state:**") + for ps in pkg_states[:_POLICY_MAX_PACKAGE_STATE_ITEMS]: + parts = [] + if ps.product_name: + parts.append(ps.product_name) + if ps.package_name: + parts.append(f"pkg={ps.package_name}") + if ps.fix_state: + parts.append(f"fix_state={ps.fix_state}") + if parts: + lines.append(f" - {' | '.join(parts)}") + if len(pkg_states) > _POLICY_MAX_PACKAGE_STATE_ITEMS: + lines.append(f" - (+ {len(pkg_states) - _POLICY_MAX_PACKAGE_STATE_ITEMS} more)") + + return "\n".join(lines) + + +class CVEPackageCodeAgentConfig(FunctionBaseConfig, name="cve_package_code_agent"): + """ + Level 1 Package Code Agent. Investigates each CVE using extracted source + code and the scoped Tantivy code index built by checker_segmentation. + + Phases: Identify -> Locate -> Verify (see HLD-standalone-checker.md §5). + """ + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="Shared RPM cache directory (for BrewDownloader).", + ) + rpm_user_type: str = Field( + default="internal", + description=( + "Brew profile for reference-patch SRPM lookup: internal or external. " + "Overridden by RPM_USER_TYPE environment variable when set." + ), + ) + llm_name: str = Field(description="The LLM model to use with the L1 code agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to L1 code agent") + max_iterations: int = Field(default=10, description="The maximum number of iterations for the agent.") + context_window_token_limit: int = Field(default=5000, description="Token limit for context window before pruning old messages.") + + +async def create_graph_code_agent(config: CVEPackageCodeAgentConfig, builder: Builder, state: AgentMorpheusEngineInput, tracer): + # Node name constants + THOUGHT_NODE = "think_node" + TOOL_NODE = "tool" + FORCED_FINISH_NODE = "forced_finish" + OBSERVATION_NODE = "observation_node" + DOWNSTREAM_SEARCH_NODE = "downstream_search" + GATHER_MORE_INFO_NODE = "gather_more_info" + L1_AGENT_NODE = "L1_agent" + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + thought_llm = llm.with_structured_output(CheckerThought) + comprehension_llm = llm.with_structured_output(CodeFindings) + observation_llm = llm.with_structured_output(Observation) + vulnerability_intel_llm = llm.with_structured_output(VulnerabilityIntel) + # Get tool names after filtering for dynamic guidance + enabled_tool_names = [tool.name for tool in tools] + tool_descriptions_list = [t.name + ": " + t.description for t in tools] + tools_node = ToolNode(tools, handle_tool_errors=True) + tool_strategy = _build_tool_strategy(enabled_tool_names) + tools_str = "\n".join(tool_descriptions_list) + + vuln_id = state.input.scan.vulns[0].vuln_id + ctx = state.info.checker_context + intel = state.info.intel + target_package = state.input.image.target_package + source_key = ctx.source_key + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + def _count_tokens(text: str) -> int: + """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: + """Estimate the token count thought_node will send to the LLM.""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return _count_tokens("\n".join(parts)) + # -- Locate setup: fix info + BrewDownloader + paths ----------------------- + aIntel = intel[0] + fix_info = _parse_fix_info_from_context(ctx, target_package.name, target_package.release) + checker_dir = Path(config.base_checker_dir) / source_key + source_dir = checker_dir / "source" + patch_dir = checker_dir / "patch" + + brew_downloader = None + if fix_info: + try: + brew_downloader = BrewDownloader( + resolve_brew_profile(config.rpm_user_type), + config.base_rpm_dir, + str(checker_dir), + ) + brew_downloader.connect() + except BrewDownloaderError as e: + logger.warning("locate: BrewDownloader init failed (%s), diff path unavailable", e) + brew_downloader = None + + descriptions: list[tuple[str, str]] = [] + if aIntel.ghsa: + cve_text = aIntel.ghsa.description or aIntel.ghsa.summary or "" + if cve_text: + descriptions.append(("ghsa", cve_text)) + if aIntel.ubuntu and aIntel.ubuntu.description: + descriptions.append(("ubuntu", aIntel.ubuntu.description)) + + cve_description = "\n".join(f"[{src}] {txt}" for src, txt in descriptions) + + + async def L1_agent(state: CodeAgentState) -> dict: + logger.info("L1_agent: starting") + downstream_report = state.get("downstream_report") + upstream_report = state.get("upstream_report") + # Extract potential commit URLs from intel references for analysis + commit_url_candidates = extract_commit_url_candidates(aIntel) + with tracer.push_active_function("Initial_Intelligence_Gathering", input_data={"commit_url_candidates": commit_url_candidates}) as span: + + if downstream_report and downstream_report.is_patch_file_available: + parsed_patch = downstream_report.parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + elif upstream_report and upstream_report.is_fixed_srpm_is_needed: + parsed_patch = upstream_report.fixed_parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + else: + parsed_patch = None + patch_data = "" + + # Extract vendor mitigations from intel (RHSA, etc.) + vendor_mitigations = "" + if aIntel and aIntel.rhsa: + mitigation = getattr(aIntel.rhsa, 'mitigation', None) + if mitigation: + mit_text = mitigation.get('value', '') if isinstance(mitigation, dict) else str(mitigation) + if mit_text: + vendor_mitigations = mit_text + + vul_prompt = VULNERABILITY_INTEL_EXTRACTION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package.name, + cve_description=cve_description, + vendor_mitigations=vendor_mitigations or "No vendor mitigations available.", + patch_data=patch_data, + ) + vulnerability_intel: VulnerabilityIntel = await vulnerability_intel_llm.ainvoke( + [SystemMessage(content=vul_prompt)], + ) + vulnerability_intel = VulnerabilityIntelSanitizer(parsed_patch).apply( + vulnerability_intel + ) + # Preserve vendor mitigations in the intel object if not already extracted + if vendor_mitigations and not vulnerability_intel.known_mitigations: + vulnerability_intel.known_mitigations = vendor_mitigations + + if downstream_report: + vulnerability_intel.is_downstream_patch_available = downstream_report.is_patch_file_available + vulnerability_intel.is_patch_applied_in_build = downstream_report.is_patch_applied_in_build + vulnerability_intel.patch_file_name = downstream_report.patch_file_name or "" + + # Early architecture check - skip further investigation if CVE doesn't apply + target_arch = target_package.arch + target_bitness = ARCH_TO_BITNESS.get(target_arch) + target_family = ARCH_TO_FAMILY.get(target_arch) + + arch_mismatch_reason = None + if target_bitness and vulnerability_intel.affected_bitness != "both": + if target_bitness != vulnerability_intel.affected_bitness: + arch_mismatch_reason = ( + f"Bitness mismatch: CVE affects {vulnerability_intel.affected_bitness}, " + f"target is {target_bitness} ({target_arch})" + ) + + if not arch_mismatch_reason and target_family: + if vulnerability_intel.affected_architectures is not None: + if target_family not in vulnerability_intel.affected_architectures: + arch_mismatch_reason = ( + f"Architecture mismatch: CVE affects {vulnerability_intel.affected_architectures}, " + f"target is {target_family} ({target_arch})" + ) + + if arch_mismatch_reason: + logger.info( + "L1_agent: %s - %s. Skipping further investigation.", + vuln_id, arch_mismatch_reason + ) + span.set_output({ + "vulnerability_intel": vulnerability_intel.model_dump(), + "arch_mismatch_reason": arch_mismatch_reason, + }) + return { + "vulnerability_intel": vulnerability_intel, + "arch_mismatch_reason": arch_mismatch_reason, + "runtime_prompt": "", + "messages": [], + } + + span.set_output({ + "vulnerability_intel": vulnerability_intel.model_dump(), + }) + + # Use case 1: Downstream patch file is available + if downstream_report and downstream_report.is_patch_file_available: + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "patch_available", + "patch_filename": downstream_report.patch_file_name, + }) + # Use case 2: code is fixed by rebase + elif upstream_report and upstream_report.is_code_fixed_by_rebase == "yes": + + if upstream_report.is_fixed_srpm_is_needed and upstream_report.fixed_parsed_patch: + # Has patch context - use patch-based verification + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_FIX, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "rebase_fix_verification", + "spec_log_change": upstream_report.spec_file_log_change[:200] if upstream_report.spec_file_log_change else "", + }) + else: + # No patch context - use CVE description-based verification + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE_NO_PATCH.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "rebase_fix_cve_description", + "spec_log_change": upstream_report.spec_file_log_change[:200] if upstream_report.spec_file_log_change else "", + }) + # use case 3: in target patch was not found but patch is found in the rpm that was mention in cve that is fixed + elif upstream_report and upstream_report.fixed_parsed_patch: + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "upstream_patch_verification", + "patch_filename": upstream_report.fixed_srpm_file_name, + }) + else: + # Use case 4: Default prompt - no patch context, use VulnerabilityIntel from CVE description + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE_NO_PATCH.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + ) + span.set_output({ + "mode": "no_patch", + }) + + messages = state.get("messages", []) + remove_messages = [RemoveMessage(id=msg.id) for msg in messages if msg.id] + + return { + "runtime_prompt": runtime_prompt, + "vulnerability_intel": vulnerability_intel, + "messages": remove_messages, + } + + async def should_continue_downstream(state: CodeAgentState) -> str: + downstream_report = state.get("downstream_report") + if downstream_report and downstream_report.is_patch_file_available: + return "L1_agent" + else: + return "gather_more_info" + + async def downstream_search(state: CodeAgentState) -> dict: + logger.info("downstream_search: starting") + + + build_log = ctx.artifacts.build_log_path if ctx and ctx.artifacts else None + with tracer.push_active_function("downstream_search", input_data={}) as span: + report: DownstreamSearchReport = await downstream_search_preprocss( + llm=llm, + vuln_id=vuln_id, + descriptions=descriptions, + source_path=Path(source_dir), + build_log_path=Path(build_log) if build_log else None, + tracer=tracer, + ) + span.set_output({ + "is_patch_file_available": report.is_patch_file_available, + "is_patch_in_spec_file": report.is_patch_in_spec_file, + "spec_file_log_change": report.spec_file_log_change, + "is_patch_applied_in_build": report.is_patch_applied_in_build, + "build_log_patch_applied": report.build_log_patch_applied, + "parsed_patch": report.parsed_patch.patch_filename if report.parsed_patch else None, + }) + + return { + "downstream_report": report, + "messages": [AIMessage(content="Downstream flow preprocess completed")], + } + + + async def gather_more_info(state: CodeAgentState) -> dict: + logger.info("gather_more_info: starting") + # Extract commit URL candidates from intel references for patch fetching + candidates = extract_commit_url_candidates(aIntel) + with tracer.push_active_function("gather_more_info", input_data={}) as span: + report: UpstreamSearchReport = await upstream_search_preprocess( + vuln_id=vuln_id, + fix_info=fix_info, + brew_downloader=brew_downloader, + patch_dir=Path(patch_dir), + source_path=Path(source_dir), + target_package=target_package, + tracer=tracer, + intel=intel, + commit_url_candidates=candidates, + cve_description=cve_description, + llm=llm, + ) + + + + span.set_output({ + "is_fixed_srpm_is_needed": report.is_fixed_srpm_is_needed, + "is_rebase_fix": report.is_code_fixed_by_rebase == "yes", + }) + return { + "messages": [AIMessage(content="Gathering more information...")], + "upstream_report": report, + } + + async def thought_node(state: CodeAgentState) -> dict: + """Generate next thought/action using the LLM.""" + step_num = state.get("step", 0) + logger.info("thought_node: starting step %d", step_num) + runtime_prompt = state.get("runtime_prompt") or "You are a security analyst investigating a CVE." + messages = [SystemMessage(content=runtime_prompt)] + state["messages"] + with tracer.push_active_function("thought_node", input_data={}) as span: + obs = state.get("observation", None) + if obs is not None: + memory_list = obs.memory if obs.memory else ["No prior knowledge."] + recent_findings = obs.results if obs.results else ["No recent findings."] + memory_context = "\n".join(f"- {m}" for m in memory_list) + findings_context = "\n".join(f"- {f}" for f in recent_findings) + context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" + messages.append(SystemMessage(content=context_block)) + response: CheckerThought = await thought_llm.ainvoke(messages) + if response.mode == "finish": + ai_message = AIMessage(content=response.final_answer) + else: + tool_name = response.actions.tool + arguments = response.actions.query + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{"name": tool_name, "args": {"query": arguments}, "id": tool_call_id}] + ) + span.set_output({ + "thought": response.thought, + "mode": response.mode, + "actions": response.actions, + "final_answer": response.final_answer, + }) + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": config.max_iterations, + } + + async def forced_finish_node(state: CodeAgentState) -> dict: + """Force finish when max iterations reached. + + Invokes the LLM with FORCED_FINISH_PROMPT to generate a final answer + based on evidence gathered so far. + """ + step_num = state.get("step", 0) + with tracer.push_active_function("forced_finish_node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) + + obs = state.get("observation") + if obs is not None and obs.memory: + memory_context = "\n".join(f"- {m}" for m in obs.memory) + messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) + + response: CheckerThought = await thought_llm.ainvoke(messages) + + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Failed to generate a final answer within the maximum allowed steps." + ai_message = AIMessage(content=final_answer) + response = CheckerThought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer, + ) + + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", config.max_iterations), + "observation": state.get("observation"), + "output": final_answer, + } + except Exception as e: + logger.exception("forced_finish_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def observation_node(state: CodeAgentState) -> dict: + """Process tool output: comprehension -> memory update with VulnerabilityIntel context.""" + logger.info("observation_node: starting") + tool_message = state["messages"][-1] + last_thought = state.get("thought") + if not last_thought: + return { + "messages": [AIMessage(content="No thought found")], + } + last_thought_text = last_thought.thought + tool_used = last_thought.actions.tool + tool_input_detail = last_thought.actions.query + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + + vulnerability_intel = state.get("vulnerability_intel") + intel_formatted = vulnerability_intel.format_for_prompt() if vulnerability_intel else "No intel available" + target_package_name = target_package.name if target_package else "unknown" + + with tracer.push_active_function("observation node", input_data=f"tool used:{tool_used} + {tool_input_detail}") as span: + tool_output_for_llm = tool_message.content + + # Check for empty/error outputs - bypass LLM if so to prevent hallucination + empty_findings = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) + if empty_findings: + code_findings = empty_findings + else: + # Get parsed_patch from state for raw diff context + # Reports may be Pydantic models or dicts depending on state serialization + downstream_report = state.get("downstream_report") + upstream_report = state.get("upstream_report") + parsed_patch = None + + if downstream_report: + if isinstance(downstream_report, dict): + parsed_patch = downstream_report.get('parsed_patch') + else: + parsed_patch = getattr(downstream_report, 'parsed_patch', None) + + if not parsed_patch and upstream_report: + if isinstance(upstream_report, dict): + parsed_patch = upstream_report.get('fixed_parsed_patch') + else: + parsed_patch = getattr(upstream_report, 'fixed_parsed_patch', None) + + # If parsed_patch is a dict, convert it to ParsedPatch model + if parsed_patch and isinstance(parsed_patch, dict): + try: + parsed_patch = ParsedPatch(**parsed_patch) + except Exception as e: + logger.warning("Failed to parse parsed_patch dict: %s", e) + parsed_patch = None + + logger.debug("observation_node: parsed_patch=%s, downstream=%s, upstream=%s", + parsed_patch is not None, + downstream_report is not None, + upstream_report is not None) + + # Extract relevant hunks based on grep target file + raw_patch_diff = "" + if tool_used == "Source Grep" and parsed_patch: + raw_patch_diff = get_relevant_hunks(parsed_patch, tool_input_detail) + + # Step 1: Comprehension - extract key findings from raw tool output + comp_prompt = L1_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=intel_formatted, + raw_patch_diff=raw_patch_diff, + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), + ) + code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + findings_text = "\n".join(f"- {f}" for f in code_findings.findings) + + # Step 2: Memory update - merge findings into cumulative memory + mem_prompt = L1_MEMORY_UPDATE_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + previous_memory="\n".join(f"- {m}" for m in previous_memory) if isinstance(previous_memory, list) else previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") + estimated = _estimate_tokens(active_prompt, messages, new_observation) + prune_messages = [] + orig_estimated = estimated + + if estimated > config.context_window_token_limit: + l_tool_count = _count_tokens(tool_output_for_llm) + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, config.context_window_token_limit, + ) + + span.set_output({ + "last_thought_text": last_thought_text, + "tool_output_for_llm": tool_output_for_llm[:500], + "findings": code_findings.findings, + "tool_outcome": code_findings.tool_outcome, + "new_memory": new_observation.memory, + "amount_of_orig_tokens": orig_estimated, + "amount_of_estimated_tokens": estimated, + }) + return { + "messages": prune_messages, + "observation": new_observation, + } + + async def should_continue(state: CodeAgentState) -> str: + thought = state.get("thought", None) + if thought is not None and thought.mode == "finish": + return END + if state.get("step", 0) >= state.get("max_steps", config.max_iterations): + return FORCED_FINISH_NODE + return TOOL_NODE + + def should_continue_after_intel(state: CodeAgentState) -> str: + """Route after L1_agent: skip ReAct loop if architecture mismatch detected.""" + if state.get("arch_mismatch_reason"): + return END + return THOUGHT_NODE + + flow = StateGraph(CodeAgentState) + + flow.add_node(DOWNSTREAM_SEARCH_NODE, downstream_search) + flow.add_node(GATHER_MORE_INFO_NODE, gather_more_info) + flow.add_node(L1_AGENT_NODE, L1_agent) + flow.add_node(THOUGHT_NODE, thought_node) + flow.add_node(TOOL_NODE, tools_node) + flow.add_node(FORCED_FINISH_NODE, forced_finish_node) + flow.add_node(OBSERVATION_NODE, observation_node) + + flow.add_edge(START, DOWNSTREAM_SEARCH_NODE) + flow.add_conditional_edges(DOWNSTREAM_SEARCH_NODE, should_continue_downstream, { + L1_AGENT_NODE: L1_AGENT_NODE, + GATHER_MORE_INFO_NODE: GATHER_MORE_INFO_NODE, + }) + flow.add_edge(GATHER_MORE_INFO_NODE, L1_AGENT_NODE) + flow.add_conditional_edges(L1_AGENT_NODE, should_continue_after_intel, { + END: END, + THOUGHT_NODE: THOUGHT_NODE, + }) + flow.add_conditional_edges( + THOUGHT_NODE, + should_continue, + {END: END, TOOL_NODE: TOOL_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE} + ) + flow.add_edge(TOOL_NODE, OBSERVATION_NODE) + flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) + flow.add_edge(FORCED_FINISH_NODE, END) + + + app = flow.compile() + return app + + +@register_function(config_type=CVEPackageCodeAgentConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_package_code_agent(config: CVEPackageCodeAgentConfig, builder: Builder): + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Run L1 investigation and return intermediate result for routing to L2 or report generation.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + # Set ctx_state so tools (e.g., Source Grep, Lexical Search) can access checker_context + from types import SimpleNamespace + workflow_state = SimpleNamespace(original_input=message, info=message.info) + ctx_state.set(workflow_state) + + logger.info("package_code_agent: starting L1 investigation") + + l1_agent_graph = await create_graph_code_agent(config, builder, message, tracer) + initial_state: CodeAgentState = { + "messages": [HumanMessage(content="Begin L1 CVE investigation")], + "step": 0, + "max_steps": config.max_iterations, + } + + with tracer.push_active_function("l1_agent_graph", input_data=initial_state["messages"][0].content): + result = await l1_agent_graph.ainvoke( + initial_state, + config={"recursion_limit": config.max_iterations * 4}, + ) + + logger.info("package_code_agent: L1 investigation finished") + + # Check for early architecture mismatch exit + arch_mismatch_reason = result.get("arch_mismatch_reason") + if arch_mismatch_reason: + logger.info( + "package_code_agent: Architecture mismatch detected - %s", + arch_mismatch_reason + ) + l1_result = L1InvestigationResult( + downstream_report=None, + upstream_report=None, + l1_agent_answer=arch_mismatch_reason, + vulnerability_intel=result.get("vulnerability_intel"), + preliminary_verdict="not_present", + confidence=1.0, + ) + if message.info.checker_context is not None: + message.info.checker_context.l1_result = l1_result + return message + + final_answer = None + thought = result.get("thought") + if thought and thought.mode == "finish": + final_answer = thought.final_answer + + vuln_id = message.input.scan.vulns[0].vuln_id + target_package = message.input.image.target_package + target_package_name = target_package.name if target_package else "unknown" + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + verdict_extraction = await extract_l1_verdict( + llm=llm, + vuln_id=vuln_id, + target_package=target_package_name, + final_answer=final_answer or "No final answer produced.", + tracer=tracer, + ) + preliminary_verdict = verdict_extraction.preliminary_verdict + confidence = verdict_extraction.confidence + + downstream_report: DownstreamSearchReport | None = result.get("downstream_report") + upstream_report: UpstreamSearchReport | None = result.get("upstream_report") + vulnerability_intel: VulnerabilityIntel | None = result.get("vulnerability_intel") + + l1_result = L1InvestigationResult( + downstream_report=downstream_report.model_dump() if downstream_report else None, + upstream_report=upstream_report.model_dump() if upstream_report else None, + l1_agent_answer=final_answer, + vulnerability_intel=vulnerability_intel, + preliminary_verdict=preliminary_verdict, + confidence=confidence, + ) + + with tracer.push_active_function( + "l1_agent_finish", + input_data={"preliminary_verdict": preliminary_verdict}, + ) as span: + span.set_output({ + "l1_agent_answer": final_answer[:500] if final_answer else None, + "vulnerability_intel": vulnerability_intel, + "confidence": l1_result.confidence, + }) + + if message.info.checker_context is not None: + message.info.checker_context.l1_result = l1_result + else: + logger.warning("package_code_agent: checker_context is None, cannot store l1_result") + logger.info( + "package_code_agent: L1 result - verdict=%s, confidence=%.2f", + preliminary_verdict, + l1_result.confidence, + ) + return message + + yield FunctionInfo.from_fn( + _arun, + description="Level 1 Package Code Agent: investigates CVEs using extracted source and Tantivy code index", + ) diff --git a/src/vuln_analysis/functions/cve_source_acquisition.py b/src/vuln_analysis/functions/cve_source_acquisition.py new file mode 100644 index 000000000..eabb2ded4 --- /dev/null +++ b/src/vuln_analysis/functions/cve_source_acquisition.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +from datetime import datetime, timezone +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +import shutil +from pathlib import Path +from pathlib import PurePath + +from exploit_iq_commons.data_models.checker_status import PackageCheckerContext, PackageCheckerStatus, PackageIdentifyResult +from exploit_iq_commons.data_models.checker_status import AcquiredArtifacts +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager, SourceRPMDownloader +from vuln_analysis.utils.package_identifier import PackageIdentifier +from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile +from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig + +logger = LoggingFactory.get_agent_logger(__name__) + + +def _artifacts_from_cache( + target_dir: Path, + source_dir: Path, + log_file: Path, + metadata_file: Path, +) -> AcquiredArtifacts: + """Populate AcquiredArtifacts from an on-disk checker cache directory.""" + artifacts = AcquiredArtifacts() + artifacts.srpm_path = source_dir + if log_file.exists(): + artifacts.build_log_path = log_file + binaries_dir = target_dir / "binaries" + if binaries_dir.exists(): + artifacts.binary_rpm_path = binaries_dir + if metadata_file.exists(): + try: + metadata = json.loads(metadata_file.read_text(encoding="utf-8")) + artifacts.source_url = metadata.get("source_url") + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to read metadata.json: %s", e) + return artifacts + + +def _update_metadata_arch(metadata_file: Path, arch: str) -> None: + """Update metadata.json to add a new arch entry to arch_artifacts.""" + if not metadata_file.exists(): + return + + try: + metadata = json.loads(metadata_file.read_text(encoding="utf-8")) + if "arch_artifacts" not in metadata: + metadata["arch_artifacts"] = {} + + metadata["arch_artifacts"][arch] = { + "build_log_downloaded_at": datetime.now(timezone.utc).isoformat(), + } + + metadata_file.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + logger.info("Updated metadata with arch=%s", arch) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to update metadata.json with arch=%s: %s", arch, e) + + +class CVESourceAcquisitionConfig(FunctionBaseConfig, name="cve_source_acquisition"): + """ + Downloads source containers, extracts layers, and locates package sources + by purl and ecosystem. Populates the pipeline state with source paths for + downstream checker segmentation and investigation nodes. + """ + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="The directory for storing pulled git repositories used for code analysis.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="The directory used for storing pickled document cache files.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="The directory used for storing rpm files.", + ) + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts (extracted sources, diffs, results).", + ) + rpm_user_type: str = Field( + default="internal", + description=( + "Brew profile: internal (Red Hat VPN) or external (Fedora public Koji). " + "Overridden by RPM_USER_TYPE environment variable when set." + ), + ) + + +@register_function(config_type=CVESourceAcquisitionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_source_acquisition(config: CVESourceAcquisitionConfig, builder: Builder): + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + logger.info("source_acquisition: starting source code acquisition") + + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + message.info.checker_context = PackageCheckerContext() + intel_list = message.info.intel or [] + vulns = message.input.scan.vulns + + intel_by_vuln = {i.vuln_id: i for i in intel_list} + target_package = message.input.image.target_package + + identifier = PackageIdentifier( + target_package=target_package, + ) + + status = PackageCheckerStatus.OK + per_vuln_results: dict[str, PackageIdentifyResult] = {} + + intel_score_config = builder.get_function_config("cve_calculate_intel_score") + assert isinstance(intel_score_config, CVECalculateIntelScoreConfig) + + for vuln_info in vulns: + intel = intel_by_vuln.get(vuln_info.vuln_id) + + if intel_score_config.generate_intel_score and intel: + score = intel.get_intel_score() + if score < intel_score_config.intel_low_score and not intel_score_config.insist_analysis: + logger.info("Intel score %d below threshold %d for %s - skipping", + score, intel_score_config.intel_low_score, vuln_info.vuln_id) + status = PackageCheckerStatus.PKG_INTEL_LOW_SCORE + break + + status, result = identifier.identify(intel) + message.info.checker_context.identify_result = result + break + + + message.info.checker_context.status = status + if status != PackageCheckerStatus.OK: + return message + + # create identifier key + str_identifier_key = f"{target_package.name}-{target_package.version}-{target_package.release}" + identifier_key = hashlib.sha256(str_identifier_key.encode()).hexdigest()[:16] + message.info.checker_context.source_key = identifier_key + + target_dir = Path(config.base_checker_dir) / identifier_key + arch = target_package.arch + + source_dir = target_dir / "source" + log_file = target_dir / "logs" / arch / "build.log" + metadata_file = target_dir / "metadata.json" + + source_exists = source_dir.exists() and any(source_dir.iterdir()) + log_exists = log_file.exists() + + brew_profile = resolve_brew_profile(config.rpm_user_type) + brew_downloader = BrewDownloader(brew_profile, config.base_rpm_dir, str(target_dir)) + + if source_exists and (log_exists or not brew_downloader.auto_fetch_build_log): + logger.info("Full cache hit for %s (arch=%s): %s", identifier_key, arch, target_dir) + message.info.checker_context.artifacts = _artifacts_from_cache( + target_dir, source_dir, log_file, metadata_file, + ) + return message + + if source_exists and not log_exists and brew_downloader.auto_fetch_build_log: + logger.info("Partial cache hit for %s - downloading log for arch=%s", identifier_key, arch) + artifacts = _artifacts_from_cache(target_dir, source_dir, log_file, metadata_file) + + try: + brew_downloader.connect() + build = brew_downloader.search_build( + target_package.name, target_package.version, target_package.release, + ) + if build: + artifacts.build_log_path = brew_downloader.try_download_build_log(build, arch) + if artifacts.build_log_path: + _update_metadata_arch(metadata_file, arch) + logger.info("Downloaded build log for arch=%s", arch) + except BrewDownloaderError as e: + logger.warning("Failed to fetch build for build log arch=%s: %s", arch, e) + + message.info.checker_context.artifacts = artifacts + return message + + # Full cache miss - download everything + logger.info("Full cache miss for %s (arch=%s)", identifier_key, arch) + target_dir.mkdir(parents=True, exist_ok=True) + try: + brew_downloader.connect() + artifacts = brew_downloader.download_target_artifacts( + target_package.name, target_package.version, target_package.release, arch, + ) + message.info.checker_context.artifacts = artifacts + + nvr = f"{target_package.name}-{target_package.version}-{target_package.release}" + arch_entry: dict[str, str] = {} + if artifacts.build_log_path: + arch_entry["build_log_downloaded_at"] = datetime.now(timezone.utc).isoformat() + metadata = { + "source_url": artifacts.source_url, + "nvr": nvr, + "downloaded_at": datetime.now(timezone.utc).isoformat(), + "arch_artifacts": {arch: arch_entry} if arch_entry else {}, + } + metadata_file.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + logger.info("Wrote metadata to %s", metadata_file) + except BrewDownloaderError as e: + logger.error("Failed to download patched SRPM: %s", e) + message.info.checker_context.status = PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM + return message + + + + return message + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Downloads source containers and locates package sources by purl and ecosystem.", + ) diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 803400675..192b08e4c 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -62,6 +62,34 @@ class Thought(BaseModel): max_length=3000, ) + +class CheckerToolCall(BaseModel): + """Tool call for RPM checker flow - simpler schema with just query.""" + tool: str = Field(description="Exact tool name from AVAILABLE_TOOLS") + query: str = Field(description="Search pattern for Source Grep or Code Keyword Search") + reason: str = Field(description="Briefly explain why this tool helps the investigation") + + +class CheckerThought(BaseModel): + """Thought model for RPM checker flow with simplified tool call schema.""" + thought: str = Field( + description="Brief reasoning about next step (max 3-4 sentences)", + max_length=3000, + ) + mode: Literal["act", "finish"] = Field( + description="'act' to call tools, 'finish' to return final answer" + ) + actions: CheckerToolCall | None = Field( + default=None, + description="When mode is 'act', the tool to execute" + ) + final_answer: str | None = Field( + default=None, + description="When mode is 'finish', concise answer with evidence", + max_length=3000, + ) + + class CodeFindings(BaseModel): """Compressed code comprehension output from raw tool results.""" findings: list[str] = Field( @@ -73,6 +101,47 @@ class CodeFindings(BaseModel): ) +def check_empty_output( + tool_output: str | list, + tool_used: str, + tool_input: str, +) -> CodeFindings | None: + """Check if tool output is empty or an error, returning factual CodeFindings if so. + + This bypasses LLM comprehension for empty/error outputs to prevent hallucination. + + Returns: + CodeFindings with factual empty/error message, or None if output has content. + """ + is_empty = ( + not tool_output + or (isinstance(tool_output, str) and tool_output.strip() in ("[]", "")) + or (isinstance(tool_output, list) and len(tool_output) == 0) + ) + + is_error = ( + isinstance(tool_output, str) + and any(m in tool_output for m in ["Error:", "error:", "Failed:", "Exception:", "Traceback"]) + ) + + if is_empty: + return CodeFindings( + findings=[f"{tool_used} for '{tool_input}' returned empty - no matches found"], + tool_outcome=f"CALLED: {tool_used} with {tool_input} -> EMPTY (no results)" + ) + + if is_error: + return CodeFindings( + findings=[ + f"FAILED: {tool_used} [{tool_input}] - tool error", + f"Details: {str(tool_output)[:150]}" + ], + tool_outcome=f"FAILED: {tool_used} with {tool_input} -> ERROR" + ) + + return None + + class Observation(BaseModel): results: list[str] = Field( description="3-5 key technical facts from this tool output. Each fact must describe what the code DOES and how it relates to the investigation goal, not just that it was found." @@ -90,6 +159,25 @@ class Classification(BaseModel): ) +class L1VerdictExtraction(BaseModel): + """Lightweight structured output for extracting verdict from L1 final answer.""" + preliminary_verdict: str = Field( + description=( + "Classify the L1 agent's conclusion: " + "'protected' if fix/patch applied or code mitigated, " + "'not_present' if vulnerable code not found in this version, " + "'vulnerable' if vulnerable code confirmed present, " + "'uncertain' if evidence is insufficient or conflicting" + ) + ) + confidence: float = Field( + description="Confidence in the verdict (0.0 to 1.0) based on evidence strength in the answer" + ) + reasoning: str = Field( + description="Brief explanation of why this verdict was chosen" + ) + + class PackageSelection(BaseModel): """Structured output for selecting the most relevant package from multiple candidates.""" selected_package: str = Field( diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index fe3b4b755..03ca69448 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -23,9 +23,12 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field +from exploit_iq_commons.data_models.common import PipelineMode +from exploit_iq_commons.data_models.checker_status import PackageCheckerStatus, PACKAGE_CHECKER_STATUS_DESCRIPTIONS from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from exploit_iq_commons.data_models.input import AgentMorpheusInput -from vuln_analysis.data_models.output import AgentMorpheusOutput +from exploit_iq_commons.data_models.info import AgentMorpheusInfo +from vuln_analysis.data_models.output import AgentMorpheusEngineOutput, AgentMorpheusOutput, JustificationOutput, OutputPayload from vuln_analysis.data_models.state import AgentMorpheusEngineState # pylint: disable=unused-import from vuln_analysis.functions import cve_agent @@ -36,8 +39,13 @@ from vuln_analysis.functions import cve_generate_vdbs from vuln_analysis.functions import cve_http_output from vuln_analysis.functions import cve_justify +from vuln_analysis.functions import cve_package_code_agent +from vuln_analysis.functions import cve_checker_segmentation +from vuln_analysis.functions import cve_source_acquisition from vuln_analysis.functions import cve_process_sbom from vuln_analysis.functions import cve_summarize +from vuln_analysis.functions import cve_checker_report +from vuln_analysis.functions import cve_build_agent from vuln_analysis.functions import cve_generate_cvss from vuln_analysis.functions import cve_generate_vex from vuln_analysis.functions import health_endpoint @@ -49,6 +57,7 @@ from vuln_analysis.tools import serp from vuln_analysis.tools import configuration_scanner from vuln_analysis.tools import import_usage_analyzer +from vuln_analysis.tools import source_grep from vuln_analysis.utils.error_handling_decorator import catch_pipeline_errors_async # pylint: enable=unused-import from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input @@ -77,6 +86,26 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Function to output workflow results " "(e.g. cve_file_output, cve_http_output). " " If None, only prints to console") + cve_source_acquisition_name: str | None = Field( + default=None, + description="Function name for source acquisition (downloads source containers, locates package sources)", + ) + cve_checker_segmentation_name: str | None = Field( + default=None, + description="Function name for scoped code indexing of extracted checker sources (Tantivy only)", + ) + cve_package_code_agent_name: str | None = Field( + default=None, + description="Function name for the Level 1 Package Code Agent (source-level CVE investigation)", + ) + cve_checker_report_name: str | None = Field( + default=None, + description="Function name for the checker report generation (L1/L2 report synthesis)", + ) + cve_build_agent_name: str | None = Field( + default=None, + description="Function name for the Level 2 Build Agent (build compilation and hardening check)", + ) description: str = Field(default="Vulnerability analysis for container security workflow", description="Workflow function description") @@ -101,6 +130,26 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): cve_generate_vex_fn = builder.get_function(name=config.cve_generate_vex_name) cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None + cve_source_acquisition_fn = ( + builder.get_function(name=config.cve_source_acquisition_name) + if config.cve_source_acquisition_name else None + ) + cve_checker_segmentation_fn = ( + builder.get_function(name=config.cve_checker_segmentation_name) + if config.cve_checker_segmentation_name else None + ) + cve_package_code_agent_fn = ( + builder.get_function(name=config.cve_package_code_agent_name) + if config.cve_package_code_agent_name else None + ) + cve_checker_report_fn = ( + builder.get_function(name=config.cve_checker_report_name) + if config.cve_checker_report_name else None + ) + cve_build_agent_fn = ( + builder.get_function(name=config.cve_build_agent_name) + if config.cve_build_agent_name else None + ) # Define langgraph node functions @catch_pipeline_errors_async @@ -185,7 +234,24 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput """Outputs results using configured output function""" return await cve_output_fn.ainvoke(state.model_dump()) if cve_output_fn else state - + + # --- Package checker path nodes --- + + @catch_pipeline_errors_async + async def checker_init_state_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """Bridges AgentMorpheusInput -> AgentMorpheusEngineInput with empty info (skips VDB generation).""" + return AgentMorpheusEngineInput(input=state, info=AgentMorpheusInfo()) + + @catch_pipeline_errors_async + async def checker_fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Fetch intel for CVE input (package checker path). Reuses the same fetch_intel function.""" + return await cve_fetch_intel_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async + async def checker_calculate_intel_score_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Calculate intel score for CVE input (package checker path).""" + return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) + async def check_vdbs_success(state: AgentMorpheusInput) -> str: """Checks if the VDBs were successfully generated""" if state.code_index_success: @@ -198,7 +264,138 @@ async def failure_node(state: AgentMorpheusInput) -> AgentMorpheusOutput: from exploit_iq_commons.data_models.info import AgentMorpheusInfo from vuln_analysis.data_models.output import OutputPayload return AgentMorpheusOutput(input=state, info=AgentMorpheusInfo(), output=OutputPayload(analysis=[], vex=None)) - # define langgraph + + + + @catch_pipeline_errors_async + async def source_acquisition_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Acquires source code for the target package (source containers, git fallback).""" + if cve_source_acquisition_fn: + state = await cve_source_acquisition_fn.ainvoke(state.model_dump()) + else: + logger.warning("Source acquisition function not configured, passing state through") + + if state.info.checker_context and state.info.checker_context.status is not None: + logger.info( + "PackageIdentify aggregate status: %s", + state.info.checker_context.status.name, + ) + return state + + @catch_pipeline_errors_async + async def checker_segmentation_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Builds scoped Tantivy code index from extracted checker sources.""" + if cve_checker_segmentation_fn: + state = await cve_checker_segmentation_fn.ainvoke(state.model_dump()) + else: + logger.warning("Checker segmentation not configured, skipping indexing") + return state + + @catch_pipeline_errors_async + async def l1_code_agent_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Level 1 Package Code Agent: investigates CVEs using extracted source and Tantivy code index. + + Returns AgentMorpheusEngineInput with l1_result populated on checker_context. + """ + if cve_package_code_agent_fn: + return await cve_package_code_agent_fn.ainvoke(state.model_dump()) + logger.warning("Package code agent function not configured, passing state through") + return state + + def route_after_l1(state: AgentMorpheusEngineInput) -> str: + """Route to L2 Build Agent if vulnerable or uncertain, else to report generation.""" + ctx = state.info.checker_context + if ctx and ctx.l1_result: + verdict = ctx.l1_result.preliminary_verdict + if verdict in ("vulnerable", "uncertain"): + return "l2_build_agent" + return "generate_report" + + @catch_pipeline_errors_async + async def l2_build_agent_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Level 2 Build Agent: BuildCompilationCheck + HardeningCheck. + + Returns AgentMorpheusEngineInput with l2_result populated on checker_context. + """ + if cve_build_agent_fn: + return await cve_build_agent_fn.ainvoke(state.model_dump()) + logger.warning("Build agent function not configured, passing state through") + return state + + @catch_pipeline_errors_async + async def generate_report_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Generate the final checker report from L1/L2 investigation results.""" + if cve_checker_report_fn: + return await cve_checker_report_fn.ainvoke(state.model_dump()) + logger.warning("Checker report function not configured, producing empty output") + return AgentMorpheusOutput( + input=state.input, + info=state.info, + output=OutputPayload(analysis=[], vex=None), + ) + + @catch_pipeline_errors_async + async def checker_early_exit_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Produces a proper output when source_acquisition exits with a non-OK status.""" + ctx = state.info.checker_context + status = ctx.status if ctx else None + + # Build detailed reason from identification result when available + detailed_reason = "" + if ctx and ctx.identify_result and ctx.identify_result.conclusion_reason: + detailed_reason = ctx.identify_result.conclusion_reason + + base_reason = ( + PACKAGE_CHECKER_STATUS_DESCRIPTIONS[status] + if status is not None and status in PACKAGE_CHECKER_STATUS_DESCRIPTIONS + else f"Checker exited early with status {status}" + ) + + # Combine base reason with detailed conclusion if available + reason = f"{base_reason}. {detailed_reason}" if detailed_reason else base_reason + logger.info("checker_early_exit: status=%s reason=%s", status, reason) + def _get_justification_label(s: PackageCheckerStatus | None) -> str: + if s in (PackageCheckerStatus.PKG_IDENT_NOT_VUL, PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH): + return "not_vulnerable" + if s == PackageCheckerStatus.PKG_INTEL_LOW_SCORE: + return "poor_quality_intel" + return "error" + + analysis = [ + AgentMorpheusEngineOutput( + vuln_id=v.vuln_id, + checklist=[], + summary=reason, + justification=JustificationOutput( + label=_get_justification_label(status), + reason=reason, + status="FALSE" if status in ( + PackageCheckerStatus.PKG_IDENT_NOT_VUL, + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH, + ) else "UNKNOWN", + ), + intel_score=0, + cvss=None, + ) + for v in state.input.scan.vulns + ] + return AgentMorpheusOutput( + input=state.input, info=state.info, + output=OutputPayload(analysis=analysis, vex=None), + ) + + def route_after_source_acquisition(state: AgentMorpheusEngineInput): + """Route to checker_segmentation (happy path) or early exit on non-OK status.""" + ctx = state.info.checker_context + if ctx and ctx.status == PackageCheckerStatus.OK: + return "checker_segmentation" + return "checker_early_exit" + + def route_after_add_start_time(state: AgentMorpheusInput): + """Route to full pipeline or package checker based on pipeline_mode.""" + if state.image.pipeline_mode == PipelineMode.PACKAGE_CHECKER: + return "checker_init_state" + return "generate_vdbs" # build llm engine subgraph subgraph_builder = StateGraph(AgentMorpheusEngineState) @@ -242,8 +439,28 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph_builder.add_node("add_completed_time", add_completed_time_node) graph_builder.add_node("output_results", output_results_node) graph_builder.add_node("failure", failure_node) +# -- Package checker nodes -- + graph_builder.add_node("checker_init_state", checker_init_state_node) + graph_builder.add_node("checker_fetch_intel", checker_fetch_intel_node) + graph_builder.add_node("checker_calculate_intel_score", checker_calculate_intel_score_node) + graph_builder.add_node("source_acquisition", source_acquisition_node) + graph_builder.add_node("checker_early_exit", checker_early_exit_node) + graph_builder.add_node("checker_segmentation", checker_segmentation_node) + graph_builder.add_node("l1_code_agent", l1_code_agent_node) + graph_builder.add_node("l2_build_agent", l2_build_agent_node) + graph_builder.add_node("generate_report", generate_report_node) + graph_builder.add_edge(START, "add_start_time") - graph_builder.add_edge("add_start_time", "generate_vdbs") + # Conditional: route to full pipeline or package checker after add_start_time + graph_builder.add_conditional_edges( + "add_start_time", + route_after_add_start_time, + { + "generate_vdbs": "generate_vdbs", + "checker_init_state": "checker_init_state", + }, + ) + graph_builder.add_conditional_edges("generate_vdbs", check_vdbs_success,{"fetch_intel": "fetch_intel", "failure": "failure"}) graph_builder.add_edge("failure", "add_completed_time") #graph_builder.add_edge("generate_vdbs", "fetch_intel") @@ -252,10 +469,39 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): graph_builder.add_edge("process_sbom", "check_vuln_deps") graph_builder.add_edge("check_vuln_deps", "llm_engine") graph_builder.add_edge("llm_engine", "add_completed_time") + + # Package checker path + graph_builder.add_edge("checker_init_state", "checker_fetch_intel") + graph_builder.add_edge("checker_fetch_intel", "checker_calculate_intel_score") + graph_builder.add_edge("checker_calculate_intel_score", "source_acquisition") + + graph_builder.add_conditional_edges( + "source_acquisition", + route_after_source_acquisition, + { + "checker_segmentation": "checker_segmentation", + "checker_early_exit": "checker_early_exit", + }, + ) + graph_builder.add_edge("checker_early_exit", "add_completed_time") + graph_builder.add_edge("checker_segmentation", "l1_code_agent") + graph_builder.add_conditional_edges( + "l1_code_agent", + route_after_l1, + { + "l2_build_agent": "l2_build_agent", + "generate_report": "generate_report", + }, + ) + graph_builder.add_edge("l2_build_agent", "generate_report") + graph_builder.add_edge("generate_report", "add_completed_time") + + # Shared tail graph_builder.add_edge("add_completed_time", "output_results") graph_builder.add_edge("output_results", END) graph = graph_builder.compile() - + #graph.get_graph().draw_mermaid_png(output_file_path="checker_flow.png") + def convert_str_to_agent_morpheus_input(input: str) -> AgentMorpheusInput: logger.debug("Converting JSON string input to AgentMorpheusInput (length: %d)", len(input)) try: diff --git a/src/vuln_analysis/tools/brew_downloader.py b/src/vuln_analysis/tools/brew_downloader.py new file mode 100644 index 000000000..fb36b5d50 --- /dev/null +++ b/src/vuln_analysis/tools/brew_downloader.py @@ -0,0 +1,353 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Production Brew Downloader -- fetch SRPMs, build logs, and binary RPMs from Brew (Koji). + +Evolved from the PoC at docs/package_analyzer/standalone_checker/brew/brew_downloader.py. +Storage is split: SRPMs go to the shared rpms/ cache, everything else to checker-specific dirs. +""" + +from __future__ import annotations + +import os +import re +import shutil +from enum import Enum +from pathlib import Path + +import koji +import requests +import yaml + +from exploit_iq_commons.data_models.checker_status import AcquiredArtifacts +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.source_rpm_downloader import SourceRPMDownloader + +logger = LoggingFactory.get_agent_logger(__name__) + +_CONFIGS_DIR = Path(__file__).resolve().parent.parent / "configs" / "brew" +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + +class BrewDownloaderError(Exception): + """Base for all Brew downloader errors.""" + + +class BrewConnectionError(BrewDownloaderError): + """Raised when the Brew hub is unreachable or session creation fails.""" + + +class BrewBuildNotFoundError(BrewDownloaderError): + """Raised when getBuild returns None for the requested NVR.""" + + +class BrewDownloadError(BrewDownloaderError): + """Raised when an HTTP download of an artifact fails.""" + + +class BrewProfileNotImplementedError(BrewDownloaderError): + """Raised when a profile type is not yet implemented.""" + + +# --------------------------------------------------------------------------- +# Profile types +# --------------------------------------------------------------------------- + +class BrewProfileType(Enum): + INTERNAL = "internal" + EXTERNAL = "external" + + +_PROFILE_PATHS: dict[BrewProfileType, Path] = { + BrewProfileType.INTERNAL: _CONFIGS_DIR / "internal-user-profile.yml", + BrewProfileType.EXTERNAL: _CONFIGS_DIR / "external-user-profile.yml", +} + + +_ENV_DEFAULT_RE = re.compile(r"^\$\{([^:}]+):-([^}]+)\}$") + + +def resolve_brew_profile(config_value: str) -> BrewProfileType: + """Resolve brew profile from deployment config. + + Supports ``rpm_user_type: ${RPM_USER_TYPE:-internal}`` in YAML (expanded at + deploy, or parsed here when the literal is still present). ``RPM_USER_TYPE`` + environment variable always wins when set. + """ + env_override = os.environ.get("RPM_USER_TYPE") + if env_override is not None: + name = env_override.strip().lower() + else: + raw = config_value.strip() + match = _ENV_DEFAULT_RE.match(raw) + if match: + var_name, default = match.group(1), match.group(2) + name = os.environ.get(var_name, default).strip().lower() + else: + name = raw.lower() + try: + return BrewProfileType(name) + except ValueError as exc: + raise BrewProfileNotImplementedError( + f"Unknown brew profile '{name}' (expected internal or external)" + ) from exc + + +# --------------------------------------------------------------------------- +# BrewDownloader +# --------------------------------------------------------------------------- + +class BrewDownloader: + """Downloads RPM artifacts and build logs from Brew (Koji) using a profile YAML. + + Storage destinations: + - SRPMs -> ``rpm_cache_dir/{NVR}.src.rpm`` (shared with SourceRPMDownloader) + - Build logs -> ``checker_dir/logs/{arch}/build.log`` + - Binary RPMs -> ``checker_dir/binaries/{NVR}/{NVRA}.rpm`` + """ + + def __init__(self, profile_type: BrewProfileType, rpm_cache_dir: str, checker_dir: str) -> None: + profile_path = _PROFILE_PATHS.get(profile_type) + if profile_path is None: + raise BrewProfileNotImplementedError( + f"Profile type '{profile_type.value}' is not configured" + ) + self._profile = self._load_profile(str(profile_path)) + + hosts = self._profile["hosts"]["rpm"] + self._brew_hub: str = hosts["brew_hub"] + self._brew_download: str = hosts["brew_download"] + self._default_arch: str = self._profile.get("default_arch", "x86_64") + self._download_binary_rpm_enabled: bool = self._profile.get("download_binary_rpm", False) + self._auto_fetch_build_log: bool = self._profile.get("build_log", {}).get("auto_fetch", True) + self._ssl_verify: bool = self._profile.get("ssl_verify", True) + self._ssl_verify_path: str | None = self._profile.get("verify_path") + self._http_verify: bool | str = self._resolve_http_verify() + + self._rpm_cache_dir = Path(rpm_cache_dir) + self._rpm_cache_dir.mkdir(parents=True, exist_ok=True) + + self._checker_dir = Path(checker_dir) + self._checker_dir.mkdir(parents=True, exist_ok=True) + + self._session: koji.ClientSession | None = None + self._pathinfo: koji.PathInfo | None = None + self._http = requests.Session() + + # -- properties -------------------------------------------------------- + + @property + def download_binary_rpm_enabled(self) -> bool: + return self._download_binary_rpm_enabled + + @property + def default_arch(self) -> str: + return self._default_arch + + @property + def auto_fetch_build_log(self) -> bool: + return self._auto_fetch_build_log + + # -- setup ------------------------------------------------------------- + + @staticmethod + def _load_profile(path: str) -> dict: + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + def _resolve_http_verify(self) -> bool | str: + if not self._ssl_verify: + return False + if self._ssl_verify_path: + return self._ssl_verify_path + return True + + def connect(self) -> None: + """Create a Koji client session and PathInfo helper from the profile.""" + logger.info("Connecting to Brew hub: %s", self._brew_hub) + try: + opts: dict = {} + if not self._ssl_verify: + opts["no_ssl_verify"] = True + elif self._ssl_verify_path: + opts["serverca"] = self._ssl_verify_path + self._session = koji.ClientSession(self._brew_hub, opts=opts) + self._pathinfo = koji.PathInfo(topdir=self._brew_download) + self._http.verify = self._http_verify + + except Exception as exc: + raise BrewConnectionError( + f"Failed to connect to Brew hub {self._brew_hub}: {exc}" + ) from exc + + # -- query ------------------------------------------------------------- + + def search_build(self, name: str, version: str, release: str) -> dict | None: + """Look up a build by NVR. Returns the build-info dict or ``None``.""" + nvr = f"{name}-{version}-{release}" + logger.info("Searching for build: %s", nvr) + build = self._session.getBuild(nvr) + if build is None: + logger.warning("Build not found: %s", nvr) + return None + logger.info( + "Found build %s (id=%s, volume=%s, task=%s)", + build["nvr"], build["id"], build.get("volume_name"), build.get("task_id"), + ) + return build + + # -- downloads --------------------------------------------------------- + + def _download_file(self, url: str, dest: Path) -> Path: + """Stream-download *url* to *dest*. Returns the destination path.""" + logger.info("Downloading %s -> %s", url, dest) + dest.parent.mkdir(parents=True, exist_ok=True) + try: + resp = self._http.get( + url, stream=True, timeout=120, verify=self._http_verify + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise BrewDownloadError(f"Failed to download {url}: {exc}") from exc + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=1 << 18): # 256 KB + fh.write(chunk) + logger.info("Saved %s (%d bytes)", dest.name, dest.stat().st_size) + return dest + + def _get_srpm_url(self, build: dict) -> str: + """Compute the download URL for the source RPM of *build*.""" + rpms = self._session.listRPMs(buildID=build["id"], arches="src") + if not rpms: + raise BrewDownloadError(f"No source RPM found for build {build['nvr']}") + rpm_info = rpms[0] + return f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + + def download_srpm(self, build: dict) -> Path: + """Download the .src.rpm for *build* into the shared RPM cache. + + Skips the download when the destination file already exists and is non-empty. + """ + rpms = self._session.listRPMs(buildID=build["id"], arches="src") + if not rpms: + raise BrewDownloadError(f"No source RPM found for build {build['nvr']}") + + rpm_info = rpms[0] + dest = self._rpm_cache_dir / f"{rpm_info['nvr']}.src.rpm" + + if dest.exists() and dest.stat().st_size > 0: + logger.info("SRPM cache hit: %s", dest) + return dest + + url = f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + return self._download_file(url, dest) + + def download_build_log(self, build: dict, arch: str | None = None) -> Path: + """Download ``build.log`` for the given arch into ``checker_dir/logs/{arch}/``.""" + arch = arch or self._default_arch + url = f"{self._pathinfo.build(build)}/data/logs/{arch}/build.log" + dest = self._checker_dir / "logs" / arch / "build.log" + return self._download_file(url, dest) + + def try_download_build_log(self, build: dict, arch: str | None = None) -> Path | None: + """Download build log when ``build_log.auto_fetch`` is enabled; return None on failure.""" + if not self._auto_fetch_build_log: + return None + arch = arch or self._default_arch + try: + return self.download_build_log(build, arch) + except BrewDownloadError as exc: + logger.warning( + "Build log unavailable for %s arch=%s: %s", + build.get("nvr", "?"), + arch, + exc, + ) + return None + + def download_binary_rpm(self, build: dict, arch: str | None = None) -> Path | None: + """Download all binary RPMs for the given arch (excludes debuginfo/debugsource). + + Saves to ``checker_dir/binaries/{NVR}/``. Returns an empty list when no + matching RPMs are found. + """ + arch = arch or self._default_arch + rpms = self._session.listRPMs(buildID=build["id"], arches=arch) + if not rpms: + logger.warning("No %s RPMs found for build %s", arch, build["nvr"]) + return None + + nvr = build["nvr"] + build_dir = self._checker_dir / "binaries" / nvr + + downloaded: list[Path] = [] + for rpm_info in rpms: + rpm_name: str = rpm_info["name"] + if rpm_name.endswith(("-debuginfo", "-debugsource")): + continue + url = f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + nvra = f"{rpm_info['name']}-{rpm_info['version']}-{rpm_info['release']}.{rpm_info['arch']}" + dest = build_dir / f"{nvra}.rpm" + self._download_file(url, dest) + downloaded.append(dest) + return build_dir + + def download_patched_srpm(self, name: str, version: str, release: str) -> Path | None: + """Download the SRPM for a patched version (from CVE fix info). + + Returns the cached SRPM path, or ``None`` if the patched build is not + found in Brew. + """ + build = self.search_build(name, version, release) + if build is None: + return None + return self.download_srpm(build) + + def download_patched_srpm_by_nevra(self, nevra: str) -> Path | None: + """Download the SRPM for a patched version (from NEVRA). + + Returns the cached SRPM path, or ``None`` if the patched build is not + found in Brew. + """ + build = self._session.getBuild(nevra) + if build is None: + logger.warning("Build not found: %s", nevra) + return None + logger.info( + "Found build %s (id=%s, volume=%s, task=%s)", + build["nvr"], build["id"], build.get("volume_name"), build.get("task_id"), + ) + return self.download_srpm(build) + + def download_target_artifacts(self, name: str, version: str, release: str, arch: str) -> AcquiredArtifacts | None: + artifacts = AcquiredArtifacts() + build = self.search_build(name, version, release) + if build is None: + raise BrewBuildNotFoundError(f"Build not found for {name}-{version}-{release}") + + artifacts.source_url = self._get_srpm_url(build) + cache_srpm_path = self.download_srpm(build) + + srpm_target_path = self._checker_dir / "source" + srpm_target_path.mkdir(parents=True, exist_ok=True) + shutil.copy2(cache_srpm_path, srpm_target_path) + SourceRPMDownloader.extract_src_rpm(cache_srpm_path, srpm_target_path) + artifacts.srpm_path = srpm_target_path + + artifacts.build_log_path = self.try_download_build_log(build, arch) + if self._download_binary_rpm_enabled: + artifacts.binary_rpm_path = self.download_binary_rpm(build, arch) + return artifacts diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 3fdc0dde6..a01292b25 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -21,6 +21,7 @@ from pydantic import Field from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.data_models.input import PipelineMode from vuln_analysis.utils.error_handling_decorator import catch_tool_errors LEXICAL_CODE_SEARCH = "lexical_code_search" @@ -32,6 +33,10 @@ class LexicalSearchToolConfig(FunctionBaseConfig, name=LEXICAL_CODE_SEARCH): Lexical search tool used to search source code. """ top_k: int = Field(default=5, description="Top K to use for the lexical search") + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) @register_function(config_type=LexicalSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -42,8 +47,16 @@ async def lexical_search(config: LexicalSearchToolConfig, builder: Builder): # @catch_tool_errors(LEXICAL_CODE_SEARCH) async def _arun(query: str) -> str: workflow_state = ctx_state.get() - code_index_path = workflow_state.code_index_path - full_text_search = FullTextSearch.get_instance(cache_path=code_index_path) + + pipeline_mode = getattr(workflow_state.original_input.input.image, 'pipeline_mode', None) + + if pipeline_mode == PipelineMode.PACKAGE_CHECKER: + source_key = workflow_state.original_input.info.checker_context.source_key + code_index_path = str(FullTextSearch.get_index_directory(config.base_code_index_dir, source_key)) + else: + code_index_path = workflow_state.code_index_path + + full_text_search = FullTextSearch(cache_path=code_index_path) if full_text_search.is_empty(): logger.debug("Lexical search: index is empty at %s", code_index_path) diff --git a/src/vuln_analysis/tools/source_grep.py b/src/vuln_analysis/tools/source_grep.py new file mode 100644 index 000000000..38b25cf98 --- /dev/null +++ b/src/vuln_analysis/tools/source_grep.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native Unix grep tool for fast source code searching. + +Provides an LLM-callable tool that uses native grep subprocess for +faster searching compared to Python-based regex scanning. +""" + +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.tools.source_inspector import SourceInspector +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +SOURCE_GREP = "source_grep" + +logger = LoggingFactory.get_agent_logger(__name__) + + +class SourceGrepToolConfig(FunctionBaseConfig, name=SOURCE_GREP): + """Fast grep search using native Unix grep subprocess.""" + + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + max_results: int = Field( + default=50, + description="Maximum number of grep results to return.", + ) + context_lines: int = Field( + default=3, + description="Number of context lines around each match.", + ) + + +VALID_TARGETS = ("source", "logs", "patch") + +TARGET_EXTENSIONS: dict[str, list[str]] = { + "source": ["*.c", "*.h", "*.cpp", "*.hpp", "*.py", "*.go", "*.java", "*.spec", "*.cmake", "Makefile", "*.mk", "*.config"], + "logs": [], # empty = search all files + "patch": ["*.patch", "*.diff"], +} + + +def _parse_query(query: str) -> tuple[str | list[str], str | None, str, bool]: + """Parse query string into (pattern(s), file_glob, target, word_boundary). + + Supports formats: + - "pattern" -> search source (default) + - "pattern,*.c" -> search source, only .c files + - "target:pattern" -> search specific target + - "target:pattern,file_glob" -> search target with file filter + - "pattern -w" -> search with word boundary (whole words only) + - "target:pattern,file_glob -w" -> full format with word boundary + - "pattern1;pattern2,file.c" -> multiple patterns (only with file_glob) + + Valid targets: source, logs, patch + + Note: Multiple patterns (separated by ';') are only supported when + a file_glob is provided. This prevents overly broad multi-pattern searches. + """ + query = query.strip().strip('"').strip("'") + + word_boundary = False + if query.endswith(" -w"): + word_boundary = True + query = query[:-3].strip() + + target = "source" + if ":" in query: + prefix, rest = query.split(":", 1) + if prefix in VALID_TARGETS: + target = prefix + query = rest + + if "," in query: + parts = query.split(",", 1) + pattern_part = parts[0].strip() + file_glob = parts[1].strip() if len(parts) > 1 else None + + # Multi-pattern support: only when file_glob is provided + if file_glob and ";" in pattern_part: + patterns = [p.strip() for p in pattern_part.split(";") if p.strip()] + return patterns, file_glob, target, word_boundary + + return pattern_part, file_glob, target, word_boundary + + return query, None, target, word_boundary + + +def _format_results(pattern: str, matches: list, root: Path) -> str: + """Format grep results for LLM consumption.""" + if not matches: + return f"No matches found for '{pattern}'" + + lines = [f"Found {len(matches)} match(es) for '{pattern}':\n"] + for i, match in enumerate(matches, 1): + try: + rel_path = match.file_path.relative_to(root) + except ValueError: + rel_path = match.file_path + lines.append(f"{i}. {rel_path}:{match.match_line_number}") + lines.append(f" {match.full_text.strip()}") + lines.append("") + + return "\n".join(lines) + + +@register_function(config_type=SourceGrepToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def source_grep(config: SourceGrepToolConfig, builder: Builder): # pylint: disable=unused-argument + from vuln_analysis.runtime_context import ctx_state + + @catch_tool_errors(SOURCE_GREP) + async def _arun(query: str) -> str: + """Search source code, build logs, or patches using native Unix grep. + + Query format: '[target:]pattern[,file_glob][ -w]' + + Targets: + - source (default): Package source code + - logs: Build compilation logs + - patch: Fixed patches from newer RPM version + + Options: + - -w: Match whole words only (word boundary) + - Multiple patterns: use ';' separator ONLY with a specific file + + Examples: + - 'archive_read_open' - search source files + - 'archive_read_open,*.c' - search only .c source files + - 'archive_read_open -w' - search for whole word only + - 'unsigned int cursor;unsigned int nodes,archive_read.c' - multiple patterns in one file + - 'logs:undefined reference' - search build logs for link errors + - 'logs:error:' - search build logs for error messages + - 'patch:CVE-2026-5121' - find patch for specific CVE + - 'patch:archive_read,*.patch' - search in patch files + """ + workflow_state = ctx_state.get() + + checker_context = None + if workflow_state.original_input and workflow_state.original_input.info: + checker_context = workflow_state.original_input.info.checker_context + + if checker_context is None or not checker_context.source_key: + raise ValueError("Checker context or source_key not available in workflow state") + + source_key = checker_context.source_key + pattern, file_glob, target, word_boundary = _parse_query(query) + + # For logs target, use arch-specific subdirectory + if target == "logs": + target_package = workflow_state.original_input.input.image.target_package + if target_package is None or not target_package.arch: + raise ValueError("logs target requires target_package.arch in workflow state") + + arch = target_package.arch + target_dir = (Path(config.base_checker_dir) / source_key / "logs" / arch).resolve() + + if not target_dir.is_dir(): + return f"No build log available for architecture '{arch}'. Build log may not have been fetched for this package." + else: + target_dir = (Path(config.base_checker_dir) / source_key / target).resolve() + + if not target_dir.is_dir(): + raise ValueError(f"Target directory does not exist: {target_dir}") + + inspector = SourceInspector(target_dir) + default_extensions = TARGET_EXTENSIONS.get(target, []) + + logger.info("Source grep: searching for '%s' in %s (target: %s, glob: %s, word_boundary: %s)", + pattern, target_dir, target, file_glob or "default extensions", word_boundary) + + matches = await inspector.grep_native( + patterns=pattern, + file_glob=file_glob, + word_boundary=word_boundary, + context_lines=config.context_lines, + max_results=config.max_results, + default_extensions=default_extensions, + ) + + logger.info("Source grep: found matches for '%s' in target '%s'", pattern, target) + return matches + + yield FunctionInfo.from_fn( + _arun, + description=( + "Fast grep search using native Unix grep. " + "Query format: '[target:]pattern[,file_glob][ -w]'. " + "Targets: 'source' (default) for package source code, " + "'logs' for build compilation logs, " + "'patch' for fixed patches from newer RPM. " + "Add ' -w' suffix for whole-word matching. " + "Multiple patterns: use ';' separator ONLY with a specific file, e.g., " + "'pattern1;pattern2,filename.c' searches for both patterns in that file. " + "Examples: 'archive_read_open' searches source, " + "'archive_read_open,*.c' searches only C source files, " + "'archive_read_open -w' searches for whole word only, " + "'unsigned int cursor;unsigned int nodes,archive_read.c' searches multiple patterns in one file, " + "'logs:undefined reference' searches build logs, " + "'patch:CVE-2026-5121' searches patch files." + ), + ) diff --git a/src/vuln_analysis/tools/source_inspector.py b/src/vuln_analysis/tools/source_inspector.py new file mode 100644 index 000000000..2df030882 --- /dev/null +++ b/src/vuln_analysis/tools/source_inspector.py @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generic filesystem utility for inspecting extracted RPM source trees. + +Provides low-level primitives (find, grep, read) that can be composed by +pipeline code or called by an LLM agent in the future. +""" + +from __future__ import annotations + +import asyncio +import re +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from nat.builder.context import Context + + +@dataclass +class GrepMatch: + file_path: Path + line_number: int + line_content: str + + +class SourceInspector: + """Filesystem inspector scoped to a root directory. + + All returned paths are absolute. The class carries no domain-specific + logic (RPM, spec, changelog); callers compose the primitives for that. + """ + + def __init__(self, source_dir: Path) -> None: + if ".." in source_dir.parts: + raise ValueError(f"Path contains traversal components: {source_dir}") + self._root = source_dir.resolve() + if not self._root.is_dir(): + raise FileNotFoundError(f"source_dir does not exist: {self._root}") + + @property + def root(self) -> Path: + return self._root + + def find_files(self, pattern: str, recursive: bool = True) -> list[Path]: + """Glob over the source tree. + + Parameters + ---------- + pattern: + Shell glob pattern, e.g. ``"*.spec"`` or ``"*.patch"``. + recursive: + If *True* use ``**/`` (deep search). + If *False* use ```` (root-level only). + """ + glob_expr = f"**/{pattern}" if recursive else pattern + return sorted(self._root.glob(glob_expr)) + + def grep_content( + self, + pattern: str, + file_path: Path | None = None, + *, + recursive: bool = False, + ) -> list[GrepMatch]: + """Search file contents for a regex *pattern*. + + Parameters + ---------- + pattern: + Regular expression (case-sensitive by default). + file_path: + If given, search that file only, or (if it is a directory) every file + in that directory (one level, regular files only). + If the path does not exist, return no matches. + If *None*, search every file under *source_dir* + (depth controlled by *recursive*). + recursive: + Only used when *file_path* is ``None``. + ``False`` searches only root-level files; ``True`` walks the tree. + """ + regex = re.compile(pattern) + matches: list[GrepMatch] = [] + + if file_path is not None: + resolved = file_path.resolve() + if resolved.is_file(): + targets = [resolved] + elif resolved.is_dir(): + targets = sorted(p for p in resolved.iterdir() if p.is_file()) + else: + targets = [] + elif recursive: + targets = sorted(p for p in self._root.rglob("*") if p.is_file()) + else: + targets = sorted(p for p in self._root.iterdir() if p.is_file()) + + for fp in targets: + try: + lines = fp.read_text(encoding="utf-8", errors="replace").splitlines() + except (OSError, UnicodeDecodeError): + continue + for idx, line in enumerate(lines, start=1): + if regex.search(line): + matches.append(GrepMatch(file_path=fp, line_number=idx, line_content=line)) + return matches + + async def grep_native( + self, + patterns: str | list[str], + file_glob: str | None = None, + *, + case_insensitive: bool = False, + word_boundary: bool = False, + context_lines: int = 0, + max_results: int = 50, + default_extensions: list[str] | None = None, + ) -> str: + """Fast grep using native Unix grep subprocess. + + Parameters + ---------- + patterns: + Search pattern(s). Can be a single string or list of patterns. + When multiple patterns are provided, matches ANY of them (OR logic). + file_glob: + Optional file pattern (e.g., ``"*.c"``, ``"*.h"``). If provided, + overrides default_extensions. + case_insensitive: + If *True*, perform case-insensitive matching (``-i`` flag). + word_boundary: + If *True*, match whole words only (``-w`` flag). + context_lines: + Lines of context around match (``-C`` flag). Default 0. + max_results: + Stop after this many matches (``-m`` flag). Default 50. + default_extensions: + List of file extensions to search when file_glob is not provided. + If *None*, searches ALL files (no --include filter). + If empty list ``[]``, searches ALL files (no --include filter). + + Returns + ------- + str + Raw grep output with matches found. + """ + cmd = ["grep", "-rn", "-I"] + + if case_insensitive: + cmd.append("-i") + if word_boundary: + cmd.append("-w") + if context_lines > 0: + cmd.extend(["-C", str(context_lines)]) + + # If file_glob contains a path (e.g., "lib/connect.c"), split into: + # - path_filter: directory portion for post-filtering results (e.g., "lib/") + # - file_glob: filename only for --include (e.g., "connect.c") + path_filter = None + if file_glob and "/" in file_glob: + path_filter = file_glob.rsplit("/", 1)[0] + "/" + file_glob = file_glob.rsplit("/", 1)[1] + + if file_glob: + cmd.extend(["--include", file_glob]) + elif default_extensions is None: + pass # No filtering - search all files (caller should pass extensions explicitly) + elif default_extensions: + for ext in default_extensions: + cmd.extend(["--include", ext]) + + cmd.extend(["-m", str(max_results)]) + + # Handle single or multiple patterns + if isinstance(patterns, list): + for p in patterns: + cmd.extend(["-e", p]) + else: + cmd.extend(["-e", patterns]) + + cmd.append(".") + + def _run_grep() -> tuple[str, int]: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + errors="replace", + cwd=self._root, + timeout=30, + ) + return result.stdout, result.returncode + + try: + tracer = Context.get() + except Exception: + tracer = None + + if tracer: + with tracer.push_active_function("grep_native", input_data={"command": shlex.join(cmd), "path_filter": path_filter}) as span: + stdout, returncode = await asyncio.to_thread(_run_grep) + span.set_output({ + "return_code": returncode, + "match_count": stdout.count('\n') if stdout else 0, + }) + else: + stdout, returncode = await asyncio.to_thread(_run_grep) + + # Post-filter results to only include paths matching path_filter + if path_filter and stdout: + filtered_lines = [line for line in stdout.splitlines() if path_filter in line] + stdout = "\n".join(filtered_lines) + if filtered_lines: + stdout += "\n" + + return stdout + + def read_file( + self, + file_path: Path, + offset: int = 0, + max_lines: int | None = None, + ) -> str: + """Read file content starting from a line *offset*. + + Parameters + ---------- + file_path: + Absolute or relative path (resolved against *source_dir*). + offset: + 0-based line offset to start reading from. + max_lines: + Maximum number of lines to return. ``None`` means read to EOF. + """ + resolved = file_path if file_path.is_absolute() else (self._root / file_path) + if not resolved.resolve().is_relative_to(self._root.resolve()): + raise ValueError(f"Path escapes root directory: {file_path}") + lines = resolved.read_text(encoding="utf-8", errors="replace").splitlines() + end = (offset + max_lines) if max_lines is not None else len(lines) + return "\n".join(lines[offset:end]) diff --git a/src/vuln_analysis/tools/tool_names.py b/src/vuln_analysis/tools/tool_names.py index f07d40d50..02125e9f2 100644 --- a/src/vuln_analysis/tools/tool_names.py +++ b/src/vuln_analysis/tools/tool_names.py @@ -65,6 +65,9 @@ class ToolNames: IMPORT_USAGE_ANALYZER = "Import Usage Analyzer" """Finds all imports and usage patterns of a specific package across indexed sources.""" + SOURCE_GREP = "Source Grep" + """Fast grep search in source code using native Unix grep""" + # Module-level constants for convenience imports CODE_SEMANTIC_SEARCH = ToolNames.CODE_SEMANTIC_SEARCH @@ -78,6 +81,7 @@ class ToolNames: FUNCTION_LIBRARY_VERSION_FINDER = ToolNames.FUNCTION_LIBRARY_VERSION_FINDER CONFIGURATION_SCANNER = ToolNames.CONFIGURATION_SCANNER IMPORT_USAGE_ANALYZER = ToolNames.IMPORT_USAGE_ANALYZER +SOURCE_GREP = ToolNames.SOURCE_GREP @@ -94,4 +98,5 @@ class ToolNames: 'FUNCTION_LIBRARY_VERSION_FINDER', 'CONFIGURATION_SCANNER', 'IMPORT_USAGE_ANALYZER', + 'SOURCE_GREP', ] diff --git a/src/vuln_analysis/utils/clients/nvd_client.py b/src/vuln_analysis/utils/clients/nvd_client.py index 1b13ff51c..fd6a43d54 100644 --- a/src/vuln_analysis/utils/clients/nvd_client.py +++ b/src/vuln_analysis/utils/clients/nvd_client.py @@ -125,19 +125,21 @@ async def _get_cwe_elements(self, cve_obj: dict) -> dict: those CWEs. """ # Get CWE name - cwe_id = None + raw_cwe_id = None weaknesses = cve_obj.get('weaknesses', []) - cwe_id = self._get_cwe(weaknesses) + raw_cwe_id = self._get_cwe(weaknesses) cwe_link = None cwe_name = None cwe_description = None cwe_extended_description = None - if cwe_id is not None: - if cwe_id.startswith('CWE-'): - cwe_id = cwe_id.replace('CWE-', '', 1) + cwe_id_numeric = None + if raw_cwe_id is not None: + cwe_id_numeric = raw_cwe_id + if cwe_id_numeric.startswith('CWE-'): + cwe_id_numeric = cwe_id_numeric.replace('CWE-', '', 1) - if cwe_id.isnumeric(): - cwe_link = self._cwe_details_url_template.format(CWE_ID=cwe_id) + if cwe_id_numeric.isnumeric(): + cwe_link = self._cwe_details_url_template.format(CWE_ID=cwe_id_numeric) if cwe_link is not None: soup = await self._get_soup(cwe_link) @@ -155,7 +157,9 @@ async def _get_cwe_elements(self, cve_obj: dict) -> dict: if extended_description_div: cwe_extended_description = extended_description_div.find('div', class_='indent').text.strip() + cwe_id = f"CWE-{cwe_id_numeric}" if cwe_id_numeric and cwe_id_numeric.isnumeric() else raw_cwe_id return { + "cwe_id": cwe_id, "cwe_name": cwe_name, "cwe_description": cwe_description, "cwe_extended_description": cwe_extended_description, @@ -330,6 +334,7 @@ async def get_intel(self, cve_id: str) -> CveIntelNvd: cvss_vector=cvss_vector, cvss_base_score=cvss_base_score, cvss_severity=cvss_severity, + cwe_id=cwe_elements["cwe_id"], cwe_name=cwe_elements["cwe_name"], cwe_description=cwe_elements["cwe_description"], cwe_extended_description=cwe_elements["cwe_extended_description"], diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 9ace44b9e..87247b67b 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -284,7 +284,7 @@ def add_documents_from_code_path(self, for root, _, files in os.walk(code_path): for file in files: - if any(file.endswith(ext) for ext in include_extensions): + if any(file.endswith(ext) for ext in include_extensions) or file in no_extension: file_path = os.path.join(root, file) try: with open(file_path, "r") as f: diff --git a/src/vuln_analysis/utils/gerrit_client.py b/src/vuln_analysis/utils/gerrit_client.py new file mode 100644 index 000000000..7a32489dd --- /dev/null +++ b/src/vuln_analysis/utils/gerrit_client.py @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gerrit API client for Chromium bug resolution. + +This module provides functions to: +- Search Chromium Gerrit for CLs associated with a bug ID +- Filter for MERGED CLs only +- Select the correct CL when multiple exist (via LLM) +- Get commit SHA from a CL for Gitiles patch fetching +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import aiohttp +from pydantic import BaseModel + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# Chromium Gerrit API base URL +GERRIT_BASE_URL = "https://chromium-review.googlesource.com" +GERRIT_TIMEOUT_SECONDS = 30 + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + +class GerritChangeCandidate(BaseModel): + """Represents a candidate Gerrit CL for LLM selection.""" + submission_id: int # _number field from Gerrit API + project: str # e.g., "angle/angle", "chromium/src" + subject: str # Commit subject line + + +class GerritChangeSelection(BaseModel): + """LLM response for selecting the correct Gerrit CL.""" + submission_id: int | None # None = no matching fix found + reason: str # Explanation for the selection + + +# --------------------------------------------------------------------------- +# Response Parsing +# --------------------------------------------------------------------------- + +def parse_gerrit_response(text: str) -> list[dict] | dict: + """Parse Gerrit API response, stripping the XSSI prevention prefix. + + Gerrit API responses start with ")]}'" to prevent XSSI attacks. + + Args: + text: Raw response text from Gerrit API + + Returns: + Parsed JSON (list for search results, dict for single item endpoints) + """ + # Strip the XSSI prefix if present + if text.startswith(")]}'"): + text = text[4:].lstrip() + + return json.loads(text) + + +def list_merged_changes(raw_changes: list[dict]) -> list[GerritChangeCandidate]: + """Filter Gerrit changes to only MERGED status. + + Args: + raw_changes: List of change dicts from Gerrit API + + Returns: + List of GerritChangeCandidate for MERGED CLs only + """ + candidates = [] + has_seen = set() + dates = [] + for change in raw_changes: + if change.get("status") == "MERGED": + if change.get("_number") not in has_seen: + + candidates.append(GerritChangeCandidate( + submission_id=change.get("_number", 0), + project=change.get("project", ""), + subject=change.get("subject", ""), + )) + has_seen.add(change.get("_number")) + dates.append(change.get("updated")) + return candidates + + +# --------------------------------------------------------------------------- +# Gerrit API Functions +# --------------------------------------------------------------------------- + +async def search_changes_by_bug( + session: aiohttp.ClientSession, + bug_id: str, + n: int = 25, +) -> list[dict]: + """Search Gerrit for CLs associated with a Chromium bug ID. + + Args: + session: aiohttp session + bug_id: Chromium bug ID (digits only) + n: Maximum number of results to return + + Returns: + List of change dicts from Gerrit API + """ + url = f"{GERRIT_BASE_URL}/changes/" + params = { + "q": f"bug:{bug_id}", + "n": str(n), + } + + timeout = aiohttp.ClientTimeout(total=GERRIT_TIMEOUT_SECONDS) + try: + async with request_with_retry( + session=session, + request_kwargs={ + "method": "GET", + "url": url, + "params": params, + "timeout": timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + text = await response.text() + data = parse_gerrit_response(text) + # Search endpoint returns a list + return data if isinstance(data, list) else [] + except aiohttp.ClientResponseError as e: + logger.warning("Gerrit search failed for bug %s: %s", bug_id, e) + return [] + except Exception as e: + logger.warning("Gerrit search error for bug %s: %s", bug_id, e) + return [] + + +async def get_current_commit_sha( + session: aiohttp.ClientSession, + change_number: int, +) -> str | None: + """Get the commit SHA for the current revision of a Gerrit CL. + + Args: + session: aiohttp session + change_number: Gerrit change number (_number field) + + Returns: + Commit SHA string, or None if not found + """ + url = f"{GERRIT_BASE_URL}/changes/{change_number}/revisions/current/commit" + + timeout = aiohttp.ClientTimeout(total=GERRIT_TIMEOUT_SECONDS) + try: + async with request_with_retry( + session=session, + request_kwargs={ + "method": "GET", + "url": url, + "timeout": timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + text = await response.text() + data = parse_gerrit_response(text) + if isinstance(data, dict): + return data.get("commit") + return None + except aiohttp.ClientResponseError as e: + logger.warning("Failed to get commit SHA for CL %d: %s", change_number, e) + return None + except Exception as e: + logger.warning("Error getting commit SHA for CL %d: %s", change_number, e) + return None + + +# --------------------------------------------------------------------------- +# Gitiles URL Building +# --------------------------------------------------------------------------- + +def project_to_gitiles_repo_url(project: str) -> str: + """Convert Gerrit project path to Gitiles repository URL. + + Args: + project: Gerrit project path (e.g., "angle/angle", "chromium/src") + + Returns: + Full Gitiles repository URL + """ + return f"https://chromium.googlesource.com/{project}" + + +def build_gitiles_patch_url(repo_url: str, sha: str) -> str: + """Build a Gitiles URL for fetching a patch in TEXT format. + + The ^! suffix means "this commit only" (not including parents). + format=TEXT returns base64-encoded patch content. + + Args: + repo_url: Gitiles repository URL + sha: Commit SHA + + Returns: + Gitiles patch URL + """ + # URL-encode the ^! as %5E%21 + return f"{repo_url}/+/{sha}%5E%21?format=TEXT" + + +# --------------------------------------------------------------------------- +# LLM Selection +# --------------------------------------------------------------------------- + +def build_gerrit_cl_select_prompt( + candidates: list[GerritChangeCandidate], + cve_id: str, + cve_description: str, +) -> str: + """Build prompt for LLM to select the correct Gerrit CL. + + Args: + candidates: List of MERGED CL candidates + cve_id: CVE identifier + cve_description: CVE description for context + + Returns: + Formatted prompt string + """ + candidate_list = "\n".join( + f"- submission_id: {c.submission_id}, project: {c.project}, subject: {c.subject}" + for c in candidates + ) + + return f"""You are a security researcher analyzing Chromium CLs to find the fix for a CVE. + +CVE: {cve_id} +Description: {cve_description} + +The following MERGED Chromium CLs are associated with the bug tracker issue. +Select the one that implements the security fix. + +Candidates: +{candidate_list} + +Guidelines: +1. REJECT CLs with subjects starting with "Roll" - these are dependency updates, not fixes +2. REJECT CLs from "chromium/src" that only update DEPS - these are roll commits +3. PREFER CLs from upstream component projects (angle/angle, WebKit, etc.) + when the CVE mentions ANGLE, WebKit, etc. +4. Look for CLs with subjects mentioning the fix (bounds checks, memory handling, etc.) +5. If none of the CLs appear to be the actual security fix, return submission_id: null + +Return ONLY the submission_id of the best matching CL, or null if none are appropriate.""" + + +async def select_gerrit_change( + candidates: list[GerritChangeCandidate], + cve_id: str, + cve_description: str, + llm: "BaseChatModel | None" = None, +) -> int | None: + """Select the correct Gerrit CL from multiple MERGED candidates. + + Args: + candidates: List of MERGED CL candidates + cve_id: CVE identifier + cve_description: CVE description for context + llm: Optional LangChain LLM (base model, not pre-configured with structured output) + + Returns: + submission_id of selected CL, or None if no valid selection + """ + if not candidates: + return None + + if len(candidates) == 1: + # Only one candidate - return it directly + return candidates[0].submission_id + + if llm is None: + # No LLM available - apply heuristics + # Filter out Roll commits and chromium/src DEPS updates + non_roll = [ + c for c in candidates + if not c.subject.lower().startswith("roll ") + and not (c.project == "chromium/src" and "roll" in c.subject.lower()) + ] + + if len(non_roll) == 1: + return non_roll[0].submission_id + + # Prefer upstream component projects over chromium/src + upstream = [c for c in non_roll if c.project != "chromium/src"] + if len(upstream) == 1: + return upstream[0].submission_id + + # Can't decide without LLM - return None to fail safely + logger.warning( + "Multiple MERGED CLs for %s, no LLM available for selection: %s", + cve_id, + [c.submission_id for c in candidates] + ) + return None + + # Use LLM for selection with structured output + prompt = build_gerrit_cl_select_prompt(candidates, cve_id, cve_description) + + try: + # Configure LLM with structured output for GerritChangeSelection + gerrit_llm = llm.with_structured_output(GerritChangeSelection) + result = await gerrit_llm.ainvoke(prompt) + # with_structured_output guarantees the response type matches the schema + selection = result if isinstance(result, GerritChangeSelection) else GerritChangeSelection.model_validate(result) + + # Validate selection is in candidate list + if selection.submission_id is None: + logger.info("LLM returned no selection for %s: %s", cve_id, selection.reason) + return None + + valid_ids = {c.submission_id for c in candidates} + if selection.submission_id not in valid_ids: + logger.warning( + "LLM selected invalid submission_id %d for %s (valid: %s)", + selection.submission_id, cve_id, valid_ids + ) + return None + + logger.info( + "LLM selected CL %d for %s: %s", + selection.submission_id, cve_id, selection.reason + ) + return selection.submission_id + + except Exception as e: + logger.warning("LLM selection failed for %s: %s", cve_id, e) + return None diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index f7e6175ca..6a47db78b 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -22,10 +22,11 @@ from pydpkg import Dpkg from pydpkg.exceptions import DpkgVersionError -from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd,CveIntel from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path + from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) @@ -600,6 +601,84 @@ def _strip_rejected_package_token(entry: str, rn: str) -> str: return re.sub(_package_token_boundary_pattern(rn), "", entry) +_COMMIT_URL_KEYWORDS = frozenset({ + "github.com/", "gitlab.com/", "gitlab.", "bitbucket.org/", + "/commit/", "/commits/", "/pull/", "/merge_requests/", + ".git", "git.kernel.org", "git.savannah", "cgit", +}) + +# Chromium issue tracker URL pattern - captures the bug ID +CHROMIUM_ISSUE_PATTERN = re.compile(r"https?://issues\.chromium\.org/issues/(\d+)") + + +def extract_commit_url_candidates(intel: CveIntel) -> dict[str, list[str]]: + """Extract URLs from intel references that may contain commit/patch information. + + Scans GHSA, NVD, RHSA, and Ubuntu references for URLs containing keywords + that suggest they may point to source code commits or patches. Also includes + Chromium issue tracker URLs (issues.chromium.org/issues/) which can be + resolved to patches via Gerrit/Gitiles. + + Args: + intel: CveIntel object containing intel from various providers + + Returns: + Dict mapping provider name to list of matching URLs. + Example: {"ghsa": ["https://github.com/foo/bar/commit/abc123"], "nvd": []} + """ + + def _matches_keywords(url: str) -> bool: + url_lower = url.lower() + return any(kw in url_lower for kw in _COMMIT_URL_KEYWORDS) + + def _is_chromium_issue(url: str) -> bool: + return bool(CHROMIUM_ISSUE_PATTERN.match(url)) + + def _extract_refs(refs: list[str] | None) -> list[str]: + if not refs: + return [] + return [r for r in refs if isinstance(r, str) and (_matches_keywords(r) or _is_chromium_issue(r))] + + result: dict[str, list[str]] = {} + + # GHSA references (extra field, may not exist) + if intel.ghsa: + ghsa_refs = getattr(intel.ghsa, "references", None) + result["ghsa"] = _extract_refs(ghsa_refs) + + # NVD references (explicit field) + if intel.nvd: + result["nvd"] = _extract_refs(intel.nvd.references) + + # RHSA references (extra field, may not exist) + if intel.rhsa: + rhsa_refs = getattr(intel.rhsa, "references", None) + # RHSA references may be newline-separated strings + if isinstance(rhsa_refs, list): + flat_refs = [] + for ref in rhsa_refs: + if isinstance(ref, str) and "\n" in ref: + flat_refs.extend(ref.split("\n")) + else: + flat_refs.append(ref) + result["rhsa"] = _extract_refs(flat_refs) + else: + result["rhsa"] = [] + + # Ubuntu references (explicit field) + if intel.ubuntu: + ubuntu_refs = getattr(intel.ubuntu, "references", None) + result["ubuntu"] = _extract_refs(ubuntu_refs) + # Also include patches field URLs + if intel.ubuntu.patches: + patch_urls = [] + for pkg_patches in intel.ubuntu.patches.values(): + patch_urls.extend(pkg_patches or []) + result["ubuntu_patches"] = _extract_refs(patch_urls) + + return result + + def filter_context_to_package(critical_context: list[str], selected: str, all_candidates: list[dict]) -> list[str]: """Narrow CVE intel strings to the user-selected package after disambiguation. Intel lines are built from GHSA, RHSA... and may mention several modules; this removes diff --git a/src/vuln_analysis/utils/osv_patch_retriever.py b/src/vuln_analysis/utils/osv_patch_retriever.py new file mode 100644 index 000000000..4eaaaf105 --- /dev/null +++ b/src/vuln_analysis/utils/osv_patch_retriever.py @@ -0,0 +1,570 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OSV Patch Retriever - fetch upstream fix patches from OSV when RPM patches are unavailable. + +.. deprecated:: + This module is deprecated. Use :mod:`vuln_analysis.utils.web_patch_fetcher` instead. + The new module provides a unified interface for fetching patches from: + - Intel references (GHSA, NVD, RHSA, Ubuntu) + - OSV API + - kernel.org (in addition to GitHub) + + Migration guide: + - Replace `OSVPatchRetriever` with `OSVClient` from `web_patch_fetcher` + - Replace `fetch_ubuntu_patch` with `WebPatchFetcher.fetch_from_intel_refs` +""" + +import warnings + +warnings.warn( + "osv_patch_retriever is deprecated, use web_patch_fetcher instead", + DeprecationWarning, + stacklevel=2, +) + +from __future__ import annotations + +import os +import re +from typing import TYPE_CHECKING + +import aiohttp +from pydantic import BaseModel +from unidiff import PatchSet + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry +from vuln_analysis.functions.code_agent_graph_defs import OSVPatchResult + +if TYPE_CHECKING: + from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch + +logger = LoggingFactory.get_agent_logger(__name__) + +_OSV_API_URL = os.environ.get("OSV_API_URL", "https://api.osv.dev/v1/vulns/") +_OSV_TIMEOUT_SECONDS = int(os.environ.get("OSV_TIMEOUT_SECONDS", "10")) +_GITHUB_PATCH_TIMEOUT_SECONDS = int(os.environ.get("GITHUB_PATCH_TIMEOUT_SECONDS", "30")) + +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu', '.uue', '.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', +}) + +_GITHUB_REPO_PATTERN = re.compile(r"https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$") + + +class OSVAffectedRange(BaseModel): + """Represents a Git range from an OSV affected block.""" + repo_url: str | None = None + fixed_commit: str | None = None + introduced_commit: str | None = None + + + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def _version_in_range(version: str, introduced: str | None, fixed: str | None) -> bool: + """Check if provided upstream version falls within [introduced, fixed) range. + + Returns True if version >= introduced (or introduced is None) AND version < fixed (or fixed is None). + """ + try: + from packaging.version import parse as parse_version + v = parse_version(version) + if introduced: + try: + if v < parse_version(introduced): + return False + except Exception: + pass + if fixed: + try: + if v >= parse_version(fixed): + return False + except Exception: + pass + return True + except Exception: + return True + + +def _parse_patch_content(patch_content: str, patch_filename: str) -> "ParsedPatch | None": + """Parse patch content string into structured ParsedPatch model. + + Reuses the same logic as code_agent_graph_defs.parse_patch_file but works on string content. + """ + from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile, PatchHunk + + try: + patch_set = PatchSet.from_string(patch_content) + except Exception: + logger.warning("_parse_patch_content: failed to parse patch content") + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_filename, files=files) + + +def _extract_commit_metadata(patch_content: str) -> tuple[str | None, str | None, str | None]: + """Extract commit message, author, and date from GitHub .patch format. + + GitHub .patch format starts with: + From Mon Sep 17 00:00:00 2001 + From: Author Name + Date: Tue, 1 Jan 2024 12:00:00 +0000 + Subject: [PATCH] Commit message + + Extended commit message... + --- + + """ + lines = patch_content.split('\n') + author = None + date = None + subject_lines = [] + in_subject = False + + for line in lines: + if line.startswith('From:'): + author = line[5:].strip() + elif line.startswith('Date:'): + date = line[5:].strip() + elif line.startswith('Subject:'): + in_subject = True + subject_part = line[8:].strip() + if subject_part.startswith('[PATCH'): + idx = subject_part.find(']') + if idx != -1: + subject_part = subject_part[idx + 1:].strip() + subject_lines.append(subject_part) + elif in_subject: + if line.startswith('---') or line.startswith('diff --git'): + break + if line.strip() == '': + in_subject = False + else: + subject_lines.append(line.strip()) + + commit_message = ' '.join(subject_lines).strip() if subject_lines else None + return commit_message, author, date + + +_UBUNTU_PATCH_URL_PATTERN = re.compile( + r"(?:upstream:\s*)?(https://github\.com/[^/]+/[^/]+/commit/([a-f0-9]+))" +) + + +async def fetch_ubuntu_patch( + session: aiohttp.ClientSession, + cve_id: str, + package_name: str, + patch_refs: list[str], + timeout: int = _GITHUB_PATCH_TIMEOUT_SECONDS, +) -> OSVPatchResult | None: + """Fetch patch from Ubuntu intel patches field. + + .. deprecated:: + Use :class:`vuln_analysis.utils.web_patch_fetcher.WebPatchFetcher.fetch_from_intel_refs` instead. + + Parses Ubuntu patch refs like: "upstream: https://github.com/curl/curl/commit/39d1976b7f..." + + Args: + session: aiohttp ClientSession for HTTP requests + cve_id: CVE identifier (e.g., "CVE-2024-1234") + package_name: Package name (for logging) + patch_refs: List of patch reference strings from Ubuntu intel + timeout: Timeout for GitHub fetch in seconds + + Returns: + OSVPatchResult with parsed patch data, or None if no valid patch found + """ + github_timeout = aiohttp.ClientTimeout(total=timeout) + + for patch_ref in patch_refs: + match = _UBUNTU_PATCH_URL_PATTERN.search(patch_ref) + if not match: + logger.debug("Ubuntu patch ref does not match GitHub commit pattern: %s", patch_ref) + continue + + commit_url = match.group(1) + commit_sha = match.group(2) + patch_url = f"{commit_url}.patch" + repo_url = commit_url.rsplit("/commit/", 1)[0] + + logger.info("Ubuntu: Fetching patch for %s from %s", cve_id, patch_url) + + try: + async with request_with_retry( + session=session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': github_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + patch_content = await response.text() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("Ubuntu: GitHub patch not found: %s", patch_url) + else: + logger.warning("Ubuntu: GitHub patch fetch failed: %s - %s", patch_url, e) + continue + except Exception as e: + logger.warning("Ubuntu: GitHub patch fetch failed: %s - %s", patch_url, e) + continue + + if not patch_content: + continue + + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content(patch_content, f"{cve_id}_{commit_sha[:8]}.patch") + + if parsed_patch: + logger.info("Ubuntu: Successfully fetched and parsed patch for %s", cve_id) + return OSVPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8], + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + ) + + logger.info("Ubuntu: No valid GitHub commit patch found for %s in %d refs", cve_id, len(patch_refs)) + return None + + +class OSVPatchRetriever: + """Retrieve upstream fix patches from OSV when RPM patches are unavailable. + + .. deprecated:: + Use :class:`vuln_analysis.utils.web_patch_fetcher.OSVClient` instead. + The new class provides the same functionality with additional support + for kernel.org and better integration with intel references. + + Usage: + async with aiohttp.ClientSession() as session: + retriever = OSVPatchRetriever(session=session) + result = await retriever.get_fix_patch("CVE-2024-1234", "3.0.7", "openssl") + if result and result.parsed_patch: + # Use result.parsed_patch for agent context + pass + """ + + def __init__( + self, + session: aiohttp.ClientSession, + osv_timeout: int = _OSV_TIMEOUT_SECONDS, + github_timeout: int = _GITHUB_PATCH_TIMEOUT_SECONDS, + ): + warnings.warn( + "OSVPatchRetriever is deprecated, use OSVClient from web_patch_fetcher instead", + DeprecationWarning, + stacklevel=2, + ) + self._session = session + self._osv_timeout = aiohttp.ClientTimeout(total=osv_timeout) + self._github_timeout = aiohttp.ClientTimeout(total=github_timeout) + + async def get_fix_patch( + self, + cve_id: str, + upstream_version: str, + package_name: str | None = None, + ) -> OSVPatchResult | None: + """Main entry point - orchestrates the full workflow. + + Args: + cve_id: CVE identifier (e.g., "CVE-2024-1234") + upstream_version: Upstream version from TargetPackage.version (e.g., "3.0.7") + package_name: Optional package name to help match the correct affected block + + Returns: + OSVPatchResult with patch data, or None if no fix found + """ + try: + osv_data = await self._query_osv(cve_id) + if not osv_data: + return None + + # 1. Try to get the highly-specific patch URL from references first + patch_url = self._extract_commit_from_references(osv_data) + fixed_commit = None + repo_url = None + if patch_url: + # Extract repo_url and fixed_commit from the patch_url for the result object + repo_url = patch_url.split('/commit/')[0] if '/commit/' in patch_url else patch_url.split('/pull/')[0] + fixed_commit = patch_url.split('/')[-1].replace('.patch', '') + logger.info("OSV: Found precise fix commit in references for %s", cve_id) + else: + # second try to find the fix commit from the affected block + affected = self._find_matching_affected(osv_data, package_name) + if not affected: + logger.info("OSV: No affected block with fix found for %s", cve_id) + return None + + range_info = self._extract_fix_commit(affected) + if not range_info.fixed_commit or not range_info.repo_url: + logger.info("OSV: No fixed commit found for %s", cve_id) + return None + + patch_url = self._build_patch_url(range_info.repo_url, range_info.fixed_commit) + fixed_commit = range_info.fixed_commit[:8] + repo_url = range_info.repo_url + if not patch_url: + logger.info("OSV: Could not build patch URL for %s (non-GitHub repo?)", cve_id) + return None + + patch_content = await self._fetch_github_patch(patch_url) + if not patch_content: + return None + + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content(patch_content, f"{cve_id}_{fixed_commit}.patch") + + return OSVPatchResult( + cve_id=cve_id, + fixed_commit=fixed_commit, + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + ) + + except Exception: + logger.warning("OSV patch retrieval failed for %s", cve_id, exc_info=True) + return None + + async def _query_osv(self, cve_id: str) -> dict | None: + """Query OSV API for CVE data. + + Args: + cve_id: CVE identifier + + Returns: + OSV vulnerability data dict, or None on failure + """ + url = f"{_OSV_API_URL}{cve_id}" + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': url, + 'timeout': self._osv_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.json() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("OSV: CVE %s not found", cve_id) + else: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + except Exception as e: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + + def _extract_commit_from_references(self, osv_data: dict) -> str | None: + """Attempt to find the exact fix commit URL from the OSV references array. + + Args: + osv_data: OSV vulnerability data dict + + Returns: + The patch URL if found, otherwise None + """ + references = osv_data.get("references", []) + + for ref in references: + if ref.get("type") == "FIX": + url = ref.get("url", "") + # We look for GitHub URLs containing either /commit/ or /pull/ + if "github.com" in url and ("/commit/" in url or "/pull/" in url): + if not url.endswith(".patch"): + return f"{url}.patch" + return url + + return None + + def _find_matching_affected( + self, + osv_data: dict, + package_name: str | None = None, + ) -> dict | None: + """Find an affected block that has a GIT range with a fixed commit. + + Args: + osv_data: OSV vulnerability data + package_name: Optional package name to filter affected blocks + + Returns: + Matching affected block dict, or None if no match + """ + + + for affected in osv_data.get("affected", []): + + + for range_block in affected.get("ranges", []): + if range_block.get("type") == "GIT": + for event in range_block.get("events", []): + if "fixed" in event: + return affected + + return None + + def _extract_fix_commit(self, affected: dict) -> OSVAffectedRange: + """Extract the fixed commit hash and repo URL from an affected block. + + Args: + affected: OSV affected block + + Returns: + OSVAffectedRange with repo_url and fixed_commit + """ + result = OSVAffectedRange() + + ranges = affected.get("ranges", []) + for range_block in ranges: + if range_block.get("type") != "GIT": + continue + + repo = range_block.get("repo") + if repo: + result.repo_url = repo + + events = range_block.get("events", []) + for event in events: + if "introduced" in event and event["introduced"] != "0": + result.introduced_commit = event["introduced"] + if "fixed" in event: + result.fixed_commit = event["fixed"] + + if result.fixed_commit: + break + + return result + + def _build_patch_url(self, repo_url: str, commit_sha: str) -> str | None: + """Build GitHub patch URL from repo URL and commit SHA. + + Args: + repo_url: Git repository URL (e.g., "https://github.com/openssl/openssl") + commit_sha: Git commit hash + + Returns: + Patch URL (e.g., "https://github.com/openssl/openssl/commit/.patch"), + or None if not a GitHub repo + """ + match = _GITHUB_REPO_PATTERN.match(repo_url) + if not match: + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + repo_path = '/'.join(parts[-2:]).replace('.git', '') + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + logger.debug("Non-GitHub repo URL: %s", repo_url) + return None + + repo_path = match.group(1) + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + + async def _fetch_github_patch(self, patch_url: str) -> str | None: + """Download patch content from GitHub. + + Args: + patch_url: URL to the .patch file + + Returns: + Patch content string, or None on failure + """ + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': self._github_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.text() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("GitHub patch not found: %s", patch_url) + else: + logger.warning("GitHub patch fetch failed: %s - %s", patch_url, e) + return None + except Exception as e: + logger.warning("GitHub patch fetch failed: %s - %s", patch_url, e) + return None diff --git a/src/vuln_analysis/utils/output_formatter.py b/src/vuln_analysis/utils/output_formatter.py index 5bbbe5e60..6d3acf591 100644 --- a/src/vuln_analysis/utils/output_formatter.py +++ b/src/vuln_analysis/utils/output_formatter.py @@ -109,7 +109,7 @@ def _add_header(markdown_content, model_dict: AgentMorpheusOutput): markdown_content[cve_id].append(f"# Vulnerability Analysis Report for {cve_id}") markdown_content[cve_id].append(f"> **Container Analyzed:** `{input_image.name}:{input_image.tag}`\n\n") # Only add SBOM info if it is a file location - if input_image.sbom_info.type == "file": + if input_image.sbom_info and input_image.sbom_info.type == "file": markdown_content[cve_id].append(f"> **SBOM Info:** `{input_image.sbom_info}`\n\n") markdown_content[cve_id].append(f"> **Status:** {_get_expoiltability_text(output.justification.status)}") diff --git a/src/vuln_analysis/utils/package_identifier.py b/src/vuln_analysis/utils/package_identifier.py new file mode 100644 index 000000000..42346d2ab --- /dev/null +++ b/src/vuln_analysis/utils/package_identifier.py @@ -0,0 +1,495 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from univers import versions + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult, PackageCheckerStatus, PackageIdentifyResult +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.string_utils import package_names_match +from exploit_iq_commons.data_models.common import TargetPackage + +logger = LoggingFactory.get_agent_logger(__name__) + +# NEVRA = Name-Epoch-Version-Release-Architecture, the standard RPM package naming format. +_RPM_NEVRA_RE = re.compile(r"^(.+?)-(?:(\d+):)?(\d\S*?)-(\S+)$") +_DIST_TAG_RE = re.compile(r"(el\d+)") +_ARCH_SUFFIXES = frozenset({"x86_64", "aarch64", "i686", "noarch", "s390x", "ppc64le", "armv7hl", "src"}) + + +def _strip_arch_suffix(release_arch: str) -> str: + """Remove .arch suffix if present, preserving dist tags like .el6_10.""" + if "." in release_arch: + base, suffix = release_arch.rsplit(".", 1) + if suffix in _ARCH_SUFFIXES: + return base + return release_arch + + +def _extract_dist_tag(release: str) -> str | None: + """Extract the RHEL dist-tag family (e.g. 'el8') from a release string.""" + m = _DIST_TAG_RE.search(release) + return m.group(1) if m else None + + +_RHEL_VERSION_RE = re.compile(r"el(\d+)") + +_AFFECTED_FIX_STATES = frozenset({ + "affected", "fix deferred", "under investigation", +}) +_NOT_AFFECTED_FIX_STATES = frozenset({ + "not affected", + "will not fix", + "out of support scope", +}) + + +def _extract_rhel_version(distro_tag: str | None) -> str | None: + """Extract RHEL major version from dist-tag (e.g., 'el7' -> '7', 'el10' -> '10').""" + if not distro_tag: + return None + m = _RHEL_VERSION_RE.match(distro_tag) + return m.group(1) if m else None + + +def _match_package_state_for_distro( + package_states: list, + target_name: str, + target_distro: str | None, +): + """Find the PackageState entry matching target package name and distro. + + Returns the matching PackageState or None if no match found. + """ + rhel_version = _extract_rhel_version(target_distro) + + for ps in package_states: + if not ps.package_name or not package_names_match(target_name, ps.package_name): + continue + if rhel_version is None: + return ps # No distro info, return first name match + # Match by CPE (e.g., "cpe:/o:redhat:enterprise_linux:7") + if ps.cpe and f":enterprise_linux:{rhel_version}" in ps.cpe: + return ps + # Match by product_name (e.g., "Red Hat Enterprise Linux 7") + if ps.product_name and re.search(rf"\b{rhel_version}\b", ps.product_name): + return ps + return None + + +def _interpret_fix_state(fix_state: str | None) -> EnumIdentifyResult | None: + """Interpret RHSA fix_state into an identification result (PackageIdentify step 2). + + Returns: + EnumIdentifyResult.NO for not affected / will not fix / out of support scope + EnumIdentifyResult.YES for affected, fix deferred, under investigation + None if fix_state is unknown or missing (fall through to NVD checks) + """ + if not fix_state: + return None + state = fix_state.lower().strip() + if state in _NOT_AFFECTED_FIX_STATES: + return EnumIdentifyResult.NO + if state in _AFFECTED_FIX_STATES: + return EnumIdentifyResult.YES + return None # Unknown state, fall through to other checks + + +class PackageIdentifier: + """ + Deterministic PackageIdentify phase: resolves package identity from intel, + cross-references the SBOM, checks version ranges, and locates RPMs in cache. + """ + + def __init__( + self, + target_package: TargetPackage, + ): + self._target_package = target_package + + + def identify(self, intel: CveIntel | None) -> tuple[PackageCheckerStatus, PackageIdentifyResult]: + """Run PackageIdentify for a single CVE. + + Step 1 — RHSA scope: target must appear in ``package_state`` or + ``affected_release`` when Red Hat published either list; else + ``PKG_IDENT_CVE_MISMATCH``. + Step 2 — Vulnerability posture: ``PKG_IDENT_NOT_VUL`` when RHSA/NVD + shows the target is not affected or already fixed. + """ + + package_identify = PackageIdentifyResult() + status = PackageCheckerStatus.OK + if intel is None: + status = PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL + return status, package_identify + + if not self._is_cve_for_target_package(intel): + status = PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH + package_identify.is_target_package_affected = EnumIdentifyResult.NO + return status, package_identify + + package_identify.is_target_package_affected = self._is_target_package_affected(intel,package_identify) + + package_identify.is_target_package_fixed = self._is_target_package_fixed(intel,package_identify) + + if package_identify.is_target_package_affected == EnumIdentifyResult.NO or package_identify.is_target_package_fixed == EnumIdentifyResult.YES: + status = PackageCheckerStatus.PKG_IDENT_NOT_VUL + + return status, package_identify + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _find_and_locate_rpm(self, intel: CveIntel) -> list[str]: + """Extract deduplicated RPM package names from RHSA package_state.""" + packages = self._extract_rhsa(intel) + packages = [p for p in packages if "/" not in p.get("package_name", "/")] + seen: set[str] = set() + names: list[str] = [] + for pkg in packages: + name = pkg.get("package_name") + if name and name not in seen: + seen.add(name) + names.append(name) + return names + + def _is_cve_for_target_package(self, intel: CveIntel) -> bool: + """Step 1: target is in scope for this CVE per Red Hat package lists. + + Returns True if RHSA has no package lists or the target appears in + ``package_state`` (any fix_state) or ``affected_release`` (patched builds). + Returns False when RHSA lists packages but the target matches neither bucket. + """ + if not intel.rhsa: + return True + + has_package_state = bool(intel.rhsa.package_state) + has_affected_release = bool( + getattr(intel.rhsa, "affected_release", None) + ) + if not has_package_state and not has_affected_release: + return True + + target_name = self._target_package.name + if has_package_state: + for ps in intel.rhsa.package_state: + if ps.package_name and package_names_match(target_name, ps.package_name): + return True + if has_affected_release and self._target_in_affected_release(intel): + return True + return False + + def _target_in_affected_release(self, intel: CveIntel) -> bool: + """True when target package name matches an RHSA affected_release NEVRA.""" + target_name = self._target_package.name + for entry in intel.rhsa.affected_release or []: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if "/" in name: + continue + if package_names_match(target_name, name): + return True + return False + + def _is_target_package_affected( + self, intel: CveIntel, package_identify: PackageIdentifyResult, + ) -> EnumIdentifyResult: + """Determine whether the target package is affected by this CVE. + + Priority 1: Check RHSA fix_state (distro-specific vendor assessment). + Priority 2: Fall back to NVD version range check. + Only returns NO with definitive proof; defaults to UNKNOWN otherwise. + """ + rpm_names = self._find_and_locate_rpm(intel) + if not rpm_names: + return EnumIdentifyResult.UNKNOWN + package_identify.affected_rpm_list = rpm_names + + target_name = self._target_package.name + target_version = self._target_package.version + target_release = self._target_package.release + target_distro = _extract_dist_tag(target_release) if target_release else None + + # Priority 1: Check RHSA fix_state (distro-specific vendor assessment) + if intel.rhsa and intel.rhsa.package_state: + matched_ps = _match_package_state_for_distro( + intel.rhsa.package_state, target_name, target_distro + ) + if matched_ps: + result = _interpret_fix_state(matched_ps.fix_state) + if result is not None: + logger.debug( + "RHSA fix_state=%s for %s on %s -> %s", + matched_ps.fix_state, target_name, target_distro, result.value + ) + if result == EnumIdentifyResult.NO: + package_identify.conclusion_reason = ( + f"RHSA fix_state indicates package is not vulnerable for analysis. " + f"Package: {target_name}, Distro: {target_distro or 'unknown'}, " + f"fix_state: '{matched_ps.fix_state}'" + ) + return result + + # Priority 2: Fall back to NVD version range check + name_matched = any(package_names_match(target_name, name) for name in rpm_names) + + if name_matched: + if target_version: + in_range = self._version_in_affected_range(target_version, intel) + if not in_range: + version_range_str = self._format_nvd_version_range(intel, target_name) + package_identify.conclusion_reason = ( + f"Target version is outside NVD affected version range. " + f"Package: {target_name}-{target_version}, " + f"Affected range: {version_range_str}" + ) + return EnumIdentifyResult.NO + return EnumIdentifyResult.YES + return EnumIdentifyResult.YES + + if target_version and intel.nvd and intel.nvd.configurations: + in_range = self._version_in_affected_range(target_version, intel) + if not in_range: + version_range_str = self._format_nvd_version_range(intel, target_name) + package_identify.conclusion_reason = ( + f"Target version is outside NVD affected version range. " + f"Package: {target_name}-{target_version}, " + f"Affected range: {version_range_str}" + ) + return EnumIdentifyResult.NO + return EnumIdentifyResult.UNKNOWN + + return EnumIdentifyResult.UNKNOWN + + def _format_nvd_version_range(self, intel: CveIntel, target_name: str) -> str: + """Format NVD version range for human-readable output.""" + if intel.nvd is None or not intel.nvd.configurations: + return "unknown" + + ranges = [] + for config in intel.nvd.configurations: + if not package_names_match(target_name, config.package): + continue + parts = [] + if config.versionStartIncluding: + parts.append(f">={config.versionStartIncluding}") + if config.versionStartExcluding: + parts.append(f">{config.versionStartExcluding}") + if config.versionEndIncluding: + parts.append(f"<={config.versionEndIncluding}") + if config.versionEndExcluding: + parts.append(f"<{config.versionEndExcluding}") + if parts: + ranges.append(" && ".join(parts)) + + return " OR ".join(ranges) if ranges else "any version (no range specified)" + + def _is_target_package_fixed(self, intel: CveIntel, package_identify: PackageIdentifyResult) -> EnumIdentifyResult: + """Determine whether the target package is already running the fixed version. + + Task 1: populate fixed_rpm_list from RHSA affected_release. + Task 2: compare target version+release against fix NVR. + """ + fix_entries = self._extract_fixed_rpms(intel) + if not fix_entries: + return EnumIdentifyResult.UNKNOWN + package_identify.fixed_rpm_list = [e["nevra"] for e in fix_entries] + + target_name = self._target_package.name + matching = [e for e in fix_entries if package_names_match(target_name, e["name"])] + if not matching: + return EnumIdentifyResult.UNKNOWN + + # NOTE: Version comparison disabled to test Option A (rely entirely on Verify phase). + # fixed_rpm_list is still populated for reference/logging. + # To re-enable, uncomment the block below. + return EnumIdentifyResult.UNKNOWN + + # --- DISABLED: Version comparison logic --- + # target_version = self._target_package.version + # target_release = self._target_package.release + # + # fix = matching[0] + # try: + # target_nvr = f"{target_version}-{target_release}" + # fix_nvr = f"{fix['version']}-{fix['release']}" + # + # target_dist = _extract_dist_tag(target_release) if target_release else None + # fix_dist = _extract_dist_tag(fix["release"]) + # if target_dist and fix_dist and target_dist != fix_dist: + # logger.debug( + # "Cross-stream fix comparison skipped: target=%s fix=%s", + # target_dist, fix_dist, + # ) + # return EnumIdentifyResult.UNKNOWN + # + # if versions.RpmVersion(target_nvr) >= versions.RpmVersion(fix_nvr): + # package_identify.conclusion_reason = ( + # f"Target package version is at or above the fix version. " + # f"Target: {target_name}-{target_nvr}, Fix: {fix_nvr}" + # ) + # return EnumIdentifyResult.YES + # return EnumIdentifyResult.NO + # except Exception as exc: + # logger.debug("Fix version comparison failed: %s", exc) + # return EnumIdentifyResult.UNKNOWN + + + def _version_in_affected_range(self, target_version: str, intel: CveIntel) -> bool: + """Check if target_version falls within any NVD configuration affected range.""" + if intel.nvd is None or not intel.nvd.configurations: + return True # no range data -> conservatively assume affected + + target_name = self._target_package.name + matched_any_config = False + for config in intel.nvd.configurations: + if not package_names_match(target_name, config.package): + continue + matched_any_config = True + version_range = [ + config.versionStartExcluding, + config.versionEndExcluding, + config.versionStartIncluding, + config.versionEndIncluding, + ] + if all(v is None for v in version_range): + continue + try: + if self._check_version_in_range(target_version, version_range): + return True + except Exception as exc: + logger.debug("Version comparison failed for %s: %s", target_version, exc) + return True # conservative: assume affected on error + + if not matched_any_config: + return True # no NVD data for this package -> conservatively assume affected + return False + + @staticmethod + def _check_version_in_range(version_to_check: str, version_range: list[str | None]) -> bool: + """Reuse the same logic as VulnerableDependencyChecker._check_version_in_range.""" + ver_start_excl, ver_end_excl, ver_start_incl, ver_end_incl = version_range + + all_versions = [v for v in version_range if v is not None] + [version_to_check] + has_el = any("el" in str(v) for v in all_versions) + has_deb = any("deb" in str(v) or "ubuntu" in str(v) for v in all_versions) + + if has_el: + vfunc = versions.RpmVersion + elif has_deb: + vfunc = versions.DebianVersion + else: + vfunc = versions.GenericVersion + + vtc = vfunc(version_to_check) + vsi = vfunc(ver_start_incl) if ver_start_incl else None + vse = vfunc(ver_start_excl) if ver_start_excl else None + vei = vfunc(ver_end_incl) if ver_end_incl else None + vee = vfunc(ver_end_excl) if ver_end_excl else None + + if vsi: + if not (vsi <= vtc): + return False + elif vse: + if not (vse < vtc): + return False + + if vei: + if not (vtc <= vei): + return False + elif vee: + if not (vtc < vee): + return False + + return True + + # ------------------------------------------------------------------ + # Intel extraction + # ------------------------------------------------------------------ + + @staticmethod + def _extract_rhsa(intel: CveIntel) -> list[dict]: + if intel.rhsa is None or not intel.rhsa.package_state: + return [] + packages = [] + for ps in intel.rhsa.package_state: + if ps.package_name: + packages.append({"package_name": ps.package_name}) + return packages + + @staticmethod + def _extract_fixed_rpms(intel: CveIntel) -> list[dict]: + """Extract all fix entries from RHSA affected_release. + + Returns a list of dicts with keys: nevra, name, version, release. + """ + if intel.rhsa is None or not hasattr(intel.rhsa, "affected_release"): + return [] + releases = intel.rhsa.affected_release + if not releases: + return [] + results: list[dict] = [] + for entry in releases: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if "/" in name: + continue + version = m.group(3) + release_arch = m.group(4) + release = _strip_arch_suffix(release_arch) + results.append({"nevra": raw, "name": name, "version": version, "release": release}) + return results + + @staticmethod + def _extract_fix_info(intel: CveIntel | None, resolved_name: str) -> dict: + """Extract fix NVR from RHSA affected_release for the resolved package. + + Returns a dict with keys nevra, name, version, release when a matching + fix entry is found; empty dict otherwise. + """ + if intel is None or intel.rhsa is None or not hasattr(intel.rhsa, "affected_release"): + return {} + releases = intel.rhsa.affected_release + if not releases: + return {} + for entry in releases: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if name.lower() != resolved_name.lower(): + continue + version = m.group(3) + release_arch = m.group(4) + release = _strip_arch_suffix(release_arch) + return {"nevra": raw, "name": name, "version": version, "release": release} + return {} + diff --git a/src/vuln_analysis/utils/rpm_checker_prompts.py b/src/vuln_analysis/utils/rpm_checker_prompts.py new file mode 100644 index 000000000..bdbcff0e2 --- /dev/null +++ b/src/vuln_analysis/utils/rpm_checker_prompts.py @@ -0,0 +1,1671 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Prompt templates for the RPM Checker L1 (Code Agent) and L2 (Build Agent) phases. + +L1 prompts handle source code verification for CVE patch status. +L2 prompts handle build-time configuration and hardening flag verification. +""" + +# =========================================================================== +# L1 CODE AGENT PROMPTS +# =========================================================================== + +# --------------------------------------------------------------------------- +# L1 Verdict Extraction +# --------------------------------------------------------------------------- + +L1_VERDICT_EXTRACTION_PROMPT = """\ +Extract the security verdict from this L1 agent investigation conclusion. + +CVE: {vuln_id} +Package: {target_package} + +L1 Agent Final Answer: +{final_answer} + +Classify the conclusion into one of these categories: +- "protected": The package is protected (patch applied, fix backported, or mitigating control present) +- "not_present": The vulnerable code/function is not present in this version +- "vulnerable": The vulnerable code is confirmed present and unpatched +- "uncertain": Insufficient evidence or conflicting findings + +Provide your confidence level (0.0-1.0) based on the strength of evidence in the answer. +""" + +# --------------------------------------------------------------------------- +# L1 Vulnerability Intel Extraction +# --------------------------------------------------------------------------- + +VULNERABILITY_INTEL_EXTRACTION_PROMPT = """\ +Extract structured vulnerability intelligence from the CVE data and patch content. +Your output will be used to guide source code searches, so focus on grep-able patterns. + + +CVE ID: {vuln_id} +Package: {target_package} +CVE Description: {cve_description} + + + +{vendor_mitigations} + + + +{patch_data} + + + +1. affected_files: Extract file paths from patch headers (strip a/ b/ prefixes) +2. vulnerable_functions: Extract function names from: + - Removed lines (- lines) in patch + - Function names mentioned in CVE description + - Functions or buffers mentioned in VENDOR_MITIGATIONS +3. vulnerable_variables: Extract variable names from: + - Removed lines that are key to the vulnerability + - Variables/buffers explicitly mentioned in VENDOR_MITIGATIONS (e.g., "sum2 buffer") +4. vulnerable_patterns: Extract distinctive code snippets from removed lines (- lines) + - Focus on patterns that can be grepped + - Include enough context to be unique +5. fix_patterns: Extract distinctive code snippets from added lines (+ lines) + - These indicate the fix is present +6. root_cause: Explain WHY the code is vulnerable in 1-2 sentences + - Incorporate insights from VENDOR_MITIGATIONS if provided +7. vulnerability_type: Classify as one of: buffer_overflow, integer_overflow, use_after_free, + null_deref, format_string, race_condition, path_traversal, injection, uninitialized_memory, other +8. search_keywords: List 3-5 grep patterns ordered by specificity: + - Start with most specific (unique variable/function names from patch or mitigations) + - End with broader patterns (file names, component names) +9. affected_bitness: Determine which bitness is affected: + - "32-bit": Look for "32-bit systems", "i386", "i686", "on 32-bit" + - "64-bit": Look for "64-bit only", "x86_64 only" (rare) + - "both": DEFAULT when not explicitly stated + NOTE: Do NOT assume bitness based on the vulnerability type. Default to "both" unless explicitly stated. +10. affected_architectures: Determine which CPU families are affected (or null for all): + - Look for: "x86", "Intel", "AMD" -> ["x86"] + - Look for: "ARM", "aarch64", "arm64" -> ["arm"] + - Look for: "PowerPC", "POWER", "ppc64" -> ["ppc"] + - Look for: "s390", "z/Architecture", "IBM Z" -> ["s390"] + - If none mentioned or "all architectures" -> null (affects all) + NOTE: Most CVEs affect all architectures. Only extract specific families if explicitly mentioned. +11. known_mitigations: Copy vendor-provided mitigations verbatim if present (e.g., compiler flags like "-ftrivial-auto-var-init=zero", configuration changes). Empty if none provided. + + + +- If no patch is provided, extract what you can from the CVE description and VENDOR_MITIGATIONS +- VENDOR_MITIGATIONS often contain specific variable names, buffers, or compiler flags that are highly relevant +- For search_keywords, prefer identifiers over natural language +- Patterns should be grep-friendly (avoid regex special chars unless escaped) + +""" + +# --------------------------------------------------------------------------- +# L1 System Prompts +# --------------------------------------------------------------------------- + +L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE = ( + "You are a security analyst investigating whether a CVE fix has been applied to a package.\n" + "A downstream patch file exists and has been analyzed.\n\n" + "VULNERABILITY_INTEL contains DOWNSTREAM_PATCH_STATUS and extracted patterns from the patch.\n" + "The source code index contains the UNPATCHED tarball; the patch is applied at BUILD time.\n\n" + "YOUR TASK: Verify (1) vulnerable code exists in source, (2) fix pattern is absent.\n" + "Both outcomes are EXPECTED when DOWNSTREAM_PATCH_STATUS is APPLIED.\n\n" + "CRITICAL RULES:\n" + "- If DOWNSTREAM_PATCH_STATUS is APPLIED, the package is PATCHED (patch applied at build time).\n" + "- Finding vulnerable code in source is EXPECTED (source is unpatched tarball).\n" + "- NOT finding fix pattern in source is EXPECTED (fix is in patch file, not tarball).\n" + "- Both findings together confirm the patch will correctly fix the code at build time.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Confirm the patch addresses the vulnerable code found.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH = ( + "You are a security analyst verifying that a package is VULNERABLE to a CVE.\n" + "The TARGET package does NOT contain a CVE-specific patch file.\n" + "However, patterns have been extracted from the patch in a FIXED RPM version.\n\n" + "VULNERABILITY_INTEL contains patterns extracted from the fixed version's patch.\n\n" + "YOUR TASK: Verify the TARGET package contains the vulnerable code and LACKS the fix.\n\n" + "VERIFICATION STRATEGY:\n" + "1. FIRST search for the VULNERABLE code pattern (from VULNERABLE_PATTERNS).\n" + " - Use function names, variable names, or unique code snippets.\n" + " - The vulnerable code SHOULD exist in the target package.\n" + "2. If vulnerable code is found, search for the FIX code pattern (from FIX_PATTERNS).\n" + " - The fix code should NOT exist in the target package.\n" + "3. CONCLUSION:\n" + " - If vulnerable code EXISTS and fix is ABSENT → Package is VULNERABLE.\n" + " - If fix code IS found → Package may be patched via rebase (investigate further).\n" + " - If neither is found → Use file paths from AFFECTED_FILES to locate relevant code.\n\n" + "CRITICAL RULES:\n" + "- The patch is from a FIXED version - expect the target to have vulnerable code.\n" + "- Use file paths and function names from VULNERABILITY_INTEL to locate code.\n" + "- Search for distinctive code patterns, not generic keywords.\n" + "- Base conclusions ONLY on tool results, not assumptions.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Compare found code against both vulnerable and fix patterns.\n" + "- Clearly state whether vulnerable code exists and whether fix is absent.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_REBASE_FIX = ( + "You are a security analyst verifying that a CVE fix is PRESENT in a rebased package.\n" + "The TARGET package was REBASED to a newer upstream version that claims to fix this CVE.\n\n" + "VULNERABILITY_INTEL contains patterns extracted from the upstream fix.\n\n" + "YOUR TASK: Verify the TARGET package contains the FIX code (proving rebase was effective).\n\n" + "VERIFICATION STRATEGY:\n" + "1. FIRST search for the FIX code pattern (from FIX_PATTERNS).\n" + " - Use function names, variable names, or unique code snippets.\n" + " - The fix code SHOULD exist in the target package (proving rebase worked).\n" + "2. If fix code is found, optionally confirm VULNERABLE code is ABSENT.\n" + " - The vulnerable code should NOT exist (was replaced by the fix).\n" + "3. CONCLUSION:\n" + " - If fix code EXISTS → Package is PATCHED via rebase.\n" + " - If vulnerable code still EXISTS and fix is ABSENT → Rebase may be incomplete.\n" + " - If neither is found → Use file paths from AFFECTED_FILES to locate relevant code.\n\n" + "CRITICAL RULES:\n" + "- The patch is from a FIXED version - expect the target to have the fix code.\n" + "- Use file paths and function names from VULNERABILITY_INTEL to locate code.\n" + "- Search for distinctive code patterns, not generic keywords.\n" + "- Base conclusions ONLY on tool results, not assumptions.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Compare found code against both vulnerable and fix patterns.\n" + "- Clearly state whether fix code exists, confirming the rebase.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH = """You are a security analyst verifying that a CVE fix is PRESENT in a rebased package. +The TARGET package was REBASED to a newer upstream version that claims to fix this CVE. +NO PATCH FILE IS AVAILABLE - you must use the CVE description to guide your search. + +YOUR TASK: Verify the TARGET package contains the fix by searching for: +1. Code patterns mentioned in the CVE description +2. Defensive code that would mitigate the vulnerability +3. Function/symbol names related to the CVE + +VERIFICATION STRATEGY (No Patch Mode): +1. EXTRACT key identifiers from the CVE description: + - Function names, API calls, variable names + - Vulnerable code constructs described + - Fixed/secure code patterns described +2. SEARCH for these patterns in the target source code +3. ANALYZE the code to determine if it shows the fix behavior +4. CONCLUDE based on presence of defensive code and absence of vulnerability indicators""" + +# --------------------------------------------------------------------------- +# L1 Prompt Templates +# --------------------------------------------------------------------------- + +L1_AGENT_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +L1_AGENT_PROMPT_TEMPLATE_NO_PATCH = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + + + +{tools} + + +{tool_selection_strategy} + +{tool_instructions}""" + +# --------------------------------------------------------------------------- +# L1 Thought Instructions +# --------------------------------------------------------------------------- + +L1_AGENT_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 0 - CHECK PATCH STATUS (PRIORITY): + FIRST check VULNERABILITY_INTEL for DOWNSTREAM_PATCH_STATUS. + If DOWNSTREAM_PATCH_STATUS is APPLIED: + - The source code index contains the UNPATCHED tarball + - The patch file is applied at BUILD time, not in the indexed source + - Do 2 verification searches, then FINISH with verdict PATCHED + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - DOWNSTREAM_PATCH_STATUS: APPLIED means package is patched at build time + - PATCH_FILE: Name of the patch file + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix (will be ABSENT in source) + +PHASE 2 - SOURCE CODE INSPECTION (when DOWNSTREAM_PATCH_STATUS is APPLIED): + Do exactly 2 verification searches: + 1. Search for vulnerable function/pattern → should FIND it (source is unpatched) + 2. Search for fix pattern → should NOT find it (fix is in separate patch file) + Both outcomes are EXPECTED and confirm the patch is correct. + After both searches, FINISH immediately with PATCHED verdict. + +PHASE 3 - VERDICT: + If DOWNSTREAM_PATCH_STATUS is APPLIED: + - Found vulnerable code + fix absent = PATCHED (patch will fix it at build time) + - This is the EXPECTED outcome, not a failure + Conclude after 2 searches - do NOT keep searching. + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. If DOWNSTREAM_PATCH_STATUS is APPLIED, do max 2 searches then conclude PATCHED. +5. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +6. When patch is APPLIED: finding vulnerable code = GOOD, not finding fix = GOOD (expected). +7. If a pattern contains special regex characters, escape them or use literal substrings. + + + +If DOWNSTREAM_PATCH_STATUS is APPLIED: +- Search 1: Find vulnerable function → EXPECTED to find (source is unpatched) +- Search 2: Check fix pattern → EXPECTED to NOT find (fix is in patch file) +- After both: FINISH with PATCHED verdict +If a search returned results: +- If vulnerable code found and patch is APPLIED, proceed to verify fix is absent +- After both checks complete, FINISH +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) + + + +{{"thought": "DOWNSTREAM_PATCH_STATUS is APPLIED. Search for vulnerable function first", "mode": "act", "actions": {{"tool": "Source Grep", "query": "parse_rockridge", "reason": "Verify vulnerable function exists in unpatched source"}}, "final_answer": null}} + + +{{"thought": "Found vulnerable function. Now verify fix pattern is absent (expected since fix is in patch file)", "mode": "act", "actions": {{"tool": "Source Grep", "query": "if (file->pz_log2_bs < 15", "reason": "Confirm fix pattern is absent from source"}}, "final_answer": null}} + + +{{"thought": "Vulnerable code found, fix absent as expected. DOWNSTREAM_PATCH_STATUS is APPLIED so package is PATCHED.", "mode": "finish", "actions": null, "final_answer": "The package is PATCHED. Found vulnerable function at file.c:123. Fix pattern absent from source (expected - fix is in patch file applied at build time). DOWNSTREAM_PATCH_STATUS confirms patch is applied."}} + + +{{"thought": "KNOWLEDGE has sufficient evidence: vulnerable code at X, fix absent", "mode": "finish", "actions": null, "final_answer": "The package is [PATCHED/VULNERABLE]. Found [evidence] at [file:line]. The code [matches/differs from] the patch because [reason]."}} +""" + +L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + For EACH item in VULNERABLE_FUNCTIONS and AFFECTED_FILES: + 1. Search for vulnerable pattern - it SHOULD exist in unpatched target + 2. Search for fix pattern - it should NOT exist in unpatched target + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - ALL VULNERABLE_FUNCTIONS have been located + - Evidence is sufficient for confident verdict + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. FIRST search for VULNERABLE code - it SHOULD exist in target. +8. THEN search for FIX code - it should NOT exist in target. +9. If a pattern contains special regex characters, escape them or use literal substrings. +10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. Search for the vulnerable code pattern from the patch", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate vulnerable code that should exist in unpatched target"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows function found at iso9660.c:2074. Now verify the fix is NOT present", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Check if fix code is absent (confirms vulnerability)"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix pattern not found but need more evidence. Search for key variable in the found file", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "Examine how the vulnerable variable is handled"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows Source Grep failed. Try Code Keyword Search for the file from patch", "mode": "act", "actions": {{"tool": "Code Keyword Search", "query": "", "reason": "Verify we are looking at the correct file"}}, "final_answer": null}} + + +{{"thought": "Vulnerable code found, fix absent", "mode": "finish", "actions": null, "final_answer": "VULNERABLE. Found vulnerable code at [file:line]. Fix not present."}} + + +{{"thought": "FIX_APPLIED_AT_CALL_SITE found despite no CVE patch", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix applied at [file:line]. Included via upstream update."}} + + +{{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} +""" + +L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + For EACH item in VULNERABLE_FUNCTIONS and AFFECTED_FILES: + 1. Search for fix pattern - it SHOULD exist in rebased target + 2. Verify vulnerable pattern is ABSENT from target + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - ALL VULNERABLE_FUNCTIONS have been located + - Evidence is sufficient for confident verdict + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. FIRST search for FIX code - it SHOULD exist in rebased target. +8. THEN verify VULNERABLE code is ABSENT from target. +9. If a pattern contains special regex characters, escape them or use literal substrings. +10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. Search for the fix code pattern from the patch", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate fix code that should exist after rebase"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix pattern not found. Try searching for key variable from the fix", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Find how the fix-related variable is handled"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows variable found at file.c:100. Search for the full fix pattern in that file", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",file.c", "reason": "Check if fix exists in the located file"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix confirmed. Now verify the vulnerable code is absent", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Check if vulnerable code was removed (confirms fix)"}}, "final_answer": null}} + + +{{"thought": "FIX_APPLIED_AT_CALL_SITE confirmed, vulnerable code absent", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix at [file:line]. Vulnerable code absent."}} + + +{{"thought": "Vulnerable code still present despite rebase", "mode": "finish", "actions": null, "final_answer": "INCOMPLETE rebase. Vulnerable code at [file:line]. Manual review required."}} + + +{{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} +""" + +L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify (may be inferred from CVE description) + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + - ROOT_CAUSE: Description of the vulnerability mechanism + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + For EACH item in VULNERABLE_FUNCTIONS and SEARCH_KEYWORDS: + 1. Search for vulnerable code patterns + 2. Search for defensive/fix patterns (bounds checks, validation, etc.) + IMPORTANT: Do NOT stop after finding the first file. Check ALL potential locations. + +PHASE 3 - VERDICT: + Only conclude when: + - Key files have been searched + - Vulnerable functions have been located + - Evidence is sufficient for confident verdict + + +RESPONSE FORMAT (JSON): +You must respond with a JSON object with these fields: +- thought: Your reasoning based on KNOWLEDGE and VULNERABILITY_INTEL (reference what was already found) +- mode: "act" (to use a tool) or "finish" (to provide final answer) +- actions: (only if mode="act") {{"tool": "Tool Name", "query": "search term", "reason": "why this search"}} +- final_answer: (only if mode="finish") Your conclusion about patch status + + +1. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +2. If KNOWLEDGE shows a search was done, your next action must be DIFFERENT. +3. Output valid JSON only. thought < 100 words. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or defensive patterns in the found code +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for SEARCH_KEYWORDS from VULNERABILITY_INTEL + + + +{{"thought": "No prior searches in KNOWLEDGE. Search for key function from CVE description", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate code related to CVE vulnerability"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows defensive code found, no vulnerability indicators", "mode": "finish", "actions": null, "final_answer": "The package is LIKELY PATCHED. Found defensive code at [file:line]: [quote code]. The fix behavior described in the CVE appears to be present."}} + + +{{"thought": "KNOWLEDGE shows insufficient evidence to determine patch status", "mode": "finish", "actions": null, "final_answer": "INCONCLUSIVE. Could not find definitive evidence of fix or vulnerability. Manual review recommended."}} +""" + +# --------------------------------------------------------------------------- +# L1 Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L1_COMPREHENSION_PROMPT = """Analyze the tool output and extract key findings for CVE patch verification. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} + +**CRITICAL FIRST CHECK - DO THIS BEFORE ANYTHING ELSE:** +Examine NEW OUTPUT below. If it is EMPTY, contains only whitespace, or shows an error: +- findings MUST be: ["FAILED: {tool_used} '{tool_input}' returned empty/no matches"] +- tool_outcome MUST be: "{tool_used} [{tool_input}] -> NO MATCHES" +- DO NOT fabricate, infer, or assume any results. STOP HERE. + + +{vulnerability_intel} + + + +{raw_patch_diff} + + + +If RAW_PATCH_DIFF contains content: +- Lines starting with "-" are REMOVED by the fix (vulnerable code) +- Lines starting with "+" are ADDED by the fix (patched code) +- CRITICAL: A pattern appearing in BOTH "-" and "+" lines is NOT a unique indicator +- Only patterns EXCLUSIVELY in "-" lines indicate vulnerable code +- Only patterns EXCLUSIVELY in "+" lines indicate the fix is present +- Compare grep results against these exact lines, not the summarized patterns above + +If RAW_PATCH_DIFF is empty: +- Fall back to using VULNERABLE_PATTERNS and FIX_PATTERNS from VULNERABILITY_INTEL + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +**ANTI-HALLUCINATION RULES:** +1. You can ONLY report findings based on text that ACTUALLY APPEARS in NEW OUTPUT above. +2. Every finding claiming code was "found" MUST include a direct quote from NEW OUTPUT. +3. If NEW OUTPUT is empty, you CANNOT claim any code was found - report FAILED. +4. The tool_outcome MUST accurately reflect what NEW OUTPUT shows, not what you expect. + +CODE ANALYSIS RULES (only if NEW OUTPUT has content): +1. READ the actual code snippets in NEW OUTPUT. Compare against VULNERABLE_PATTERNS and FIX_PATTERNS. +2. For each match found: + - Quote the actual line from NEW OUTPUT + - State the file:line where it was found + - Determine if it matches VULNERABLE or FIX pattern +3. If RAW_PATCH_DIFF is available: + - Check if the grep match appears in "-" lines (removed/vulnerable) + - Check if the grep match appears in "+" lines (added/fix) + - Report accordingly: matches "-" lines = VULNERABLE_CODE_FOUND + - A match that appears in BOTH "-" and "+" lines is NOT a unique indicator - note this +4. RECORD file paths and line numbers for all relevant matches. +5. FIX LOCATION: Tag FIX_DEFINITION_FOUND if found outside AFFECTED_FILES or is a function definition. Tag FIX_APPLIED_AT_CALL_SITE only when fix is CALLED and result USED in an AFFECTED_FILE. + + +Based on VULNERABILITY_INTEL above, assess investigation completeness: +- Have you searched in ALL files listed in AFFECTED_FILES? +- Have you found ALL instances of VULNERABLE_FUNCTIONS? +- Are there OTHER files containing the same vulnerable pattern? +If coverage is incomplete, note which files/functions remain unchecked. + + +OUTPUT RULES: +- findings: 2-4 observations. Each positive finding MUST quote actual content from NEW OUTPUT. +- tool_outcome: "{tool_used} [pattern] -> found in file.c:123" OR "{tool_used} [pattern] -> NO MATCHES" +RESPONSE: +{{""" + +L1_MEMORY_UPDATE_PROMPT = """Merge new findings into the CVE patch investigation memory. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS (from tool analysis): +{findings} +TOOL CALL RECORD: {tool_outcome} + +**CRITICAL: HANDLE FAILURES CORRECTLY** +If NEW FINDINGS contains "FAILED:" or TOOL CALL RECORD shows "NO MATCHES": +- Add the failure/no-match to memory verbatim +- Do NOT convert a failed search into a positive finding +- "NO MATCHES" for a fix pattern means FIX_CODE_ABSENT, not FIX_CODE_FOUND + +MEMORY RULES: +1. Start from PREVIOUS MEMORY. Append new facts from NEW FINDINGS. No duplicates. +2. Add TOOL CALL RECORD verbatim so future steps know what was already searched. +3. If NEW FINDINGS report a failure or no matches, record it as-is. Do NOT infer positive findings. + +PATCH VERIFICATION TRACKING: +- Vulnerable code FOUND: add "VULNERABLE_CODE_FOUND: [pattern] in [file:line]" +- Fix at definition only: add "FIX_DEFINITION_FOUND: [pattern] in [file:line]" +- Fix called+used in AFFECTED_FILE: add "FIX_APPLIED_AT_CALL_SITE: [pattern] in [file:line]" +- No vulnerable code matches: add "VULNERABLE_CODE_ABSENT: [pattern] not found" +- No fix code matches: add "FIX_CODE_ABSENT: [pattern] not found" + +VERDICT EVIDENCE: +- PATCHED: FIX_APPLIED_AT_CALL_SITE in AFFECTED_FILES or vulnerable code absent +- VULNERABLE: vulnerable code found, fix absent from call sites +- UNCERTAIN: FIX_DEFINITION_FOUND only, no call site evidence +- INCONCLUSIVE: neither pattern found, or conflicting evidence + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with search results and evidence tags. +RESPONSE: +{{""" + +# --------------------------------------------------------------------------- +# L1 Observation Prompts (CVE-Description Mode - No Patch Available) +# --------------------------------------------------------------------------- + +L1_COMPREHENSION_PROMPT_CVE_DESC = """Analyze the tool output for CVE patch verification using CVE description context. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} + +**CRITICAL FIRST CHECK - DO THIS BEFORE ANYTHING ELSE:** +Examine NEW OUTPUT below. If it is EMPTY, contains only whitespace, or shows an error: +- findings MUST be: ["FAILED: {tool_used} '{tool_input}' returned empty/no matches"] +- tool_outcome MUST be: "{tool_used} [{tool_input}] -> NO MATCHES" +- DO NOT fabricate, infer, or assume any results. STOP HERE. + +CVE DESCRIPTION: +{cve_description} + +SPEC CHANGELOG (rebase info): +{spec_log_change} + +NOTE: No patch file available. Extract search terms from CVE description. + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +**ANTI-HALLUCINATION RULES:** +1. You can ONLY report findings based on text that ACTUALLY APPEARS in NEW OUTPUT above. +2. Every finding claiming code was "found" MUST include a direct quote from NEW OUTPUT. +3. If NEW OUTPUT is empty, you CANNOT claim any code was found - report FAILED. +4. The tool_outcome MUST accurately reflect what NEW OUTPUT shows, not what you expect. + +CODE ANALYSIS RULES (only if NEW OUTPUT has content): +1. EXTRACT key identifiers from the CVE description: + - Function names, variable names, API calls + - File paths or component names mentioned + +2. For each code match in NEW OUTPUT: + - Quote the actual line from NEW OUTPUT + - Does it relate to the vulnerability described? + - Does it show defensive patterns (bounds checking, null validation)? + - Record file path and line number as evidence + +3. DEFENSIVE PATTERNS indicating a fix: + - Input validation, bounds checking, null guards + - Resource cleanup, error handling + +OUTPUT: +- findings: 2-4 observations. Each positive finding MUST quote actual content from NEW OUTPUT. +- tool_outcome: "{tool_used} [pattern] -> found in file.c:123" OR "{tool_used} [pattern] -> NO MATCHES" +RESPONSE: +{{""" + +L1_MEMORY_UPDATE_PROMPT_CVE_DESC = """Merge findings into CVE patch investigation memory. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} +MODE: CVE-description based (no patch patterns) + +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append new facts from NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. + +CVE-BASED TRACKING: +- CVE-related code FOUND: "CVE_CODE_FOUND: [symbol] in [file:line]" +- Defensive pattern FOUND: "DEFENSIVE_CODE_FOUND: [pattern] in [file:line]" +- Search no match: "SEARCH_NO_MATCH: [pattern]" + +VERDICT (CVE-description mode): +- LIKELY_PATCHED: defensive code found, no vulnerability indicators +- LIKELY_VULNERABLE: vulnerability patterns found, no defensive code +- INCONCLUSIVE: insufficient evidence + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with evidence tags. +RESPONSE: +{{""" + + +# =========================================================================== +# L2 BUILD AGENT PROMPTS +# =========================================================================== + +# --------------------------------------------------------------------------- +# L2 Configuration Investigation Prompts +# --------------------------------------------------------------------------- + +L2_CONFIG_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is DISABLED at build time.\n\n" + "GOAL: Determine if the CVE-affected feature/component is compiled into the binary.\n\n" + "EVIDENCE SOURCES:\n" + "1. BUILD_HARVEST section below - disabled/enabled features ALREADY extracted (analyze in thought)\n" + "2. Build log (searchable with 'logs:' prefix) - verify affected source files were compiled\n\n" + "INVESTIGATION FLOW:\n" + "1. Analyze BUILD_HARVEST in your thought (no tool needed - just read and decide)\n" + "2. If feature IS DISABLED -> mode='finish' with NOT_COMPILED verdict immediately\n" + "3. If feature IS ENABLED -> mode='act' with 'logs:' prefix to verify affected files in build log\n" + "4. If feature NOT listed -> mode='act' with 'logs:' prefix to search build log for compilation evidence\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: Feature is disabled OR affected files not in build log\n" + "- COMPILED: Feature is enabled AND affected files are compiled\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_CONFIG_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +** CHECK THESE FIRST - No tool call needed! ** + +Disabled Features (from build log -D defines): +{disabled_features} + +Disabled Features (from spec configure flags): +{spec_disabled_features} + +Enabled Features (from build log -D defines): +{enabled_features} + +Enabled Features (from spec configure flags): +{spec_enabled_features} + +Linked Libraries (from build -l flags): +{linked_libraries} + +Built Subpackages (from spec %package): +{built_subpackages} + +Excluded Subpackages (from spec %bcond_without, ExcludeArch): +{excluded_subpackages} + +Kernel Config (CONFIG_*=y/n/m, kernel packages only): +{kernel_config} + +DECISION GUIDE: +- If CVE-affected feature in DISABLED lists -> verdict NOT_COMPILED (no tool needed) +- If CVE-affected feature in ENABLED lists -> strong signal for COMPILED (verify affected files in build log) +- If CVE-affected module in EXCLUDED subpackages -> verdict NOT_COMPILED +- If CVE-affected library NOT in linked_libraries -> check for alternative backend +- For kernel CVEs: if CONFIG_X=n in kernel_config -> NOT_COMPILED +- If lists are empty or feature not listed -> search build log with 'logs:' prefix + +IMPORTANT - Library vs Capability: +- Disabled features often name a specific LIBRARY, not the capability itself +- Example: 'ssl' disabled = OpenSSL library disabled, NOT TLS capability +- Always check if an ALTERNATIVE library for the same capability is ENABLED +- If alternative enabled, the capability IS compiled (just different implementation) +- Common patterns: ssl->nss/gnutls (TLS), zlib->zstd (compression), libidn->libidn2 (IDN) +- Check linked_libraries for actual library linkage evidence + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_CONFIG_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. NEVER use mode="act" with null actions. +4. Analyze BUILD_HARVEST in your thought, then immediately decide: finish (if disabled) or act with tool (to verify). +5. If feature in DISABLED lists -> check if alternative library for same capability is ENABLED first. +6. If DISABLED library has alternative ENABLED (e.g., ssl disabled but nss enabled) -> capability IS compiled. +7. If feature in ENABLED lists -> strong signal for COMPILED, verify affected files in build log. +8. If feature NOT in BUILD_HARVEST, search build log using 'logs:' prefix (e.g., 'logs:filename.c'). +9. NEVER grep source code - use 'logs:' prefix to search build logs for compilation evidence. +10. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects SM2 crypto. BUILD_HARVEST shows 'sm2' in spec_disabled_features. SM2 is disabled at build time.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "CVE affects zisofs. BUILD_HARVEST disabled features are empty - zisofs not disabled. Need to verify affected file was compiled.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:archive_read_support_format_iso9660", "reason": "Check if affected file appears in build compilation log"}}, "final_answer": null}} + + + +{{"thought": "CVE affects LDAP. BUILD_HARVEST shows 'ldap' in spec_enabled_features. LDAP explicitly enabled - strong signal feature is compiled. Verifying affected files in build log.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:ldap.c", "reason": "Verify LDAP-related files appear in build compilation log"}}, "final_answer": null}} + + + +{{"thought": "CVE affects TLS. 'ssl' in DISABLED but 'nss' in ENABLED. ssl=OpenSSL library, nss=NSS library - both provide TLS. Alternative library enabled, so TLS capability IS compiled. Verifying TLS files.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:vtls", "reason": "Verify TLS abstraction layer files compiled in build log"}}, "final_answer": null}} + + + +{{"thought": "Feature not disabled. Now verify affected file crypto/sm2/sm2.c was compiled.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:sm2.c", "reason": "Check if affected source file appears in build log"}}, "final_answer": null}} + + + +{{"thought": "Found no-sm2 in spec_disabled_features. SM2 code is not compiled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "SM2 not disabled. Found sm2.c compilation in build log.", "mode": "finish", "actions": null, "final_answer": "COMPILED. SM2 is not in disabled features. Build log shows 'gcc -c crypto/sm2/sm2_crypt.c -o sm2_crypt.o', confirming the vulnerable code is compiled into the binary."}} + + + +{{"thought": "Cannot find evidence either way. Affected files not in build log but feature not disabled.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. The feature is not explicitly disabled, but the affected files do not appear in the build log. Cannot determine compilation status."}} +""" + +# --------------------------------------------------------------------------- +# L2 Configuration Investigation Prompts - SPEC-ONLY MODE (no build log) +# --------------------------------------------------------------------------- + +L2_CONFIG_SPEC_ONLY_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is DISABLED at build time.\n\n" + "GOAL: Determine if the CVE-affected feature/component is compiled into the binary.\n\n" + "** SPEC-ONLY MODE - No build log available **\n" + "Confidence is LOWER because we cannot verify actual compilation output.\n" + "Conclusions are based on BUILD INTENT from spec configure flags, not actual build output.\n\n" + "EVIDENCE SOURCES:\n" + "1. BUILD_HARVEST section below - disabled/enabled features from spec %build section\n" + "2. Source/build system files (CMakeLists.txt, Makefile.am, configure.ac) - searchable with Source Grep\n\n" + "INVESTIGATION FLOW:\n" + "1. Analyze BUILD_HARVEST in your thought (no tool needed - just read and decide)\n" + "2. If feature IS DISABLED -> mode='finish' with NOT_COMPILED verdict immediately\n" + "3. If feature IS ENABLED -> mode='finish' with COMPILED verdict (lower confidence)\n" + "4. If feature NOT listed -> mode='act' to search build system files (CMakeLists.txt, Makefile.am)\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: Feature explicitly disabled in spec configure flags\n" + "- COMPILED: Feature not disabled (lower confidence - based on spec, not build output)\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +** SPEC-ONLY MODE - No build log available ** +Confidence: Lower - conclusions based on spec configure flags only + +Disabled Features (from spec configure flags): +{spec_disabled_features} + +Enabled Features (from spec configure flags): +{spec_enabled_features} + +Built Subpackages (from spec %package): +{built_subpackages} + +Excluded Subpackages (from spec %bcond_without, ExcludeArch): +{excluded_subpackages} + +Kernel Config (CONFIG_*=y/n/m, kernel packages only): +{kernel_config} + +DECISION GUIDE (spec-only): +- If CVE-affected feature in DISABLED list -> verdict NOT_COMPILED +- If CVE-affected feature in ENABLED list -> strong signal for COMPILED +- If CVE-affected module in EXCLUDED subpackages -> verdict NOT_COMPILED +- For kernel CVEs: if CONFIG_X=n in kernel_config -> NOT_COMPILED +- If feature not listed -> search build system files (CMakeLists.txt, Makefile.am, configure.ac) +- Cannot verify actual compilation - verdicts are based on BUILD INTENT, not build output + +IMPORTANT - Library vs Capability: +- Disabled features often name a specific LIBRARY, not the capability itself +- Example: 'ssl' disabled = OpenSSL library disabled, NOT TLS capability +- Always check if an ALTERNATIVE library for the same capability is ENABLED +- If alternative enabled, the capability IS compiled (just different implementation) +- Common patterns: ssl->nss/gnutls (TLS), zlib->zstd (compression), libidn->libidn2 (IDN) + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. NEVER use mode="act" with null actions. +4. Analyze BUILD_HARVEST in your thought, then immediately decide: finish (if disabled) or act with tool (to verify). +5. If feature in DISABLED lists -> check if alternative library for same capability is ENABLED first. +6. If DISABLED library has alternative ENABLED (e.g., ssl disabled but nss enabled) -> capability IS compiled. +7. If feature in ENABLED lists -> strong signal for COMPILED. Can finish or search build system files for confirmation. +8. If feature NOT in BUILD_HARVEST, search build system files (CMakeLists.txt, Makefile.am, configure.ac). +9. Use Source Grep to search source code and build system files. Do NOT use 'logs:' prefix (no build log available). +10. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects SM2 crypto. BUILD_HARVEST shows 'sm2' in spec_disabled_features. SM2 is disabled at build time.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "CVE affects zisofs. BUILD_HARVEST disabled features are empty - zisofs not disabled. Searching build system for zisofs configuration.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "zisofs", "reason": "Search CMakeLists.txt or Makefile.am for zisofs feature configuration"}}, "final_answer": null}} + + + +{{"thought": "CVE affects LDAP. BUILD_HARVEST shows 'ldap' in spec_enabled_features. LDAP explicitly enabled in spec - feature is compiled (lower confidence, spec-only mode).", "mode": "finish", "actions": null, "final_answer": "COMPILED. The spec file explicitly enables LDAP with '--with-ldap' configure flag. Based on spec configure flags (no build log available), the vulnerable code is likely compiled into the binary."}} + + + +{{"thought": "CVE affects TLS. 'ssl' in DISABLED but 'nss' in ENABLED. ssl=OpenSSL library, nss=NSS library - both provide TLS. Alternative library enabled, so TLS capability IS compiled.", "mode": "finish", "actions": null, "final_answer": "COMPILED. While OpenSSL (ssl) is disabled, NSS is enabled as an alternative TLS library. The TLS capability is compiled with a different implementation."}} + + + +{{"thought": "Feature not in spec flags. Searching CMakeLists.txt for feature configuration.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "option.*SM2|SM2.*OFF|SM2.*ON", "reason": "Search CMakeLists.txt for SM2 feature toggle"}}, "final_answer": null}} + + + +{{"thought": "Found no-sm2 in spec_disabled_features. SM2 code is not compiled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "SM2 not disabled in spec. No evidence of disabling in build system files either.", "mode": "finish", "actions": null, "final_answer": "COMPILED. SM2 is not in disabled features in the spec file. No build system configuration disables it. Based on spec configure flags (no build log available), the vulnerable code is likely compiled into the binary."}} + + + +{{"thought": "Cannot find evidence either way. Feature not mentioned in spec flags or build system files.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. The feature is not explicitly configured in the spec file or build system files. Cannot determine compilation status without build log evidence."}} +""" + +# --------------------------------------------------------------------------- +# L2 Hardening Investigation Prompts +# --------------------------------------------------------------------------- + +L2_HARDENING_SYS_PROMPT = ( + "You are an L2 Build Agent investigating COMPILER HARDENING mitigations.\n\n" + "GOAL: Determine if hardening flags relevant to this CVE's vulnerability class are present.\n\n" + "CONTEXT: Investigation 1 determined the vulnerable code IS compiled. Now check if\n" + "compiler/linker hardening makes exploitation significantly harder.\n\n" + "CRITICAL - CWE-SPECIFIC MATCHING:\n" + "- ONLY flags listed in EXPECTED_HARDENING can justify a MITIGATED verdict\n" + "- General hardening flags (stack protector, FORTIFY_SOURCE) do NOT mitigate all CWEs\n" + "- Example: -fstack-protector helps CWE-121 (stack overflow), NOT CWE-190 (integer overflow)\n" + "- You MUST match the EXACT flags from EXPECTED_HARDENING table to the build output\n\n" + "EVIDENCE SOURCES:\n" + "1. EXPECTED_HARDENING table (CWE-specific flags from knowledge base) - THIS IS YOUR CHECKLIST\n" + "2. Build log (searchable with 'logs:' prefix) - contains CFLAGS/CXXFLAGS/LDFLAGS definitions\n\n" + "EFFICIENT SEARCH STRATEGY:\n" + "- Search 'logs:FLAGS=' to get ALL compiler/linker flags in ONE call (matches CFLAGS=, LDFLAGS=, etc.)\n" + "- Grep supports regex OR: 'logs:CFLAGS\\|LDFLAGS' combines patterns\n" + "- Analyze the output to check for expected hardening flags - avoid multiple individual searches\n\n" + "IMPORTANT - RHEL/Fedora Specs Files:\n" + "When you see these specs files in build logs, hardening flags are IMPLICITLY enabled:\n" + "- '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' => -fPIE (position-independent code for ASLR)\n" + "- '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld' => -pie + -z now (PIE linking + BIND_NOW/Full RELRO)\n" + "These flags will NOT appear explicitly in the build log - the specs file injects them.\n" + "If you see these specs files, count the corresponding protections as PRESENT.\n\n" + "INVESTIGATION STEPS:\n" + "1. Review EXPECTED_HARDENING table - these are the ONLY flags that matter for this CWE\n" + "2. Search 'logs:FLAGS=' to get all compiler/linker flag definitions at once\n" + "3. For EACH flag in EXPECTED_HARDENING, check if present in build output\n" + "4. Verdict based ONLY on EXPECTED_HARDENING flags (ignore unrelated hardening)\n\n" + "VERDICTS:\n" + "- MITIGATED: One or more flags from EXPECTED_HARDENING are present in build\n" + "- NOT_MITIGATED: NONE of the EXPECTED_HARDENING flags are present (even if other hardening exists)\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_HARDENING_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} +CWE: {cwe_id} + + + +The following compiler/linker flags mitigate this vulnerability class: + +{expected_hardening_table} + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_HARDENING_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. EFFICIENT SEARCH: First search for 'logs:FLAGS=' to get ALL compiler/linker flags in one call. +5. Grep supports regex OR: use 'logs:CFLAGS\\|LDFLAGS' to combine patterns. +6. Analyze the FLAGS output to check for expected hardening flags - avoid multiple individual searches. +7. Do NOT call the same tool with the same input twice. +8. STRICT MATCHING: Only flags from EXPECTED_HARDENING justify MITIGATED verdict. +9. IGNORE UNRELATED HARDENING: -fstack-protector, -D_FORTIFY_SOURCE do NOT mitigate all CWEs. Check the table! + + + +{{"thought": "Need to check hardening flags for this CWE. Search for all compiler/linker flag definitions at once.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:FLAGS=", "reason": "Get all CFLAGS/CXXFLAGS/LDFLAGS definitions in one search"}}, "final_answer": null}} + + + +{{"thought": "CWE-121 stack overflow. EXPECTED_HARDENING lists -fstack-protector-strong. Found -fstack-protector-strong in CFLAGS. This matches EXPECTED_HARDENING.", "mode": "finish", "actions": null, "final_answer": "MITIGATED. Found -fstack-protector-strong which is listed in EXPECTED_HARDENING for CWE-121. Stack buffer overflows will be detected at runtime."}} + + + +{{"thought": "CWE-190 integer overflow. EXPECTED_HARDENING lists -ftrapv, -fsanitize=signed-integer-overflow. Found -fstack-protector-strong and -D_FORTIFY_SOURCE=2, but these are for memory safety NOT integer overflow. Checking for expected flags: -ftrapv MISSING, -fsanitize=signed-integer-overflow MISSING.", "mode": "finish", "actions": null, "final_answer": "NOT_MITIGATED. Build has general hardening (-fstack-protector-strong, -D_FORTIFY_SOURCE=2) but NONE of the CWE-190 specific flags from EXPECTED_HARDENING (-ftrapv, -fsanitize=signed-integer-overflow) are present. Integer overflow is not mitigated."}} + + + +{{"thought": "CWE-693 control flow. EXPECTED_HARDENING lists PIE. Found -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 which implies -fPIE. This matches EXPECTED_HARDENING.", "mode": "finish", "actions": null, "final_answer": "MITIGATED. Build uses RHEL hardened specs files which implicitly enable PIE (listed in EXPECTED_HARDENING for CWE-693). Control flow exploitation is significantly harder."}} + + + +{{"thought": "CWE-190 integer overflow. EXPECTED_HARDENING lists -ftrapv, -fsanitize=*-integer-overflow. None found in FLAGS output.", "mode": "finish", "actions": null, "final_answer": "NOT_MITIGATED. None of the EXPECTED_HARDENING flags for CWE-190 (-ftrapv, -fsanitize=signed-integer-overflow, -fsanitize=unsigned-integer-overflow) are present in the build."}} + + + +{{"thought": "Build log does not contain FLAGS= definitions. Cannot determine if EXPECTED_HARDENING flags are present.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. Build log does not contain CFLAGS/LDFLAGS definitions. Cannot determine if EXPECTED_HARDENING mitigations are present."}} +""" + +# --------------------------------------------------------------------------- +# L2 Kernel-Specific Configuration Prompts +# --------------------------------------------------------------------------- + +L2_KERNEL_CONFIG_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is COMPILED in a KERNEL package.\n\n" + "KERNEL BUILD CHARACTERISTICS:\n" + "- Kernel builds use 'make -s' (silent) - individual .c file compilation is NOT visible in build logs\n" + "- Compilation is controlled by CONFIG_* options in kernel config files (kernel-{arch}-rhel.config)\n" + "- CONFIG_X=y means built-in (COMPILED), CONFIG_X=m means module (COMPILED), CONFIG_X=n means NOT COMPILED\n" + "- Source-to-module mapping requires parsing Makefile to find obj-$(CONFIG_X) directives\n\n" + "GOAL: Determine if the CVE-affected kernel code is compiled by checking CONFIG_* settings.\n\n" + "INVESTIGATION STRATEGY:\n" + "1. Identify the affected source file(s) from VULNERABILITY_INTEL (e.g., net/netfilter/nf_tables_api.c)\n" + "2. Read the Makefile in the same directory to find which CONFIG_* controls compilation\n" + " - Look for patterns like: nf_tables-objs := ... nf_tables_api.o\n" + " - Then find: obj-$(CONFIG_NF_TABLES) += nf_tables.o\n" + "3. Grep the kernel config file for that CONFIG symbol to get its value (y/m/n)\n" + "4. Determine compilation status based on CONFIG value\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: CONFIG_X=n or CONFIG symbol not defined (feature disabled)\n" + "- COMPILED: CONFIG_X=y (built-in) or CONFIG_X=m (loadable module)\n" + "- UNKNOWN: Cannot determine CONFIG symbol or config file unavailable" +) + +L2_KERNEL_CONFIG_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +Target Architecture Config File: {kernel_config_path} +Kernel Source Root: {kernel_source_root} + +IMPORTANT: Different architectures (x86_64, aarch64, s390x, ppc64le) have DIFFERENT config files. +A CONFIG option may be enabled on x86_64 but disabled on s390x. +Always search the TARGET ARCHITECTURE config file specified above. + +INVESTIGATION STEPS: +1. From VULNERABILITY_INTEL, identify affected source file(s) + Example: net/netfilter/nf_tables_api.c + +2. Grep the Makefile in the same directory as the affected .c file to find the CONFIG symbol + Source Grep format is pattern[,file_glob] — it searches FILE CONTENTS, not file paths. + WRONG: "drivers/nvme/host/Makefile" (searches for that path string inside files → no matches) + RIGHT: "nvme-tcp,drivers/nvme/host/Makefile" (search for "nvme-tcp" only in that Makefile) + Look for: nf_tables-objs := ... nf_tables_api.o OR nvme-tcp-y += tcp.o + obj-$(CONFIG_NF_TABLES) += nf_tables.o OR obj-$(CONFIG_NVME_TCP) += nvme-tcp.o + → This tells you which CONFIG_* controls the affected .c file + +3. Grep the TARGET ARCHITECTURE kernel config file for that CONFIG symbol + Use the basename from "Target Architecture Config File" above (e.g. kernel-x86_64-rhel.config). + Example: "CONFIG_NF_TABLES=,kernel-x86_64-rhel.config" + Result: CONFIG_NF_TABLES=m + +4. Interpret the result: + - CONFIG_X=y → Built into kernel (COMPILED) + - CONFIG_X=m → Built as module (COMPILED) + - CONFIG_X=n or "# CONFIG_X is not set" → NOT_COMPILED + - Symbol not found → Likely NOT_COMPILED (feature not configured) + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_KERNEL_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +Source Grep query: [target:]pattern[,file_glob] +- pattern = text to find INSIDE files (function name, CONFIG_*, module name like nvme-tcp) +- file_glob = optional path/filename filter after a comma (Makefile, *.c, kernel-x86_64-rhel.config) +- NEVER pass a bare file path as the pattern (e.g. "drivers/nvme/host/Makefile" will NOT open that file) +- Use specific patterns, not short strings like "tcp" (matches unrelated files like kernel.spec / mptcp) + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. +4. For kernel packages: FIRST grep the directory Makefile for CONFIG/obj-* lines, THEN grep the arch config file. +5. Makefile step: use pattern[,relative/path/Makefile] — e.g. "nf_tables,net/netfilter/Makefile" or "nvme-tcp,drivers/nvme/host/Makefile". +6. Config step: use CONFIG_SYMBOL=,config-basename — e.g. "CONFIG_NVME_TCP=,kernel-x86_64-rhel.config" (basename only, not full path). +7. Do NOT search build logs for .c compilation - kernel uses make -s (silent). +8. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects net/netfilter/nf_tables_api.c. Grep the directory Makefile for nf_tables / CONFIG lines.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "nf_tables,net/netfilter/Makefile", "reason": "Find obj-$(CONFIG_*) controlling nf_tables in that Makefile only"}}, "final_answer": null}} + + + +{{"thought": "CVE affects drivers/nvme/host/tcp.c. Grep the host Makefile for nvme-tcp and CONFIG_NVME_TCP.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "nvme-tcp,drivers/nvme/host/Makefile", "reason": "Find obj-$(CONFIG_NVME_TCP) and nvme-tcp-y lines linking tcp.c to a CONFIG symbol"}}, "final_answer": null}} + + + +{{"thought": "Makefile shows obj-$(CONFIG_NF_TABLES) += nf_tables.o. Grep arch config for CONFIG_NF_TABLES value.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NF_TABLES=,kernel-x86_64-rhel.config", "reason": "Check if CONFIG_NF_TABLES is y/m (compiled) or n (not compiled)"}}, "final_answer": null}} + + + +{{"thought": "Makefile shows obj-$(CONFIG_NVME_TCP) += nvme-tcp.o and nvme-tcp-y += tcp.o. Check CONFIG_NVME_TCP in arch config.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NVME_TCP=,kernel-x86_64-rhel.config", "reason": "Confirm CONFIG_NVME_TCP=y/m/n for compilation verdict"}}, "final_answer": null}} + + + +{{"thought": "CONFIG_NF_TABLES=m found in kernel config. Module is compiled.", "mode": "finish", "actions": null, "final_answer": "COMPILED. CONFIG_NF_TABLES=m in kernel config means nf_tables is built as a loadable module. The vulnerable code in net/netfilter/nf_tables_api.c IS compiled into the kernel."}} + + + +{{"thought": "CONFIG_NF_TABLES=y found in kernel config. Built-in to kernel.", "mode": "finish", "actions": null, "final_answer": "COMPILED. CONFIG_NF_TABLES=y in kernel config means nf_tables is built into the kernel. The vulnerable code in net/netfilter/nf_tables_api.c IS compiled into the kernel."}} + + + +{{"thought": "CONFIG_NF_TABLES is not set (n) in kernel config. Feature disabled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. CONFIG_NF_TABLES=n (or not set) in kernel config means nf_tables is disabled. The vulnerable code in net/netfilter/nf_tables_api.c is NOT compiled into the kernel."}} + + + +{{"thought": "Could not find CONFIG symbol in Makefile. Cannot determine compilation status.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. Could not determine which CONFIG symbol controls the affected code. Manual review of kernel configuration required."}} +""" + +# --------------------------------------------------------------------------- +# L2 Verdict Extraction Prompts +# --------------------------------------------------------------------------- + +L2_COMPILATION_VERDICT_PROMPT = """Extract the compilation verdict from the L2 Configuration investigation. + + +{final_answer} + + +Extract: +1. compilation_status: "compiled", "not_compiled", or "unknown" +2. confidence: 0.0 to 1.0 based on evidence strength +3. reasoning: Brief explanation (1-2 sentences) + +Output JSON only: +{{"compilation_status": "...", "confidence": 0.X, "reasoning": "..."}}""" + +L2_HARDENING_VERDICT_PROMPT = """Extract the hardening verdict from the L2 Hardening investigation. + + +{final_answer} + + +Extract: +1. hardening_status: "mitigated", "not_mitigated", "not_applicable", or "unknown" + - "not_applicable": This CWE class has no compiler-level mitigations available +2. hardening_flags: List of specific compiler/linker flags that provide protection (e.g., ["-fstack-protector-strong", "-D_FORTIFY_SOURCE=2", "RELRO", "PIE"]) + - Extract the actual flag names mentioned in the investigation + - Empty list if no relevant flags found +3. confidence: 0.0 to 1.0 based on evidence strength +4. reasoning: Brief explanation (1-2 sentences) + +Output JSON only: +{{"hardening_status": "...", "hardening_flags": ["..."], "confidence": 0.X, "reasoning": "..."}}""" + +# --------------------------------------------------------------------------- +# L2 Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L2_COMPREHENSION_PROMPT = """Analyze the tool output for L2 build/compilation verification. +GOAL: Determine whether {vuln_id} vulnerable code is COMPILED in {target_package} + + +{vulnerability_intel} + + + +Disabled Features (build log): {disabled_features} +Disabled Features (spec file): {spec_disabled_features} +Enabled Features (build log): {enabled_features} +Enabled Features (spec file): {spec_enabled_features} + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +BUILD ANALYSIS RULES: +1. CHECK if tool output shows: + - Compilation commands for AFFECTED_FILES (e.g., gcc -c file.c -o file.o) + - Feature-disable flags that match the CVE-affected component + - Object files or compilation artifacts for VULNERABLE_FUNCTIONS + +2. COMPILATION EVIDENCE: + - COMPILED: Found gcc/compile commands for affected files + - NOT_COMPILED: Feature is disabled OR affected files not in build + - UNKNOWN: Insufficient evidence + +3. RECORD specific file paths, compile commands, or flag matches. + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty or error: "FAILED: [tool] [input] - [reason]" +- Source Grep: Check if matches show compilation or feature disabling +- Build log search: Look for compile commands and disabled features + +OUTPUT: +- findings: 2-4 key observations about compilation status +- tool_outcome: "Source Grep [pattern] -> found in build.log:123" +RESPONSE: +{{""" + +L2_MEMORY_UPDATE_PROMPT = """Merge findings into L2 build investigation memory. +GOAL: Determine whether {vuln_id} vulnerable code is COMPILED in {target_package} + +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. +3. If NEW FINDINGS report a failure, add the failure to memory. + +COMPILATION TRACKING: +- Affected file COMPILED: "FILE_COMPILED: [file] - evidence: [compile command]" +- Affected file NOT_COMPILED: "FILE_NOT_COMPILED: [file] - evidence: [disabled feature]" +- Feature DISABLED: "FEATURE_DISABLED: [feature] in [build_log/spec]" +- Feature ENABLED: "FEATURE_ENABLED: [feature] - no disable flag found" + +VERDICT EVIDENCE: +- NOT_COMPILED evidence: feature disabled OR affected files not compiled +- COMPILED evidence: affected files appear in compile commands +- UNKNOWN: conflicting evidence or no compilation info found + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with evidence tags. +RESPONSE: +{{""" + +# --------------------------------------------------------------------------- +# L2 Hardening Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L2_HARDENING_COMPREHENSION_PROMPT = """Analyze the tool output for L2 hardening flag verification. +GOAL: Determine whether {vuln_id} has HARDENING mitigations in {target_package} + + +CWE: {cwe_id} +Expected Hardening Flags: +{expected_hardening} + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +HARDENING ANALYSIS RULES: +1. FIRST CHECK Expected Hardening Flags above: + - If "None" or empty: This CWE has NO known compiler-level mitigations + - Mark findings as "NO_RELEVANT_HARDENING: {cwe_id} has no compiler mitigations" + - Skip searching for generic flags - they won't help this vulnerability class + +2. IF expected hardening flags exist, CHECK tool output for: + - Compiler hardening flags (e.g., -fstack-protector, -fPIE, -fstack-clash-protection) + - Preprocessor defines (e.g., -D_FORTIFY_SOURCE=2, -D_GLIBCXX_ASSERTIONS) + - Linker hardening flags (e.g., -Wl,-z,relro, -Wl,-z,now) + +3. HARDENING EVIDENCE: + - FLAG_PRESENT: Found expected hardening flag in build commands + - FLAG_ABSENT: Searched but did not find expected flag + - NOT_APPLICABLE: No compiler mitigations exist for this CWE class + - UNKNOWN: Insufficient evidence + +4. RECORD specific flags found and their context (compilation line). + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty or error: "FAILED: [tool] [input] - [reason]" +- Source Grep: Check if matches show hardening flags in gcc/clang commands +- Build log search: Look for -f*, -D*, -Wl,* patterns + +OUTPUT: +- findings: 2-4 key observations about hardening flags +- tool_outcome: "Source Grep [pattern] -> found in build.log:123" +RESPONSE: +{{""" + +L2_HARDENING_MEMORY_UPDATE_PROMPT = """Merge findings into L2 hardening investigation memory. +GOAL: Determine whether {vuln_id} has HARDENING mitigations in {target_package} + +CWE: {cwe_id} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. +3. If NEW FINDINGS report a failure, add the failure to memory. + +HARDENING TRACKING: +- No relevant hardening: "NO_RELEVANT_HARDENING: [CWE] has no compiler mitigations" +- Flag FOUND: "HARDENING_PRESENT: [flag] - evidence: [build command excerpt]" +- Flag NOT FOUND: "HARDENING_ABSENT: [flag] - searched but not found" +- Critical mitigation: "CRITICAL_MITIGATION: [flag] for [CWE] - [present/absent]" + +VERDICT EVIDENCE: +- NOT_APPLICABLE evidence: this CWE class has no compiler-level mitigations +- MITIGATED evidence: key hardening flags present that reduce exploitability +- NOT_MITIGATED evidence: expected hardening flags absent +- UNKNOWN: build log incomplete or no compilation commands found + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with hardening evidence tags. +RESPONSE: +{{""" + + +# =========================================================================== +# FINAL REPORT PROMPTS +# =========================================================================== + +CODE_AGENT_REPORT_PROMPT = """\ + +You are a security analyst generating the final checker investigation report. +Synthesize the results from the target package analysis, additional intel (target rebase check + reference package), code agent analysis (source code), +and optionally build agent analysis (compilation, linking, hardening) into a comprehensive, auditable report with a clear +justification and supporting evidence. + + + +CVE: {vuln_id} +Target Package: {target_package} +CVE Description: {cve_description} + + +{policy_context_section} + +## Target Package Analysis +(Checked the target package being scanned for CVE-specific patch files) +{downstream_section} + +## Additional Intel (Target Rebase + Reference Package) +This section contains TWO distinct checks: +- TARGET REBASE CHECK: Searched the target package's spec file for CVE mentions indicating a rebase fix +- REFERENCE PACKAGE: Downloaded a known-fixed package version from intel to extract patch patterns +{upstream_section} + +## Code Agent Analysis +{l1_agent_section} + + +{build_agent_status_section} +{l2_context_section} +{override_notice_section} + +Generate a structured report following these requirements: + +1. JUSTIFICATION LABEL: + + FIRST: Check BUILD_AGENT_STATUS above. + - If status is "ran_with_override": You MUST use the DECISION TREE below to select the label. + The code agent may report "vulnerable" in source, but the build agent determines final exploitability. + IGNORE the code agent's "vulnerable" verdict when an override is present. + - If status is "ran_no_override": Use code agent verdict with build context. + - If status is "not_run": Use CODE AGENT PRECEDENCE RULES at the end of this section. + + LABEL SELECTION DECISION TREE (MANDATORY when BUILD_AGENT_STATUS is "ran_with_override"): + +-----------------------------------------------------------------------------+ + | Step 1: Is compilation_status == "not_compiled"? | + | YES -> STOP. Label = "code_not_present" | + | NO -> Continue to Step 2 | + +-----------------------------------------------------------------------------+ + | Step 2: Does build agent evidence mention "architecture mismatch" or | + | architecture-specific condition (32-bit/64-bit, endianness)? | + | YES -> STOP. Label = "requires_environment" | + | NO -> Continue to Step 3 | + +-----------------------------------------------------------------------------+ + | Step 3: Is build agent verdict "vulnerable_mitigated" with hardening flags? | + | YES -> STOP. Label = "protected_by_compiler" | + | NO -> Continue to Step 4 | + +-----------------------------------------------------------------------------+ + | Step 4: Is build agent verdict "not_vulnerable" for other reasons? | + | YES -> Label = "code_not_reachable" | + | NO -> Use code-agent-based rules below | + +-----------------------------------------------------------------------------+ + + CRITICAL: When the decision tree applies: + - compilation_status="not_compiled" ALWAYS means "code_not_present" + - Do NOT output "vulnerable" if build agent has an override verdict + - The build agent verdict supersedes code agent source analysis + + AVAILABLE LABELS: + - code_not_present: Vulnerable code/function is NOT COMPILED into the binary + (e.g., disabled via configure flags like --no-ssl, --without-feature, or + conditionally excluded at build time) + - code_not_reachable: Code IS compiled but cannot be reached/executed in this context + - protected_by_mitigating_control: Downstream patch or backport mitigates the vulnerability + - protected_by_compiler: Compiler hardening flags (FORTIFY_SOURCE, stack protector, etc.) + mitigate the vulnerability + - requires_environment: Vulnerability requires specific RUNTIME/ARCHITECTURE conditions + not present (e.g., 32-bit integer overflow impossible on 64-bit, big-endian specific + bug on little-endian system). NOT for build-time configuration. + - vulnerable: Package is actually vulnerable and needs patching + - uncertain: Insufficient information to determine exploitability + + When Override verdict is "none", the build agent ran but did not change the code agent + label—still apply compilation/hardening facts in the executive summary. + + Do NOT state "vulnerable" if build agent evidence contradicts it. Instead explain: + "While source contains vulnerable patterns, the build is not affected due to [build agent reason]." + + EXAMPLES (build agent scenarios): + + Example A - SSL disabled via configure: + Build agent verdict: not_vulnerable + compilation_status: not_compiled + Evidence: "configured with --no-ssl flag" + -> CORRECT: "code_not_present" (Step 1: not_compiled -> code_not_present) + -> WRONG: "vulnerable" (build agent override supersedes code agent) + -> WRONG: "requires_environment" (this is build config, not runtime/arch) + + Example B - 64-bit architecture prevents overflow: + Build agent verdict: not_vulnerable + compilation_status: compiled + Evidence: "Architecture mismatch - 32-bit overflow impossible on 64-bit" + -> CORRECT: "requires_environment" (Step 2: architecture condition) + -> WRONG: "code_not_present" (code IS compiled, just not exploitable) + + Example C - Hardening flags mitigate: + Build agent verdict: vulnerable_mitigated + compilation_status: compiled + Evidence: "FORTIFY_SOURCE=2 prevents buffer overflow exploitation" + -> CORRECT: "protected_by_compiler" (Step 3: hardening mitigation) + + CODE AGENT PRECEDENCE RULES (ONLY when BUILD_AGENT_STATUS is "not_run"): + WARNING: Skip this section if BUILD_AGENT_STATUS contains "ran". + - If a CVE-specific patch file exists AND is applied in build, use "protected_by_mitigating_control". + - If the code agent found the fix code in source, use "protected_by_mitigating_control". + - If the code agent found vulnerable code pattern still present, use "vulnerable". + - If upstream shows rebase fixed the issue, use "protected_by_mitigating_control". + - Only use "uncertain" when evidence is conflicting or insufficient. + +2. EVIDENCE CHAIN: + - Start with target package patch availability + - Include target rebase findings (if CVE mentioned in target's spec changelog, this is from the TARGET package) + - Include reference package findings (if a known-fixed package from intel was used for comparison) + - Include code analysis findings (patch targets, vulnerable vs fix patterns) + - Reference specific files, line numbers, and code snippets + - Summarize findings; the rendered report places an "Extracted facts" section **after** the Evidence chain with verbatim spec Patch lines, changelog hits, and build log lines (when available)—do not invent `PatchN:` numbers or spec quotes; only state patch indices you could derive from the investigation text below, or point readers to *Extracted facts* for exact lines + + PHRASING GUIDANCE for code analysis findings: + - GOOD: "Code analysis verified that the patch modifies `filename.c` to address the vulnerability" + - GOOD: "Patch targets the `function_name()` function in `filename.c`" + - BAD: "L1 agent found the fix code in the source" (use "code agent" — do not say L1/L2) + - BAD: "L2 says not vulnerable" (use "build agent") + - BAD: "Found fix in source" (unclear what was found) + - Use active voice: "The patch adds validation..." not "Validation was found..." + +3. EXECUTIVE SUMMARY (scenario-aware; use "code agent" and "build agent", never L1/L2): + + Read BUILD_AGENT_STATUS first: + + A) Build agent status: ran_with_override (Override verdict is not "none"): + - Write 4-5 sentences. + - Sentence 1 (Verdict): State final posture clearly + - Sentences 2-3 (Technical Context): Nature of the flaw and why the build agent negates or mitigates it + - Sentences 4-5 (Reconciliation): Why the code agent found patterns in source but the build agent changes exploitability + + B) Build agent status: ran_no_override: + - Write 3-5 sentences. + - Sentence 1: State verdict (code agent label stands) + - Include 1-2 sentences on compilation status and/or hardening from BUILD_AGENT_CONTEXT when present + - Do NOT say the build agent is absent, missing, or "context is not present" + + C) Build agent status: not_run: + - Write 3-4 sentences on code agent and patch intel only + - Do NOT mention the build agent at all + + FORBIDDEN phrases (never write these): + - "build agent context is not present" + - "L2 context is not present" + - "verdict is based solely on the code agent analysis" (when BUILD_AGENT_STATUS is ran_no_override) + - "because the build agent did not run" (when BUILD_AGENT_STATUS is ran_no_override) + + Do NOT invent RHSA IDs, function names, or technical details not present in the context. + +4. PATCH ANALYSIS (semantic fix narrative): + - When target package patch or reference patch evidence exists, briefly describe **what** the fix does: name the function(s) or file(s) and the nature of the change (e.g. "adds range validation 15–17 in parse_rockridge_ZF1"). + - Derive this from Target Package Analysis, Reference Intel, patch file names, or code agent excerpts—do NOT invent code or function names absent from investigation results. + +5. DELIVERY MODEL: + - When a CVE-named patch file is present, explicitly note that the fix is carried as a separate `%patch` directive while the upstream tarball (`Source0`) version may remain unchanged. + - Encourage citing "Extracted facts" for exact spec `PatchN:` and `Source0`/`Version` lines when shown below. + +6. CAVEATS (optional): + - Note any missing data (no patch file, no build log, etc.) + - Flag low-confidence findings that may need manual review + - Leave empty if no significant gaps exist + + + +Provide a structured JSON response with: +- justification_label: one of the labels above +- executive_summary: 3-5 sentence summary (see Instruction #3 for structure) +- evidence_chain: list of evidence items in logical order +- affected_files: list of source files involved +- patch_analysis: analysis of patches (or null if none) +- caveats: list of investigation gaps or uncertainties (empty list if none) + +Ensure all code snippets and special characters within JSON string values are properly escaped +(e.g., quotes as \", backslashes as \\, newlines as \\n) to maintain valid JSON format. + +""" diff --git a/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py new file mode 100644 index 000000000..d98ccbc35 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for VulnerabilityIntelSanitizer v1.""" + +from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel + +from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile +from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer + + +def _patch_with_util_c() -> ParsedPatch: + return ParsedPatch( + patch_filename="cve.patch", + files=[ + PatchFile( + source_path="tar/util.c", + target_path="tar/util.c", + hunks=[], + ) + ], + ) + + +class TestSanitizeAffectedFiles: + def test_clears_affected_files_when_no_patch(self): + raw = VulnerabilityIntel(affected_files=["generator.c", "tar/util.c"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.affected_files == [] + + def test_clears_affected_files_when_empty_files(self): + patch = ParsedPatch(patch_filename="empty.patch", files=[]) + raw = VulnerabilityIntel(affected_files=["util.c"]) + result = VulnerabilityIntelSanitizer(patch).apply(raw) + assert result.affected_files == [] + + def test_keeps_matching_basename_with_patch(self): + raw = VulnerabilityIntel(affected_files=["util.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["util.c"] + + def test_keeps_full_path_when_basename_matches_patch(self): + raw = VulnerabilityIntel(affected_files=["tar/util.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["tar/util.c"] + + def test_drops_non_patch_file_with_patch(self): + raw = VulnerabilityIntel(affected_files=["util.c", "generator.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["util.c"] + + +class TestFilterVulnerableFunctions: + def test_drops_function_with_spaces(self): + raw = VulnerabilityIntel( + vulnerable_functions=["parse_header", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.vulnerable_functions == ["parse_header"] + + def test_drops_function_with_spaces_without_patch(self): + raw = VulnerabilityIntel(vulnerable_functions=["rsync compares file checksums"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.vulnerable_functions == [] + + +class TestFilterSearchKeywords: + def test_drops_keyword_with_spaces_no_boolean(self): + raw = VulnerabilityIntel( + search_keywords=["s2length", "sum2", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["s2length", "sum2"] + + def test_keeps_keyword_with_or(self): + raw = VulnerabilityIntel(search_keywords=["s2length OR sum2"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["s2length OR sum2"] + + def test_keeps_keyword_with_and(self): + raw = VulnerabilityIntel(search_keywords=["foo AND bar"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["foo AND bar"] + + +class TestRsyncStyleNoPatch: + def test_strips_hallucinated_paths_and_prose(self): + raw = VulnerabilityIntel( + affected_files=["generator.c", "src/foo.c"], + vulnerable_functions=["rsync compares file checksums"], + vulnerable_variables=["s2length", "sum2"], + search_keywords=["s2length", "sum2", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.affected_files == [] + assert result.vulnerable_functions == [] + assert result.vulnerable_variables == ["s2length", "sum2"] + assert result.search_keywords == ["s2length", "sum2"] diff --git a/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py new file mode 100644 index 000000000..42f70dbaa --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py @@ -0,0 +1,426 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for web_patch_fetcher module.""" + +import pytest +from unittest.mock import MagicMock, patch + +from vuln_analysis.utils.web_patch_fetcher import ( + WebPatchFetcher, + build_patch_url_from_repo, + _GITHUB_COMMIT_PATTERN, + _GITHUB_PR_PATTERN, + _GITWEB_COMMIT_PATTERN, + _KERNEL_CGIT_COMMIT_PATTERN, + _KERNEL_SHORT_PATTERN, +) + + +class TestUrlPatterns: + """Test URL pattern matching.""" + + def test_github_commit_pattern(self): + """Test GitHub commit URL pattern matching.""" + url = "https://github.com/libarchive/libarchive/commit/a2a73a8f14b3208c7f6acbbc93265254a7c1efd0" + match = _GITHUB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "libarchive/libarchive" + assert match.group(2) == "a2a73a8f14b3208c7f6acbbc93265254a7c1efd0" + + def test_github_commit_pattern_short_sha(self): + """Test GitHub commit URL with short SHA.""" + url = "https://github.com/curl/curl/commit/39d1976b7f" + match = _GITHUB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "curl/curl" + assert match.group(2) == "39d1976b7f" + + def test_github_pr_pattern(self): + """Test GitHub PR URL pattern matching.""" + url = "https://github.com/libarchive/libarchive/pull/2934" + match = _GITHUB_PR_PATTERN.match(url) + assert match is not None + assert match.group(1) == "libarchive/libarchive" + assert match.group(2) == "2934" + + def test_kernel_cgit_commit_pattern(self): + """Test kernel.org cgit commit URL pattern.""" + url = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/commit/?id=f342de4e2f33e0e39165d8639387aa6c19dff660" + ) + match = _KERNEL_CGIT_COMMIT_PATTERN.match(url) + assert match is not None + expected_repo = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + assert match.group(1) == expected_repo + assert match.group(2) == "f342de4e2f33e0e39165d8639387aa6c19dff660" + + def test_kernel_short_pattern(self): + """Test kernel.org short URL pattern.""" + url = "https://git.kernel.org/stable/c/096bb5b43edf755bc4477e64004fa3a20539ec2f" + match = _KERNEL_SHORT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "stable" + assert match.group(2) == "096bb5b43edf755bc4477e64004fa3a20539ec2f" + + def test_gitweb_commit_pattern(self): + """Test gitweb commit URL pattern.""" + url = "https://git.samba.org/?p=rsync.git;a=commit;h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + match = _GITWEB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "https://git.samba.org" + assert match.group(2) == "rsync.git" + assert match.group(3) == "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + + +class TestWebPatchFetcherUrlResolution: + """Test URL resolution in WebPatchFetcher.""" + + @pytest.fixture + def fetcher(self): + """Create a WebPatchFetcher with a mock session.""" + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_resolve_github_commit_url(self, fetcher): + """Test resolving GitHub commit URL to patch URL.""" + url = "https://github.com/curl/curl/commit/39d1976b7f" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/curl/curl/commit/39d1976b7f.patch" + assert resolved.platform == "github" + assert resolved.url_type == "commit" + assert resolved.repo_url == "https://github.com/curl/curl" + assert resolved.commit_sha == "39d1976b7f" + + def test_resolve_github_pr_url(self, fetcher): + """Test resolving GitHub PR URL to patch URL.""" + url = "https://github.com/libarchive/libarchive/pull/2934" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/libarchive/libarchive/pull/2934.patch" + assert resolved.platform == "github" + assert resolved.url_type == "pull" + assert resolved.repo_url == "https://github.com/libarchive/libarchive" + assert resolved.commit_sha == "PR-2934" + + def test_resolve_ubuntu_style_url(self, fetcher): + """Test resolving Ubuntu-style 'upstream:' URL.""" + url = "upstream: https://github.com/curl/curl/commit/39d1976b7f" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/curl/curl/commit/39d1976b7f.patch" + assert resolved.platform == "github" + assert resolved.url_type == "commit" + + def test_resolve_kernel_cgit_commit_url(self, fetcher): + """Test resolving kernel.org cgit commit URL.""" + url = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/commit/?id=f342de4e2f33" + ) + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/patch/?id=f342de4e2f33" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + assert resolved.url_type == "commit" + assert resolved.commit_sha == "f342de4e2f33" + + def test_resolve_kernel_short_stable_url(self, fetcher): + """Test resolving kernel.org short stable URL.""" + url = "https://git.kernel.org/stable/c/096bb5b43edf" + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" + "/patch/?id=096bb5b43edf" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + assert resolved.commit_sha == "096bb5b43edf" + + def test_resolve_kernel_short_torvalds_url(self, fetcher): + """Test resolving kernel.org short torvalds URL.""" + url = "https://git.kernel.org/torvalds/c/abc123def456" + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/patch/?id=abc123def456" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + + def test_resolve_unknown_url_returns_none(self, fetcher): + """Test that unknown URLs return None.""" + url = "https://example.com/some/path" + resolved = fetcher._resolve_to_patch_url(url) + assert resolved is None + + def test_resolve_gitweb_commit_url(self, fetcher): + """Test resolving git.samba.org gitweb commit URL.""" + url = ( + "https://git.samba.org/?p=rsync.git;a=commit;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + resolved = fetcher._resolve_gitweb_url(url) + assert resolved is not None + assert resolved.platform == "gitweb" + assert resolved.patch_url == ( + "https://git.samba.org/?p=rsync.git;a=patch;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + assert resolved.repo_url == "https://git.samba.org/rsync.git" + assert resolved.commit_sha == "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + + def test_resolve_gitweb_patch_url(self, fetcher): + """Test resolving gitweb patch URL directly.""" + url = ( + "https://git.samba.org/?p=rsync.git;a=patch;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + resolved = fetcher._resolve_gitweb_url(url) + assert resolved is not None + assert resolved.platform == "gitweb" + assert resolved.patch_url == url + + +class TestBuildPatchUrlFromRepo: + """Test patch URL construction from OSV repo + commit.""" + + def test_samba_rsync_gitweb(self): + repo = "https://git.samba.org/rsync.git" + sha = "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == f"https://git.samba.org/?p=rsync.git;a=patch;h={sha}" + + def test_kernel_cgit_path_info(self): + repo = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + sha = "f342de4e2f33" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == f"{repo}/patch/?id={sha}" + + def test_github_repo(self): + repo = "https://github.com/openssl/openssl" + sha = "abc123def456" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == "https://github.com/openssl/openssl/commit/abc123def456.patch" + + +class TestPrioritizeAndDedupe: + """Test URL prioritization and deduplication.""" + + @pytest.fixture + def fetcher(self): + """Create a WebPatchFetcher with a mock session.""" + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_prioritize_commit_over_pull(self, fetcher): + """Test that commit URLs are prioritized over PR URLs.""" + candidates = { + "ghsa": [ + "https://github.com/foo/bar/pull/123", + "https://github.com/foo/bar/commit/abc123", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][2] == "commit" # commit first + assert result[1][2] == "pull" # PR second + + def test_prioritize_ubuntu_patches_first(self, fetcher): + """Test that ubuntu_patches source is prioritized for commits.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/ghsa123"], + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/ubuntu123"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][1] == "ubuntu_patches" # ubuntu_patches first + assert result[1][1] == "ghsa" + + def test_deduplicate_same_url(self, fetcher): + """Test that duplicate URLs are deduplicated.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/abc123"], + "nvd": ["https://github.com/foo/bar/commit/abc123"], + "rhsa": ["https://github.com/foo/bar/commit/abc123"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 1 # Only one unique URL + + def test_deduplicate_with_trailing_slash(self, fetcher): + """Test deduplication handles trailing slashes.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/abc123"], + "nvd": ["https://github.com/foo/bar/commit/abc123/"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 1 + + def test_kernel_org_urls_prioritized_as_commits(self, fetcher): + """Test that kernel.org URLs are recognized as commit URLs.""" + candidates = { + "nvd": [ + "https://git.kernel.org/stable/c/abc123", + "https://github.com/foo/bar/pull/456", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][2] == "commit" # kernel.org first (commit) + assert result[1][2] == "pull" # PR second + + def test_empty_candidates(self, fetcher): + """Test handling of empty candidates.""" + candidates = {} + result = fetcher._prioritize_and_dedupe(candidates) + assert result == [] + + def test_skip_non_matching_urls(self, fetcher): + """Test that non-commit/PR URLs are skipped.""" + candidates = { + "ghsa": [ + "https://github.com/advisories/GHSA-xxxx-xxxx-xxxx", + "https://example.com/something", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert result == [] # Advisory URLs don't match commit/PR patterns + + +class TestNormalizeUrl: + """Test URL normalization for deduplication.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_normalize_removes_trailing_slash(self, fetcher): + url = "https://github.com/foo/bar/commit/abc123/" + normalized = fetcher._normalize_url_for_dedupe(url) + assert not normalized.endswith("/") + + def test_normalize_lowercases(self, fetcher): + url = "https://GitHub.com/Foo/Bar/Commit/ABC123" + normalized = fetcher._normalize_url_for_dedupe(url) + assert normalized == normalized.lower() + + def test_normalize_removes_patch_suffix(self, fetcher): + url = "https://github.com/foo/bar/commit/abc123.patch" + normalized = fetcher._normalize_url_for_dedupe(url) + assert not normalized.endswith(".patch") + + +class TestIsCommitUrl: + """Test commit URL detection.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_github_commit_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://github.com/foo/bar/commit/abc123") + + def test_kernel_cgit_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://git.kernel.org/pub/scm/linux.git/commit/?id=abc123") + + def test_kernel_short_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://git.kernel.org/stable/c/abc123") + + def test_github_pr_is_not_commit_url(self, fetcher): + assert not fetcher._is_commit_url("https://github.com/foo/bar/pull/123") + + def test_advisory_is_not_commit_url(self, fetcher): + assert not fetcher._is_commit_url("https://github.com/advisories/GHSA-xxxx") + + def test_gitweb_commit_is_commit_url(self, fetcher): + url = "https://git.samba.org/?p=rsync.git;a=commit;h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + assert fetcher._is_commit_url(url) + + +class TestIsPrUrl: + """Test PR URL detection.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_github_pr_is_pr_url(self, fetcher): + assert fetcher._is_pr_url("https://github.com/foo/bar/pull/123") + + def test_github_commit_is_not_pr_url(self, fetcher): + assert not fetcher._is_pr_url("https://github.com/foo/bar/commit/abc123") + + +@pytest.mark.asyncio +class TestFetchFromIntelRefs: + """Test fetch_from_intel_refs integration.""" + + @pytest.fixture + def mock_session(self): + return MagicMock() + + async def test_fetch_from_intel_refs_empty_candidates(self, mock_session): + """Test handling of empty candidates.""" + fetcher = WebPatchFetcher(session=mock_session) + result = await fetcher.fetch_from_intel_refs({}, "CVE-2024-1234") + assert result is None + + async def test_fetch_from_intel_refs_returns_first_success(self, mock_session): + """Test that first successful fetch is returned.""" + fetcher = WebPatchFetcher(session=mock_session) + + # Mock the fetch_from_url method + with patch.object(fetcher, 'fetch_from_url') as mock_fetch: + mock_result = MagicMock() + mock_result.parsed_patch = MagicMock() + mock_fetch.return_value = mock_result + + candidates = { + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/abc123"], + } + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + + assert result is mock_result + mock_fetch.assert_called_once() + + async def test_fetch_from_intel_refs_tries_multiple_urls(self, mock_session): + """Test that multiple URLs are tried on failure.""" + fetcher = WebPatchFetcher(session=mock_session) + + with patch.object(fetcher, 'fetch_from_url') as mock_fetch: + # First call returns None (failure), second returns success + mock_result = MagicMock() + mock_result.parsed_patch = MagicMock() + mock_fetch.side_effect = [None, mock_result] + + candidates = { + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/abc123"], + "ghsa": ["https://github.com/foo/bar/commit/def456"], + } + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + + assert result is mock_result + assert mock_fetch.call_count == 2 diff --git a/src/vuln_analysis/utils/token_utils.py b/src/vuln_analysis/utils/token_utils.py index c3e435892..3624e3e80 100644 --- a/src/vuln_analysis/utils/token_utils.py +++ b/src/vuln_analysis/utils/token_utils.py @@ -79,6 +79,21 @@ def truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400 kept_lines.append(f"[... truncated {token_count - kept_tokens} tokens ...]") return '\n'.join(kept_lines) + if tool_name == ToolNames.SOURCE_GREP: + blocks = tool_output.split('\n--\n') + kept_blocks, kept_tokens = [], 0 + for block in blocks: + block_tokens = count_tokens(block) + if kept_tokens + block_tokens > max_tokens: + break + kept_blocks.append(block) + kept_tokens += block_tokens + truncated_count = len(blocks) - len(kept_blocks) + result = '\n--\n'.join(kept_blocks) + if truncated_count > 0: + result += f"\n[... truncated {truncated_count} more matches ...]" + return result + head_budget = int(max_tokens * 0.7) tail_budget = max_tokens - head_budget head_lines, head_tokens = [], 0 diff --git a/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py new file mode 100644 index 000000000..a703d0e96 --- /dev/null +++ b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sanitize LLM-extracted VulnerabilityIntel after L1 extraction.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel +from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch + +_BOOLEAN_OP_RE = re.compile(r"\s+(?:OR|AND)\s+", re.IGNORECASE) + + +def _patch_basenames(parsed_patch: ParsedPatch) -> set[str]: + names: set[str] = set() + for patch_file in parsed_patch.files: + for path in (patch_file.source_path, patch_file.target_path): + if path: + names.add(Path(path).name.lower()) + return names + + +def _has_boolean_operator(keyword: str) -> bool: + return bool(_BOOLEAN_OP_RE.search(keyword)) + + +class VulnerabilityIntelSanitizer: + """Apply shape rules to L1 VulnerabilityIntel; extensible one method per rule.""" + + def __init__(self, parsed_patch: ParsedPatch | None = None) -> None: + self._parsed_patch = parsed_patch + + @property + def _has_trusted_patch(self) -> bool: + return self._parsed_patch is not None and bool(self._parsed_patch.files) + + def apply(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + intel = self.sanitize_affected_files(intel) + intel = self.filter_vulnerable_functions(intel) + return self.filter_search_keywords(intel) + + def sanitize_affected_files(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + parsed_patch = self._parsed_patch + if parsed_patch is None or not parsed_patch.files: + return intel.model_copy(update={"affected_files": []}) + + allowed = _patch_basenames(parsed_patch) + kept = [ + path + for path in intel.affected_files + if Path(path).name.lower() in allowed + ] + return intel.model_copy(update={"affected_files": kept}) + + def filter_vulnerable_functions(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + kept = [name for name in intel.vulnerable_functions if " " not in name] + return intel.model_copy(update={"vulnerable_functions": kept}) + + def filter_search_keywords(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + kept = [ + kw + for kw in intel.search_keywords + if " " not in kw or _has_boolean_operator(kw) + ] + return intel.model_copy(update={"search_keywords": kept}) diff --git a/src/vuln_analysis/utils/web_patch_fetcher.py b/src/vuln_analysis/utils/web_patch_fetcher.py new file mode 100644 index 000000000..cd0d2018b --- /dev/null +++ b/src/vuln_analysis/utils/web_patch_fetcher.py @@ -0,0 +1,1073 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Web Patch Fetcher - unified patch fetching from intel references and OSV API. + +This module provides a unified interface for fetching upstream fix patches from: +- Intel references (GHSA, NVD, RHSA, Ubuntu) +- Chromium issue tracker (via Gerrit/Gitiles) +- OSV API (fallback) + +Supported platforms: +- GitHub (/commit/, /pull/) +- kernel.org (cgit format, short /stable/c/ format) +- gitweb (e.g. git.samba.org — ``?p=repo.git;a=patch;h=``) +- gitiles (Chromium/Google source, base64-encoded patches) +""" + +from __future__ import annotations + +import base64 +import os +import re +from typing import Literal, TYPE_CHECKING + +from urllib.parse import urlparse, unquote + +import aiohttp +from pydantic import BaseModel +from unidiff import PatchSet + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry +from vuln_analysis.utils.intel_utils import CHROMIUM_ISSUE_PATTERN +from vuln_analysis.utils.gerrit_client import ( + search_changes_by_bug, + list_merged_changes, + select_gerrit_change, + get_current_commit_sha, + project_to_gitiles_repo_url, + build_gitiles_patch_url, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + OSVPatchResult, + ParsedPatch, + PatchFile, + PatchHunk, +) + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# Re-export for backwards compatibility +WebPatchResult = OSVPatchResult + +# Environment configuration +_GITHUB_TOKEN = os.environ.get("GHSA_API_KEY") +_OSV_API_URL = os.environ.get("OSV_API_URL", "https://api.osv.dev/v1/vulns/") +_OSV_TIMEOUT_SECONDS = int(os.environ.get("OSV_TIMEOUT_SECONDS", "10")) +_PATCH_FETCH_TIMEOUT_SECONDS = int(os.environ.get("PATCH_FETCH_TIMEOUT_SECONDS", "30")) + +# Binary file extensions to skip when parsing patches +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu', '.uue', '.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', +}) + +# URL patterns for platform detection and resolution +_GITHUB_COMMIT_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+)/commit/([a-f0-9]+)" +) +_GITHUB_PR_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+)/pull/(\d+)" +) +_GITHUB_REPO_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$" +) + +# kernel.org cgit patterns +_KERNEL_CGIT_COMMIT_PATTERN = re.compile( + r"(https://git\.kernel\.org/.+\.git)/commit/\?id=([a-f0-9]+)" +) +_KERNEL_CGIT_PATCH_PATTERN = re.compile( + r"https://git\.kernel\.org/.+\.git/patch/\?id=[a-f0-9]+" +) +_KERNEL_SHORT_PATTERN = re.compile( + r"https://git\.kernel\.org/(\w+)/c/([a-f0-9]+)" +) + +# Mapping from kernel.org short paths to full repo paths +_KERNEL_SHORT_PATH_MAP = { + "stable": "linux/kernel/git/stable/linux.git", + "torvalds": "linux/kernel/git/torvalds/linux.git", + "next": "linux/kernel/git/next/linux-next.git", +} + +# Ubuntu patch reference pattern (e.g., "upstream: https://github.com/.../commit/...") +_UBUNTU_PATCH_URL_PATTERN = re.compile( + r"(?:upstream:\s*)?(https://github\.com/[^/]+/[^/]+/commit/([a-f0-9]+))" +) + +# gitweb (Samba, etc.): ?p=repo.git;a=commit|patch;h= +_GITWEB_COMMIT_PATTERN = re.compile( + r"(https?://[^/]+)/\?p=([^;&]+\.git);a=commit;h=([a-f0-9]+)" +) +_GITWEB_PATCH_PATTERN = re.compile( + r"(https?://[^/]+)/\?p=([^;&]+\.git);a=patch;h=([a-f0-9]+)" +) + +# Gitiles (Chromium/Google source): /+/ with optional ^! and ?format=TEXT +# Matches: https://chromium.googlesource.com/angle/angle/+/abc123 +# https://chromium.googlesource.com/chromium/src/+/abc123%5E%21?format=TEXT +_GITILES_COMMIT_PATTERN = re.compile( + r"(https://[^/]+\.googlesource\.com/[^+]+)/\+/([a-f0-9]+)(?:%5E%21|\^!)?" +) + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + +class OSVAffectedRange(BaseModel): + """Represents a Git range from an OSV affected block.""" + repo_url: str | None = None + fixed_commit: str | None = None + introduced_commit: str | None = None + + +class ResolvedUrl(BaseModel): + """Result of resolving a URL to a patch download URL.""" + patch_url: str + platform: Literal["github", "kernel.org", "gitweb", "cgit", "gitiles"] + url_type: Literal["commit", "pull", "chromium_issue"] + repo_url: str | None = None + commit_sha: str | None = None + + +# --------------------------------------------------------------------------- +# Helper Functions +# --------------------------------------------------------------------------- + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def _parse_patch_content(patch_content: str, patch_filename: str) -> ParsedPatch | None: + """Parse patch content string into structured ParsedPatch model.""" + try: + patch_set = PatchSet.from_string(patch_content) + except Exception: + logger.warning("_parse_patch_content: failed to parse patch content") + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_filename, files=files) + + +def _extract_commit_metadata(patch_content: str) -> tuple[str | None, str | None, str | None]: + """Extract commit message, author, and date from .patch format. + + Works with both GitHub and kernel.org patch formats. + """ + lines = patch_content.split('\n') + author = None + date = None + subject_lines = [] + in_subject = False + + for line in lines: + if line.startswith('From:'): + author = line[5:].strip() + elif line.startswith('Date:'): + date = line[5:].strip() + elif line.startswith('Subject:'): + in_subject = True + subject_part = line[8:].strip() + if subject_part.startswith('[PATCH'): + idx = subject_part.find(']') + if idx != -1: + subject_part = subject_part[idx + 1:].strip() + subject_lines.append(subject_part) + elif in_subject: + if line.startswith('---') or line.startswith('diff --git'): + break + if line.strip() == '': + in_subject = False + else: + subject_lines.append(line.strip()) + + commit_message = ' '.join(subject_lines).strip() if subject_lines else None + return commit_message, author, date + + +def build_patch_url_from_repo(repo_url: str, commit_sha: str) -> str | None: + """Build a patch download URL from a git repo URL and commit SHA. + + Supports GitHub, gitweb (shallow ``host/repo.git`` paths), and cgit path-info. + """ + match = _GITHUB_REPO_PATTERN.match(repo_url) + if match: + repo_path = match.group(1) + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + repo_path = '/'.join(parts[-2:]).replace('.git', '') + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + + parsed = urlparse(repo_url.rstrip('/')) + if not parsed.path.endswith('.git'): + return None + + path_parts = [p for p in parsed.path.split('/') if p] + if not path_parts: + return None + + project = path_parts[-1] + base = f"{parsed.scheme}://{parsed.netloc}" + + # Shallow repo path (e.g. git.samba.org/rsync.git) → gitweb + if len(path_parts) == 1: + return f"{base}/?p={project};a=patch;h={commit_sha}" + + # Deep path (e.g. git.kernel.org/pub/scm/.../linux.git) → cgit path-info + return f"{repo_url.rstrip('/')}/patch/?id={commit_sha}" + + +# --------------------------------------------------------------------------- +# WebPatchFetcher - Unified Patch Fetcher +# --------------------------------------------------------------------------- + +class WebPatchFetcher: + """Unified patch fetcher for intel references and direct URLs. + + Supports: + - GitHub commit URLs (/commit/) + - GitHub PR URLs (/pull/) + - kernel.org cgit URLs (/commit/?id=, /patch/?id=) + - kernel.org short URLs (/stable/c/, /torvalds/c/) + - gitweb URLs (?p=repo.git;a=commit|patch;h=) + + Usage: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session) + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + if result: + # Use result.parsed_patch + pass + """ + + def __init__( + self, + session: aiohttp.ClientSession, + timeout: int = _PATCH_FETCH_TIMEOUT_SECONDS, + ): + self._session = session + self._timeout = aiohttp.ClientTimeout(total=timeout) + self._headers = {"User-Agent": "vuln-analysis/1.0"} + if _GITHUB_TOKEN: + self._headers["Authorization"] = f"token {_GITHUB_TOKEN}" + self._cache: dict[str, WebPatchResult] = {} + + async def fetch_from_intel_refs( + self, + candidates: dict[str, list[str]], + cve_id: str, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, + ) -> WebPatchResult | None: + """Fetch patch from intel reference URLs in priority order. + + Priority: + 1. /commit/ URLs (ubuntu_patches first, then ghsa, nvd, rhsa, ubuntu) + 2. /pull/ URLs (ghsa, nvd, rhsa, ubuntu) + 3. Chromium issue URLs (require Gerrit/Gitiles resolution) + + Args: + candidates: Dict mapping source to list of URLs + Example: {"ghsa": ["https://github.com/.../commit/..."], "nvd": [...]} + cve_id: CVE identifier for logging and result + cve_description: Optional CVE description for Chromium CL disambiguation + llm: Optional LangChain LLM for Chromium CL selection when multiple MERGED CLs exist + + Returns: + WebPatchResult on success, None if no valid patch found + """ + prioritized = self._prioritize_and_dedupe(candidates) + + if not prioritized: + logger.debug("No candidate URLs found for %s", cve_id) + return None + + urls_tried = 0 + for url, source, url_type in prioritized: + urls_tried += 1 + + if url_type == "chromium_issue": + # Handle Chromium issue URLs via Gerrit/Gitiles + result = await self._fetch_chromium_issue( + url, cve_id, source, cve_description, llm + ) + else: + # Standard commit/PR URL handling + result = await self.fetch_from_url(url, cve_id, source=source, url_type_hint=url_type) + + if result and result.parsed_patch: + logger.info( + "Intel refs: Found patch for %s from %s (%s, %s)", + cve_id, source, url_type, result.platform + ) + return result + + logger.info("Intel refs: No valid patch found for %s after trying %d URLs", cve_id, urls_tried) + return None + + async def fetch_from_url( + self, + url: str, + cve_id: str, + source: str | None = None, + url_type_hint: str | None = None, + ) -> WebPatchResult | None: + """Fetch and parse a patch from a single URL. + + Args: + url: URL to fetch (commit, PR, or patch URL) + cve_id: CVE identifier + source: Source provider (for result metadata) + url_type_hint: URL type hint (commit/pull) + + Returns: + WebPatchResult on success, None on failure + """ + # Check cache first + cache_key = url.lower().rstrip("/") + if cache_key in self._cache: + logger.debug("Cache hit for %s", url) + return self._cache[cache_key] + + # Resolve URL to patch download URL + resolved = self._resolve_to_patch_url(url) + if not resolved: + logger.debug("Could not resolve URL to patch format: %s", url) + return None + + # Fetch patch content + patch_content = await self._fetch_patch_content(resolved.patch_url) + if not patch_content: + return None + + # Extract metadata and parse + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + commit_sha = resolved.commit_sha or "unknown" + parsed_patch = _parse_patch_content( + patch_content, + f"{cve_id}_{commit_sha[:8]}.patch" + ) + + if not parsed_patch: + logger.warning("Failed to parse patch content from %s", url) + return None + + result = WebPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8] if len(commit_sha) > 8 else commit_sha, + repo_url=resolved.repo_url or "", + patch_url=resolved.patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + source=source, + url_type=url_type_hint or resolved.url_type, + platform=resolved.platform, + ) + + # Cache successful result + self._cache[cache_key] = result + return result + + async def _fetch_chromium_issue( + self, + issue_url: str, + cve_id: str, + source: str, + cve_description: str | None, + llm: "BaseChatModel | None", + ) -> WebPatchResult | None: + """Fetch patch for a Chromium issue via Gerrit/Gitiles. + + Flow: + 1. Extract bug ID from issue URL + 2. Search Gerrit for CLs with that bug ID + 3. Filter to MERGED CLs only + 4. Select the correct CL (single = use it, multiple = LLM selection) + 5. Get commit SHA from selected CL + 6. Fetch patch from Gitiles + + Args: + issue_url: Chromium issue URL (e.g., https://issues.chromium.org/issues/466192044) + cve_id: CVE identifier + source: Source provider (ghsa, nvd, etc.) + cve_description: CVE description for LLM disambiguation + llm: Optional LangChain LLM for CL selection + + Returns: + WebPatchResult on success, None on failure + """ + # Extract bug ID from URL + match = CHROMIUM_ISSUE_PATTERN.match(issue_url) + if not match: + logger.debug("Invalid Chromium issue URL: %s", issue_url) + return None + + bug_id = match.group(1) + logger.info("Chromium issue: Searching Gerrit for bug %s", bug_id) + + # Search Gerrit for CLs with this bug ID + raw_changes = await search_changes_by_bug(self._session, bug_id) + if not raw_changes: + logger.info("Chromium issue: No CLs found for bug %s", bug_id) + return None + + # Filter to MERGED only + merged = list_merged_changes(raw_changes) + if not merged: + logger.info("Chromium issue: No MERGED CLs found for bug %s", bug_id) + return None + + logger.info( + "Chromium issue: Found %d MERGED CLs for bug %s: %s", + len(merged), bug_id, [c.submission_id for c in merged] + ) + + # Select the correct CL + selected_id = await select_gerrit_change( + merged, cve_id, cve_description or "", llm + ) + if selected_id is None: + logger.info("Chromium issue: Could not select CL for bug %s", bug_id) + return None + + # Find the selected candidate to get project info + selected = next((c for c in merged if c.submission_id == selected_id), None) + if not selected: + logger.warning("Chromium issue: Selected ID %d not found in candidates", selected_id) + return None + + logger.info( + "Chromium issue: Selected CL %d (%s) for bug %s", + selected_id, selected.project, bug_id + ) + + # Get commit SHA from the selected CL + commit_sha = await get_current_commit_sha(self._session, selected_id) + if not commit_sha: + logger.warning("Chromium issue: Could not get commit SHA for CL %d", selected_id) + return None + + # Build Gitiles URL and fetch patch + repo_url = project_to_gitiles_repo_url(selected.project) + patch_url = build_gitiles_patch_url(repo_url, commit_sha) + + logger.info("Chromium issue: Fetching patch from %s", patch_url) + + # Fetch patch content (Gitiles base64 decoding handled automatically) + patch_content = await self._fetch_patch_content(patch_url) + if not patch_content: + logger.warning("Chromium issue: Failed to fetch patch from Gitiles") + return None + + # Parse patch + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content( + patch_content, + f"{cve_id}_{commit_sha[:8]}.patch" + ) + + if not parsed_patch: + logger.warning("Chromium issue: Failed to parse patch from Gitiles") + return None + + return WebPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8] if len(commit_sha) > 8 else commit_sha, + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message or selected.subject, + commit_author=commit_author, + commit_date=commit_date, + source=source, + url_type="chromium_issue", + platform="gitiles", + ) + + def _prioritize_and_dedupe( + self, + candidates: dict[str, list[str]], + ) -> list[tuple[str, str, str]]: + """Prioritize and deduplicate candidate URLs. + + Returns list of (url, source, url_type) tuples in priority order. + + Priority order: + 1. /commit/ URLs (direct patches, highest confidence) + 2. /pull/ URLs (may contain multiple commits) + 3. Chromium issue URLs (require Gerrit/Gitiles resolution, lowest priority) + """ + seen: set[str] = set() + result: list[tuple[str, str, str]] = [] + + # Priority 1: /commit/ URLs (and kernel.org commit URLs) + # Order: ubuntu_patches first (curated), then other sources + commit_sources = ["ubuntu_patches", "ghsa", "nvd", "rhsa", "ubuntu"] + for source in commit_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_commit_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "commit")) + + # Priority 2: /pull/ URLs + pr_sources = ["ghsa", "nvd", "rhsa", "ubuntu"] + for source in pr_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_pr_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "pull")) + + # Priority 3: Chromium issue URLs (lowest priority - require Gerrit/Gitiles resolution) + chromium_sources = ["ghsa", "nvd", "rhsa", "ubuntu"] + for source in chromium_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_chromium_issue_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "chromium_issue")) + + return result + + def _normalize_url_for_dedupe(self, url: str) -> str: + """Normalize URL for deduplication.""" + # Remove trailing slash, convert to lowercase + normalized = url.rstrip("/").lower() + # Remove .patch suffix for comparison + if normalized.endswith(".patch"): + normalized = normalized[:-6] + # Remove query string for commit URLs (but keep for kernel.org) + if "github.com" in normalized and "?" in normalized: + normalized = normalized.split("?")[0] + return normalized + + def _is_commit_url(self, url: str) -> bool: + """Check if URL is a commit URL (GitHub, kernel.org, or gitweb).""" + url_lower = url.lower() + return ( + "/commit/" in url_lower or + "/c/" in url_lower or # kernel.org short form + "?id=" in url_lower or # cgit path-info form + ";a=commit;" in url_lower # gitweb form + ) + + def _is_pr_url(self, url: str) -> bool: + """Check if URL is a PR URL.""" + return "/pull/" in url.lower() + + def _is_chromium_issue_url(self, url: str) -> bool: + """Check if URL is a Chromium issue tracker URL. + + These URLs point to issues.chromium.org/issues/ and require + resolution via Gerrit/Gitiles to fetch the actual patch. + """ + return bool(CHROMIUM_ISSUE_PATTERN.match(url)) + + def _resolve_to_patch_url(self, url: str) -> ResolvedUrl | None: + """Resolve a URL to its patch download URL. + + Dispatches to platform-specific handlers. + """ + # Try GitHub first + resolved = self._resolve_github_url(url) + if resolved: + return resolved + + # Try gitweb (e.g. git.samba.org) + resolved = self._resolve_gitweb_url(url) + if resolved: + return resolved + + # Try kernel.org + resolved = self._resolve_kernel_org_url(url) + if resolved: + return resolved + + # Try Gitiles (Chromium/Google) + resolved = self._resolve_gitiles_url(url) + if resolved: + return resolved + + return None + + def _resolve_github_url(self, url: str) -> ResolvedUrl | None: + """Resolve GitHub commit or PR URL to patch URL.""" + # GitHub commit URL + match = _GITHUB_COMMIT_PATTERN.match(url) + if match: + repo_path, commit_sha = match.groups() + patch_url = f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="commit", + repo_url=f"https://github.com/{repo_path}", + commit_sha=commit_sha, + ) + + # GitHub PR URL + match = _GITHUB_PR_PATTERN.match(url) + if match: + repo_path, pr_number = match.groups() + patch_url = f"https://github.com/{repo_path}/pull/{pr_number}.patch" + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="pull", + repo_url=f"https://github.com/{repo_path}", + commit_sha=f"PR-{pr_number}", + ) + + # Ubuntu-style "upstream: https://github.com/.../commit/..." + match = _UBUNTU_PATCH_URL_PATTERN.search(url) + if match: + commit_url, commit_sha = match.groups() + patch_url = f"{commit_url}.patch" + repo_url = commit_url.rsplit("/commit/", 1)[0] + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + return None + + def _resolve_gitweb_url(self, url: str) -> ResolvedUrl | None: + """Resolve gitweb commit or patch URL (e.g. git.samba.org).""" + match = _GITWEB_PATCH_PATTERN.match(url) + if match: + base, project, commit_sha = match.groups() + repo_url = f"{base}/{project}" + return ResolvedUrl( + patch_url=url, + platform="gitweb", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + match = _GITWEB_COMMIT_PATTERN.match(url) + if match: + base, project, commit_sha = match.groups() + patch_url = f"{base}/?p={project};a=patch;h={commit_sha}" + repo_url = f"{base}/{project}" + return ResolvedUrl( + patch_url=patch_url, + platform="gitweb", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + return None + + def _resolve_kernel_org_url(self, url: str) -> ResolvedUrl | None: + """Resolve kernel.org URL to patch URL.""" + # Already a patch URL + if _KERNEL_CGIT_PATCH_PATTERN.match(url): + # Extract commit SHA from ?id= parameter + sha_match = re.search(r"\?id=([a-f0-9]+)", url) + commit_sha = sha_match.group(1) if sha_match else "unknown" + repo_url = url.split("/patch/")[0] if "/patch/" in url else None + return ResolvedUrl( + patch_url=url, + platform="kernel.org", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + # cgit commit URL: .../commit/?id= + match = _KERNEL_CGIT_COMMIT_PATTERN.match(url) + if match: + repo_base, commit_sha = match.groups() + patch_url = f"{repo_base}/patch/?id={commit_sha}" + return ResolvedUrl( + patch_url=patch_url, + platform="kernel.org", + url_type="commit", + repo_url=repo_base, + commit_sha=commit_sha, + ) + + # Short form: /stable/c/, /torvalds/c/, etc. + match = _KERNEL_SHORT_PATTERN.match(url) + if match: + tree, commit_sha = match.groups() + repo_path = _KERNEL_SHORT_PATH_MAP.get(tree) + if repo_path: + patch_url = f"https://git.kernel.org/pub/scm/{repo_path}/patch/?id={commit_sha}" + repo_url = f"https://git.kernel.org/pub/scm/{repo_path}" + return ResolvedUrl( + patch_url=patch_url, + platform="kernel.org", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + else: + logger.warning("Unknown kernel.org tree: %s", tree) + + return None + + def _resolve_gitiles_url(self, url: str) -> ResolvedUrl | None: + """Resolve Gitiles commit URL to patch URL. + + Gitiles is used by Chromium and other Google projects. + The patch is fetched via /+/^! with ?format=TEXT which returns base64-encoded content. + """ + # Decode URL-encoded characters for pattern matching + decoded_url = unquote(url) + + match = _GITILES_COMMIT_PATTERN.match(decoded_url) + if not match: + # Try with the original URL (might have literal ^!) + match = _GITILES_COMMIT_PATTERN.match(url) + if not match: + return None + + repo_url, commit_sha = match.groups() + # Build patch URL with ^! suffix (pre-encoded) and format=TEXT + # Must use %5E%21 with yarl.URL(encoded=True) to avoid mixed encoding + patch_url = f"{repo_url}/+/{commit_sha}%5E%21?format=TEXT" + + return ResolvedUrl( + patch_url=patch_url, + platform="gitiles", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + async def _fetch_patch_content(self, patch_url: str) -> str | None: + """Fetch patch content from URL with rate limit handling. + + Automatically handles Gitiles base64-encoded responses. + """ + is_gitiles = ".googlesource.com" in patch_url and "format=TEXT" in patch_url + + # For Gitiles URLs, use a fresh session with yarl.URL(encoded=True) + # to preserve the exact %5E%21 encoding and avoid re-encoding issues + if is_gitiles: + content = await self._fetch_gitiles_patch(patch_url) + if content: + return self._decode_gitiles_response(content) + else: + return None + + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': self._timeout, + 'headers': self._headers, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + content = await response.text() + return content + + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.debug("Patch not found: %s", patch_url) + elif e.status == 429: + logger.warning("Rate limited on %s, skipping", patch_url) + else: + logger.warning("Patch fetch failed: %s - %s", patch_url, e) + return None + except Exception as e: + logger.warning("Patch fetch failed: %s - %s", patch_url, e) + return None + + async def _fetch_gitiles_patch(self, patch_url: str) -> str | None: + """Fetch patch from Gitiles using a fresh session. + + Gitiles requires exact URL encoding for ^! suffix (%5E%21). + Using a fresh session avoids any URL re-encoding issues from shared sessions. + """ + import requests + # 2. Fetch the encoded data + response = requests.get(patch_url,timeout=(3.05, 15)) + + if response.status_code == 200: + return response.text + else: + logger.warning("Gitiles patch fetch failed: %s - %s", patch_url, response.status_code) + return None + + + def _decode_gitiles_response(self, content: str) -> str | None: + """Decode Gitiles base64-encoded patch response. + + Args: + content: Base64-encoded content from Gitiles ?format=TEXT endpoint + + Returns: + Decoded patch content, or None on decode failure + """ + try: + decoded_bytes = base64.b64decode(content) + return decoded_bytes.decode("utf-8") + except Exception as e: + logger.warning("Failed to decode Gitiles base64 response: %s", e) + return None + + +# --------------------------------------------------------------------------- +# OSVClient - OSV API Client +# --------------------------------------------------------------------------- + +class OSVClient: + """Client for querying OSV API and fetching fix patches. + + Usage: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session) + client = OSVClient(session, fetcher) + result = await client.get_fix_patch("CVE-2024-1234", "3.0.7", "openssl") + """ + + def __init__( + self, + session: aiohttp.ClientSession, + patch_fetcher: WebPatchFetcher | None = None, + osv_timeout: int = _OSV_TIMEOUT_SECONDS, + ): + self._session = session + self._patch_fetcher = patch_fetcher or WebPatchFetcher(session) + self._osv_timeout = aiohttp.ClientTimeout(total=osv_timeout) + + async def get_fix_patch( + self, + cve_id: str, + upstream_version: str | None = None, # pylint: disable=unused-argument + package_name: str | None = None, # pylint: disable=unused-argument + ) -> WebPatchResult | None: + """Query OSV API and fetch the fix patch. + + Args: + cve_id: CVE identifier (e.g., "CVE-2024-1234") + upstream_version: Upstream version (e.g., "3.0.7") - reserved for future filtering + package_name: Optional package name - reserved for future filtering + + Returns: + WebPatchResult with patch data, or None if no fix found + """ + try: + osv_data = await self._query_osv(cve_id) + if not osv_data: + return None + + # Try to get fix commit URL from references first (most specific) + patch_url = self._extract_commit_from_references(osv_data) + repo_url = None + + if patch_url: + # Extract repo_url from the patch URL + if '/commit/' in patch_url: + repo_url = patch_url.split('/commit/')[0] + elif '/pull/' in patch_url: + repo_url = patch_url.split('/pull/')[0] + logger.info("OSV: Found fix commit in references for %s", cve_id) + else: + return None + #OSV affected GIT fixed events mark version boundaries (e.g. release tags), + #not security patches, so using them as + # fix diffs sends the checker the wrong upstream patch. + # Find fix commit from affected block + #affected = self._find_matching_affected(osv_data) + #if not affected: + # logger.info("OSV: No affected block with fix found for %s", cve_id) + # return None + + #range_info = self._extract_fix_commit(affected) + #if not range_info.fixed_commit or not range_info.repo_url: + # logger.info("OSV: No fixed commit found for %s", cve_id) + # return None + + #patch_url = self._build_patch_url(range_info.repo_url, range_info.fixed_commit) + #repo_url = range_info.repo_url + + if not patch_url: + logger.info("OSV: Could not build patch URL for %s (unsupported repo host?)", cve_id) + return None + + # Use WebPatchFetcher to fetch and parse + url_for_resolution = patch_url + if patch_url.endswith('.patch') and 'github.com' in patch_url: + url_for_resolution = patch_url[:-6] + result = await self._patch_fetcher.fetch_from_url( + url_for_resolution, + cve_id, + source="osv", + ) + + if result: + # Ensure we have the correct repo_url + if repo_url and not result.repo_url: + result.repo_url = repo_url + return result + + return None + + except Exception: + logger.warning("OSV patch retrieval failed for %s", cve_id, exc_info=True) + return None + + async def _query_osv(self, cve_id: str) -> dict | None: + """Query OSV API for CVE data.""" + url = f"{_OSV_API_URL}{cve_id}" + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': url, + 'timeout': self._osv_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.json() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("OSV: CVE %s not found", cve_id) + else: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + except Exception as e: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + + def _extract_commit_from_references(self, osv_data: dict) -> str | None: + """Extract fix commit URL from OSV references.""" + references = osv_data.get("references", []) + + for ref in references: + if ref.get("type") == "FIX": + url = ref.get("url", "") + if "github.com" in url and ("/commit/" in url or "/pull/" in url): + if not url.endswith(".patch"): + return f"{url}.patch" + return url + if ";a=commit;" in url: + return url.replace(";a=commit;", ";a=patch;") + if ";a=patch;" in url: + return url + + return None + + def _find_matching_affected(self, osv_data: dict) -> dict | None: + """Find an affected block with a GIT range containing a fixed commit.""" + for affected in osv_data.get("affected", []): + for range_block in affected.get("ranges", []): + if range_block.get("type") == "GIT": + for event in range_block.get("events", []): + if "fixed" in event: + return affected + return None + + def _extract_fix_commit(self, affected: dict) -> OSVAffectedRange: + """Extract fixed commit hash and repo URL from affected block.""" + result = OSVAffectedRange() + + ranges = affected.get("ranges", []) + for range_block in ranges: + if range_block.get("type") != "GIT": + continue + + repo = range_block.get("repo") + if repo: + result.repo_url = repo + + events = range_block.get("events", []) + for event in events: + if "introduced" in event and event["introduced"] != "0": + result.introduced_commit = event["introduced"] + if "fixed" in event: + result.fixed_commit = event["fixed"] + + if result.fixed_commit: + break + + return result + + def _build_patch_url(self, repo_url: str, commit_sha: str) -> str | None: + """Build patch download URL from repo URL and commit SHA.""" + patch_url = build_patch_url_from_repo(repo_url, commit_sha) + if not patch_url: + logger.debug("Unsupported repo URL: %s", repo_url) + return patch_url diff --git a/tests/test_brew_downloader.py b/tests/test_brew_downloader.py new file mode 100644 index 000000000..819e810c0 --- /dev/null +++ b/tests/test_brew_downloader.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for BrewDownloader profile resolution and artifact orchestration.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from vuln_analysis.tools.brew_downloader import ( + BrewDownloader, + BrewProfileNotImplementedError, + BrewProfileType, + resolve_brew_profile, +) + + +class TestResolveBrewProfile: + def test_resolves_internal_from_config(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("internal") == BrewProfileType.INTERNAL + + def test_resolves_external_from_config(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("external") == BrewProfileType.EXTERNAL + + def test_env_overrides_config(self, monkeypatch): + monkeypatch.setenv("RPM_USER_TYPE", "external") + assert resolve_brew_profile("internal") == BrewProfileType.EXTERNAL + + def test_yaml_env_default_syntax_without_expansion(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("${RPM_USER_TYPE:-internal}") == BrewProfileType.INTERNAL + + def test_yaml_env_default_syntax_uses_env(self, monkeypatch): + monkeypatch.setenv("RPM_USER_TYPE", "external") + assert resolve_brew_profile("${RPM_USER_TYPE:-internal}") == BrewProfileType.EXTERNAL + + def test_unknown_profile_raises(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + with pytest.raises(BrewProfileNotImplementedError): + resolve_brew_profile("unknown") + + +class TestBrewProfileFlags: + def test_external_profile_disables_build_log_fetch(self, tmp_path): + downloader = BrewDownloader( + BrewProfileType.EXTERNAL, + str(tmp_path / "rpms"), + str(tmp_path / "checker"), + ) + assert downloader.auto_fetch_build_log is False + assert downloader.download_binary_rpm_enabled is False + + def test_internal_profile_enables_build_log_fetch(self, tmp_path): + downloader = BrewDownloader( + BrewProfileType.INTERNAL, + str(tmp_path / "rpms"), + str(tmp_path / "checker"), + ) + assert downloader.auto_fetch_build_log is True + + +class TestDownloadTargetArtifacts: + def test_skips_build_log_when_auto_fetch_disabled(self, tmp_path): + rpm_cache = tmp_path / "rpms" + checker_dir = tmp_path / "checker" + downloader = BrewDownloader( + BrewProfileType.EXTERNAL, + str(rpm_cache), + str(checker_dir), + ) + build = {"id": 1, "nvr": "curl-8.11.1-8.fc42"} + srpm_file = rpm_cache / "curl-8.11.1-8.fc42.src.rpm" + srpm_file.write_bytes(b"fake-srpm") + + downloader._session = MagicMock() + downloader._pathinfo = MagicMock() + + with ( + patch.object(downloader, "search_build", return_value=build), + patch.object(downloader, "_get_srpm_url", return_value="https://example/srpm"), + patch.object(downloader, "download_srpm", return_value=srpm_file), + patch.object(downloader, "download_build_log") as mock_build_log, + patch( + "vuln_analysis.tools.brew_downloader.SourceRPMDownloader.extract_src_rpm", + ), + ): + artifacts = downloader.download_target_artifacts( + "curl", "8.11.1", "8.fc42", "x86_64", + ) + + mock_build_log.assert_not_called() + assert artifacts.build_log_path is None + assert artifacts.srpm_path == checker_dir / "source" + + +class TestSourceAcquisitionCacheCondition: + """Mirror the cache-hit predicate used in cve_source_acquisition.""" + + @staticmethod + def _is_full_cache_hit(source_exists: bool, log_exists: bool, auto_fetch_build_log: bool) -> bool: + return source_exists and (log_exists or not auto_fetch_build_log) + + def test_source_only_hit_when_auto_fetch_false(self): + assert self._is_full_cache_hit( + source_exists=True, + log_exists=False, + auto_fetch_build_log=False, + ) + + def test_requires_log_when_auto_fetch_true(self): + assert not self._is_full_cache_hit( + source_exists=True, + log_exists=False, + auto_fetch_build_log=True, + ) + assert self._is_full_cache_hit( + source_exists=True, + log_exists=True, + auto_fetch_build_log=True, + ) diff --git a/tests/test_package_identifier.py b/tests/test_package_identifier.py new file mode 100644 index 000000000..993a7a0f2 --- /dev/null +++ b/tests/test_package_identifier.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult, PackageCheckerStatus +from exploit_iq_commons.data_models.common import TargetPackage +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelRhsa, CveIntelNvd + +from vuln_analysis.utils.package_identifier import ( + PackageIdentifier, + _extract_rhel_version, + _interpret_fix_state, + _match_package_state_for_distro, +) + + +class TestExtractRhelVersion: + """Tests for _extract_rhel_version helper.""" + + def test_el7_extracts_7(self): + assert _extract_rhel_version("el7") == "7" + + def test_el8_extracts_8(self): + assert _extract_rhel_version("el8") == "8" + + def test_el10_extracts_10(self): + assert _extract_rhel_version("el10") == "10" + + def test_none_returns_none(self): + assert _extract_rhel_version(None) is None + + def test_empty_string_returns_none(self): + assert _extract_rhel_version("") is None + + def test_invalid_format_returns_none(self): + assert _extract_rhel_version("rhel7") is None + assert _extract_rhel_version("centos7") is None + + +class TestInterpretFixState: + """Tests for _interpret_fix_state helper.""" + + def test_not_affected_returns_no(self): + assert _interpret_fix_state("Not affected") == EnumIdentifyResult.NO + assert _interpret_fix_state("not affected") == EnumIdentifyResult.NO + assert _interpret_fix_state("NOT AFFECTED") == EnumIdentifyResult.NO + + def test_will_not_fix_returns_no(self): + assert _interpret_fix_state("Will not fix") == EnumIdentifyResult.NO + assert _interpret_fix_state("will not fix") == EnumIdentifyResult.NO + + def test_out_of_support_scope_returns_no(self): + assert _interpret_fix_state("Out of support scope") == EnumIdentifyResult.NO + assert _interpret_fix_state("out of support scope") == EnumIdentifyResult.NO + + def test_affected_returns_yes(self): + assert _interpret_fix_state("Affected") == EnumIdentifyResult.YES + assert _interpret_fix_state("affected") == EnumIdentifyResult.YES + + def test_fix_deferred_returns_yes(self): + assert _interpret_fix_state("Fix deferred") == EnumIdentifyResult.YES + assert _interpret_fix_state("fix deferred") == EnumIdentifyResult.YES + + def test_under_investigation_returns_yes(self): + assert _interpret_fix_state("Under investigation") == EnumIdentifyResult.YES + + def test_none_returns_none(self): + assert _interpret_fix_state(None) is None + + def test_empty_returns_none(self): + assert _interpret_fix_state("") is None + + def test_unknown_state_returns_none(self): + assert _interpret_fix_state("Fixed") is None + assert _interpret_fix_state("Unknown") is None + assert _interpret_fix_state("Pending") is None + + +class TestMatchPackageStateForDistro: + """Tests for _match_package_state_for_distro helper.""" + + @pytest.fixture + def rhel6_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ) + + @pytest.fixture + def rhel7_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ) + + @pytest.fixture + def rhel8_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state="Affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:8", + ) + + def test_matches_by_cpe(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el7") + assert result is not None + assert result.fix_state == "Will not fix" + + def test_matches_el6(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el6") + assert result is not None + assert result.fix_state == "Not affected" + + def test_no_distro_returns_first_name_match(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", None) + assert result is not None + assert result == rhel6_package_state + + def test_no_package_match_returns_none(self, rhel7_package_state): + states = [rhel7_package_state] + + result = _match_package_state_for_distro(states, "curl", "el7") + assert result is None + + def test_no_distro_match_returns_none(self, rhel7_package_state): + states = [rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el9") + assert result is None + + def test_matches_by_product_name(self): + state = CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe=None, # No CPE, should match by product_name + ) + + result = _match_package_state_for_distro([state], "libarchive", "el7") + assert result is not None + assert result.fix_state == "Will not fix" + + +class TestPackageIdentifierWithFixState: + """Integration tests for PackageIdentifier with RHSA fix_state.""" + + def test_cve_2016_8687_rhel7_will_not_fix(self): + """RHSA Will not fix on RHEL 7 -> NOT_VUL even when NVD version would match.""" + target = TargetPackage( + name="libarchive", + version="3.1.2", + release="14.el7_9.1", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2016-8687", + nvd=CveIntelNvd( + cve_id="CVE-2016-8687", + configurations=[ + CveIntelNvd.Configuration( + package="libarchive", + vendor="libarchive", + versionStartIncluding="3.2.1", + versionEndIncluding="3.2.1", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ), + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + assert "Will not fix" in result.conclusion_reason + + def test_cve_2016_8687_rhel6_not_affected(self): + """CVE-2016-8687 on RHEL 6 should be NO (Not affected).""" + target = TargetPackage( + name="libarchive", + version="3.0.3", + release="6.el6_10", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2016-8687", + nvd=CveIntelNvd( + cve_id="CVE-2016-8687", + configurations=[ + CveIntelNvd.Configuration( + package="libarchive", + vendor="libarchive", + versionStartIncluding="3.2.1", + versionEndIncluding="3.2.1", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ), + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + # Verify conclusion_reason is populated with RHSA details + assert "RHSA fix_state" in result.conclusion_reason + assert "Not affected" in result.conclusion_reason + assert "libarchive" in result.conclusion_reason + assert "el6" in result.conclusion_reason + + def test_falls_back_to_nvd_when_no_fix_state(self): + """When RHSA has no fix_state, should fall back to NVD version check.""" + target = TargetPackage( + name="somelib", + version="1.0.0", + release="1.el8", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0001", + nvd=CveIntelNvd( + cve_id="CVE-2024-0001", + configurations=[ + CveIntelNvd.Configuration( + package="somelib", + vendor="somevendor", + versionStartIncluding="1.0.0", + versionEndIncluding="1.5.0", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state=None, # No fix_state, should fall back to NVD + package_name="somelib", + cpe="cpe:/o:redhat:enterprise_linux:8", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + # Should be YES because 1.0.0 is in NVD range [1.0.0, 1.5.0] + assert result.is_target_package_affected == EnumIdentifyResult.YES + assert status == PackageCheckerStatus.OK + + def test_nvd_version_outside_range_populates_conclusion_reason(self): + """When target version is outside NVD range, conclusion_reason should explain.""" + target = TargetPackage( + name="somelib", + version="2.0.0", # Outside the affected range + release="1.el8", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0001", + nvd=CveIntelNvd( + cve_id="CVE-2024-0001", + configurations=[ + CveIntelNvd.Configuration( + package="somelib", + vendor="somevendor", + versionStartIncluding="1.0.0", + versionEndExcluding="1.5.0", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state=None, # No fix_state, should fall back to NVD + package_name="somelib", + cpe="cpe:/o:redhat:enterprise_linux:8", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + # Should be NO because 2.0.0 is outside NVD range [1.0.0, 1.5.0) + assert result.is_target_package_affected == EnumIdentifyResult.NO + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + # Verify conclusion_reason is populated with NVD version range details + assert "NVD" in result.conclusion_reason + assert "version" in result.conclusion_reason.lower() + assert "somelib" in result.conclusion_reason + assert "2.0.0" in result.conclusion_reason + assert ">=1.0.0" in result.conclusion_reason + assert "<1.5.0" in result.conclusion_reason + + +class TestPackageIdentifierRhsaScope: + """Step 1: RHSA package_state + affected_release scope gate.""" + + def test_affected_release_only_passes_scope(self): + target = TargetPackage( + name="curl", + version="7.76.1", + release="26.el9", + arch="x86_64", + ) + intel = CveIntel( + vuln_id="CVE-2024-TEST", + rhsa=CveIntelRhsa( + package_state=[], + affected_release=[ + {"package": "curl-7.76.1-23.el9", "product_name": "Red Hat Enterprise Linux 9"}, + ], + ), + ) + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert status == PackageCheckerStatus.OK + assert result.fixed_rpm_list + + def test_not_in_either_rhsa_bucket_is_cve_mismatch(self): + target = TargetPackage( + name="curl", + version="7.76.1", + release="26.el9", + arch="x86_64", + ) + intel = CveIntel( + vuln_id="CVE-2024-TEST", + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + package_name="webkit2gtk3", + fix_state="Affected", + product_name="Red Hat Enterprise Linux 9", + ), + ], + affected_release=[ + {"package": "webkit2gtk3-2.42.5-1.el9"}, + ], + ), + ) + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert status == PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH + assert result.is_target_package_affected == EnumIdentifyResult.NO From 6f40c87d57e970d6232befc194f87f15ea9ad270 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:33:51 +0300 Subject: [PATCH 243/286] Fix Python ecosystem PyPI-to-import name mismatch, Go stdlib handling, and integration test accuracy (#243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix Python ecosystem PyPI-to-import name mismatch and code review follow-ups Python name normalization: - Add _resolve_tree_key in CCA to fall back to is_same_package for tree lookups - Re-key dep tree from PyPI names to import names via _find_module_dirs/top_level.txt - Add PEP 503 is_same_package override in PythonLanguageFunctionsParser - Wire dependency builder into parser via lang_functions_parsers_factory - Use parser is_same_package in FL _is_package_available instead of inline lambda - Use PEP 503 re.sub(r'[-_.]', '-') for root dep matching instead of replace('-', '_') - Replace pip install with uv pip install for deptree setup CCA query parsing: - Fix dotted function name splitting with rpartition instead of split - Fix __determine_doc_package_name to use _resolve_tree_key with fallback JavaScript parser: - Guard is_function check before get_function_name in search_for_called_function * Fixed bugs from prior commit * Invoke deptree as python -m deptree instead of bin/deptree entry point * Fix Python ecosystem PyPI-to-import name mismatch, Go stdlib handling, and integration test accuracy Python CCA name resolution: - Add _resolve_tree_key with is_same_package fallback for normalized tree lookups - Re-key dep tree from PyPI names to import names via _find_module_dirs/top_level.txt - Add PEP 503 is_same_package override in PythonLanguageFunctionsParser - Wire dependency builder into parser via lang_functions_parsers_factory - Use parser is_same_package in FL instead of inline lambda - Invoke deptree as python -m deptree instead of bin/deptree entry point - Replace pip install with uv pip install for deptree setup Go stdlib package selection: - Add _find_go_stdlib_candidate to bypass LLM package filter for Go stdlib packages - Scan both candidate_packages and critical_context hints for stdlib module paths - Use _GO_STDLIB_ROOTS frozenset for reliable detection CCA and parser fixes: - Fix dotted functionion and move Go '/' check into correct bl - Fix Python parser Nt in code_documents - Guard is_function c search_for_called_fun Prompt and rule impro - Merge reachability le Rule 6 (package/function pri - Add patch hint preftry guidance - Reduce rule count: - Update all runtime rtions to match new numbering Patch enrichment: - Add JS/Python test FUNC_NAMES (describe,it, beforeEach, etc.) * Fixed regression * Fix Python deptree failure when pickle cache survives pod restart - build_tree() fails when pickle cache is warm and install_dependencies() is skipped - transitive_env/bin/python is a broken symlink from a previous pod - uv fails with Permission denied on ~/.cache/uv when UV_CACHE_DIR is not set - Empty dep tree causes FL to report "Package not found" for valid packages - Add _ensure_uv_cache_dir: sets UV_CACHE_DIR fallback to writable path in cloned repo - Add _ensure_venv_python: re-creates venv when bin/python is missing or broken - Add _build_flat_tree_from_manifest: fallback flat tree from requirements.txt - Call helpers from both build_tree and install_dependencies - Set UV_CACHE_DIR=/tmp/uv-cache in .tekton sidecar env for on-pull-request and on-cm-runner * Fix venv Python version mismatch when re-creating broken venv - uv downloaded Python 3.13 but packages live under python3.12 site-packages - Add _detect_existing_site_packages_version: finds installed Python version from transitive_env/lib/python*/ - Prefer detected version over project metadata when re-creating venv * Fix _ensure_venv_python defaulting to Python 3.13 when no version hint exists - When no existing site-packages and no project metadata, uv downloads latest Python (3.13) - Packages installed by lint-and-test under python3.12 become invisible to python3.13 deptree - Add sys.version_info fallback as last resort so venv matches the running container's Python - Always pass explicit --python to uv venv, never let uv pick the latest * Fix _ensure_venv_python to share venv creation logic with install_dependencies - Extract venv creation from install_dependencies into _ensure_venv with existence check - Both build_tree and install_dependencies now call _ensure_venv for consistent Python version - Remove _detect_existing_site_packages_version — got confused by polluted python3.13 directories - Remove sys.version_info fallback — unnecessary when using determine_python_version consistently - Both lint-and-test (UBI) and integration-test (sidecar) produce same Python version via determine_python_version * Fix _ensure_venv_python defaulting to Python 3.13 when no version hint exists - When determine_python_version returns None, uv venv without --python downloads latest Python (3.13) - Sidecar image has no system Python so uv can't find 3.12 on PATH - Add sys.version_info fallback so --python is always passed explicitly - Prevents version mismatch between lint-and-test (python3.12) and sidecar (python3.13) * Store uv-managed Python on shared PVC to fix cross-container symlink breakage - Set UV_PYTHON_INSTALL_DIR on PVC for both lint-and-test and integration-test sidecar - Both containers find Python at the same PVC path, bin/python symlink no longer breaks - lint-and-test: UV_PYTHON_INSTALL_DIR points to .cache/am_cache/python (PVC via symlink) - integration-test sidecar: UV_PYTHON_INSTALL_DIR points to /exploit-iq-data/python (same PVC) * Switch lint-and-test to ubi-minimal so uv stores Python on shared PVC - Replace ubi9/python-312 with ubi9/ubi-minimal (no system Python) - uv downloads standalone Python to UV_PYTHON_INSTALL_DIR on PVC - bin/python symlink points to PVC path, survives across containers * dd system packages to ubi-minimal for lint-and-test - ubi-minimal lacks git, curl, tar, make, gcc needed by the build script - Add microdnf install step before git config * Remove curl from microdnf install — ubi-minimal ships curl-minimal which conflicts * Source uv env after install — ubi-minimal doesn't have it on PATH by default * Install Node.js 20 and npm on ubi-minimal for JavaScript transitive tests - python-312 image had Node v20 pre-installed, ubi-minimal doesn't - Enable nodejs:20 module stream to match the major version - npm ls --json --all needs Node.js to build JavaScript dependency trees * PVC clean — git repo, pickle, transitive_env, and python dir all deleted. Fix shared PVC Python path for cross-container venv symlinks - Mount PVC at /exploit-iq-data in lint-test step (same path as sidecar) - Set UV_PYTHON_INSTALL_DIR=/exploit-iq-data/python in both containers - Run uv python install 3.12 explicitly so Python is stored on PVC - uv venv --python 3.12 now symlinks to PVC path that exists in both containers * Add Python-specific reachability prompt to use class names for CCA - Agent was calling CCA with formparser.MultiPartParser.parse (method) instead of formparser.MultiPartParser (class) - CCA searches for the last component — parse() isn't directly called from app code, MultiPartParser is - Add REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON with Rule 7: use class name, not method name for CCA - Wire up ecosystem-specific prompt selection: Go, Python, or default * Replace C/C++ examples with per-ecosystem examples and add dummy-branch warning - FL tool description: replace static libxml2 example with {fl_input_format} template - Add FL_INPUT_FORMATS and FL_EXAMPLES dicts in prompt_factory.py for per-ecosystem examples - Substitute ecosystem-specific input format in _build_tool_guidance_for_ecosystem - FL error message: use ecosystem-specific example from FL_EXAMPLES instead of libxml2 - CCA: warn agent when package not in dependency tree (dummy-branch fallback used) * Add dummy-branch warning to CCA when package not in dependency tree - When CCA falls back to import scanning (package not in tree_dict), prepend NOTE to first result - Helps agent distinguish real call chain analysis from import-based fallback - Applies to all ecosystems: Go stdlib, Java missing source JARs, C/C++ system libs - Prepend to existing entry instead of appending to preserve list length for assertions * Fix CCA same-artifact false negatives, Rule 6/7 split, and reachability-unverified summarize gate CCA same-artifact fix (java_functions_parsers.py, java_chain_of_calls_retriever.py): - Add _is_same_artifact() to allow polymorphic call matching within the same JAR. The early exit optimization and _is_concrete_or_paired filter were rejecting legitimate interface-dispatched calls between classes in different Java packages but the same Maven artifact (e.g. TreeMarshaller.convert → CollectionConverter.marshal in xstream). - Fixes CVE-2021-43859 (infinispan/xstream) false negative — CCA now traces intra-library call chains through interfaces. Rule 6/7 split (react_internals.py): - Split merged Rule 6 back into separate rules: Rule 6 for target package enforcement, Rule 7 for function name prioritization. - The merged rule was too complex for the LLM to parse when a violation fired, causing the Python CVE-2024-49769 (waitress) regression where the agent couldn't recover from a wrong package name. - Updated all three prompt variants (default, Go, Python) and enforcement messages. Reachability-unverified summarize gate (cve_summarize.py, cve_agent.py): - Add _reachability_without_cca() gate that detects when a reachability question was asked but CCA was never called (agent timeout, package name mismatch, tool errors). - Injects "REACHABILITY UNVERIFIED" warning into summarize prompt so the LLM weighs the absence of reachability evidence instead of defaulting to "vulnerable" based on library presence alone. - Propagate is_reachability from agent state to checklist item output. * Fix RCE via untrusted mvnw in cloned repositories(f001) * Address PR #243 review comments - Reuse _is_same_artifact in _is_concrete_or_paired instead of inline duplicate - Guard against empty candidates in __determine_doc_package_name - Move _pep503 from closure to @staticmethod on PythonLanguageFunctionsParser - Pin ubi-minimal image to 9.8 * Add krb5-devel to ubi-minimal for gssapi build Signed-off-by: Theodor Mihalache --- .tekton/on-cm-runner.yaml | 2 + .tekton/on-pull-request.yaml | 25 +++- ci/it/integration-tests-input.json | 6 +- .../utils/chain_of_calls_retriever.py | 63 ++++++-- src/exploit_iq_commons/utils/dep_tree.py | 134 +++++++++++++++--- .../java_functions_parsers.py | 27 +++- .../javascript_functions_parser.py | 15 +- .../lang_functions_parsers_factory.py | 5 +- .../python_functions_parser.py | 43 +++++- .../utils/functions_parsers/tests/__init__.py | 0 .../tests/test_python_is_same_package.py | 77 ++++++++++ .../utils/java_chain_of_calls_retriever.py | 5 + .../utils/tests/__init__.py | 0 .../utils/tests/test_python_build_tree.py | 116 +++++++++++++++ .../functions/base_graph_agent.py | 39 +++++ src/vuln_analysis/functions/cve_agent.py | 6 +- src/vuln_analysis/functions/cve_summarize.py | 23 +++ .../functions/reachability_agent.py | 18 ++- .../functions/react_internals.py | 68 +++++---- .../tools/tests/mock_documents.py | 37 +++++ .../tests/test_transitive_code_search.py | 110 +++++++++++--- .../tools/transitive_code_search.py | 26 ++-- .../utils/function_name_locator.py | 23 +-- src/vuln_analysis/utils/intel_utils.py | 2 + src/vuln_analysis/utils/prompt_factory.py | 31 ++++ .../test_function_name_locator_python.py | 43 ++++++ tests/test_python_version_detection.py | 6 +- tests/test_react_internals_rules.py | 12 +- uv.lock | 68 +++++++++ 29 files changed, 904 insertions(+), 126 deletions(-) create mode 100644 src/exploit_iq_commons/utils/functions_parsers/tests/__init__.py create mode 100644 src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py create mode 100644 src/exploit_iq_commons/utils/tests/__init__.py create mode 100644 src/exploit_iq_commons/utils/tests/test_python_build_tree.py create mode 100644 src/vuln_analysis/utils/tests/test_function_name_locator_python.py diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 2adfd29ba..17c42105d 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -188,6 +188,8 @@ spec: value: "$(params.TRIGGER_COMMENT)" - name: GOMODCACHE value: "/exploit-iq-data/go/pkg/mod" + - name: UV_CACHE_DIR + value: "/tmp/uv-cache" - name: SERPAPI_BASE_URL value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi - name: CVE_DETAILS_BASE_URL diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index a1b085f0d..34c219807 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -192,23 +192,36 @@ spec: value: "{{target_branch}}" - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH value: $(workspaces.source.path)/kustomize/base/settings.xml - image: registry.access.redhat.com/ubi9/python-312:9.6 + - name: UV_PYTHON_INSTALL_DIR + value: "/exploit-iq-data/python" + volumeMounts: + - name: $(workspaces.unit-test-cache.volume) + mountPath: /exploit-iq-data + image: registry.access.redhat.com/ubi9/ubi-minimal:9.8 workingDir: $(workspaces.source.path) script: | #!/bin/bash set -eou pipefail - + print_banner() { echo echo "----------- ${1} -----------" } - + + # ubi-minimal has no dev tools — install the essentials + print_banner "Installing system packages" + microdnf install -y git tar gzip findutils make gcc gcc-c++ krb5-devel + microdnf module enable -y nodejs:20 + microdnf install -y nodejs npm + # Mark the workspace as safe before any tool (uv, git, etc.) touches it. git config --global --add safe.directory /workspace/source - # Install uv + # Install uv and Python 3.12 to shared PVC print_banner "Installing uv" curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.local/bin/env + uv python install 3.12 print_banner "CREATING AND ACTIVATING TEST ENV" uv venv --python 3.12 .venv @@ -352,6 +365,10 @@ spec: # Pass the raw comment text into the container - name: GOMODCACHE value: "/exploit-iq-data/go/pkg/mod" + - name: UV_CACHE_DIR + value: "/tmp/uv-cache" + - name: UV_PYTHON_INSTALL_DIR + value: "/exploit-iq-data/python" - name: SERPAPI_BASE_URL value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi - name: CVE_DETAILS_BASE_URL diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json index 6d01cb7e3..971cb50f6 100644 --- a/ci/it/integration-tests-input.json +++ b/ci/it/integration-tests-input.json @@ -54,9 +54,9 @@ "ref": "ab9e2ade2f45890033a140b314ef87e367317b51" }, "use_sbom": false, - "expected_label": "vulnerable", - "expected_result": "Exploitable", - "allowed_deviation_labels" : ["vulnerable"], + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], "skip": false }, { diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index 198dfe8b6..f627b145d 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -162,13 +162,39 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_pat if not self.language_parser.is_search_algo_dfs(): self.sort_docs = self.__group_docs_by_pkg() + def _resolve_tree_key(self, package: str, ctx: _SearchCtx) -> str | None: + """Find the canonical tree_dict key for a package name. + Tries exact match first, then falls back to is_same_package + for ecosystems with name normalization (e.g. Python PEP 503).""" + if package in self.tree_dict or package in ctx.tree_additions: + logger.debug("_resolve_tree_key: '%s' exact match", package) + return package + for tree_key in self.tree_dict: + if self.language_parser.is_same_package(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_same_package (tree_dict)", package, tree_key) + return tree_key + for tree_key in ctx.tree_additions: + if self.language_parser.is_same_package(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_same_package (tree_additions)", package, tree_key) + return tree_key + logger.debug("_resolve_tree_key: '%s' NOT found in tree_dict (%d keys) or tree_additions (%d keys)", + package, len(self.tree_dict), len(ctx.tree_additions)) + return None + def _get_parents(self, package: str, ctx: _SearchCtx) -> list | None: """Look up parents for a package, checking both the immutable tree_dict and any packages added during the current search (ctx.tree_additions).""" - parents = self.tree_dict.get(package) + resolved = self._resolve_tree_key(package, ctx) + if resolved is None: + logger.debug("_get_parents: '%s' has no resolved key, returning None", package) + return None + parents = self.tree_dict.get(resolved) if parents is not None: + logger.debug("_get_parents: '%s' resolved='%s' -> tree_dict parents=%s", package, resolved, parents) return parents - return ctx.tree_additions.get(package) + parents = ctx.tree_additions.get(resolved) + logger.debug("_get_parents: '%s' resolved='%s' -> tree_additions parents=%s", package, resolved, parents) + return parents def __group_docs_by_pkg(self) -> dict[str, list[Document]]: """ @@ -312,6 +338,8 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack parents = self._get_parents(function_package, ctx) if parents is not None: direct_parents.extend(parents) + logger.debug("__find_caller_functions_bfs: function_package='%s' parents=%s direct_parents=%s", + function_package, parents, direct_parents) # Add same package itself to search path. # direct_parents.extend([function_package]) # gets list of documents to search in only from parents of function' package. @@ -498,13 +526,15 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: ctx = _SearchCtx() query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() (package_name, function) = tuple(query.split(",")) - class_name = None - splitters = [splitter for splitter in ['.'] if splitter in function] - if splitters: - class_name, function = function.split(splitters[0]) + before, sep, after = function.rpartition('.') + if sep: + class_name = before.rsplit('.', 1)[-1] + function = after if class_name and '/' in class_name: package_name = f"{package_name}/{class_name}" class_name = None + else: + class_name = None found_package = False matching_documents = [] standard_libs_cache = StandardLibraryCache.get_instance() @@ -582,9 +612,24 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: return matching_documents, ctx.found_path def __determine_doc_package_name(self, target_function_doc, ctx: _SearchCtx): - return [package_name for package_name in - self.language_parser.get_package_names(target_function_doc) - if package_name in self.tree_dict or package_name in ctx.tree_additions][0] + candidates = self.language_parser.get_package_names(target_function_doc) + for package_name in candidates: + if package_name in self.tree_dict or package_name in ctx.tree_additions: + logger.debug("__determine_doc_package_name: candidates=%s, exact match '%s'", candidates, package_name) + return package_name + for package_name in candidates: + resolved = self._resolve_tree_key(package_name, ctx) + if resolved is not None: + logger.debug("__determine_doc_package_name: candidates=%s, '%s' resolved to '%s' via is_same_package", + candidates, package_name, resolved) + return resolved + if not candidates: + logger.warning("__determine_doc_package_name: no candidates returned for doc %s", target_function_doc) + return None + fallback = candidates[0] + logger.debug("__determine_doc_package_name: no match in tree, fallback='%s' from candidates=%s", + fallback, candidates) + return fallback def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], ctx: _SearchCtx, class_name: str = None) -> Document: diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index e4ba1d55d..09b91789e 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -856,6 +856,7 @@ def extract_package_name(self, package_name: str) -> str: class JavaDependencyTreeBuilder(DependencyTreeBuilder): DEP_SOURCE_DIR = "dependencies-sources" + _MAVEN_VERSION_RE = re.compile(r'^\d+\.\d+\.\d+$') def __init__(self, query: str): self._query = query @@ -867,13 +868,38 @@ def __check_file_exists(self, dir_path: str | Path, filename: str) -> bool: p = Path(dir_path) / filename return p.is_file() + @staticmethod + def _extract_mvnw_maven_version(manifest_path: Path) -> str | None: + """Extract and validate the Maven version from the wrapper properties file.""" + props = manifest_path / ".mvn" / "wrapper" / "maven-wrapper.properties" + if not props.is_file(): + return None + try: + for line in props.read_text().splitlines(): + if line.startswith("distributionUrl="): + match = re.search(r'apache-maven-(\d+\.\d+\.\d+)', line) + if match and JavaDependencyTreeBuilder._MAVEN_VERSION_RE.match(match.group(1)): + return match.group(1) + except OSError: + pass + return None + def install_dependencies(self, manifest_path: Path): mvn_command = "mvn" settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml') source_path = self.DEP_SOURCE_DIR if self.__check_file_exists(manifest_path, "mvnw"): - mvn_command = "./mvnw" + version = self._extract_mvnw_maven_version(manifest_path) + if version: + logger.info("Regenerating Maven wrapper with version %s", version) + result = subprocess.run(["mvn", "wrapper:wrapper", f"-Dmaven={version}"], cwd=manifest_path) + if result.returncode == 0: + mvn_command = "./mvnw" + else: + logger.warning("Failed to regenerate Maven wrapper, falling back to system mvn") + else: + logger.warning("Could not extract valid Maven version from wrapper properties, using system mvn") process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) @@ -1127,14 +1153,56 @@ def looks_like_version(v: str) -> bool: class PythonDependencyTreeBuilder(DependencyTreeBuilder): DEP_SOURCE_DIR = TRANSITIVE_ENV_NAME - def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: + def _ensure_uv_cache_dir(self, manifest_path: Path): + """Set UV_CACHE_DIR if not already configured. + + In OpenShift containers the default ~/.cache/uv is often read-only. + This is normally set during install_dependencies(), but when the pickle + cache is warm that method is skipped entirely. + """ + if "UV_CACHE_DIR" not in os.environ: + fallback_cache = manifest_path / ".uv_cache" + try: + fallback_cache.mkdir(exist_ok=True) + os.environ["UV_CACHE_DIR"] = str(fallback_cache) + logger.info("Set UV_CACHE_DIR=%s (home dir may be read-only)", fallback_cache) + except OSError: + logger.warning("Could not create UV_CACHE_DIR at %s", fallback_cache) + + def _ensure_venv(self, manifest_path: Path) -> str: + """Ensure transitive_env exists with a working python binary. + + Checks if bin/python exists; if not, creates the venv using the + same version-detection logic as install_dependencies. This is + called from both build_tree (where the pickle cache may be warm + and install_dependencies was skipped) and install_dependencies + itself. Using the same determine_python_version ensures both + callers produce venvs with the same Python version, so + site-packages directories are consistent across pods. + """ venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' - run_command(f'{venv_python} -m pip install "setuptools<81" deptree') - cmd = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/deptree' - dependencies = run_command(cmd) + if Path(venv_python).exists(): + return venv_python + logger.warning("Venv python not found at %s — creating venv", venv_python) + self._ensure_uv_cache_dir(manifest_path) + python_version = self.determine_python_version(str(manifest_path)) + if not python_version: + import sys + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + logger.info("Python version undetermined; using current interpreter %s", python_version) + logger.info("Creating transitive_env with Python %s using uv", python_version) + cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" + run_command(cmd) + return venv_python + + def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: + venv_python = self._ensure_venv(manifest_path) + run_command(f'uv pip install "setuptools<81" deptree --python {venv_python}') + dependencies = run_command(f'{venv_python} -m deptree') if not dependencies or not dependencies.strip(): logger.error("deptree returned empty output — third-party dependency tree will be incomplete. " "Check that the virtual environment at %s has packages installed.", manifest_path) + dependencies = self._build_flat_tree_from_manifest(manifest_path) parent_stack = [] tree = defaultdict(set) ROOT_PROJECT = 'root_project' @@ -1145,7 +1213,6 @@ def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: level += 1 line = line[2:] package = line.split('==')[0].strip().lower() - package = package.replace('-', '_') if level < len(parent_stack): parent_stack = parent_stack[:level] try: @@ -1154,17 +1221,57 @@ def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: pass parent_stack.append(package) - installed_dependencies = [] + # Normalize manifest names with PEP 503 for comparison against tree keys + installed_dependencies = set() with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: for line in manifest: if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): - installed_dependencies.append(re.split(r"[=>< ]", line.strip())[0]) + raw_name = re.split(r"[=>< ]", line.strip())[0] + installed_dependencies.add(re.sub(r'[-_.]', '-', raw_name.lower())) for dependency, parents in tree.items(): - if dependency in installed_dependencies: + if re.sub(r'[-_.]', '-', dependency) in installed_dependencies: parents.add(ROOT_PROJECT) tree[dependency] = list(parents) + + # Re-key from PyPI names to import names using dist-info/top_level.txt + self._import_to_pypi = {} + site_packages = self._find_site_packages(manifest_path) + if site_packages: + for pypi_key in list(tree.keys()): + if pypi_key == ROOT_PROJECT: + continue + import_names = self._find_module_dirs(pypi_key, site_packages) + if import_names: + primary_import = import_names[0].lower() + if primary_import != pypi_key: + # Re-key: move parents from PyPI key to import key + tree[primary_import] = tree.pop(pypi_key) + self._import_to_pypi[primary_import] = pypi_key + # Update parent references in other packages + for other_key, parents in tree.items(): + if pypi_key in parents: + idx = parents.index(pypi_key) + parents[idx] = primary_import + return tree + def _build_flat_tree_from_manifest(self, manifest_path: Path) -> str: + """Build a flat deptree-format string from requirements.txt as a fallback + when deptree cannot run (e.g. venv binary missing after pickle cache hit).""" + lines = [] + try: + with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: + for line in manifest: + line = line.strip() + if line and not PythonLanguageFunctionsParser.is_comment_line(line): + lines.append(line) + except FileNotFoundError: + logger.error("requirements.txt not found at %s — cannot build fallback tree", manifest_path) + return "" + if lines: + logger.warning("Using requirements.txt fallback for dependency tree (%d packages)", len(lines)) + return os.linesep.join(lines) + def extract_version_from_specifier(self, specifier_str: str) -> str | None: """Extract the most likely runtime Python version from a PEP 440 specifier string. @@ -1483,14 +1590,7 @@ def install_dependencies(self, manifest_path: Path): manifest_path: Absolute path to the root of the cloned repository, which is expected to contain a ``requirements.txt`` manifest. """ - python_version = self.determine_python_version(str(manifest_path)) - if python_version: - logger.info("Creating transitive_env with Python %s using uv", python_version) - cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" - else: - logger.warning("Python version undetermined for %s; using uv default interpreter", manifest_path) - cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME}" - run_command(cmd) + self._ensure_venv(manifest_path) site_packages = self._find_site_packages(manifest_path) with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: for line in tqdm(manifest): diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index 971d50508..1a442c634 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -365,6 +365,15 @@ def get_comment_line_notation(self) -> str: def dir_name_for_3rd_party_packages(self) -> str: return "dependencies-sources" + def _is_same_artifact(self, source_a: str, source_b: str) -> bool: + """Check if two source paths are from the same third-party JAR artifact.""" + prefix = self.dir_name_for_3rd_party_packages() + "/" + if not (source_a and source_b and source_a.startswith(prefix) and source_b.startswith(prefix)): + return False + jar_a = source_a[len(prefix):].split("/", 1)[0] + jar_b = source_b[len(prefix):].split("/", 1)[0] + return jar_a == jar_b + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: return True @@ -1050,7 +1059,9 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: # 1. Simple class name appears in the source text (direct usage or import) # 2. Wildcard import of the declaring class's package (import pkg.*) # 3. Caller is in the same package as the declaring class - # In large repos, ~90%+ of source files fail all three checks, avoiding + # 4. Caller and callee are in the same JAR (intra-library polymorphic + # calls through interfaces don't require the concrete class name) + # In large repos, ~90%+ of source files fail all checks, avoiding # expensive regex/type resolution for each one. if declaring_fqcn: _caller_src = caller_function.metadata.get('source') @@ -1062,7 +1073,8 @@ def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: if f"import {_declaring_pkg}.*" not in _full_text: _pkg_m = re.search(r'^\s*package\s+([\w.]+)\s*;', _full_text, re.MULTILINE) if not _pkg_m or _pkg_m.group(1) != _declaring_pkg: - return False + if not self._is_same_artifact(_caller_src, callee_function_file_name): + return False # CHANGED: support ctor targeting by simple name, no-package inner name, or fqcn. def _strip_package_prefix(fqcn_dot: str) -> str: @@ -1542,14 +1554,21 @@ def _is_concrete_or_paired(candidate_fqcn: str) -> bool: This function accepts a candidate only if it is: - "concrete": the candidate FQCN IS the callee's declaring class, OR - "paired": the callee's declaring class is explicitly imported in - the same file, meaning the caller is aware of the concrete type. + the same file, meaning the caller is aware of the concrete type, OR + - "same-artifact": caller and callee are in the same JAR/artifact. + Within the same artifact, classes freely call each other through + interfaces without needing explicit imports of concrete types. """ cand_dot = candidate_fqcn.replace('$', '.') decl_dot = callee_declaring_fqcn.replace('$', '.') if cand_dot == decl_dot: return True callee_simple = decl_dot.rsplit('.', 1)[-1] - return callee_simple in _explicit_imports + if callee_simple in _explicit_imports: + return True + if self._is_same_artifact(caller_src or "", callee_function_file_name or ""): + return True + return False def _strip_type_syntax(token: str) -> str: """Strip generics, arrays, and wildcard bounds from a type-ish token (CHANGED).""" diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py index 3c6017d77..6ed0895d9 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -228,13 +228,14 @@ def search_for_called_function(self, caller_function: Document, callee_function_ return True if callee_class_name := self.get_class_name_from_class_function(callee_function): - caller_func_name = self.get_function_name(caller_function) - func_key = f"{caller_func_name}@{caller_source}" - local_vars = functions_local_variables_index.get(func_key, {}) - var_info = local_vars.get(identifier, {}) - var_type = var_info.get('type') - if var_type == callee_class_name or self._is_subclass_of(var_type, callee_class_name, code_documents): - return True + if self.is_function(caller_function): + caller_func_name = self.get_function_name(caller_function) + func_key = f"{caller_func_name}@{caller_source}" + local_vars = functions_local_variables_index.get(func_key, {}) + var_info = local_vars.get(identifier, {}) + var_type = var_info.get('type') + if var_type == callee_class_name or self._is_subclass_of(var_type, callee_class_name, code_documents): + return True if self.is_package_imported(caller_content, identifier, callee_function_package): return True diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py index 755455324..e4f9e693c 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py @@ -34,7 +34,10 @@ def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | No if ecosystem == Ecosystem.GO: return GoLanguageFunctionsParser() elif ecosystem == Ecosystem.PYTHON: - return PythonLanguageFunctionsParser() + parser = PythonLanguageFunctionsParser() + if tree is not None and tree.builder is not None: + parser.set_dependency_builder(tree.builder) + return parser elif ecosystem == Ecosystem.JAVASCRIPT: return JavaScriptFunctionsParser() elif ecosystem == Ecosystem.C_CPP: diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py index 8aa46f681..e2599191c 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -46,6 +46,33 @@ class PythonLanguageFunctionsParser(LanguageFunctionsParser): def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}(): pass" + def set_dependency_builder(self, builder): + """Store a reference to the PythonDependencyTreeBuilder so is_same_package + can consult the PyPI-to-import mapping built during build_tree.""" + self._builder = builder + + @property + def _import_to_pypi(self) -> dict[str, str]: + if hasattr(self, '_builder') and self._builder and hasattr(self._builder, '_import_to_pypi'): + return self._builder._import_to_pypi + return {} + + @staticmethod + def _pep503(name: str) -> str: + return re.sub(r'[-_.]', '-', name.lower()) + + def is_same_package(self, package_name_from_input, package_name_from_tree): + if self._pep503(package_name_from_input) == self._pep503(package_name_from_tree): + return True + + mapping = self._import_to_pypi + if mapping: + pypi_name = mapping.get(package_name_from_tree.lower()) + if pypi_name and self._pep503(package_name_from_input) == self._pep503(pypi_name): + return True + + return False + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: mappings = dict() for func_method in functions_methods_documents: @@ -168,7 +195,6 @@ def search_for_called_function(self, caller_function: Document, callee_function_ callee_function_doc = code_documents[callee_function_file_name] callee_function_package = self.get_package_names(callee_function_doc)[0] except KeyError: - # Third-party package without source files - use file name as package name callee_function_package = callee_function_file_name caller_function_package = self.get_package_names(caller_function)[0] @@ -428,6 +454,21 @@ def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: re.compile(rf"from\s+({escaped}[\w.]*)\s+import\s+", re.IGNORECASE | re.MULTILINE), ] + def get_package_name(self, function: Document, package_name: str) -> str: + package_names = self.get_package_names(function) + for package in package_names: + if self.is_same_package(package_name, package): + return package.lower() + return '' + + def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: + source_form = re.sub(r'[-.]', '_', package_name) + return [ + doc for doc in documents + if source_form in doc.metadata.get('source', '') and + function_name in doc.page_content + ] + def is_a_package(self, package_name: str, doc: Document) -> bool: return (not self.is_root_package(doc) and self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/__init__.py b/src/exploit_iq_commons/utils/functions_parsers/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py b/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py new file mode 100644 index 000000000..6d9099a41 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py @@ -0,0 +1,77 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from exploit_iq_commons.utils.dep_tree import Ecosystem, DependencyTree +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser + + +class TestPythonIsSamePackage: + """Test PythonLanguageFunctionsParser.is_same_package with PEP 503 normalization + and PyPI-to-import mapping lookup.""" + + def setup_method(self): + self.parser = PythonLanguageFunctionsParser() + + def test_exact_match(self): + assert self.parser.is_same_package("urllib3", "urllib3") is True + + def test_case_insensitive(self): + assert self.parser.is_same_package("Flask", "flask") is True + + def test_hyphen_underscore_equivalence(self): + assert self.parser.is_same_package("my-package", "my_package") is True + + def test_dot_equivalence(self): + assert self.parser.is_same_package("my.package", "my-package") is True + + def test_no_match(self): + assert self.parser.is_same_package("urllib3", "requests") is False + + def test_mapping_pypi_to_import(self): + """Agent sends PyPI name 'python-dateutil', tree key is import name 'dateutil'.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("python-dateutil", "dateutil") is True + + def test_mapping_normalized_pypi_to_import(self): + """Agent sends 'python_dateutil' (underscore), tree key is 'dateutil'.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("python_dateutil", "dateutil") is True + + def test_mapping_pillow_to_pil(self): + builder = MagicMock() + builder._import_to_pypi = {"pil": "pillow"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("Pillow", "pil") is True + + def test_no_builder_falls_back_to_normalization(self): + assert self.parser.is_same_package("my-pkg", "my_pkg") is True + assert self.parser.is_same_package("python-dateutil", "dateutil") is False + + def test_mapping_does_not_false_positive(self): + """Mapping for one package should not match a different package.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("requests", "dateutil") is False + + +class TestParserFactoryWiring: + def test_python_parser_receives_builder(self): + """Factory should call set_dependency_builder for Python parsers.""" + tree = DependencyTree(ecosystem=Ecosystem.PYTHON) + parser = get_language_function_parser(Ecosystem.PYTHON, tree) + assert hasattr(parser, '_builder') + assert parser._builder is tree.builder + + def test_python_parser_handles_none_tree(self): + parser = get_language_function_parser(Ecosystem.PYTHON, None) + assert parser._import_to_pypi == {} + + def test_go_parser_unaffected(self): + """Go parser should not have _builder attribute.""" + parser = get_language_function_parser(Ecosystem.GO, None) + assert not hasattr(parser, '_builder') \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index 8e305a4d3..e624fa99b 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -502,6 +502,8 @@ def __find_caller_function(self, document_function: Document, function_package: # Search for caller functions only at parents according to dependency tree. # Iterate candidate docs lazily via generators instead of collecting into a list first. + logger.debug("__find_caller_function: searching for callers of '%s' in '%s', fqcn='%s'", + function_name_to_search, function_package, fqcn) def _candidate_docs(): for package in direct_parents[last_visited_package_index:]: sources_location_packages = True @@ -516,6 +518,8 @@ def _candidate_docs(): method_exclusions, ctx.root_docs, ctx.jar_to_docs) + logger.debug("__find_caller_function: parent='%s' possible_docs=%d", package, len(possible_docs)) + jar_name = convert_from_maven_artifact(package) yield from self.get_functions_for_package(package_name=jar_name, documents=possible_docs, @@ -556,6 +560,7 @@ def _candidate_docs(): return doc # If didn't find a matching caller function document, returns None. + logger.debug("__find_caller_function: no caller found for '%s' in '%s'", function_name_to_search, function_package) return None def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: diff --git a/src/exploit_iq_commons/utils/tests/__init__.py b/src/exploit_iq_commons/utils/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/tests/test_python_build_tree.py b/src/exploit_iq_commons/utils/tests/test_python_build_tree.py new file mode 100644 index 000000000..5e3e5c9eb --- /dev/null +++ b/src/exploit_iq_commons/utils/tests/test_python_build_tree.py @@ -0,0 +1,116 @@ +import pytest +from unittest.mock import patch, MagicMock +from pathlib import Path + +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder + + +DEPTREE_OUTPUT = ( + 'flask==3.1.2 # flask\n' + ' werkzeug==3.1.3 # werkzeug>=3.1.0\n' + ' click==8.3.0 # click>=8.1.3\n' + 'python-dateutil==2.9.0 # python-dateutil\n' + ' six==1.17.0 # six>=1.5\n' + 'Pillow==11.2.1 # Pillow\n' +) + +REQUIREMENTS_TXT = ( + "flask==3.1.2\n" + "python-dateutil==2.9.0\n" + "Pillow==11.2.1\n" +) + + +class TestPythonBuildTree: + def setup_method(self): + self.builder = PythonDependencyTreeBuilder() + + def _mock_open_side_effect(self, *args, **kwargs): + file_path = str(args[0]) if args else "" + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'requirements.txt' in file_path: + mock_file.__iter__ = MagicMock(return_value=iter(REQUIREMENTS_TXT.splitlines(keepends=True))) + return mock_file + + def _build(self, deptree_output=DEPTREE_OUTPUT, import_mapping=None): + """Run build_tree with mocked deptree output and _find_module_dirs.""" + if import_mapping is None: + import_mapping = { + "python-dateutil": ["dateutil"], + "pillow": ["PIL"], + "flask": ["flask"], + "werkzeug": ["werkzeug"], + "click": ["click"], + "six": ["six"], + } + + fake_site_packages = Path("/fake/site-packages") + + def fake_find_module_dirs(pkg_name, site_packages): + return import_mapping.get(pkg_name, [pkg_name.replace('-', '_')]) + + with patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=deptree_output), \ + patch('builtins.open', side_effect=self._mock_open_side_effect), \ + patch.object(self.builder, '_find_site_packages', return_value=fake_site_packages), \ + patch.object(self.builder, '_find_module_dirs', side_effect=fake_find_module_dirs): + return self.builder.build_tree(Path("/fake/repo")) + + def test_no_hyphen_underscore_normalization(self): + """Hyphens should NOT be replaced with underscores in tree keys.""" + tree = self._build() + assert "python_dateutil" not in tree + assert "python-dateutil" not in tree # re-keyed to import name + + def test_import_name_rekey_dateutil(self): + """python-dateutil should be re-keyed to 'dateutil' (its import name).""" + tree = self._build() + assert "dateutil" in tree + + def test_import_name_rekey_pillow(self): + """Pillow should be re-keyed to 'pil' (PIL lowercased).""" + tree = self._build() + assert "pil" in tree + + def test_same_name_unchanged(self): + """Packages where PyPI name == import name should keep their key.""" + tree = self._build() + assert "flask" in tree + assert "werkzeug" in tree + + def test_root_project_for_direct_deps(self): + """Direct dependencies from requirements.txt should have ROOT_PROJECT as parent.""" + tree = self._build() + assert "root_project" in tree["flask"] + assert "root_project" in tree["dateutil"] + assert "root_project" in tree["pil"] + + def test_transitive_dep_no_root_project(self): + """Transitive-only deps should NOT have ROOT_PROJECT.""" + tree = self._build() + assert "root_project" not in tree.get("six", []) + assert "root_project" not in tree.get("werkzeug", []) + + def test_parent_relationships_preserved(self): + """Transitive deps should still point to correct parents.""" + tree = self._build() + # werkzeug is a dep of flask + assert "flask" in tree["werkzeug"] + # six is a dep of python-dateutil -> re-keyed to dateutil + assert "dateutil" in tree["six"] + + def test_import_to_pypi_mapping_stored(self): + """Builder should store _import_to_pypi after build_tree.""" + self._build() + assert self.builder._import_to_pypi.get("dateutil") == "python-dateutil" + assert self.builder._import_to_pypi.get("pil") == "pillow" + + def test_no_site_packages_skips_rekey(self): + """If site-packages not found, tree keeps lowercased PyPI names.""" + with patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=DEPTREE_OUTPUT), \ + patch('builtins.open', side_effect=self._mock_open_side_effect), \ + patch.object(self.builder, '_find_site_packages', return_value=None): + tree = self.builder.build_tree(Path("/fake/repo")) + assert "python-dateutil" in tree + assert "pillow" in tree \ No newline at end of file diff --git a/src/vuln_analysis/functions/base_graph_agent.py b/src/vuln_analysis/functions/base_graph_agent.py index 51bbf4f7d..1a3532938 100644 --- a/src/vuln_analysis/functions/base_graph_agent.py +++ b/src/vuln_analysis/functions/base_graph_agent.py @@ -89,6 +89,37 @@ def _load_all_tools(builder, config) -> list: from aiq.builder.framework_enum import LLMFrameworkEnum return builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + _GO_STDLIB_ROOTS = frozenset({ + "archive", "bufio", "builtin", "bytes", "cmp", "compress", "container", + "context", "crypto", "database", "debug", "embed", "encoding", "errors", + "expvar", "flag", "fmt", "go", "hash", "html", "image", "index", "io", + "iter", "log", "maps", "math", "mime", "net", "os", "path", "plugin", + "reflect", "regexp", "runtime", "slices", "sort", "strconv", "strings", + "structs", "sync", "syscall", "testing", "text", "time", "unicode", + "unique", "unsafe", + }) + + _GO_VULN_DB_HINT_RE = re.compile(r"Vulnerable module \(Go vuln DB hint\): (\S+)") + + @classmethod + def _find_go_stdlib_candidate(cls, ecosystem: str, candidate_packages: list[dict], + critical_context: list[str] | None = None) -> str | None: + """If any candidate is a Go standard library package, return it directly. + Go stdlib packages start with a known root (e.g. crypto/x509, net/http) + unlike third-party packages (github.com/..., golang.org/x/...). + Also checks critical_context for Go vuln DB hints (OSV enrichment).""" + if ecosystem != "go": + return None + for pkg in candidate_packages: + name = pkg.get("name", "") + if name and name.split("/")[0] in cls._GO_STDLIB_ROOTS: + return name + for ctx in critical_context or []: + m = cls._GO_VULN_DB_HINT_RE.search(ctx) + if m and m.group(1).split("/")[0] in cls._GO_STDLIB_ROOTS: + return m.group(1) + return None + async def _select_package( self, ecosystem: str, candidate_packages: list[dict], critical_context: list[str], workflow_state, @@ -101,6 +132,14 @@ async def _select_package( selected_package = None if len(candidate_packages) > 1: + stdlib_match = self._find_go_stdlib_candidate(ecosystem, candidate_packages, critical_context) + if stdlib_match: + selected_package = stdlib_match + logger.info("Package filter selected Go stdlib '%s' directly (no LLM call needed, %d candidates)", + selected_package, len(candidate_packages)) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + return critical_context, selected_package + image_input = workflow_state.original_input.input.image image_name = image_input.name image_repo = image_input.source_info[0].git_repo if image_input.source_info else None diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 19f6cb9b4..512478c4b 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -192,7 +192,8 @@ def _postprocess_results(results: list[tuple], replace_exceptions: bool, replace question_text = questions[j] if j < len(questions) else checklist_questions[i][j] outputs[i].append({"input": question_text, "output": replace_exceptions_value, "intermediate_steps": None, "cca_results": [], - "package_validated": None}) + "package_validated": None, + "is_reachability": None}) if isinstance(answer, ToolRaisedException): logger.warning( "Tool execution exception in result[%d][%d], replacing with default output: %s", @@ -206,7 +207,8 @@ def _postprocess_results(results: list[tuple], replace_exceptions: bool, replace outputs[i].append({"input": answer["input"], "output": answer["output"], "intermediate_steps": None, "cca_results": answer.get("cca_results", []), - "package_validated": answer.get("package_validated")}) + "package_validated": answer.get("package_validated"), + "is_reachability": answer.get("is_reachability")}) return outputs diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index 3bd61f794..6c0644ac6 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -46,6 +46,18 @@ def _all_cca_not_reachable(checklist_items: list[dict]) -> bool: return len(all_cca) > 0 and not any(all_cca) +def _reachability_without_cca(checklist_items: list[dict]) -> bool: + """Return True if a reachability question was asked but CCA was never called. + This happens when the agent exhausts iterations due to package name + mismatches or tool errors — absence of CCA evidence means reachability + was never verified. + """ + for item in checklist_items: + if item.get("is_reachability") == "yes" and not item.get("cca_results"): + return True + return False + + def _package_not_found(checklist_items: list[dict]) -> bool: """Return True if Function Locator confirmed the target package is absent. Fires when any thread has package_validated=False and no thread has True. @@ -94,6 +106,17 @@ async def summarize_cve(results, ecosystem: str = ""): + response ) logger.info("Reachability gate activated: all CCA results are negative") + elif _reachability_without_cca(checklist_items): + response = ( + "REACHABILITY UNVERIFIED: A reachability question was investigated but " + "Call Chain Analyzer was NEVER called — reachability could not be verified. " + "Library presence in dependencies alone does NOT prove exploitability. " + "Without Call Chain Analyzer confirmation, there is NO evidence that the " + "vulnerable function is reachable from application code. Weigh this lack " + "of reachability evidence heavily when determining the verdict.\n\n" + + response + ) + logger.info("Reachability unverified gate activated: reachability question without CCA") summary_prompt = SUMMARY_PROMPT if ecosystem.lower() == "java": diff --git a/src/vuln_analysis/functions/reachability_agent.py b/src/vuln_analysis/functions/reachability_agent.py index 6bda95f9f..152272e91 100644 --- a/src/vuln_analysis/functions/reachability_agent.py +++ b/src/vuln_analysis/functions/reachability_agent.py @@ -29,6 +29,7 @@ build_classification_prompt, FORCED_FINISH_PROMPT, REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO, + REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON, REACHABILITY_AGENT_NON_REACH_SYS_PROMPT, REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS, ) @@ -41,6 +42,7 @@ TOOL_SELECTION_STRATEGY, TOOL_SELECTION_STRATEGY_NON_REACHABILITY, FEW_SHOT_EXAMPLES, + FL_INPUT_FORMATS, ) logger = LoggingFactory.get_agent_logger(__name__) AGENT_TRACER = Context.get() @@ -92,10 +94,14 @@ def _build_tool_guidance_for_ecosystem(self, ecosystem: str, available_tools: li (t.name != ToolNames.FUNCTION_LIBRARY_VERSION_FINDER or ecosystem == "java") ] list_of_tool_names = [t.name for t in filtered_tools] - list_of_tool_descriptions = [t.name + ": " + t.description for t in filtered_tools] + lang = ecosystem.lower() if ecosystem else "" + fl_format = FL_INPUT_FORMATS.get(lang, "Input: 'package_name,function_name'.") + list_of_tool_descriptions = [ + t.name + ": " + t.description.replace("{fl_input_format}", fl_format) + for t in filtered_tools + ] strategy = TOOL_SELECTION_STRATEGY if is_reachability == "yes" else TOOL_SELECTION_STRATEGY_NON_REACHABILITY - lang = ecosystem.lower() if ecosystem else "" if lang in strategy: tool_guidance_local = strategy[lang] if lang == "java": @@ -164,8 +170,12 @@ async def pre_process_node(self, state: AgentState) -> AgentState: if is_reachability == "yes": tool_guidance_local, descriptions_local = self._build_tool_guidance_for_ecosystem(ecosystem, self.tools) - go_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO} if ecosystem == "go" else {} - runtime_prompt = build_reachability_system_prompt(descriptions_local, tool_guidance_local, **go_instructions) + ecosystem_instructions = {} + if ecosystem == "go": + ecosystem_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO} + elif ecosystem == "python": + ecosystem_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON} + runtime_prompt = build_reachability_system_prompt(descriptions_local, tool_guidance_local, **ecosystem_instructions) active_tool_names = [t.name for t in self.tools] else: reachability_tool_names = {ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER} diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 192b08e4c..371c069cf 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -252,7 +252,7 @@ def check_thought_behavior(self, action: str, action_input: str, output) -> tupl "Check KNOWLEDGE for what was already tried." ) if self._rule_number_7(action, action_input, output): - return True, ("You are NOT following Rule 7. Your query contains dots and returned " + return True, ("You are NOT following Rule 5. Your query contains dots and returned " "no results. You MUST retry with just the final component. Follow the rules.") if self._rule_use_allowed_tools(action): return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") @@ -341,7 +341,7 @@ def _rule_number_9(self, action: str, action_input: str) -> tuple[bool, str]: pending = [fn for fn, checked in self.target_functions.items() if not checked] if pending: return True, ( - f"You are NOT following Rule 9. The CVE lists specific vulnerable functions " + f"You are NOT following Rule 7. The CVE lists specific vulnerable functions " f"that you MUST investigate first: {', '.join(pending)}. " f"Check these functions before investigating other functions." ) @@ -356,18 +356,18 @@ def check_thought_behavior(self, action: str, action_input: str, output) -> tupl "Check KNOWLEDGE for what was already tried." ) if self._rule_number_7(action, action_input, output): - logger.debug("Reachability Rule 7 triggered: dotted query with empty results for tool '%s'", action) - return True, ("You are NOT following Rule 7. Your query contains dots and returned " + logger.debug("Reachability Rule 5 triggered: dotted query with empty results for tool '%s'", action) + return True, ("You are NOT following Rule 5. Your query contains dots and returned " "no results. You MUST retry with just the final component. Follow the rules.") if self._rule_number_8(action, action_input, output): - logger.debug("Reachability Rule 8 triggered: wrong package for tool '%s', expected '%s'", action, self.target_package) - return True, (f"You are NOT following Rule 8. You are using the wrong package name. You MUST use the target package name {self.target_package} see KNOWLEDGE as the package_name before trying alternative packages. Follow the rules.") + logger.debug("Reachability Rule 6 triggered: wrong package for tool '%s', expected '%s'", action, self.target_package) + return True, (f"You are NOT following Rule 6. You are using the wrong package name. You MUST use the target package name {self.target_package} as the package_name before trying alternative packages. Follow the rules.") if self._rule_use_allowed_tools(action): logger.debug("Reachability allowed-tools rule triggered: '%s' not in %s", action, self.allowed_tools) return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") rule9, msg9 = self._rule_number_9(action, action_input) if rule9: - logger.debug("Reachability Rule 9 triggered: must investigate target functions first for tool '%s'", action) + logger.debug("Reachability Rule 7 triggered: must investigate target functions first for tool '%s'", action) return True, msg9 self.add_action(action, action_input, output) return False, "" @@ -443,15 +443,13 @@ class AgentState(MessagesState): {{""" REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS = """ -1. Output valid JSON only. thought < 100 words. final_answer < 150 words. -2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. -3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. -4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. -5. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. -6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. -7. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. -8. When using Function Locator, Call Chain Analyzer, or Function Caller Finder, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. -9. If KNOWLEDGE lists "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)", you MUST investigate those specific functions FIRST before checking any other functions in the same package. +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. {{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} @@ -464,16 +462,14 @@ class AgentState(MessagesState): """ REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO = """ -1. Output valid JSON only. thought < 100 words. final_answer < 150 words. -2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. -3. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Do NOT use query. -4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. -5. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. -6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. -7. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. -8. When using Function Locator, Call Chain Analyzer, or Function Caller Finder, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. -9. If KNOWLEDGE lists "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)", you MUST investigate those specific functions FIRST before checking any other functions in the same package. -10. After Function Locator validates the package, use Function Caller Finder to identify callers. If FCF finds callers: the function IS reachable -- you MAY conclude and finish. If FCF returns empty: this does NOT mean unreachable. You MUST proceed to Call Chain Analyzer. Do NOT finish or conclude without calling Call Chain Analyzer when FCF returned empty. +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. +8. After Function Locator validates the package, use Function Caller Finder to identify callers. If FCF finds callers: the function IS reachable -- you MAY conclude and finish. If FCF returns empty: this does NOT mean unreachable. You MUST proceed to Call Chain Analyzer. Do NOT finish or conclude without calling Call Chain Analyzer when FCF returned empty. {{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} @@ -489,6 +485,26 @@ class AgentState(MessagesState): {{"thought": "Function Caller Finder returned no callers, but this does not prove unreachable. MUST call Call Chain Analyzer to confirm reachability", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} """ +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON = """ +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. +8. For Call Chain Analyzer, use the class name (e.g. formparser.MultiPartParser) NOT the method name (e.g. formparser.MultiPartParser.parse). Python imports are at the class/module level, so reachability must be traced at that level. If Function Locator returned a module.ClassName.method match, drop the method part for Call Chain Analyzer. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before Call Chain Analyzer"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now trace reachability with Call Chain Analyzer using the class name", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": ".", "query": null, "tool_input": null, "reason": "Check if class is reachable from application code"}}, "final_answer": null}} +""" + REACHABILITY_AGENT_NON_REACH_SYS_PROMPT = ( "You are a security analyst investigating CVE exploitability in container images.\n" "This is NOT a reachability question -- do NOT trace call chains.\n" diff --git a/src/vuln_analysis/tools/tests/mock_documents.py b/src/vuln_analysis/tools/tests/mock_documents.py index e205da6eb..fdf733382 100644 --- a/src/vuln_analysis/tools/tests/mock_documents.py +++ b/src/vuln_analysis/tools/tests/mock_documents.py @@ -243,3 +243,40 @@ metadata={'source': 'transitive_env/lib/python3.12/site-packages/mock_package/mock_file.py', 'content_type': 'simplified_code', 'language': 'python'}) + +# Mock documents for python-dateutil (PyPI name) / dateutil (import name) +# Simulates a package where PyPI name != import name +python_dateutil_function = Document( + page_content='def parse(timestr, parserinfo=None, **kwargs):\n' + ' return _parse(timestr, parserinfo, **kwargs)', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/dateutil/parser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) + +python_dateutil_full_source = Document( + page_content='from dateutil.parser import parse\n\n' + 'import datetime\n', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/dateutil/parser.py', + 'content_type': 'simplified_code', + 'language': 'python'}) + +python_app_using_dateutil = Document( + page_content='from dateutil.parser import parse\n\n' + 'def process_date(raw: str) -> str:\n' + ' dt = parse(raw)\n' + ' return dt.isoformat()', + type='Document', + metadata={'source': 'src/app.py', + 'content_type': 'simplified_code', + 'language': 'python'}) + +python_app_function_using_dateutil = Document( + page_content='def process_date(raw: str) -> str:\n' + ' dt = parse(raw)\n' + ' return dt.isoformat()', + type='Document', + metadata={'source': 'src/app.py', + 'content_type': 'functions_classes', + 'language': 'python'}) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index b638d2498..48276726e 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -6,16 +6,21 @@ from exploit_iq_commons.data_models.common import AnalysisType from vuln_analysis.data_models.state import AgentMorpheusEngineState -from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig +from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig, _searcher_cache from exploit_iq_commons.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput , SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) from vuln_analysis.runtime_context import ctx_state from vuln_analysis.tools.tests.mock_documents import (python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, - python_mock_function_in_use, python_mock_file) + python_mock_function_in_use, python_mock_file, + python_dateutil_function, python_dateutil_full_source, + python_app_using_dateutil, python_app_function_using_dateutil + ) from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder +from pathlib import Path transitive_code_search_runner_coroutine = None @@ -47,6 +52,7 @@ async def test_transitive_search_golang_2(): assert path_found is False assert len(list_path) == 1 + assert "NOTE" in list_path[0], "Should warn about dummy-branch fallback for stdlib package" @pytest.mark.asyncio @@ -206,7 +212,7 @@ def mock_file_open(*args, **kwargs): "name": "python_2", "search_query": "werkzeug,formparser.MultiPartParser.parse", "expected_path_found": False, - "expected_list_length": 0, + "expected_list_length": 1, "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example] }, { @@ -217,11 +223,11 @@ def mock_file_open(*args, **kwargs): "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file] } ]) -@pytest.mark.skip @patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) @patch('builtins.open', side_effect=mock_file_open) async def test_transitive_search_python_parameterized(mock_open, mock_run_command,test_case): """Parameterized test that runs all existing test cases with their respective configurations.""" + _searcher_cache.clear() transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() logging.basicConfig(level=logging.DEBUG) @@ -1055,29 +1061,79 @@ async def test_javascript_object_method_direct_call(mock_open, mock_npm_ls): @pytest.mark.parametrize("raw_query, expected_cleaned", [ - # Standard ASCII quotes ("'commons-beanutils:commons-beanutils:1.9.4'", "commons-beanutils:commons-beanutils:1.9.4"), ('"commons-beanutils:commons-beanutils:1.9.4"', "commons-beanutils:commons-beanutils:1.9.4"), - # Unicode smart quotes (left/right single) - ("\u2018commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), - # Unicode smart quotes (left/right double) - ("\u201ccommons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), - # Mixed: ASCII left, unicode right - ("'commons-beanutils:commons-beanutils:1.9.4\u2019", "commons-beanutils:commons-beanutils:1.9.4"), - ("\"commons-beanutils:commons-beanutils:1.9.4\u201d", "commons-beanutils:commons-beanutils:1.9.4"), - # No quotes + ("‘commons-beanutils:commons-beanutils:1.9.4’", "commons-beanutils:commons-beanutils:1.9.4"), + ("“commons-beanutils:commons-beanutils:1.9.4”", "commons-beanutils:commons-beanutils:1.9.4"), + ("'commons-beanutils:commons-beanutils:1.9.4’", "commons-beanutils:commons-beanutils:1.9.4"), + ("\"commons-beanutils:commons-beanutils:1.9.4”", "commons-beanutils:commons-beanutils:1.9.4"), ("commons-beanutils:commons-beanutils:1.9.4", "commons-beanutils:commons-beanutils:1.9.4"), - # Whitespace + quotes (" 'commons-beanutils:commons-beanutils:1.9.4' ", "commons-beanutils:commons-beanutils:1.9.4"), - # Trailing newline junk from LLM ("'commons-beanutils:commons-beanutils:1.9.4'\nPlease wait...", "commons-beanutils:commons-beanutils:1.9.4"), ]) def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): """Test that query cleaning handles both ASCII and Unicode smart quotes.""" - cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"\u2018\u2019\u201c\u201d").strip() + cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"‘’“”").strip() assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" +python_hyphenated_deptree_output = ( + 'flask==3.1.2 # flask\n' + 'python-dateutil==2.9.0 # python-dateutil\n' + ' six==1.17.0 # six>=1.5\n' +) + + +def mock_file_open_with_dateutil(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "PYTHON" + elif 'requirements.txt' in str(file_path): + mock_file.__iter__ = MagicMock(return_value=iter([ + "flask==3.1.2\n", + "python-dateutil==2.9.0\n", + ])) + else: + mock_file.read.return_value = "" + return mock_file + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_hyphenated_deptree_output) +@patch('builtins.open', side_effect=mock_file_open_with_dateutil) +@patch.object(PythonDependencyTreeBuilder, '_find_site_packages', return_value=Path("/fake/site-packages")) +@patch.object(PythonDependencyTreeBuilder, '_find_module_dirs', side_effect=lambda pkg, sp: {"python-dateutil": ["dateutil"]}.get(pkg, [pkg])) +async def test_transitive_search_python_hyphenated_package(mock_find_module, mock_find_sp, mock_open, mock_run_command): + """Python packages with hyphens (python-dateutil) should be found via import name (dateutil).""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/test/test', + git_ref='abc123', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + mock_documents = [ + python_dateutil_function, + python_dateutil_full_source, + python_app_using_dateutil, + python_app_function_using_dateutil, + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', + return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("python-dateutil,parse") + + (path_found, list_path) = result + assert path_found is True, f"Expected reachable path for python-dateutil,parse but got False. Path: {list_path}" + assert len(list_path) >= 2 + + # --------------------------------------------------------------------------- # Go CCA sub-package disambiguation tests # @@ -1167,3 +1223,25 @@ async def test_go_cca_yaml_unmarshal_not_reachable_in_egress_router(): assert not last_in_chain.startswith("vendor/"), ( f"yaml.Unmarshal chain should reach app code, not stop in vendor/. Path: {list_path}" ) + + +@pytest.mark.asyncio +async def test_go_cca_quicgo_detectAndRemoveAckedPackets_reachable_in_caddy(): + """CVE-2025-29785: quic-go detectAndRemoveAckedPackets IS reachable + in caddy — caddy uses quic-go for HTTP/3 transport, and the function + is called internally when processing ACK frames from QUIC clients.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/caddyserver/caddy", + git_ref="8dc76676fb5156f971db49c50a5c509d1c93613b", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("github.com/quic-go/quic-go,detectAndRemoveAckedPackets") + (path_found, list_path) = result + assert path_found is True, ( + f"detectAndRemoveAckedPackets should be reachable in caddy — " + f"caddy uses quic-go for HTTP/3 and the function processes ACK frames. " + f"Path: {list_path}" + ) + assert len(list_path) > 1 diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index 3aea7d6df..de5216114 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -379,6 +379,23 @@ async def _arun(query: str) -> tuple: # Return concise call chain summary instead of full Document objects # to avoid blowing up the agent's context window with source code. path_summary = _summarize_call_chain(call_hierarchy_list) + # When a package isn't in the dependency tree (e.g. stdlib like crypto/x509), + # the retriever falls back to a "dummy package" branch that synthesizes a + # function document and searches imports instead of walking the real call chain. + # This can produce misleading results — warn the agent so it doesn't treat + # import-based scanning as equivalent to full CCA reachability proof. + package_name = validation_result.split(",")[0].strip() + coc = transitive_code_searcher.chain_of_calls_retriever + if package_name not in coc.supported_packages and not any( + coc.language_parser.is_same_package(package_name, sp) for sp in coc.supported_packages): + if path_summary: + path_summary[0] = ("NOTE (import-scan fallback, package not in dependency tree): " + + path_summary[0]) + else: + path_summary.append( + "NOTE: Package was not found in the dependency tree. " + "No results from import scanning." + ) return found_path, path_summary yield FunctionInfo.from_fn( @@ -457,14 +474,7 @@ async def _arun(query: str) -> dict: _arun, description=(""" Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, and provides ecosystem type (GO/Python/Java/JavaScript/C/C++). - Make sure the input format is matching exactly one of the following formats: - - Input format 1: 'package_name,function_name' or 'package_name,class_name.method_name'. - Example 1: 'libxml2,xmlParseDocument'. - - Input format 2(java): 'maven_gav,class_name.method_name' or 'maven_gav,fqcn.method_name' (preferred). - Example 2(java): 'commons-beanutils:commons-beanutils:x.y.z,a.b.c.ClassA.foo'. - + {fl_input_format} Returns: {'ecosystem': str, 'package_msg': str, 'result': [function_names]}. """)) diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index ccdd35798..a7f43260e 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -23,7 +23,7 @@ from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager -from exploit_iq_commons.data_models.input import SBOMPackage +from vuln_analysis.utils.prompt_factory import FL_EXAMPLES logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") @@ -232,22 +232,11 @@ def search_in_third_party_packages(self, package: str) -> bool: Returns: True if package is found in third party packages, False otherwise """ - def is_same_package(package_name_from_input, package_name_from_tree): - return package_name_from_input.lower() == package_name_from_tree.lower() - # Check in supported_packages list (handles None case) - long_path = False - for supported_package in self.coc_retriever.supported_packages: - if is_same_package(package, supported_package): - long_path = True - break + if self.lang_parser.is_same_package(package, supported_package): + return True - - # Check in short_go_package_name dict (keys) - short_path = package in (self.short_go_package_name or {}) - if long_path: - return True - if short_path: + if package in (self.short_go_package_name or {}): return True if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: fqdn_match = self._resolve_go_fqdn_to_module(package) @@ -273,11 +262,13 @@ async def locate_functions(self, query: str) -> list[str]: self.is_package_valid = False # Validate input format if len(parts_input) < 2: + eco_key = self.coc_retriever.ecosystem.value if self.coc_retriever.ecosystem else "" + example = FL_EXAMPLES.get(eco_key, "'package_name,function_name'") error_msg = ( f"ERROR: Invalid input format. Expected format: 'package_name,function_name' " f"but received: '{query}'. " f"Please provide input in the correct format with a comma separating package name and function name. " - f"Example: 'libxml2,xmlParseDocument'" + f"Example: {example}" ) logger.error(error_msg) return [error_msg] # Return error message that LLM can see diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 6a47db78b..4a7f87a41 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -60,6 +60,8 @@ _NOISE_FUNC_NAMES = frozenset({ "main", "init", "test", "setup", "teardown", "if", "else", "for", "while", "do", "switch", "return", "sizeof", "typeof", + "describe", "it", "beforeEach", "afterEach", "beforeAll", "afterAll", + "expect", "assert", }) _JAVA_TEST_RE = re.compile(r"^test[A-Z]") diff --git a/src/vuln_analysis/utils/prompt_factory.py b/src/vuln_analysis/utils/prompt_factory.py index 93c70ec16..45b07ef99 100644 --- a/src/vuln_analysis/utils/prompt_factory.py +++ b/src/vuln_analysis/utils/prompt_factory.py @@ -128,6 +128,37 @@ } # Optional: Add language-specific usage hints +FL_INPUT_FORMATS: dict[str, str] = { + "python": ( + "Input: 'package_name,function_name' or 'package_name,module.ClassName'.\n" + " Example: 'werkzeug,formparser.MultiPartParser'." + ), + "go": ( + "Input: 'package_path,FunctionName'.\n" + " Example: 'github.com/pkg/errors,New'." + ), + "java": ( + "Input: 'maven_gav,class_name.method_name' or 'maven_gav,fqcn.method_name' (preferred).\n" + " Example: 'commons-beanutils:commons-beanutils:1.9.4,PropertyUtilsBean.getProperty'." + ), + "javascript": ( + "Input: 'package_name,function_name'.\n" + " Example: 'express,Router'." + ), + "c": ( + "Input: 'library_name,function_name'.\n" + " Example: 'openssl,EVP_EncryptInit_ex2'." + ), +} + +FL_EXAMPLES: dict[str, str] = { + "python": "'werkzeug,formparser.MultiPartParser'", + "go": "'github.com/pkg/errors,New'", + "java": "'commons-beanutils:commons-beanutils:1.9.4,PropertyUtilsBean.getProperty'", + "javascript": "'express,Router'", + "c": "'openssl,EVP_EncryptInit_ex2'", +} + FEW_SHOT_EXAMPLES: dict[str, str] = { "python": "Example: Function Locator input 'urllib,parse'; Code Keyword Search for 'urllib.parse'.", "go": "Example: Function Caller Finder 'pkg,errors.New(\"x\")'; Function Locator 'github.com/pkg,New'.", diff --git a/src/vuln_analysis/utils/tests/test_function_name_locator_python.py b/src/vuln_analysis/utils/tests/test_function_name_locator_python.py new file mode 100644 index 000000000..5cd6f0ca5 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_function_name_locator_python.py @@ -0,0 +1,43 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from vuln_analysis.utils.function_name_locator import FunctionNameLocator + + +class TestFLPythonPackageMatching: + def _make_locator(self, supported_packages, import_to_pypi=None): + """Build a FunctionNameLocator with a mocked Python retriever.""" + parser = PythonLanguageFunctionsParser() + if import_to_pypi: + builder = MagicMock() + builder._import_to_pypi = import_to_pypi + parser.set_dependency_builder(builder) + + retriever = MagicMock() + retriever.language_parser = parser + retriever.ecosystem = Ecosystem.PYTHON + retriever.supported_packages = supported_packages + retriever.documents_of_functions = [] + return FunctionNameLocator(retriever) + + def test_exact_match(self): + locator = self._make_locator(["flask", "requests"]) + assert locator.search_in_third_party_packages("flask") is True + + def test_hyphen_underscore_match(self): + """my-package should match my_package via PEP 503 normalization.""" + locator = self._make_locator(["my_package"]) + assert locator.search_in_third_party_packages("my-package") is True + + def test_pypi_to_import_mapping_match(self): + """python-dateutil should match 'dateutil' via mapping.""" + locator = self._make_locator( + ["dateutil"], + import_to_pypi={"dateutil": "python-dateutil"} + ) + assert locator.search_in_third_party_packages("python-dateutil") is True + + def test_no_false_match(self): + locator = self._make_locator(["flask"]) + assert locator.search_in_third_party_packages("django") is False \ No newline at end of file diff --git a/tests/test_python_version_detection.py b/tests/test_python_version_detection.py index 6f4a69420..d1cf68dbb 100644 --- a/tests/test_python_version_detection.py +++ b/tests/test_python_version_detection.py @@ -286,7 +286,9 @@ def test_uses_python_flag_when_version_known(self, builder, tmp_path, monkeypatc builder.install_dependencies(tmp_path) assert any("--python 3.9" in c for c in calls), f"calls: {calls}" - def test_omits_python_flag_when_version_unknown(self, builder, tmp_path, monkeypatch): + def test_falls_back_to_sys_version_when_version_unknown(self, builder, tmp_path, monkeypatch): + import sys + expected_version = f"{sys.version_info.major}.{sys.version_info.minor}" (tmp_path / "requirements.txt").write_text("requests\n") calls = [] monkeypatch.setattr( @@ -296,4 +298,4 @@ def test_omits_python_flag_when_version_unknown(self, builder, tmp_path, monkeyp builder.install_dependencies(tmp_path) venv_calls = [c for c in calls if "venv" in c] assert venv_calls, "venv command not called" - assert "--python" not in venv_calls[0], f"Should not pass --python: {venv_calls[0]}" + assert f"--python {expected_version}" in venv_calls[0], f"Should pass --python {expected_version}: {venv_calls[0]}" diff --git a/tests/test_react_internals_rules.py b/tests/test_react_internals_rules.py index 2440aecd5..73121eb5c 100644 --- a/tests/test_react_internals_rules.py +++ b/tests/test_react_internals_rules.py @@ -182,7 +182,7 @@ def test_duplicate_priority_over_rule7(self): violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Class", []) assert violated is True assert "already called" in msg - assert "Rule 7" not in msg + assert "Rule 5" not in msg def test_rule7_priority_over_allowed_tools(self): tracker = BaseRulesTracker() @@ -190,7 +190,7 @@ def test_rule7_priority_over_allowed_tools(self): tracker.add_action("Code Keyword Search", "org.Class", []) violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) assert violated is True - assert "Rule 7" in msg + assert "Rule 5" in msg assert "AVAILABLE_TOOLS" not in msg def test_allowed_tools_error_message_format(self): @@ -242,7 +242,7 @@ def test_rule9_vulnerable_functions_first(self): tracker.set_target_functions(["getProperty", "setProperty"]) violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,someOtherFunction") assert violated is True - assert "Rule 9" in msg + assert "Rule 7" in msg assert "getProperty" in msg or "setProperty" in msg def test_rule9_passes_after_checking_vulnerable(self): @@ -264,14 +264,14 @@ def test_reachability_check_order(self): tracker.add_action("Code Keyword Search", "org.Class", []) violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) assert violated is True - assert "Rule 7" in msg + assert "Rule 5" in msg tracker2 = ReachabilityRulesTracker() tracker2.set_allowed_tools(["Call Chain Analyzer"]) tracker2.set_target_package("commons-beanutils") violated, msg = tracker2.check_thought_behavior("Call Chain Analyzer", "wrong-package,fn", []) assert violated is True - assert "Rule 8" in msg + assert "Rule 6" in msg tracker3 = ReachabilityRulesTracker() tracker3.set_allowed_tools(["Other Tool"]) @@ -285,7 +285,7 @@ def test_reachability_check_order(self): tracker4.set_target_functions(["getProperty"]) violated, msg = tracker4.check_thought_behavior("Call Chain Analyzer", "pkg,otherFunction", []) assert violated is True - assert "Rule 9" in msg + assert "Rule 7" in msg class TestCheckFinishAllowed: diff --git a/uv.lock b/uv.lock index 152420d87..ed640a803 100644 --- a/uv.lock +++ b/uv.lock @@ -923,6 +923,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "deprecated" version = "1.2.18" @@ -1464,6 +1473,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/59/900aa2445891fc47a33f7d2f76e00ca5d6ae6584b20d19af9c06fa09bf9a/grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", size = 4490123, upload-time = "2025-07-24T18:53:39.528Z" }, ] +[[package]] +name = "gssapi" +version = "1.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/52/c1e90623c259a42ab0587078bb04f959867b970add46ff66750ead8fc7c5/gssapi-1.11.1.tar.gz", hash = "sha256:2049ee4b1d0c363163a1344b7282a363f9f4094e51d2c36de0cf01d4735e0ae2", size = 95233, upload-time = "2026-01-26T21:01:39.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/75/3cc18f2d084d19fbba38dc684588cf5f674c647e754f9cf1625bd78c39f8/gssapi-1.11.1-cp311-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2298e5909a8f2d27c29352885a24e4026cfd3fa24fc38d4a0a3743fa5a3e7667", size = 611712, upload-time = "2026-01-26T21:01:19.626Z" }, + { url = "https://files.pythonhosted.org/packages/81/f9/ac0f8c43c209d56c89655f80cd4ae43379f88370d01a7e11f264f081eef5/gssapi-1.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:5b60b1f8d8d3e36c025bd3494105de1dfccee578e8de001f423cc094468e3022", size = 642782, upload-time = "2026-01-26T21:01:21.462Z" }, + { url = "https://files.pythonhosted.org/packages/f8/97/f4ea9248bfdf5fcde2c5bf0bc0e573d212748724a32a5aa1002e11edb760/gssapi-1.11.1-cp311-abi3-win32.whl", hash = "sha256:9738fe0ba163c28ccf521de7520640bde4b135c1b6e87a5ac5a90435369e89c0", size = 699801, upload-time = "2026-01-26T21:01:23.226Z" }, + { url = "https://files.pythonhosted.org/packages/54/ca/7f839880baf7c365768884161c246a3b6201738722fc7581a995190ec431/gssapi-1.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:96a102ad1ec266e2d843468bf03149982969fc70344f303f81ea20197b80d7a1", size = 781646, upload-time = "2026-01-26T21:01:24.618Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1914,6 +1938,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/05/ed2ad330b22530772f0498431d6f589a18c5eb3bd858da577f1b2ef5980e/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7b4aa27204cb091a4ef96438773c9609f24f217fb3cd53612c41394f39b0d8b6", size = 73983, upload-time = "2025-08-10T20:22:26.498Z" }, ] +[[package]] +name = "koji" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "requests-gssapi" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/4d/6ccfb241b9b8a2809cbbd74dcc51ec2520b87222c5883aee0efc0ee88435/koji-1.36.0.tar.gz", hash = "sha256:481552b9ec9aad55e393c6c5f92e834bc2288e1da01ea67ee48ca63466c4049e", size = 225085, upload-time = "2026-03-23T22:13:34.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/8a/37cf0308e10684e36505e870b37fe22142b4cd2d2c16a894c4fda7891b72/koji-1.36.0-py3-none-any.whl", hash = "sha256:7aeb7932731106e489cd61c471b689aeb80c16583d5eded46f4566696fd5a157", size = 232156, upload-time = "2026-03-23T22:13:32.824Z" }, +] + [[package]] name = "langchain" version = "0.3.27" @@ -4400,6 +4440,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/23/7c1096731c15c83826cb0dd42078b561a838aed44c36f370aeb815168106/requests_futures-1.0.2-py2.py3-none-any.whl", hash = "sha256:a3534af7c2bf670cd7aa730716e9e7d4386497554f87792be7514063b8912897", size = 7671, upload-time = "2024-11-15T22:14:50.255Z" }, ] +[[package]] +name = "requests-gssapi" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gssapi" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/7b/f83ca4b28fbfe122461d8faf27f71ee9141e47473d5520a4f26fa0af04a6/requests_gssapi-1.4.0.tar.gz", hash = "sha256:ba27eb329f4840d965bc8fa5d360c627c74349efa6156ca501ad89afc6a134f4", size = 19090, upload-time = "2025-10-16T04:09:01.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/3e/7f76b3a1816ea2557c9d67d52f14f2fb230865a1e5d9e3b28e2fe74fa49d/requests_gssapi-1.4.0-py3-none-any.whl", hash = "sha256:146a1d1e74d7ca0b9d21af0f8d2d1ca259e9228ff57fbbf23f92e25217da4520", size = 12697, upload-time = "2025-10-16T04:08:59.942Z" }, +] + [[package]] name = "requests-toolbelt" version = "1.0.0" @@ -5247,6 +5300,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/2d/691f741ffd72b6c84438a93749ac57bf1a3f217ac4b0ea4fd0e96119e118/ujson-5.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:38665e7d8290188b1e0d57d584eb8110951a9591363316dd41cf8686ab1d0abc", size = 42211, upload-time = "2024-05-14T02:01:17.567Z" }, ] +[[package]] +name = "unidiff" +version = "0.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/48/81be0ac96e423a877754153699731ef439fd7b80b4c8b5425c94ed079ebd/unidiff-0.7.5.tar.gz", hash = "sha256:2e5f0162052248946b9f0970a40e9e124236bf86c82b70821143a6fc1dea2574", size = 20931, upload-time = "2023-03-10T01:05:39.185Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/54/57c411a6e8f7bd7848c8b66e4dcaffa586bf4c02e63f2280db0327a4e6eb/unidiff-0.7.5-py2.py3-none-any.whl", hash = "sha256:c93bf2265cc1ba2a520e415ab05da587370bc2a3ae9e0414329f54f0c2fc09e8", size = 14386, upload-time = "2023-03-10T01:05:36.594Z" }, +] + [[package]] name = "univers" version = "30.12.0" @@ -5343,6 +5405,8 @@ dependencies = [ { name = "gitpython" }, { name = "google-search-results" }, { name = "json5" }, + { name = "jsonschema" }, + { name = "koji" }, { name = "litellm" }, { name = "nbformat" }, { name = "nemollm" }, @@ -5354,6 +5418,7 @@ dependencies = [ { name = "tantivy" }, { name = "tree-sitter" }, { name = "tree-sitter-languages" }, + { name = "unidiff" }, { name = "univers" }, ] @@ -5389,6 +5454,8 @@ requires-dist = [ { name = "gitpython" }, { name = "google-search-results", specifier = "==2.4" }, { name = "json5" }, + { name = "jsonschema", specifier = ">=4.0.0,<5.0.0" }, + { name = "koji" }, { name = "litellm", specifier = "<=1.75.8" }, { name = "nbformat" }, { name = "nemollm" }, @@ -5400,6 +5467,7 @@ requires-dist = [ { name = "tantivy", specifier = "==0.22.2" }, { name = "tree-sitter", specifier = "==0.21.3" }, { name = "tree-sitter-languages", specifier = "==1.10.2" }, + { name = "unidiff", specifier = ">=0.7.5" }, { name = "univers", specifier = "==30.12" }, ] From c13b71768460c2ea5c943818f852dd47d4343ab4 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 8 Jun 2026 11:52:30 +0300 Subject: [PATCH 244/286] fix: RCE vulnerability for NPM ( from user scripts) and pip (installed dependency' risk of shell command injection fix: SSRF prevention on sbom downloaded from URL fix: password expansion/reflection on CLI fix: potential path traversal on repo_ref when reading documents cache fix: various issues that might potentially lead to shell command injections fix: path traversal prevention when reading sbom from file Signed-off-by: Zvi Grinberg --- src/exploit_iq_commons/utils/data_utils.py | 45 ++++++------ src/exploit_iq_commons/utils/dep_tree.py | 68 +++++++++++-------- .../functions/cve_process_sbom.py | 41 ++++++++++- 3 files changed, 100 insertions(+), 54 deletions(-) diff --git a/src/exploit_iq_commons/utils/data_utils.py b/src/exploit_iq_commons/utils/data_utils.py index 771d9e6b2..dac8ba241 100644 --- a/src/exploit_iq_commons/utils/data_utils.py +++ b/src/exploit_iq_commons/utils/data_utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib import json import os import pickle @@ -108,14 +109,29 @@ def merge_intel_and_plugin_data_convert_to_dataframe(intel_list: list[CveIntel]) ] ], sep="_") -def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: typing.Any, documents_name: str = ""): - cached_documents_path = \ - (f"{base_pickle_dir}/" - f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" - f"{repo_ref}") +def _sanitize_path_component(value: str) -> str: + sanitized = value.replace('//', '.').replace('/', '.').replace(':', '').replace('..', '_') + sanitized = os.path.basename(sanitized) + if not sanitized: + sanitized = hashlib.sha256(value.encode()).hexdigest()[:16] + return sanitized + +def _build_cache_path(base_pickle_dir: str, repo_url: str, repo_ref: str, documents_name: str = "") -> str: + safe_url = repo_url.replace('//', '.').replace('/', '.').replace(':', '') + safe_ref = _sanitize_path_component(repo_ref) + cached_documents_path = f"{base_pickle_dir}/{safe_url}-{safe_ref}" if documents_name: - cached_documents_path += "-" + documents_name + cached_documents_path += "-" + _sanitize_path_component(documents_name) + resolved = os.path.realpath(cached_documents_path) + base_resolved = os.path.realpath(base_pickle_dir) + if not resolved.startswith(base_resolved + os.sep): + raise ValueError(f"Cache path escapes base directory: {cached_documents_path}") + return cached_documents_path + + +def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: typing.Any, documents_name: str = ""): + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) logger.debug(f"Saved loaded Documents into Cache => {cached_documents_path}") with open(cached_documents_path, 'wb') as doc_file: # open a text file @@ -123,14 +139,7 @@ def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: def cache_exists(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> bool: """Check if a pickle cache file exists without loading it into memory.""" - cached_documents_path = \ - (f"{base_pickle_dir}/" - f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" - f"{repo_ref}") - - if documents_name: - cached_documents_path += "-" + documents_name - + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) return os.path.isfile(cached_documents_path) @@ -138,13 +147,7 @@ def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref, documents_name: str if not os.path.exists(base_pickle_dir): os.makedirs(base_pickle_dir) - cached_documents_path = \ - (f"{base_pickle_dir}/" - f"{repo_url.replace('//', '.').replace('/', '.').replace(':', '')}-" - f"{repo_ref}") - - if documents_name: - cached_documents_path += "-" + documents_name + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) if os.path.isfile(cached_documents_path): with open(cached_documents_path, 'rb') as doc_file: diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 09b91789e..00310509d 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -150,15 +150,15 @@ def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None: return None -def run_command(cmd: str) -> str: +def run_command(args: list[str], cwd: str | Path | None = None, input_data: str | None = None) -> str | None: try: - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - logger.debug(f'Successfully run command: {cmd}') + result = subprocess.run(args, capture_output=True, text=True, cwd=cwd, input=input_data) + logger.debug('Successfully ran command: %s', args) if result.stderr: - logger.debug(f"Stderr: {result.stderr}") + logger.debug("Stderr: %s", result.stderr) return result.stdout except Exception as e: - logger.error(f"Failed to run command:{cmd} with error: {e}") + logger.error("Failed to run command: %s with error: %s", args, e) class DependencyTreeBuilder(ABC): @@ -265,6 +265,14 @@ def find_all_files(self, root_dir): for fname in filenames: if fname.endswith(('.c', '.h')): full_path = Path(dirpath) / fname + try: + resolved = full_path.resolve() + root_resolved = Path(root_dir).resolve() + if not str(resolved).startswith(str(root_resolved) + os.sep): + logger.warning("Skipping rename outside root: %s", full_path) + continue + except OSError: + continue new_path = full_path.with_suffix(".k") try: full_path.rename(new_path) @@ -582,9 +590,9 @@ def _setup_skopeo_auth(self, username: str, password: str): """Setup skopeo authentication for registry.redhat.io""" logger.info("Setting up skopeo authentication...") - cmd = ['skopeo', 'login', 'registry.redhat.io', '--username', username, '--password', password] + cmd = ['skopeo', 'login', 'registry.redhat.io', '--username', username, '--password-stdin'] - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, input=password) if result.returncode != 0: raise RuntimeError(f"Skopeo login failed: {result.stderr}") @@ -634,12 +642,10 @@ def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: continue logger.info(f"Extracting: {tar_file.name}") try: - result = subprocess.run(['tar', 'xf', str(tar_file), '-C', str(container_sources_dir)], - capture_output=True, - text=True, - check=False) - if result.returncode != 0: - logger.warning(f"Failed to extract {tar_file}: {result.stderr}") + with tarfile.open(str(tar_file)) as tar: + tar.extractall(str(container_sources_dir), filter='data') + except tarfile.TarError as e: + logger.warning(f"Failed to extract {tar_file}: {e}") except Exception as e: logger.warning(f"Error extracting {tar_file}: {e}") @@ -811,18 +817,20 @@ def get_go_mod_graph_tree(manifest_path) -> str: """ process_object: subprocess.CompletedProcess formatted_error_msg: str + env = {**os.environ, 'GOWORK': 'off'} try: process_object = subprocess.run( - ["bash", "-c", f" export GOWORK=off ; go mod graph " - f"-modfile {manifest_path}/go.mod"], + ["go", "mod", "graph", "-modfile", str(Path(manifest_path) / "go.mod")], capture_output=True, - text=True) + text=True, + env=env) if process_object.returncode > 0: process_object = subprocess.run( - ["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ;" - f"go mod graph"], + ["go", "mod", "graph"], capture_output=True, - text=True) + text=True, + cwd=str(manifest_path), + env=env) if process_object.returncode > 0: formatted_error_msg = (f"Failed to generate dependencies tree from go.mod " f"manifest at {manifest_path}, error details => " @@ -843,8 +851,10 @@ def download_go_mod_vendor(self, manifest_path): it downloads it to git repo root dir where the go.mod file located. :param manifest_path: path to go.mod manifest in repo directory. """ - subprocess.run(["bash", "-c", f" export GOWORK=off ; cd {manifest_path} ; " - f"go mod vendor"]) + env = {**os.environ, 'GOWORK': 'off'} + subprocess.run(["go", "mod", "vendor"], + cwd=str(manifest_path), + env=env) def extract_package_name(self, package_name: str) -> str: if package_name.__contains__("@"): @@ -1191,14 +1201,13 @@ def _ensure_venv(self, manifest_path: Path) -> str: python_version = f"{sys.version_info.major}.{sys.version_info.minor}" logger.info("Python version undetermined; using current interpreter %s", python_version) logger.info("Creating transitive_env with Python %s using uv", python_version) - cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" - run_command(cmd) + run_command(['uv', 'venv', TRANSITIVE_ENV_NAME, '--python', python_version], cwd=str(manifest_path)) return venv_python def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: venv_python = self._ensure_venv(manifest_path) - run_command(f'uv pip install "setuptools<81" deptree --python {venv_python}') - dependencies = run_command(f'{venv_python} -m deptree') + run_command(['uv', 'pip', 'install', 'setuptools<81', 'deptree', '--python', venv_python]) + dependencies = run_command([venv_python, '-m', 'deptree']) if not dependencies or not dependencies.strip(): logger.error("deptree returned empty output — third-party dependency tree will be incomplete. " "Check that the virtual environment at %s has packages installed.", manifest_path) @@ -1605,8 +1614,8 @@ def install_dependency(self, dependency, repo_path): valid_signs = ['==', '>=', '<=', '!='] if not any(sign in dependency for sign in valid_signs): dependency = dependency.replace('=', '==') - cmd = f'uv pip install {dependency} --python {repo_path}/{TRANSITIVE_ENV_NAME}/bin/python' - res = run_command(cmd) + venv_python = str(Path(repo_path) / TRANSITIVE_ENV_NAME / 'bin' / 'python') + res = run_command(['uv', 'pip', 'install', dependency, '--python', venv_python]) if not res: logger.warning('Failed to install dependency %s', dependency) @@ -1774,7 +1783,7 @@ def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: logger.error(f"package.json not found at {package_json_path}") raise FileNotFoundError(f"package.json not found at {package_json_path}") - result = run_command(f'cd {manifest_path} && npm ls --json --all') + result = run_command(['npm', 'ls', '--json', '--all'], cwd=str(manifest_path)) try: npm_tree = json.loads(result) except json.JSONDecodeError as e: @@ -1801,8 +1810,7 @@ def _parse_npm_tree(self, node: dict, tree: dict, parent: str): def install_dependencies(self, manifest_path: Path): logger.info(f"Installing JavaScript dependencies in {manifest_path}") - cmd = f'cd {manifest_path} && npm install' - res = run_command(cmd) + res = run_command(['npm', 'install', '--ignore-scripts'], cwd=str(manifest_path)) if not res: logger.error(f"Failed to install JavaScript dependencies") else: diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index c2b78d250..39a11788b 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ipaddress +import os +import socket +from urllib.parse import urlparse + from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo @@ -25,6 +30,35 @@ from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) +_ALLOWED_SBOM_DIR = os.getenv("SBOM_ALLOWED_DIR", "/tmp/sbom") + + +def _validate_file_path(file_path: str) -> str: + resolved = os.path.realpath(file_path) + allowed = os.path.realpath(_ALLOWED_SBOM_DIR) + if not resolved.startswith(allowed + os.sep) and resolved != allowed: + raise ValueError(f"SBOM file path not within allowed directory ({_ALLOWED_SBOM_DIR}): {file_path}") + return resolved + + +def _validate_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ("https", "http"): + raise ValueError(f"URL scheme must be http or https, got: {parsed.scheme!r}") + hostname = parsed.hostname + if not hostname: + raise ValueError(f"URL has no hostname: {url}") + try: + resolved_ips = socket.getaddrinfo(hostname, None) + except socket.gaierror: + raise ValueError(f"Cannot resolve hostname: {hostname}") + for _family, _type, _proto, _canonname, sockaddr in resolved_ips: + ip = ipaddress.ip_address(sockaddr[0]) + # Allow only loopback for testing(localhost) or public http/https url locations for sbom. + if ip.is_private or ip.is_link_local or ip.is_reserved: + raise ValueError(f"URL resolves to non-public address ({ip}): {url}") + return url + class CVEProcessSBOMConfig(FunctionBaseConfig, name="cve_process_sbom"): """ @@ -83,8 +117,8 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: elif (message.input.image.sbom_info.type == FileSBOMInfoInput.static_type()): assert isinstance(message.input.image.sbom_info, FileSBOMInfoInput) - # Read the file to an object - with open(message.input.image.sbom_info.file_path, "r", encoding="utf-8") as f: + safe_path = _validate_file_path(message.input.image.sbom_info.file_path) + with open(safe_path, "r", encoding="utf-8") as f: sbom_lines = f.readlines() # Create the SBOM object @@ -94,9 +128,10 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: assert isinstance(message.input.image.sbom_info, HTTPSBOMInfoInput) try: + validated_url = _validate_url(message.input.image.sbom_info.url) _, response = http_utils.request_with_retry( request_kwargs={ - "url": message.input.image.sbom_info.url, "method": HTTPMethod.GET.value + "url": validated_url, "method": HTTPMethod.GET.value }, max_retries=config.max_retries ) From f16bc7e273ed346a890fff2828b782cb7f973ac3 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 8 Jun 2026 16:02:25 +0300 Subject: [PATCH 245/286] test: fix one python test, and remove old one not relevant anymore Signed-off-by: Zvi Grinberg --- tests/test_python_version_detection.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/test_python_version_detection.py b/tests/test_python_version_detection.py index d1cf68dbb..6032209ea 100644 --- a/tests/test_python_version_detection.py +++ b/tests/test_python_version_detection.py @@ -281,21 +281,9 @@ def test_uses_python_flag_when_version_known(self, builder, tmp_path, monkeypatc calls = [] monkeypatch.setattr( "exploit_iq_commons.utils.dep_tree.run_command", - lambda cmd: calls.append(cmd) or "", + lambda args, cwd="", input_data=None: calls.extend(args) or "", ) builder.install_dependencies(tmp_path) - assert any("--python 3.9" in c for c in calls), f"calls: {calls}" - - def test_falls_back_to_sys_version_when_version_unknown(self, builder, tmp_path, monkeypatch): - import sys - expected_version = f"{sys.version_info.major}.{sys.version_info.minor}" - (tmp_path / "requirements.txt").write_text("requests\n") - calls = [] - monkeypatch.setattr( - "exploit_iq_commons.utils.dep_tree.run_command", - lambda cmd: calls.append(cmd) or "", - ) - builder.install_dependencies(tmp_path) - venv_calls = [c for c in calls if "venv" in c] - assert venv_calls, "venv command not called" - assert f"--python {expected_version}" in venv_calls[0], f"Should pass --python {expected_version}: {venv_calls[0]}" + assert any("--python" in c for c in calls) and any("3.9" in c for c in calls) , f"calls: {calls}" +# def run_command(args: list[str], cwd: str | Path | None = None, input_data: str | None = None) -> str | None: + From 4572dc21e0b693d5d2a6790dc852272542ce1712 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 9 Jun 2026 12:19:42 +0300 Subject: [PATCH 246/286] chore: extend pipeline timeout from 1 hour to 2 hours and half Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 34c219807..e15cb65f5 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -17,6 +17,8 @@ metadata: # How many runs we want to keep. pipelinesascode.tekton.dev/max-keep-runs: "5" spec: + timeouts: + pipeline: 2h30m0s # Timeout for the entire PipelineRun taskRunTemplate: podTemplate: imagePullSecrets: @@ -210,7 +212,7 @@ spec: # ubi-minimal has no dev tools — install the essentials print_banner "Installing system packages" - microdnf install -y git tar gzip findutils make gcc gcc-c++ krb5-devel + microdnf install -y git tar gzip findutils make gcc gcc-c++ krb5-devel bsdtar microdnf module enable -y nodejs:20 microdnf install -y nodejs npm From 117f19e9990d49d61a2c0373c86bc54e76c43965 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:44:18 +0300 Subject: [PATCH 247/286] APPENG-5393 update documentation and some feedback improvment (#248) documentation for the RPM CVE Checker feature, and RHSA "Not Affected" Handling Improvement --- README.md | 26 ++ RPM-CVE-CHECKER.md | 246 ++++++++++++++++++ .../data_models/checker_status.py | 8 + .../functions/cve_checker_report.py | 55 +++- .../functions/cve_package_code_agent.py | 2 + src/vuln_analysis/utils/package_identifier.py | 16 +- tests/test_package_identifier.py | 50 +++- 7 files changed, 393 insertions(+), 10 deletions(-) create mode 100644 RPM-CVE-CHECKER.md diff --git a/README.md b/README.md index f4fab8a9f..4aff86137 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ limitations under the License. - [Customizing the embedding model](#customizing-the-embedding-model) - [Steps to configure an alternate embedding provider](#steps-to-configure-an-alternate-embedding-provider) - [Customizing the Output](#customizing-the-output) +- [RPM CVE Checker](#rpm-cve-checker) - [Limitations](#limitations) - [Troubleshooting](#troubleshooting) - [Git LFS issues](#git-lfs-issues) @@ -772,6 +773,31 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. +## RPM CVE Checker + +Determine if an RPM package is vulnerable to a specific CVE by examining source patches, changelogs, and build artifacts. + +**When to use:** +- Investigating a specific CVE in a single RHEL or Fedora RPM package +- Need source-level evidence (patches, changelogs) for vulnerability status +- Targeted package analysis vs. container-wide scanning + +**Quick example:** +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64", + "cve_id": "CVE-2026-5121" + } +} +``` + +For detailed usage, input/output formats, and verdict explanations, see [RPM CVE Checker Documentation](RPM-CVE-CHECKER.md). + ## Limitations The following limitations apply to this workflow: diff --git a/RPM-CVE-CHECKER.md b/RPM-CVE-CHECKER.md new file mode 100644 index 000000000..0b245c963 --- /dev/null +++ b/RPM-CVE-CHECKER.md @@ -0,0 +1,246 @@ +# RPM CVE Checker + +Determine if an RPM package is vulnerable to a specific CVE by examining source patches, changelogs, and build artifacts. + +This mode provides targeted, single-package vulnerability analysis for RHEL and Fedora RPMs, producing detailed evidence-based verdicts. + +## Table of Contents + +- [Supported Distributions](#supported-distributions) +- [How It Works](#how-it-works) +- [Input Format](#input-format) +- [Configuration](#configuration) +- [Verdicts](#verdicts) +- [Output Format](#output-format) +- [Example](#example) +- [Limitations](#limitations) + +## Supported Distributions + +| Distribution | L1 Source Analysis | L2 Compilation Check | L2 Hardening Check | +|--------------|-------------------|---------------------|-------------------| +| **RHEL** | Full | Full | Full | +| **Fedora** | Full | Full | Not available | + +**RHEL** receives full analysis across both levels: +- L1: Source analysis (spec files, changelogs, patches) +- L2: Compilation check (detect disabled features) + hardening flag analysis (compiler protections) + +**Fedora** receives L1 + partial L2: +- L1: Source analysis (spec files, changelogs, patches) +- L2: Compilation check only (can detect if features like LDAP/OpenSSL are disabled via configure/makefiles) +- No hardening flag analysis (different build system, no Brew build logs) + +## How It Works + +The checker performs a two-level investigation to determine vulnerability status: + +```mermaid +flowchart TD + A["Input: CVE + RPM Package"] --> B["L1: Source Analysis"] + B --> C{"Backport found?"} + C -->|Yes| D["NOT_VULNERABLE"] + C -->|No| E["L2: Compilation Check"] + E --> F{"Code compiled?"} + F -->|No| D + F -->|Yes| G["L2: Hardening Check (RHEL only)"] + G --> H{"Hardening mitigates?"} + H -->|Yes| I["VULNERABLE_MITIGATED"] + H -->|No| J["VULNERABLE"] +``` + +### L1: Source Analysis + +Examines package source artifacts in three phases: + +1. **IDENTIFY** — Is this package affected by the CVE? (version range, architecture) +2. **LOCATE** — Is the vulnerable code present in the source tree? +3. **VERIFY** — Has a fix been applied? (backport patches, changelog entries, spec file references) + +### L2: Build Analysis + +When L1 finds no source-level fix: + +1. **Compilation Check** — Is the vulnerable code actually compiled into the binary? (feature flags, configure options) +2. **Hardening Check** (RHEL only) — Do compiler flags mitigate the vulnerability? (`-fstack-protector`, RELRO, PIE, etc.) + +## Input Format + +Provide the target package and CVE via JSON input with `pipeline_mode: "package_checker"`. + +### RHEL Example + +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64", + "cve_id": "CVE-2026-5121" + } +} +``` + +### Fedora Example + +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "openssl", + "version": "3.2.1", + "release": "1.fc40", + "arch": "x86_64", + "cve_id": "CVE-2024-2511" + } +} +``` + +### Required Fields + +| Field | Description | +|-------|-------------| +| `name` | RPM package name | +| `version` | Package version | +| `release` | Release string (e.g., `14.el7_9.2` for RHEL, `1.fc40` for Fedora) | +| `arch` | Architecture (e.g., `x86_64`) | +| `cve_id` | CVE identifier to investigate | + +## Configuration + +The RPM CVE Checker uses these configuration options in the workflow YAML: + +### Brew Profile (`rpm_user_type`) + +Controls which build system is used for fetching SRPMs and build logs: + +```yaml +cve_source_acquisition: + _type: cve_source_acquisition + rpm_user_type: ${RPM_USER_TYPE:-internal} + +cve_package_code_agent: + _type: cve_package_code_agent + rpm_user_type: ${RPM_USER_TYPE:-internal} +``` + +| Profile | Hub | Build Logs | Use Case | +|---------|-----|------------|----------| +| `internal` | Red Hat Brew (`brewhub.engineering.redhat.com`) | Auto-fetched | RHEL packages (requires VPN) | +| `external` | Fedora Koji (`koji.fedoraproject.org`) | Not fetched | Fedora packages (public access) | + +**Environment variable override:** Set `RPM_USER_TYPE=external` to override the config file value. + +### Cache Directories + +Configure where artifacts are stored: + +```yaml +cve_source_acquisition: + base_rpm_dir: .cache/am_cache/rpms # Shared SRPM cache + base_checker_dir: .cache/am_cache/checker # Checker-specific artifacts + +cve_checker_segmentation: + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index # Tantivy search index +``` + +| Directory | Contents | +|-----------|----------| +| `base_rpm_dir` | Downloaded SRPMs (shared cache) | +| `base_checker_dir` | Extracted sources, build logs, patches, reports | +| `base_code_index_dir` | Tantivy full-text search index for source code | + +### Profile Details + +**Internal profile** (`internal-user-profile.yml`): +- Connects to Red Hat Brew hub +- Fetches build logs automatically (`build_log.auto_fetch: true`) +- Enables L2 hardening check via build log analysis +- Requires Red Hat VPN connection + +**External profile** (`external-user-profile.yml`): +- Connects to Fedora Koji hub +- Does not fetch build logs (`build_log.auto_fetch: false`) +- L2 hardening check unavailable (no build log) +- Public access, no authentication required + +For other configuration options (LLM models, output paths), see the main [Configuration file reference](README.md#configuration-file-reference). + +## Verdicts + +The checker produces one of three verdicts with a justification label explaining the reasoning: + +| Verdict | Justification Label | When | +|---------|---------------------|------| +| `NOT_VULNERABLE` | `code_not_present` | Package version outside affected range, architecture not applicable, or vulnerable code not compiled (feature disabled) | +| `NOT_VULNERABLE` | `protected_by_mitigating_control` | Downstream backport patch applied, confirmed in changelog/spec/build log | +| `VULNERABLE_MITIGATED` | `protected_by_compiler` | No source fix, but compiler hardening flags mitigate the exploit mechanism (RHEL only) | +| `VULNERABLE` | `vulnerable` | Package is affected, no fix or relevant mitigation found | + +Each verdict includes a **confidence score** (0-100%) indicating the strength of evidence. + +## Output Format + +The checker produces a structured report with these sections: + +### Executive Summary +One-paragraph assessment of vulnerability status with key findings. + +### Verdict and Justification +- `justification_label`: The classification (see Verdicts table) +- `confidence`: Percentage confidence in the verdict +- Status: `TRUE` (vulnerable) or `FALSE` (not vulnerable) + +### Evidence Chain +Bullet-point list of evidence supporting the verdict: +- Patch files found/not found +- Changelog mentions +- Build log confirmations +- Version comparisons + +### Extracted Facts +Verbatim excerpts from package artifacts: +- Spec `PatchN:` lines matching the CVE +- `%changelog` entries mentioning the CVE +- Build log lines showing patch application + +### Code Snippets +When available, shows vulnerable code and fix code side-by-side. + +## Example + +**Input:** CVE-2026-5121 in `libarchive-3.1.2-14.el7_9.2` + +**Verdict:** `NOT_VULNERABLE` (`protected_by_mitigating_control`, 90% confidence) + +**Executive Summary:** +The package is protected by a downstream patch that mitigates the vulnerability. The fix is carried as a separate %patch directive. + +**Evidence Chain:** +- Downstream patch file `libarchive-3.3.3-Fix-CVE-2026-5121.patch` found and applied +- Spec file contains: `Patch33: %{name}-3.3.3-Fix-CVE-2026-5121.patch` +- Changelog contains: `Security fix for CVE-2026-4424 CVE-2026-5121` +- Build log confirms: `Patch #33 (libarchive-3.3.3-Fix-CVE-2026-5121.patch)` + +**Affected File:** `libarchive/archive_read_support_format_iso9660.c` + +**Fix Applied:** Added bounds check to prevent integer overflow in zisofs block pointer allocation: +```c +if (file->pz_log2_bs < 15 || file->pz_log2_bs > 17) { + file->pz = 0; + return; +} +``` + +## Limitations + +1. **Fedora: No hardening check** — L2 compilation check works (can detect disabled features), but no compiler flag analysis due to different build system. + +2. **Requires source access** — Needs SRPM via Brew (RHEL) or Koji (Fedora). Packages without accessible source cannot be fully analyzed. + +3. **LLM-based analysis** — Verdicts are generated using LLM reasoning and should be reviewed by security analysts before action. + +4. **Single-package scope** — Designed for targeted investigation of one CVE in one package. For container-wide scanning, use the main workflow. diff --git a/src/exploit_iq_commons/data_models/checker_status.py b/src/exploit_iq_commons/data_models/checker_status.py index 029cf8245..bdfbbb8e5 100644 --- a/src/exploit_iq_commons/data_models/checker_status.py +++ b/src/exploit_iq_commons/data_models/checker_status.py @@ -28,6 +28,7 @@ class PackageCheckerStatus(IntEnum): ERROR_FAILED_TO_DOWNLOAD_SRPM = 3 PKG_IDENT_CVE_MISMATCH = 4 PKG_INTEL_LOW_SCORE = 5 + CONFLICTING_INTEL_EVIDENCE = 6 PACKAGE_CHECKER_STATUS_DESCRIPTIONS: dict[PackageCheckerStatus, str] = { @@ -43,6 +44,8 @@ class PackageCheckerStatus(IntEnum): "CVE does not apply to target package - RHSA does not list this package", PackageCheckerStatus.PKG_INTEL_LOW_SCORE: "Intel quality score below threshold - insufficient information for reliable analysis", + PackageCheckerStatus.CONFLICTING_INTEL_EVIDENCE: + "Code analysis conflicts with RHSA 'not affected' assessment - manual review required", } CHECKER_FAILURE_ERROR_TYPES: dict[PackageCheckerStatus, str] = { @@ -66,6 +69,11 @@ class PackageIdentifyResult(BaseModel): is_target_package_affected: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN is_target_package_fixed: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN + rhsa_fix_state: str | None = Field( + default=None, + description="Raw RHSA fix_state value when is_target_package_affected is based on RHSA assessment" + ) + conclusion_reason: str = Field( default="", description="Detailed explanation of why the package was determined to be vulnerable or not vulnerable" diff --git a/src/vuln_analysis/functions/cve_checker_report.py b/src/vuln_analysis/functions/cve_checker_report.py index 4e70b5ae4..ef62e580a 100644 --- a/src/vuln_analysis/functions/cve_checker_report.py +++ b/src/vuln_analysis/functions/cve_checker_report.py @@ -36,7 +36,12 @@ from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput -from exploit_iq_commons.data_models.checker_status import L1InvestigationResult, L2BuildResult +from exploit_iq_commons.data_models.checker_status import ( + EnumIdentifyResult, + L1InvestigationResult, + L2BuildResult, + PackageCheckerStatus, +) from nat.builder.context import Context from vuln_analysis.data_models.output import ( @@ -67,17 +72,36 @@ "requires_environment": "FALSE", "vulnerable": "TRUE", "uncertain": "UNKNOWN", + "conflicting_evidence": "UNKNOWN", } _POLICY_MAX_RPM_LIST_ITEMS = 5 _POLICY_RHSA_STATEMENT_CAP = 400 + _POLICY_MAX_PACKAGE_STATE_ITEMS = 8 _BUILD_LOG_EXCERPT_MAX_LINES = 8 + _EVIDENCE_CHAIN_ITEM_CAP = 8 +def _detect_intel_conflict(ctx, final_label: str) -> bool: + """Detect when final verdict contradicts RHSA 'not affected' assessment. + + Returns True if RHSA assessed the package as "not affected" but the + code/build analysis concluded it is vulnerable. This conflict requires + human review to determine if it's an RHSA error or an analysis bug. + """ + if not ctx or not ctx.identify_result: + return False + if ctx.identify_result.is_target_package_affected != EnumIdentifyResult.NO: + return False + if ctx.identify_result.rhsa_fix_state != "not affected": + return False + return final_label == "vulnerable" + + def _md_section_header(title: str) -> list[str]: """UI-friendly section break: horizontal rule + level-2 heading (no tables).""" return ["---", "", f"## {title}", ""] @@ -600,6 +624,11 @@ def _format_snippet_bullets(self, snippets: list[CodeSnippet]) -> list[str]: "Labeled uncertain because patch, spec, build, and source evidence are " "insufficient or conflicting." ), + "conflicting_evidence": ( + "Labeled conflicting evidence because RHSA assessed this package as 'not affected' " + "but code analysis found vulnerable patterns. Manual review required to determine " + "if this is an RHSA assessment error or an analysis false positive." + ), } @@ -611,9 +640,15 @@ def _all_patch_checks_failed(blocks: ReportBlocks) -> bool: ) -def _format_justification_reason(blocks: ReportBlocks) -> str: - """One-sentence VEX label rationale (outcome voice); evidence lives in details.""" - label = blocks.justification_label +def _format_justification_reason(blocks: ReportBlocks, label_override: str | None = None) -> str: + """One-sentence VEX label rationale (outcome voice); evidence lives in details. + + Args: + blocks: Report blocks containing investigation data + label_override: Optional label to use instead of blocks.justification_label + (used when label is changed due to conflict detection) + """ + label = label_override if label_override else blocks.justification_label if label == "vulnerable": if _all_patch_checks_failed(blocks): return ( @@ -936,8 +971,18 @@ def _build_analysis( label = blocks.justification_label status: _StatusLiteral = _JUSTIFICATION_LABEL_TO_STATUS.get(label, "UNKNOWN") + # Check for conflict between RHSA "not affected" assessment and analysis verdict + if _detect_intel_conflict(ctx, label): + logger.warning( + "Intel conflict detected: RHSA says 'not affected' but analysis concluded 'vulnerable'. " + "Flagging for human review." + ) + label = "conflicting_evidence" + status = "UNKNOWN" + ctx.status = PackageCheckerStatus.CONFLICTING_INTEL_EVIDENCE + summary = _strip_build_agent_absence_boilerplate(blocks.executive_summary.strip()) - reason = _format_justification_reason(blocks) + reason = _format_justification_reason(blocks, label_override=label if label != blocks.justification_label else None) details = _build_details_md(blocks) return [ diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py index 0df0b90d0..2d5f9b8b0 100644 --- a/src/vuln_analysis/functions/cve_package_code_agent.py +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -194,6 +194,8 @@ def _format_policy_context_for_l1_report( suffix = f" (+ {len(affected) - len(shown)} more)" if len(affected) > len(shown) else "" lines.append(f"**Affected NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") lines.append(f" - is_target_package_affected: `{identify_result.is_target_package_affected.value}`") + if identify_result.conclusion_reason: + lines.append(f" - conclusion_reason: `{identify_result.conclusion_reason}`") if fixed: shown = fixed[:_POLICY_MAX_RPM_LIST_ITEMS] diff --git a/src/vuln_analysis/utils/package_identifier.py b/src/vuln_analysis/utils/package_identifier.py index 42346d2ab..e355d8609 100644 --- a/src/vuln_analysis/utils/package_identifier.py +++ b/src/vuln_analysis/utils/package_identifier.py @@ -128,7 +128,9 @@ def identify(self, intel: CveIntel | None) -> tuple[PackageCheckerStatus, Packag ``affected_release`` when Red Hat published either list; else ``PKG_IDENT_CVE_MISMATCH``. Step 2 — Vulnerability posture: ``PKG_IDENT_NOT_VUL`` when RHSA/NVD - shows the target is not affected or already fixed. + shows the target is already fixed, or when fix_state is "will not fix" + or "out of support scope". When fix_state is "not affected", investigation + continues (status=OK) to verify the RHSA assessment and catch potential errors. """ package_identify = PackageIdentifyResult() @@ -146,8 +148,16 @@ def identify(self, intel: CveIntel | None) -> tuple[PackageCheckerStatus, Packag package_identify.is_target_package_fixed = self._is_target_package_fixed(intel,package_identify) - if package_identify.is_target_package_affected == EnumIdentifyResult.NO or package_identify.is_target_package_fixed == EnumIdentifyResult.YES: + # Determine if we should early-exit with PKG_IDENT_NOT_VUL + if package_identify.is_target_package_fixed == EnumIdentifyResult.YES: + # Package is already running the fixed version status = PackageCheckerStatus.PKG_IDENT_NOT_VUL + elif package_identify.is_target_package_affected == EnumIdentifyResult.NO: + # RHSA says not affected - check if we should continue investigation + # "not affected" continues to verify; other states (will not fix, out of support) early-exit + if package_identify.rhsa_fix_state != "not affected": + status = PackageCheckerStatus.PKG_IDENT_NOT_VUL + # else: status stays OK, continue investigation to verify RHSA assessment return status, package_identify @@ -243,6 +253,8 @@ def _is_target_package_affected( matched_ps.fix_state, target_name, target_distro, result.value ) if result == EnumIdentifyResult.NO: + # Store the raw fix_state for downstream decision-making + package_identify.rhsa_fix_state = matched_ps.fix_state.lower().strip() if matched_ps.fix_state else None package_identify.conclusion_reason = ( f"RHSA fix_state indicates package is not vulnerable for analysis. " f"Package: {target_name}, Distro: {target_distro or 'unknown'}, " diff --git a/tests/test_package_identifier.py b/tests/test_package_identifier.py index 993a7a0f2..67accd3cc 100644 --- a/tests/test_package_identifier.py +++ b/tests/test_package_identifier.py @@ -214,8 +214,13 @@ def test_cve_2016_8687_rhel7_will_not_fix(self): assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL assert "Will not fix" in result.conclusion_reason - def test_cve_2016_8687_rhel6_not_affected(self): - """CVE-2016-8687 on RHEL 6 should be NO (Not affected).""" + def test_cve_2016_8687_rhel6_not_affected_continues_investigation(self): + """CVE-2016-8687 on RHEL 6 'Not affected' should continue investigation (status=OK). + + When RHSA says 'not affected', we continue investigation to verify the assessment + and catch potential RHSA errors. The rhsa_fix_state is preserved for downstream + conflict detection. + """ target = TargetPackage( name="libarchive", version="3.0.3", @@ -258,13 +263,52 @@ def test_cve_2016_8687_rhel6_not_affected(self): status, result = identifier.identify(intel) assert result.is_target_package_affected == EnumIdentifyResult.NO - assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + # "not affected" continues investigation instead of early exit + assert status == PackageCheckerStatus.OK + # rhsa_fix_state is preserved for conflict detection + assert result.rhsa_fix_state == "not affected" # Verify conclusion_reason is populated with RHSA details assert "RHSA fix_state" in result.conclusion_reason assert "Not affected" in result.conclusion_reason assert "libarchive" in result.conclusion_reason assert "el6" in result.conclusion_reason + def test_out_of_support_scope_early_exits(self): + """RHSA 'out of support scope' should early-exit with PKG_IDENT_NOT_VUL. + + Unlike 'not affected', 'out of support scope' means Red Hat won't provide + further assessment, so we don't continue investigation. + """ + target = TargetPackage( + name="oldlib", + version="1.0.0", + release="1.el5", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0002", + nvd=None, + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 5", + fix_state="Out of support scope", + package_name="oldlib", + cpe="cpe:/o:redhat:enterprise_linux:5", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + # "out of support scope" still triggers early exit + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + assert result.rhsa_fix_state == "out of support scope" + def test_falls_back_to_nvd_when_no_fix_state(self): """When RHSA has no fix_state, should fall back to NVD version check.""" target = TargetPackage( From b952f2d29853dd39cef95bcede8b803042b2675c Mon Sep 17 00:00:00 2001 From: Gal Netanel Date: Thu, 11 Jun 2026 13:18:07 +0300 Subject: [PATCH 248/286] APPENG-5343: Fix LLM checklist tool_descriptions parsing (#244) The checklist generation LLM input contained the literal text '{tool_descriptions}' instead of templating it to the actual value. Co-authored-by: Gal Netanel --- src/vuln_analysis/utils/checklist_prompt_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index ff6d83b9d..dd400e5b2 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -31,7 +31,8 @@ # Format MOD_FEW_SHOT with examples, preserving {tool_descriptions} for Jinja2 rendering # Use double braces for tool_descriptions to escape it during format() _MOD_FEW_SHOT_ESCAPED = MOD_FEW_SHOT.replace('{tool_descriptions}', '{{tool_descriptions}}') -DEFAULT_CHECKLIST_PROMPT = _MOD_FEW_SHOT_ESCAPED.format(examples=get_mod_examples()) +INTERMEDIATE_CHECKLIST_PROMPT = _MOD_FEW_SHOT_ESCAPED.format(examples=get_mod_examples()) +DEFAULT_CHECKLIST_PROMPT = INTERMEDIATE_CHECKLIST_PROMPT.replace('{tool_descriptions}', '{{ tool_descriptions }}') cve_prompt2 = """Parse the following numbered checklist into a python list in the format ["x", "y", "z"], a comma separated list surrounded by square braces: {{template}}""" From 66473b2ede5c5fc4956121a9bc991f37bbce7cff Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Sun, 14 Jun 2026 12:04:38 +0300 Subject: [PATCH 249/286] Add cve_fetch_patches pipeline step to populate vulnerability patch details for all ecosystems (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add cve_fetch_patches pipeline step to populate vulnerability patch details for all ecosystems - cve_fetch_patches.py (new): Pipeline function that fetches vulnerability fix patches from intel refs + OSV API per CVE - state.py: Add patch_results field to AgentMorpheusEngineState for carrying per-CVE patch data - register.py: Wire fetch_patches_node into LLM engine subgraph (checklist → fetch_patches → agent_executor) - llm_engine_utils.py: Add _build_full_pipeline_details_md() to format patches as markdown, pass patch_result through postprocess_engine_output - web_patch_fetcher.py: Add shared fetch_patch_for_cve() helper (intel refs first, OSV fallback) - config-http-nim.yml, config-http-openai.yml, config-tracing.yml, config.yml: Add cve_fetch_patches function config - code_agent_graph_defs.py: Fix lstrip("ab/") → removeprefix("b/").removeprefix("a/") (6 occurrences) - cve_checker_report.py: Make infer_language_from_path public, add non-code extension mappings (.rst/.md/.yml/etc), change default fallback from "c" to "" - test_transitive_code_search.py: Test changes (staged from before) * Fix patch details: conditional display, DRY path cleanup, non-code filter, CVE lookup bug - Only show patch details in output when justification status is TRUE (vulnerable) - Add PatchFile.clean_target_path property to DRY removeprefix("b/").removeprefix("a/") (8 occurrences) - Filter non-code files (.rst/.md/.txt/.yml etc.) from Analysis Details UI - Fix wrong CVE patch lookup: pass vuln_id per-CVE instead of using cve_intel[0] - Remove early return that blocked OSV fallback when intel refs had no URLs - Remove dead _extract_functions_from_patch function - Move _ECO_MAP to module top under _PACKAGE_TOKEN_CHARS - Rewrite test_intel_utils.py to use WebPatchResult objects via patch_result parameter * Filter test files from Analysis Details UI - Add _TEST_FILE_RE filter to _build_full_pipeline_details_md to skip test files (e.g. tests/test_formparser.py) * - Add bsdtar to on-pull-request.yaml microdnf install (fixes C test RPM extraction failure on cluster) * Fix EPERM on stale node_modules and add Java CCA test cases - Remove stale node_modules owned by a different UID before Maven fallback build to prevent EPERM from frontend-maven-plugin (dep_tree.py) * Add vul code details improvements across all ecosystems - Remove Java standard library list from standard_libraries.json (unused) - Add is_tree_key_match to LanguageFunctionsParser with Go boundary-aware '/' override - Replace is_same_package with is_tree_key_match in CCA _resolve_tree_key and dep tree lookup - Detect uber-JARs by file count threshold (600) and disable _is_same_artifact bypass for them - Add _invoke_comprehension with LengthFinishReasonError fallback in BaseGraphAgent - Add section_header field to PatchHunk and extract function names from @@ hunk headers - Fix clean_target_path to use slicing instead of chained removeprefix - Add semaphore (5) to throttle concurrent patch fetches in cve_fetch_patches - Add Go common-root auto-resolve for package names in Function Locator - Make _TEST_FILE_RE public (TEST_FILE_RE) and remove unused GitHub commit regex constants - Use infer_language_from_path for non-code filtering in patch details markdown builder - Add _attempted set to WebPatchFetcher to skip re-fetching failed URLs - Simplify quick_standard_lib_check result normalization to str() * - Add LengthFinishReasonError fallback to cve_build_agent and cve_package_code_agent comprehension calls - Remove _attempted set from WebPatchFetcher to allow retrying transiently failed URLs from OSV fallback - Add set_uber_jar_dirs() setter on JavaLanguageFunctionsParser to replace direct private field mutation - Move NON_CODE_LANGUAGES constant to cve_checker_report.py as single source of truth for non-code filtering - Make cve_fetch_patches concurrency config-driven (max_concurrency field, per-invocation semaphore) * Deduplicate LengthFinishReasonError fallback, sync ext_map, simplify semaphore - Extract shared invoke_comprehension() helper into react_internals.py to replace triplicated try/except LengthFinishReasonError fallback in base_graph_agent, cve_build_agent, and cve_package_code_agent - Add .csv to ext_map in cve_checker_report.py so NON_CODE_LANGUAGES "csv" entry is reachable - Simplify semaphore ternary in cve_fetch_patches.py from explicit if/else to idiomatic `or` * Update semaphore test to check config default instead of source grep - Update test_semaphore_value_is_reasonable to assert CVEFetchPatchesConfig().max_concurrency == 5 instead of grepping for literal Semaphore(5) in source - Remove unused inspect and asyncio imports from test - Make uber_jar_file_threshold configurable via YAML config (default 600), threaded from CVEAgentExecutorToolConfig through state/transitive_code_search to JavaChainOfCallsRetriever - Delete hardcoded _UBER_JAR_FILE_THRESHOLD constant from java_chain_of_calls_retriever.py - Add uber_jar_file_threshold field to AgentMorpheusEngineState as single source of truth for the default - Add uber_jar_file_threshold to all 4 config YAML files - Make vuln_id a required keyword-only parameter in _process_steps - Escape markdown link/image syntax in patch details fields (patch_url, source, commit_message) to prevent injection via react-markdown rendering - Update test_concurrency.py to derive threshold from AgentMorpheusEngineState model fields - Update test_process_steps.py to pass required vuln_id keyword argument * Fix Go CCA sub-package false positive and is_tree_key_match direction - Fix sub-package disambiguation: move '/' in class_name check outside if/else so it runs for both sep and non-sep paths - Remove reverse direction from is_tree_key_match (doc-is-child-of-tree-key) to prevent false chains when sub-packages share a parent module (e.g. strvals.Parse matching ignore.Parse under helm.sh/helm/v3) - Update is_tree_key_match unit tests to assert reverse direction is rejected - Remove noisy __find_caller_function DEBUG log that fires per-parent during BFS --- .../standard_libs/standard_libraries.json | 106 ------ .../utils/chain_of_calls_retriever.py | 16 +- .../utils/chain_of_calls_retriever_factory.py | 6 +- src/exploit_iq_commons/utils/dep_tree.py | 7 + .../golang_functions_parsers.py | 17 + .../java_functions_parsers.py | 15 +- .../lang_functions_parsers.py | 6 + .../tests/test_go_is_tree_key_match.py | 245 ++++++++++++++ .../utils/java_chain_of_calls_retriever.py | 19 +- src/vuln_analysis/configs/config-http-nim.yml | 4 + .../configs/config-http-openai.yml | 4 + src/vuln_analysis/configs/config-tracing.yml | 4 + src/vuln_analysis/configs/config.yml | 4 + src/vuln_analysis/data_models/state.py | 4 +- .../functions/base_graph_agent.py | 9 +- .../functions/code_agent_graph_defs.py | 85 +++-- src/vuln_analysis/functions/cve_agent.py | 13 +- .../functions/cve_build_agent.py | 6 +- .../functions/cve_checker_report.py | 23 +- .../functions/cve_fetch_patches.py | 92 ++++++ .../functions/cve_package_code_agent.py | 6 +- .../functions/react_internals.py | 16 + .../functions/tests/test_cve_fetch_patches.py | 35 ++ src/vuln_analysis/register.py | 21 +- .../tools/tests/test_concurrency.py | 47 +-- .../tools/transitive_code_search.py | 18 +- .../utils/function_name_locator.py | 53 ++- src/vuln_analysis/utils/intel_utils.py | 223 ++++++------- src/vuln_analysis/utils/llm_engine_utils.py | 96 +++++- .../utils/osv_patch_retriever.py | 1 + .../tests/test_function_name_locator_go.py | 157 +++++++++ .../utils/tests/test_intel_utils_exports.py | 54 ++++ .../utils/tests/test_llm_engine_utils.py | 112 +++++++ .../utils/tests/test_web_patch_fetcher.py | 131 ++++++++ src/vuln_analysis/utils/web_patch_fetcher.py | 51 ++- tests/test_intel_utils.py | 301 +++++++++--------- tests/test_process_steps.py | 38 +-- 37 files changed, 1510 insertions(+), 535 deletions(-) create mode 100644 src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py create mode 100644 src/vuln_analysis/functions/cve_fetch_patches.py create mode 100644 src/vuln_analysis/functions/tests/test_cve_fetch_patches.py create mode 100644 src/vuln_analysis/utils/tests/test_function_name_locator_go.py create mode 100644 src/vuln_analysis/utils/tests/test_intel_utils_exports.py create mode 100644 src/vuln_analysis/utils/tests/test_llm_engine_utils.py diff --git a/src/exploit_iq_commons/data/standard_libs/standard_libraries.json b/src/exploit_iq_commons/data/standard_libs/standard_libraries.json index 0d0d660f6..0f5e4412f 100644 --- a/src/exploit_iq_commons/data/standard_libs/standard_libraries.json +++ b/src/exploit_iq_commons/data/standard_libs/standard_libraries.json @@ -235,112 +235,6 @@ "zipfile", "zlib" ], - "java": [ - "java.applet", - "java.awt", - "java.awt.color", - "java.awt.datatransfer", - "java.awt.event", - "java.awt.font", - "java.awt.geom", - "java.awt.im", - "java.awt.image", - "java.awt.print", - "java.beans", - "java.io", - "java.lang", - "java.lang.annotation", - "java.lang.instrument", - "java.lang.invoke", - "java.lang.management", - "java.lang.ref", - "java.lang.reflect", - "java.math", - "java.net", - "java.net.http", - "java.nio", - "java.nio.channels", - "java.nio.channels.spi", - "java.nio.charset", - "java.nio.charset.spi", - "java.nio.file", - "java.nio.file.attribute", - "java.nio.file.spi", - "java.rmi", - "java.rmi.activation", - "java.rmi.registry", - "java.rmi.server", - "java.security", - "java.security.acl", - "java.security.cert", - "java.security.interfaces", - "java.security.spec", - "java.sql", - "java.text", - "java.text.spi", - "java.time", - "java.time.chrono", - "java.time.format", - "java.time.temporal", - "java.time.zone", - "java.util", - "java.util.concurrent", - "java.util.concurrent.atomic", - "java.util.concurrent.locks", - "java.util.function", - "java.util.jar", - "java.util.logging", - "java.util.prefs", - "java.util.regex", - "java.util.spi", - "java.util.stream", - "java.util.zip", - "javax.accessibility", - "javax.crypto", - "javax.crypto.interfaces", - "javax.crypto.spec", - "javax.imageio", - "javax.imageio.event", - "javax.imageio.spi", - "javax.imageio.stream", - "javax.lang.model", - "javax.management", - "javax.naming", - "javax.naming.directory", - "javax.naming.event", - "javax.naming.ldap", - "javax.naming.spi", - "javax.net", - "javax.net.ssl", - "javax.print", - "javax.rmi", - "javax.script", - "javax.security.auth", - "javax.security.auth.callback", - "javax.security.auth.login", - "javax.security.auth.x500", - "javax.security.cert", - "javax.sound.midi", - "javax.sound.sampled", - "javax.sql", - "javax.sql.rowset", - "javax.swing", - "javax.swing.border", - "javax.swing.event", - "javax.swing.filechooser", - "javax.swing.plaf", - "javax.swing.table", - "javax.swing.text", - "javax.swing.tree", - "javax.swing.undo", - "javax.tools", - "javax.transaction.xa", - "javax.xml.parsers", - "javax.xml.transform", - "javax.xml.xpath", - "org.w3c.dom", - "org.xml.sax" - ], "javascript": [ "assert", "async_hooks", diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index f627b145d..0233564a4 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -170,12 +170,12 @@ def _resolve_tree_key(self, package: str, ctx: _SearchCtx) -> str | None: logger.debug("_resolve_tree_key: '%s' exact match", package) return package for tree_key in self.tree_dict: - if self.language_parser.is_same_package(package, tree_key): - logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_same_package (tree_dict)", package, tree_key) + if self.language_parser.is_tree_key_match(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_tree_key_match (tree_dict)", package, tree_key) return tree_key for tree_key in ctx.tree_additions: - if self.language_parser.is_same_package(package, tree_key): - logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_same_package (tree_additions)", package, tree_key) + if self.language_parser.is_tree_key_match(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_tree_key_match (tree_additions)", package, tree_key) return tree_key logger.debug("_resolve_tree_key: '%s' NOT found in tree_dict (%d keys) or tree_additions (%d keys)", package, len(self.tree_dict), len(ctx.tree_additions)) @@ -530,11 +530,11 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: if sep: class_name = before.rsplit('.', 1)[-1] function = after - if class_name and '/' in class_name: - package_name = f"{package_name}/{class_name}" - class_name = None else: class_name = None + if class_name and '/' in class_name: + package_name = f"{package_name}/{class_name}" + class_name = None found_package = False matching_documents = [] standard_libs_cache = StandardLibraryCache.get_instance() @@ -542,7 +542,7 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): # Check if input package is in dependency tree for package in self.tree_dict: - if self.language_parser.is_same_package(package_name, package): + if self.language_parser.is_tree_key_match(package_name, package): package_name = package found_package = True break diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py index 5e64f3016..c80933d38 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py @@ -25,7 +25,8 @@ from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever def get_chain_of_calls_retriever(ecosystem: Ecosystem, documents: List[Document], manifest_path: Path, query: str, - code_source_info: SourceDocumentsInfo) -> ChainOfCallsRetrieverBase: + code_source_info: SourceDocumentsInfo, + uber_jar_file_threshold: int) -> ChainOfCallsRetrieverBase: """ Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated with the ecosystem. @@ -43,6 +44,7 @@ def get_chain_of_calls_retriever(ecosystem: Ecosystem, documents: List[Document] ecosystem, manifest_path, query, - code_source_info) + code_source_info, + uber_jar_file_threshold=uber_jar_file_threshold) else: return ChainOfCallsRetriever(documents, ecosystem, manifest_path) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 00310509d..aa48755d7 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -915,6 +915,13 @@ def install_dependencies(self, manifest_path: Path): "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) if process_object.returncode > 0: + # Remove stale node_modules owned by a different UID to prevent + # EPERM when frontend-maven-plugin runs pnpm install during build. + for child in manifest_path.rglob("node_modules"): + if child.is_dir() and child.name == "node_modules": + shutil.rmtree(child, ignore_errors=True) + logger.debug("Removed stale %s", child) + process_object = subprocess.run([mvn_command, "clean", "install", "-DskipTests", "-s", settings_path], cwd=manifest_path) if process_object.returncode > 0: diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py index 54d1edcaf..c97437807 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -49,6 +49,23 @@ class GoLanguageFunctionsParser(LanguageFunctionsParser): def is_same_package(self, package_name_from_input, package_name_from_tree): return package_name_from_input.lower() in package_name_from_tree.lower() + def is_tree_key_match(self, package_from_doc: str, tree_key: str) -> bool: + """Boundary-aware matching at '/' separators for Go module paths. + + Only matches when the tree key is equal to, or a child of, the + doc package (tree_key starts with package_from_doc). The reverse + direction (doc package is a sub-path of tree key) is intentionally + excluded — it would map a specific sub-package query like + ``helm.sh/helm/v3/pkg/strvals`` to the entire module + ``helm.sh/helm/v3``, causing false reachability chains.""" + a = package_from_doc.lower() + b = tree_key.lower() + if a == b: + return True + if b.startswith(a + "/"): + return True + return False + def get_dummy_function(self, function_name): return f"{self.get_function_reserved_word()} {function_name}() {{}}" diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py index 1a442c634..0bd51f3be 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -365,14 +365,25 @@ def get_comment_line_notation(self) -> str: def dir_name_for_3rd_party_packages(self) -> str: return "dependencies-sources" + _uber_jar_dirs: frozenset[str] = frozenset() + + def set_uber_jar_dirs(self, dirs: frozenset[str]) -> None: + self._uber_jar_dirs = dirs + def _is_same_artifact(self, source_a: str, source_b: str) -> bool: - """Check if two source paths are from the same third-party JAR artifact.""" + """Check if two source paths are from the same third-party JAR artifact. + + Returns False for uber-JARs (detected by file count threshold) since + shaded libraries within an uber-JAR are unrelated artifacts. + """ prefix = self.dir_name_for_3rd_party_packages() + "/" if not (source_a and source_b and source_a.startswith(prefix) and source_b.startswith(prefix)): return False jar_a = source_a[len(prefix):].split("/", 1)[0] jar_b = source_b[len(prefix):].split("/", 1)[0] - return jar_a == jar_b + if jar_a != jar_b: + return False + return jar_a not in self._uber_jar_dirs def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: return True diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py index ab1da4064..7c7f71467 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -140,6 +140,12 @@ def is_script_language(): def is_same_package(self, package_name_from_input, package_name_from_tree): return package_name_from_input.lower() == package_name_from_tree.lower() + def is_tree_key_match(self, package_from_doc: str, tree_key: str) -> bool: + """Match a document's package name against a dependency tree key. + Used by _resolve_tree_key for tree traversal. Default: delegates to + is_same_package. Go overrides with boundary-aware '/' matching.""" + return self.is_same_package(package_from_doc, tree_key) + @staticmethod def get_constructor_method_name(): return None diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py b/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py new file mode 100644 index 000000000..fb32672e6 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py @@ -0,0 +1,245 @@ +import pytest +from unittest.mock import MagicMock, patch +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser + + +class TestGoIsTreeKeyMatch: + """Tests for GoLanguageFunctionsParser.is_tree_key_match — boundary-aware + '/' matching that prevents fan-out explosion in _resolve_tree_key.""" + + def setup_method(self): + self.parser = GoLanguageFunctionsParser() + + # --- Bug scenario --- + + def test_bug_scenario_no_false_cross_org_match(self): + """The original bug: 'github.com/hashicorp' substring-matched + 'github.com/hashicorp-terraform/foo' because 'hashicorp' is a + substring of 'hashicorp-terraform'. Boundary matching rejects this.""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp-terraform/foo" + ) is False + + def test_bug_scenario_hyphenated_suffix_no_match(self): + """'github.com/foo' must NOT match 'github.com/foo-bar'.""" + assert self.parser.is_tree_key_match( + "github.com/foo", "github.com/foo-bar" + ) is False + + def test_bug_scenario_partial_segment_no_match(self): + """'github.com/go' must NOT match 'github.com/golang/protobuf'.""" + assert self.parser.is_tree_key_match( + "github.com/go", "github.com/golang/protobuf" + ) is False + + # --- Exact match --- + + def test_exact_match(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + def test_exact_match_case_insensitive(self): + assert self.parser.is_tree_key_match( + "GitHub.Com/GoLang-JWT/JWT/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + # --- Input is prefix of tree key (doc package is parent of tree module) --- + + def test_input_prefix_of_tree_key(self): + """2-level path from get_package_names should match child module.""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp/vault" + ) is True + + def test_input_prefix_deeper_nesting(self): + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp/vault/api/sub" + ) is True + + def test_input_prefix_three_level(self): + assert self.parser.is_tree_key_match( + "github.com/hashicorp/vault", "github.com/hashicorp/vault/api" + ) is True + + # --- Tree key is prefix of input (rejected: would conflate sub-packages) --- + + def test_tree_key_prefix_of_input_rejected(self): + """Mapping a specific sub-package query to a broader tree entry + would cause false reachability chains (e.g. strvals.Parse matching + ignore.Parse because both live under helm.sh/helm/v3).""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp/vault/api", "github.com/hashicorp/vault" + ) is False + + def test_tree_key_prefix_of_input_deep_rejected(self): + assert self.parser.is_tree_key_match( + "google.golang.org/protobuf/encoding/protojson", + "google.golang.org/protobuf" + ) is False + + # --- No match cases --- + + def test_completely_different_packages(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "github.com/baz/qux" + ) is False + + def test_same_domain_different_org(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "github.com/baz/bar" + ) is False + + def test_domain_only_not_enough(self): + """Sharing only 'github.com' should not match.""" + assert self.parser.is_tree_key_match( + "github.com/orgA", "github.com/orgB" + ) is False + + def test_substring_within_segment_no_match(self): + """'net' is substring of 'netty' but not at a '/' boundary.""" + assert self.parser.is_tree_key_match( + "io.netty/net", "io.netty/netty-codec" + ) is False + + # --- Versioned modules --- + + def test_versioned_module_exact(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + def test_versioned_child(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", + "github.com/golang-jwt/jwt/v5/parser" + ) is True + + def test_different_versions_no_match(self): + """v4 and v5 are different path segments — no prefix relationship.""" + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v4" + ) is False + + # --- Edge cases --- + + def test_empty_strings(self): + assert self.parser.is_tree_key_match("", "") is True + + def test_empty_input(self): + assert self.parser.is_tree_key_match( + "", "github.com/foo/bar" + ) is False + + def test_empty_tree_key(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "" + ) is False + + def test_single_segment(self): + assert self.parser.is_tree_key_match("fmt", "fmt") is True + + def test_single_segment_prefix_no_match(self): + """'fmt' is not a prefix of 'fmtlib' at a '/' boundary.""" + assert self.parser.is_tree_key_match("fmt", "fmtlib") is False + + def test_trailing_slash_not_treated_as_boundary(self): + """Trailing slash is part of the path, not a boundary marker.""" + assert self.parser.is_tree_key_match( + "github.com/foo/", "github.com/foo/bar" + ) is False + + # --- is_same_package still does substring (unchanged) --- + + def test_is_same_package_still_substring(self): + """Verify is_same_package retains substring behavior for FL/line 545.""" + assert self.parser.is_same_package("jwt", "github.com/golang-jwt/jwt/v5") is True + assert self.parser.is_same_package( + "github.com/hashicorp", "github.com/hashicorp-terraform/foo" + ) is True + + +class TestBaseParserIsTreeKeyMatchDefault: + """Verify base class is_tree_key_match delegates to is_same_package.""" + + def test_base_class_delegates_via_java(self): + """Base class is abstract; use Java parser which inherits default + is_tree_key_match (delegates to exact-match is_same_package).""" + from exploit_iq_commons.utils.functions_parsers.java_functions_parsers import JavaLanguageFunctionsParser + parser = JavaLanguageFunctionsParser() + assert parser.is_tree_key_match("foo", "foo") is True + assert parser.is_tree_key_match("foo", "bar") is False + + def test_python_inherits_base_behavior(self): + """Python parser doesn't override is_tree_key_match — should use + is_same_package (PEP 503 normalization).""" + parser = PythonLanguageFunctionsParser() + assert parser.is_tree_key_match("my-pkg", "my_pkg") is True + assert parser.is_tree_key_match("flask", "Flask") is True + assert parser.is_tree_key_match("urllib3", "requests") is False + + +class TestResolveTreeKeyWithGoParser: + """Integration-style tests: verify _resolve_tree_key uses is_tree_key_match + for Go and prevents the fan-out explosion.""" + + def _make_retriever(self, tree_dict_keys): + """Build a minimal mock ChainOfCallsRetriever with Go parser.""" + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + retriever = object.__new__(ChainOfCallsRetriever) + retriever.language_parser = GoLanguageFunctionsParser() + retriever.tree_dict = {k: ["root"] for k in tree_dict_keys} + ctx = _SearchCtx() + return retriever, ctx + + def test_exact_match_preferred(self): + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + "github.com/hashicorp/vault/api", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp/vault", ctx) + assert result == "github.com/hashicorp/vault" + + def test_subpackage_does_not_resolve_to_parent(self): + """A sub-package query must NOT resolve to a parent tree entry — + this would conflate all sub-packages under the parent module.""" + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp/vault/api", ctx) + assert result is None + + def test_no_cross_org_fan_out(self): + """The bug scenario: short 2-level path must NOT match unrelated modules.""" + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + "github.com/hashicorp/consul", + "github.com/hashicorp-terraform/aws", + "github.com/hashicorp-labs/experiment", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp", ctx) + assert result in ("github.com/hashicorp/vault", "github.com/hashicorp/consul") + assert result != "github.com/hashicorp-terraform/aws" + assert result != "github.com/hashicorp-labs/experiment" + + def test_no_match_returns_none(self): + retriever, ctx = self._make_retriever([ + "github.com/foo/bar", + ]) + result = retriever._resolve_tree_key("github.com/baz/qux", ctx) + assert result is None + + def test_tree_additions_also_uses_boundary(self): + """Short prefix from get_package_names resolves to child module + in tree_additions, but sub-package does NOT resolve to parent.""" + retriever, ctx = self._make_retriever([]) + ctx.tree_additions["github.com/hashicorp/vault"] = ["root"] + ctx.tree_additions["github.com/hashicorp-terraform/aws"] = ["root"] + result = retriever._resolve_tree_key("github.com/hashicorp", ctx) + assert result == "github.com/hashicorp/vault" + result2 = retriever._resolve_tree_key("github.com/hashicorp/vault/api", ctx) + assert result2 is None \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py index e624fa99b..1b2a8e6d5 100644 --- a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -277,7 +277,8 @@ def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_path: Path, query: str, - code_source_info: SourceDocumentsInfo): + code_source_info: SourceDocumentsInfo, + uber_jar_file_threshold: int): """ :param documents: List of documents containing the functions/methods and classes/types of the application code + @@ -352,6 +353,20 @@ def __init__(self, documents: List[Document], logger.debug("Chain of Calls Retriever - doc index built (root_docs=%d, jar_groups=%d)", len(self._root_docs), len(self._jar_to_docs)) + # Detect uber-JARs: JARs with more unique source files than the + # threshold contain shaded libraries from unrelated artifacts. + # Disabling _is_same_artifact for these prevents catastrophic + # performance (O(all_files) type resolution instead of O(same_jar)). + uber_jars = set() + for jar_name, docs in self._jar_to_docs.items(): + unique_sources = len({doc.metadata['source'] for doc in docs}) + if unique_sources > uber_jar_file_threshold: + uber_jars.add(jar_name) + logger.info("Detected uber-JAR '%s' with %d unique source files (threshold=%d), " + "disabling same-artifact bypass", jar_name, unique_sources, uber_jar_file_threshold) + if uber_jars: + self.language_parser.set_uber_jar_dirs(frozenset(uber_jars)) + # Satisfy base class documents attribute via property (avoids duplicate list) self._documents_materialized = None logger.debug("Chain of Calls Retriever - init complete") @@ -518,8 +533,6 @@ def _candidate_docs(): method_exclusions, ctx.root_docs, ctx.jar_to_docs) - logger.debug("__find_caller_function: parent='%s' possible_docs=%d", package, len(possible_docs)) - jar_name = convert_from_maven_artifact(package) yield from self.get_functions_for_package(package_name=jar_name, documents=possible_docs, diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 1d3ae4c23..f78153ac0 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -107,6 +107,7 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false cve_web_search_enabled: true + uber_jar_file_threshold: 600 verbose: false cve_generate_cvss: _type: cve_generate_cvss @@ -151,6 +152,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches llms: checklist_llm: @@ -240,6 +243,7 @@ workflow: cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_http_output + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index 1ee69cfb4..9a42f7c7d 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -119,6 +119,7 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false cve_web_search_enabled: true + uber_jar_file_threshold: 600 verbose: false cve_generate_cvss: _type: cve_generate_cvss @@ -193,6 +194,8 @@ functions: tool_names: - Source Grep - Code Keyword Search + cve_fetch_patches: + _type: cve_fetch_patches health_check: _type: health_check @@ -289,6 +292,7 @@ workflow: cve_package_code_agent_name: cve_package_code_agent cve_checker_report_name: cve_checker_report cve_build_agent_name: cve_build_agent + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index a4d89ff9d..176f130dc 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -117,6 +117,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false cve_web_search_enabled: true + uber_jar_file_threshold: 600 verbose: false cve_generate_cvss: _type: cve_generate_cvss @@ -154,6 +155,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches health_check: _type: health_check @@ -237,6 +240,7 @@ workflow: cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 9ba5f23be..b1cbdd45c 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -87,6 +87,7 @@ functions: replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false cve_web_search_enabled: true + uber_jar_file_threshold: 600 verbose: false cve_generate_cvss: _type: cve_generate_cvss @@ -125,6 +126,8 @@ functions: generate_intel_score: true intel_low_score: 51 insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches llms: checklist_llm: @@ -206,6 +209,7 @@ workflow: cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index 38108bce3..9d147cbb6 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -36,4 +36,6 @@ class AgentMorpheusEngineState(BaseModel): cvss_results: dict[str, dict[str, str]] = {} vex: dict[str, typing.Any] | None = None current_vuln_id: str | None = None - + patch_results: dict[str, typing.Any] = {} + uber_jar_file_threshold: int = 600 + diff --git a/src/vuln_analysis/functions/base_graph_agent.py b/src/vuln_analysis/functions/base_graph_agent.py index 1a3532938..9d142a319 100644 --- a/src/vuln_analysis/functions/base_graph_agent.py +++ b/src/vuln_analysis/functions/base_graph_agent.py @@ -32,6 +32,7 @@ CodeFindings, PackageSelection, _build_tool_arguments, + invoke_comprehension, FORCED_FINISH_PROMPT, COMPREHENSION_PROMPT, MEMORY_UPDATE_PROMPT, @@ -407,6 +408,12 @@ async def forced_finish_node(self, state: AgentState) -> AgentState: span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) raise + async def _invoke_comprehension(self, prompt: str, tool_used: str, tool_input: str, tool_output: str) -> CodeFindings: + """Invoke comprehension LLM with fallback on token limit overflow.""" + return await invoke_comprehension( + self.comprehension_llm, prompt, tool_used, tool_input, tool_output, agent_label=self.agent_type, + ) + async def observation_node(self, state: AgentState) -> AgentState: tool_message = state["messages"][-1] last_thought_text = state["thought"].thought if state.get("thought") else "No previous thought." @@ -465,7 +472,7 @@ async def observation_node(self, state: AgentState) -> AgentState: "Comprehension prompt truncated: %d -> %d tokens (limit %d)", prompt_tokens, count_tokens(comp_prompt), max_input_tokens, ) - code_findings: CodeFindings = await self.comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + code_findings = await self._invoke_comprehension(comp_prompt, tool_used, tool_input_detail, truncated_output) sanitized = self.sanitize_findings(code_findings.findings, state) findings_text = "\n".join(f"- {f}" for f in sanitized) diff --git a/src/vuln_analysis/functions/code_agent_graph_defs.py b/src/vuln_analysis/functions/code_agent_graph_defs.py index 28225f021..f655ac375 100644 --- a/src/vuln_analysis/functions/code_agent_graph_defs.py +++ b/src/vuln_analysis/functions/code_agent_graph_defs.py @@ -99,6 +99,7 @@ class PatchHunk(BaseModel): source_length: int target_start: int target_length: int + section_header: str = Field(default="", description="Function context from @@ hunk header") context_lines: list[str] = Field(default_factory=list, description="Unchanged lines") removed_lines: list[str] = Field(default_factory=list, description="Deleted lines (- stripped)") added_lines: list[str] = Field(default_factory=list, description="Added lines (+ stripped)") @@ -112,6 +113,15 @@ class PatchFile(BaseModel): is_new_file: bool = False is_deleted_file: bool = False + @property + def clean_target_path(self) -> str: + """Target path with git diff prefix (b/ or a/) stripped.""" + if self.target_path.startswith("b/"): + return self.target_path[2:] + if self.target_path.startswith("a/"): + return self.target_path[2:] + return self.target_path + class ParsedPatch(BaseModel): """Structured representation of a downstream patch file.""" @@ -997,7 +1007,7 @@ def _extract_downstream_patch_code_snippets( for hunk in pf.hunks: if hunk.removed_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.source_start, code="\n".join(hunk.removed_lines[:10]), snippet_type="vulnerable", @@ -1005,7 +1015,7 @@ def _extract_downstream_patch_code_snippets( )) elif hunk.context_lines and hunk.added_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.source_start, code="\n".join(hunk.context_lines[:10]), snippet_type="vulnerable", @@ -1013,7 +1023,7 @@ def _extract_downstream_patch_code_snippets( )) if hunk.added_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.target_start, code="\n".join(hunk.added_lines[:10]), snippet_type="fix", @@ -1038,7 +1048,7 @@ def _extract_code_snippets( for hunk in pf.hunks: if hunk.removed_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.source_start, code="\n".join(hunk.removed_lines[:10]), snippet_type="vulnerable", @@ -1046,7 +1056,7 @@ def _extract_code_snippets( )) elif hunk.context_lines and hunk.added_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.source_start, code="\n".join(hunk.context_lines[:10]), snippet_type="vulnerable", @@ -1054,7 +1064,7 @@ def _extract_code_snippets( )) if hunk.added_lines: snippets.append(CodeSnippet( - file_path=pf.target_path.lstrip("ab/"), + file_path=pf.clean_target_path, line_number=hunk.target_start, code="\n".join(hunk.added_lines[:10]), snippet_type="fix", @@ -1449,6 +1459,7 @@ def parse_patch_file(patch_path: Path) -> ParsedPatch | None: source_length=hunk.source_length, target_start=hunk.target_start, target_length=hunk.target_length, + section_header=hunk.section_header or "", context_lines=context, removed_lines=removed, added_lines=added, @@ -1647,56 +1658,34 @@ async def upstream_search_preprocess( span.set_output({ "is_fixed_srpm_is_needed": report.is_fixed_srpm_is_needed}) - # Try intel references (unified: commit URLs from GHSA/NVD/RHSA/Ubuntu) - if (not patch_dir.exists() or need_to_find_code) and not report.fixed_parsed_patch: - if commit_url_candidates: - from vuln_analysis.utils.web_patch_fetcher import WebPatchFetcher - with tracer.push_active_function( - "fetch_patch_from_intel_refs", - input_data={ - "vuln_id": vuln_id, - "candidates_count": {src: len(urls) for src, urls in commit_url_candidates.items()}, - } - ) as span: - async with aiohttp.ClientSession() as session: - fetcher = WebPatchFetcher(session=session) - result = await fetcher.fetch_from_intel_refs( - commit_url_candidates, - vuln_id, - cve_description=cve_description, - llm=llm, - ) - if result and result.parsed_patch: - report.fixed_parsed_patch = result.parsed_patch - report.fixed_srpm_file_name = result.patch_url - report.is_fixed_srpm_is_needed = True - report.osv_result = result - span.set_output({ - "source_found": result.source, - "url_type": result.url_type, - "platform": result.platform, - "patch_url": result.patch_url, - "commit_message": result.commit_message, - "patch_found": True, - }) - else: - span.set_output({"patch_found": False}) - - # OSV API fallback + # Try intel references + OSV fallback if (not patch_dir.exists() or need_to_find_code) and not report.fixed_parsed_patch: - from vuln_analysis.utils.web_patch_fetcher import WebPatchFetcher, OSVClient - with tracer.push_active_function("fetch_patch_from_osv", input_data={"vuln_id": vuln_id}) as span: + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + with tracer.push_active_function( + "fetch_patch_for_cve", + input_data={ + "vuln_id": vuln_id, + "candidates_count": {src: len(urls) for src, urls in commit_url_candidates.items()} if commit_url_candidates else {}, + } + ) as span: async with aiohttp.ClientSession() as session: - fetcher = WebPatchFetcher(session=session) - client = OSVClient(session=session, patch_fetcher=fetcher) - result = await client.get_fix_patch(vuln_id, target_package.version, target_package.name) + result = await fetch_patch_for_cve( + session=session, + candidates=commit_url_candidates or {}, + vuln_id=vuln_id, + cve_description=cve_description, + llm=llm, + upstream_version=target_package.version, + package_name=target_package.name, + ) if result and result.parsed_patch: report.fixed_parsed_patch = result.parsed_patch report.fixed_srpm_file_name = result.patch_url report.is_fixed_srpm_is_needed = True report.osv_result = result span.set_output({ - "source_found": "osv", + "source_found": result.source, + "url_type": result.url_type, "platform": result.platform, "patch_url": result.patch_url, "commit_message": result.commit_message, diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 512478c4b..ac2a1d00b 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -74,9 +74,14 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): default=5000, description="Estimated token threshold for pruning old messages in observation node." ) + uber_jar_file_threshold: int = Field( + default=600, + description="Source file count above which a JAR is treated as an uber-JAR, " + "disabling same-artifact optimizations to prevent O(N^2) type resolution." + ) -async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterations: int = 10): +async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterations: int = 10, *, vuln_id: str): workflow_state = ctx_state.get() critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) @@ -91,8 +96,10 @@ async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterat if not vulnerable_functions: vf_set: set[str] = set() + pre_fetched_patch = workflow_state.patch_results.get(vuln_id) await enrich_vulnerable_functions_from_patch( workflow_state.cve_intel, critical_context, vf_set, ecosystem, + patch_result=pre_fetched_patch, ) if vf_set: vulnerable_functions = sorted(vf_set) @@ -218,6 +225,7 @@ async def cve_agent(config: CVEAgentExecutorToolConfig, builder: Builder): async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: trace_id.set(state.original_input.input.scan.id) + state.uber_jar_file_threshold = config.uber_jar_file_threshold ctx_state.set(state) checklist_plans = state.checklist_plans @@ -242,8 +250,9 @@ async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: steps, semaphore, config.max_iterations, + vuln_id=vid, ) - for steps in checklist_plans.values() + for vid, steps in checklist_plans.items() ), return_exceptions=True, ) diff --git a/src/vuln_analysis/functions/cve_build_agent.py b/src/vuln_analysis/functions/cve_build_agent.py index a0c4a65d2..bb4e8b458 100644 --- a/src/vuln_analysis/functions/cve_build_agent.py +++ b/src/vuln_analysis/functions/cve_build_agent.py @@ -41,7 +41,7 @@ from nat.builder.context import Context from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output +from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output, invoke_comprehension from vuln_analysis.functions.build_agent_graph_defs import ( BuildAgentState, @@ -427,7 +427,9 @@ async def observation_node(state: BuildAgentState) -> dict: last_thought=last_thought_text, tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), ) - code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + code_findings: CodeFindings = await invoke_comprehension( + comprehension_llm, comp_prompt, tool_used, tool_input_detail, tool_output_for_llm, agent_label="L2", + ) findings_text = "\n".join(f"- {f}" for f in code_findings.findings) # Step 2: Memory update - merge findings into cumulative memory diff --git a/src/vuln_analysis/functions/cve_checker_report.py b/src/vuln_analysis/functions/cve_checker_report.py index ef62e580a..805e8d5c2 100644 --- a/src/vuln_analysis/functions/cve_checker_report.py +++ b/src/vuln_analysis/functions/cve_checker_report.py @@ -265,8 +265,7 @@ def _summarize_parsed_patch_lines(parsed_patch: ParsedPatch | None) -> list[str] if parsed_patch.patch_filename: lines.append(f"- **Patch file:** `{parsed_patch.patch_filename}`") for pf in parsed_patch.files[:2]: - path = pf.target_path.lstrip("ab/") - lines.append(f"- **File in patch:** `{path}`") + lines.append(f"- **File in patch:** `{pf.clean_target_path}`") for hunk in pf.hunks[:1]: if hunk.removed_lines: lines.append(f" - Removed: `{hunk.removed_lines[0].strip()}`") @@ -581,7 +580,7 @@ def _format_snippet_bullets(self, snippets: list[CodeSnippet]) -> list[str]: "> **Note:** Excerpt path is from a reference/patched tree, not the target SRPM source." ) lines.append("") - lang = _infer_language_from_path(snippet.file_path) + lang = infer_language_from_path(snippet.file_path) code_body = "\n".join(snippet.code.strip().splitlines()[:12]) if code_body: lines.append(f"```{lang}") @@ -689,7 +688,10 @@ def _strip_build_agent_absence_boilerplate(text: str) -> str: return " ".join(kept).strip() -def _infer_language_from_path(file_path: str) -> str: +NON_CODE_LANGUAGES = frozenset({"markdown", "yaml", "json", "toml", "xml", "rst", "text", "ini", "csv"}) + + +def infer_language_from_path(file_path: str) -> str: """Infer programming language hint from file extension for syntax highlighting.""" ext_map = { ".c": "c", @@ -706,8 +708,19 @@ def _infer_language_from_path(file_path: str) -> str: ".ts": "typescript", ".rb": "ruby", ".sh": "bash", + ".md": "markdown", + ".rst": "rst", + ".txt": "text", + ".yml": "yaml", + ".yaml": "yaml", + ".json": "json", + ".xml": "xml", + ".toml": "toml", + ".cfg": "ini", + ".ini": "ini", + ".csv": "csv", } - return ext_map.get(Path(file_path).suffix.lower(), "c") + return ext_map.get(Path(file_path).suffix.lower(), "") def _build_details_md(blocks: ReportBlocks) -> str | None: diff --git a/src/vuln_analysis/functions/cve_fetch_patches.py b/src/vuln_analysis/functions/cve_fetch_patches.py new file mode 100644 index 000000000..a78b5acbe --- /dev/null +++ b/src/vuln_analysis/functions/cve_fetch_patches.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import contextlib + +import aiohttp +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVEFetchPatchesConfig(FunctionBaseConfig, name="cve_fetch_patches"): + """Fetches vulnerability fix patches from intel references and OSV for the details UI.""" + llm_name: str | None = Field(default=None, description="Optional LLM for Chromium CL disambiguation") + max_concurrency: int | None = Field(ge=1, default=5, description="Maximum number of concurrent patch fetch operations") + + +@register_function(config_type=CVEFetchPatchesConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_fetch_patches(config: CVEFetchPatchesConfig, builder: Builder): + + from vuln_analysis.data_models.state import AgentMorpheusEngineState + from vuln_analysis.utils.intel_utils import extract_commit_url_candidates + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + + llm = ( + await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + if config.llm_name else None + ) + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) + + intel_map = {intel.vuln_id: intel for intel in state.cve_intel} + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None + + async with aiohttp.ClientSession() as session: + + async def _throttled_fetch(vuln_id, intel): + async with semaphore or contextlib.nullcontext(): + candidates = extract_commit_url_candidates(intel) + cve_description = None + if intel.nvd and intel.nvd.cve_description: + cve_description = intel.nvd.cve_description + return await fetch_patch_for_cve( + session=session, + candidates=candidates, + vuln_id=vuln_id, + cve_description=cve_description, + llm=llm, + ) + + tasks = { + vuln_id: _throttled_fetch(vuln_id, intel) + for vuln_id, intel in intel_map.items() + } + + results = await asyncio.gather(*tasks.values(), return_exceptions=True) + + for vuln_id, result in zip(tasks.keys(), results): + if isinstance(result, Exception): + logger.warning("Patch fetch failed for %s: %s", vuln_id, result) + state.patch_results[vuln_id] = None + else: + state.patch_results[vuln_id] = result + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineState, + description="Fetches vulnerability fix patches from intel references and OSV API.", + ) diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py index 2d5f9b8b0..a9721848d 100644 --- a/src/vuln_analysis/functions/cve_package_code_agent.py +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -63,7 +63,7 @@ from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile from vuln_analysis.utils.package_identifier import _extract_dist_tag -from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output +from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output, invoke_comprehension from vuln_analysis.utils.intel_utils import extract_commit_url_candidates from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer from vuln_analysis.utils.token_utils import truncate_tool_output @@ -760,7 +760,9 @@ async def observation_node(state: CodeAgentState) -> dict: last_thought=last_thought_text, tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), ) - code_findings: CodeFindings = await comprehension_llm.ainvoke([SystemMessage(content=comp_prompt)]) + code_findings: CodeFindings = await invoke_comprehension( + comprehension_llm, comp_prompt, tool_used, tool_input_detail, tool_output_for_llm, agent_label="L1", + ) findings_text = "\n".join(f"- {f}" for f in code_findings.findings) # Step 2: Memory update - merge findings into cumulative memory diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 371c069cf..15386f61e 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from langchain_core.messages import SystemMessage +from openai import LengthFinishReasonError from pydantic import BaseModel, Field from typing import Any from typing import Literal @@ -142,6 +144,20 @@ def check_empty_output( return None +async def invoke_comprehension(llm, prompt: str, tool_used: str, tool_input: str, tool_output: str, + agent_label: str = "") -> CodeFindings: + """Invoke comprehension LLM with fallback on token limit overflow.""" + try: + return await llm.ainvoke([SystemMessage(content=prompt)]) + except LengthFinishReasonError: + logger.warning("%s comprehension LLM hit token limit (tool=%s), using fallback", agent_label, tool_used) + summary = tool_output[:500] if isinstance(tool_output, str) else str(tool_output)[:500] + return CodeFindings( + findings=[f"Tool output received but comprehension exceeded token limit. Raw excerpt: {summary}"], + tool_outcome=f"CALLED: {tool_used} with {tool_input} -> output received (comprehension truncated)", + ) + + class Observation(BaseModel): results: list[str] = Field( description="3-5 key technical facts from this tool output. Each fact must describe what the code DOES and how it relates to the investigation goal, not just that it was found." diff --git a/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py b/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py new file mode 100644 index 000000000..5f9119a2e --- /dev/null +++ b/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for cve_fetch_patches module.""" + +import asyncio +import inspect + +from vuln_analysis.functions import cve_fetch_patches as mod + + +class TestCveFetchPatchesSemaphore: + """Tests for semaphore-bounded concurrency (fix 4).""" + + def test_semaphore_exists_in_source(self): + """The module should use asyncio.Semaphore to bound concurrent fetches.""" + source = inspect.getsource(mod) + assert "Semaphore" in source + + def test_semaphore_value_is_reasonable(self): + """The default max_concurrency should be a small positive integer (not unbounded).""" + config = mod.CVEFetchPatchesConfig() + assert config.max_concurrency == 5 \ No newline at end of file diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 03ca69448..d76430e77 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -46,6 +46,7 @@ from vuln_analysis.functions import cve_summarize from vuln_analysis.functions import cve_checker_report from vuln_analysis.functions import cve_build_agent +from vuln_analysis.functions import cve_fetch_patches from vuln_analysis.functions import cve_generate_cvss from vuln_analysis.functions import cve_generate_vex from vuln_analysis.functions import health_endpoint @@ -106,6 +107,10 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): default=None, description="Function name for the Level 2 Build Agent (build compilation and hardening check)", ) + cve_fetch_patches_name: str | None = Field( + default=None, + description="Function name for fetching vulnerability fix patches from intel references and OSV", + ) description: str = Field(default="Vulnerability analysis for container security workflow", description="Workflow function description") @@ -150,6 +155,10 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): builder.get_function(name=config.cve_build_agent_name) if config.cve_build_agent_name else None ) + cve_fetch_patches_fn = ( + builder.get_function(name=config.cve_fetch_patches_name) + if config.cve_fetch_patches_name else None + ) # Define langgraph node functions @catch_pipeline_errors_async @@ -223,6 +232,11 @@ async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEn return await cve_generate_cvss_fn.ainvoke(state.model_dump()) + async def fetch_patches_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Fetches vulnerability fix patches from intel references and OSV.""" + + return await cve_fetch_patches_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async async def add_completed_time_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Adds the completed time to the output""" @@ -407,7 +421,12 @@ def route_after_add_start_time(state: AgentMorpheusInput): subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_edge(START, "checklist") - subgraph_builder.add_edge("checklist", "agent_executor") + if cve_fetch_patches_fn: + subgraph_builder.add_node("fetch_patches", fetch_patches_node) + subgraph_builder.add_edge("checklist", "fetch_patches") + subgraph_builder.add_edge("fetch_patches", "agent_executor") + else: + subgraph_builder.add_edge("checklist", "agent_executor") subgraph_builder.add_edge("agent_executor", "summarize") subgraph_builder.add_edge("summarize", "justify") subgraph_builder.add_edge("justify", "generate_vex") diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py index 0bfc98694..f2009ef44 100644 --- a/src/vuln_analysis/tools/tests/test_concurrency.py +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -16,6 +16,7 @@ from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher +from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.transitive_code_search import ( _build_or_get_cached, _searcher_cache, @@ -23,6 +24,8 @@ _repo_build_locks, ) +_DEFAULT_THRESHOLD = AgentMorpheusEngineState.model_fields["uber_jar_file_threshold"].default + # --------------------------------------------------------------------------- # Helpers @@ -136,7 +139,7 @@ def _make_slow_builder(build_log, sleep_secs=0.2, java=True): """ lock = threading.Lock() - def slow_build(si, query): + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD): tag = query.split(",")[0] start = time.monotonic() time.sleep(sleep_secs) @@ -231,8 +234,8 @@ async def test_java_same_repo_different_packages_are_serialized(): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=_make_slow_builder(build_log, java=True)): - task1 = asyncio.create_task(_build_or_get_cached(si, "pkg-a:art-a:1.0,ClassA.foo")) - task2 = asyncio.create_task(_build_or_get_cached(si, "pkg-b:art-b:2.0,ClassB.bar")) + task1 = asyncio.create_task(_build_or_get_cached(si, "pkg-a:art-a:1.0,ClassA.foo", _DEFAULT_THRESHOLD)) + task2 = asyncio.create_task(_build_or_get_cached(si, "pkg-b:art-b:2.0,ClassB.bar", _DEFAULT_THRESHOLD)) await asyncio.gather(task1, task2) assert len(build_log) == 2, f"Expected 2 Java builds (different packages), got {len(build_log)}" @@ -253,7 +256,7 @@ async def test_different_repos_can_build_concurrently(): si_b = _make_si("https://github.com/example/repo-b") build_log = [] - def slow_build(si, query): + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD): tag = si[0].git_repo.split("/")[-1] start = time.monotonic() time.sleep(0.2) @@ -263,8 +266,8 @@ def slow_build(si, query): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=slow_build): - task1 = asyncio.create_task(_build_or_get_cached(si_a, "pkg-a:art-a:1.0,Foo.bar")) - task2 = asyncio.create_task(_build_or_get_cached(si_b, "pkg-b:art-b:2.0,Baz.qux")) + task1 = asyncio.create_task(_build_or_get_cached(si_a, "pkg-a:art-a:1.0,Foo.bar", _DEFAULT_THRESHOLD)) + task2 = asyncio.create_task(_build_or_get_cached(si_b, "pkg-b:art-b:2.0,Baz.qux", _DEFAULT_THRESHOLD)) await asyncio.gather(task1, task2) assert len(build_log) == 2 @@ -285,7 +288,7 @@ async def test_same_key_deduplicates_build(): build_count = 0 count_lock = threading.Lock() - def counting_build(build_si, q): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): nonlocal build_count with count_lock: build_count += 1 @@ -294,8 +297,8 @@ def counting_build(build_si, q): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - task1 = asyncio.create_task(_build_or_get_cached(si, query)) - task2 = asyncio.create_task(_build_or_get_cached(si, query)) + task1 = asyncio.create_task(_build_or_get_cached(si, query, _DEFAULT_THRESHOLD)) + task2 = asyncio.create_task(_build_or_get_cached(si, query, _DEFAULT_THRESHOLD)) results = await asyncio.gather(task1, task2) assert build_count == 1, f"Expected 1 build (deduplicated), got {build_count}" @@ -316,14 +319,14 @@ async def test_cache_hit_skips_build(): build_count = 0 - def counting_build(build_si, q): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): nonlocal build_count build_count += 1 return _make_nonjava_searcher() with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - result = await _build_or_get_cached(si, query) + result = await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) assert build_count == 0, "Build should not run when cache hit exists" assert result is pre_cached, "Should return the pre-cached searcher" @@ -342,14 +345,14 @@ async def test_java_cache_hit_skips_build(): build_count = 0 - def counting_build(build_si, q): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): nonlocal build_count build_count += 1 return _make_java_searcher() with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - result = await _build_or_get_cached(si, query) + result = await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) assert build_count == 0, "Build should not run when Java cache hit exists" assert result is pre_cached, "Should return the pre-cached Java searcher" @@ -364,13 +367,13 @@ async def test_build_failure_cleans_up_building_marker(): full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") repo_key = ("https://github.com/example/repo", "main") - def failing_build(build_si, q): + def failing_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): raise RuntimeError("Maven failed") with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=failing_build): with pytest.raises(RuntimeError, match="Maven failed"): - await _build_or_get_cached(si, query) + await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) assert full_key not in _searcher_building, "Building marker not cleaned up after failure" assert full_key not in _searcher_cache, "Failed build should not be cached" @@ -391,7 +394,7 @@ async def test_java_repo_lock_recheck_avoids_redundant_build(): build_count = 0 count_lock = threading.Lock() - def build_that_precaches_b(build_si, q): + def build_that_precaches_b(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): nonlocal build_count with count_lock: build_count += 1 @@ -403,9 +406,9 @@ def build_that_precaches_b(build_si, q): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=build_that_precaches_b): - task1 = asyncio.create_task(_build_or_get_cached(si, query_a)) + task1 = asyncio.create_task(_build_or_get_cached(si, query_a, _DEFAULT_THRESHOLD)) await asyncio.sleep(0.01) - task2 = asyncio.create_task(_build_or_get_cached(si, query_b)) + task2 = asyncio.create_task(_build_or_get_cached(si, query_b, _DEFAULT_THRESHOLD)) await asyncio.gather(task1, task2) assert build_count == 1, ( @@ -426,7 +429,7 @@ async def test_nonjava_same_repo_different_packages_share_cache(): build_count = 0 count_lock = threading.Lock() - def counting_build(build_si, q): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): nonlocal build_count with count_lock: build_count += 1 @@ -435,8 +438,8 @@ def counting_build(build_si, q): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - task1 = asyncio.create_task(_build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey")) - task2 = asyncio.create_task(_build_or_get_cached(si, "net/http,ListenAndServe")) + task1 = asyncio.create_task(_build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey", _DEFAULT_THRESHOLD)) + task2 = asyncio.create_task(_build_or_get_cached(si, "net/http,ListenAndServe", _DEFAULT_THRESHOLD)) results = await asyncio.gather(task1, task2) # Non-Java: both should resolve to the same (repo, ref) cache entry. @@ -458,7 +461,7 @@ async def test_nonjava_caches_under_repo_key(): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", return_value=_make_nonjava_searcher()): - await _build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey") + await _build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey", _DEFAULT_THRESHOLD) assert repo_key in _searcher_cache, "Non-Java should cache under repo_key" assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" \ No newline at end of file diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index de5216114..ec5e7c281 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -146,7 +146,7 @@ class FunctionLibraryVersionFinderToolConfig(FunctionBaseConfig, name=FUNCTION_L """ -def get_call_of_chains_retriever(documents_embedder, si, query: str): +def get_call_of_chains_retriever(documents_embedder, si, query: str, uber_jar_file_threshold: int): documents: list[Document] git_repo = None code_source_info: SourceDocumentsInfo @@ -164,7 +164,8 @@ def get_call_of_chains_retriever(documents_embedder, si, query: str): documents=documents, manifest_path=git_repo, query=query, - code_source_info=code_source_info) + code_source_info=code_source_info, + uber_jar_file_threshold=uber_jar_file_threshold) # Release the raw documents list — the retriever has classified and indexed # them into its own structures. Dropping this reference allows the GC to # reclaim any Document objects not retained by the retriever's indexes. @@ -208,7 +209,7 @@ def _get_cache_keys(si, query: str) -> tuple[tuple | None, tuple | None]: return None, None -def _build_searcher(si, query: str) -> TransitiveCodeSearcher: +def _build_searcher(si, query: str, uber_jar_file_threshold: int) -> TransitiveCodeSearcher: """Synchronous helper that builds a TransitiveCodeSearcher. Separated so it can be offloaded to a thread via asyncio.to_thread(), @@ -218,11 +219,11 @@ def _build_searcher(si, query: str) -> TransitiveCodeSearcher: by (git_repo, ref), backed by pickle sub-caches on disk). """ documents_embedder = DocumentEmbedding(embedding=None) - coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query, uber_jar_file_threshold) return TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) -async def _build_or_get_cached(si, query: str) -> TransitiveCodeSearcher: +async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int) -> TransitiveCodeSearcher: """Build a TransitiveCodeSearcher, or return a cached one. Cache keys: @@ -295,7 +296,7 @@ async def _build_or_get_cached(si, query: str) -> TransitiveCodeSearcher: # other tasks can read/write the cache while this build runs, # but no concurrent build on the same repo's filesystem. logger.info("Building TransitiveCodeSearcher for %s", full_key) - searcher = await asyncio.to_thread(_build_searcher, si, query) + searcher = await asyncio.to_thread(_build_searcher, si, query, uber_jar_file_threshold) async with _searcher_cache_lock: # Cache under the appropriate key based on ecosystem @@ -330,9 +331,10 @@ async def _build_or_get_cached(si, query: str) -> TransitiveCodeSearcher: async def get_transitive_code_searcher(query: str): state: AgentMorpheusEngineState = ctx_state.get() si = state.original_input.input.image.source_info + threshold = state.uber_jar_file_threshold if state.transitive_code_searcher is None: - state.transitive_code_searcher = await _build_or_get_cached(si, query) + state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold) elif isinstance(state.transitive_code_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): # Java: different queries produce different dep trees (build_tree uses # -DtargetIncludes for GAV queries), so rebuild when the package changes. @@ -344,7 +346,7 @@ async def get_transitive_code_searcher(query: str): if cached is not None and cached is state.transitive_code_searcher: pass # Same searcher, no change needed else: - state.transitive_code_searcher = await _build_or_get_cached(si, query) + state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold) # Both Java and non-Java retrievers use per-search context objects (_JavaSearchCtx / _SearchCtx) # for mutable state, so the retriever instance is immutable after __init__ and needs no deep copy. diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py index a7f43260e..4530ec24a 100644 --- a/src/vuln_analysis/utils/function_name_locator.py +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -101,6 +101,28 @@ def _find_java_gav_auto_resolve(requested: str, matches: list[str]) -> str | Non return resolved[0] return None + @staticmethod + def _find_go_common_root_auto_resolve(package: str, matches: list[str]) -> str | None: + """For Go, find the longest common module root (at / boundaries) between + the input package and any close match. Returns the common root, not the + sibling itself, so downstream get_package_name filtering works correctly. + Requires at least 2 common path segments to avoid false matches.""" + best_root = None + best_common = 0 + for m in matches: + pkg_parts = package.lower().split("/") + m_parts = m.lower().split("/") + common = 0 + for p, mp in zip(pkg_parts, m_parts): + if p == mp: + common += 1 + else: + break + if common >= 2 and common > best_common: + best_root = "/".join(m.split("/")[:common]) + best_common = common + return best_root + def _build_java_package_error(self, package: str, close_matches: list[str]) -> str: """Build error message for Java when package is not found (after auto-resolve already failed). Detects version mismatches and returns a CONCLUSION or generic error.""" @@ -343,13 +365,22 @@ async def locate_functions(self, query: str) -> list[str]: else: error_msg = self._build_java_package_error(package, close_package_matches) else: - error_msg = ( - f"ERROR: Package '{package}' not found in available packages. " - f"Close matches are: {', '.join(close_package_matches)}. " - f"Please use one of the suggested package names. " - f"Available ecosystem: " - f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" - ) + is_go = (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.GO.value) + if is_go: + resolved_root = self._find_go_common_root_auto_resolve(package, close_package_matches) + if resolved_root: + logger.info("Go auto-resolved '%s' to common root '%s'", package, resolved_root) + package = resolved_root + self.is_package_valid = True + if not self.is_package_valid: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) else: if not is_error: error_msg = ( @@ -439,13 +470,7 @@ async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> t search = MorpheusSerpAPIWrapper(max_retries=2) result = await search.arun(f"Is '{package_name}' part of the {ecosystem.value} standard library?") logger.info("quick_standard_lib_check Standard library check result: %s", result) - # Normalize result: if list, join into single string - if isinstance(result, list): - text = " ".join(result).lower() - elif isinstance(result, str): - text = result.lower() - else: - text = str(result).lower() + text = str(result).lower() # Basic positive signals and avoidance of negative phrasing if "part of the standard library" in text or \ diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 4a7f87a41..c88e06c7e 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re from pathlib import Path @@ -37,10 +36,6 @@ _OSV_API_URL = "https://api.osv.dev/v1/vulns/" _OSV_TIMEOUT_SECONDS = 5 -_GITHUB_COMMIT_RE = re.compile(r"github\.com/([^/]+)/([^/]+)/commit/([0-9a-f]{7,40})") -_GITHUB_API_TIMEOUT = 10 -_PATCH_MAX_COMMITS = 3 - _FUNC_PATTERNS = { "go": re.compile(r"func\s+(?:\([^)]*\)\s+)?(\w+)\s*\("), "python": re.compile(r"def\s+(\w+)\s*\("), @@ -66,7 +61,7 @@ _JAVA_TEST_RE = re.compile(r"^test[A-Z]") -_TEST_FILE_RE = re.compile( +TEST_FILE_RE = re.compile( r"(?:_test\.go" r"|Test(?:s|Case)?\.java" r"|(?:^|/)test_[^/]*\.py" @@ -78,6 +73,14 @@ _PACKAGE_TOKEN_CHARS = r"A-Za-z0-9_.\/:@+\-" +_ECO_MAP = { + "go": "go", "golang": "go", + "pypi": "python", "python": "python", "pip": "python", + "npm": "javascript", "javascript": "javascript", "node": "javascript", + "maven": "java", "java": "java", + "conan": "c", "c": "c", "cpp": "c", "c++": "c", +} + def update_version(incoming_version, current_version, compare): """ @@ -441,146 +444,122 @@ def _is_fix_ref(ref) -> bool: return isinstance(ref, dict) and ref.get("type", "").upper() == "FIX" -def _extract_functions_from_patch(patch_text: str, ecosystem: str) -> set[str]: - """Extract function names from a unified diff patch using hunk headers and modified lines.""" - funcs: set[str] = set() +def _get_ecosystem_patterns(ecosystem: str) -> list[re.Pattern]: + """Return regex patterns for the given ecosystem, or all patterns if unknown.""" eco_lower = ecosystem.lower() if ecosystem else "" + key = _ECO_MAP.get(eco_lower) + if key: + return [_FUNC_PATTERNS[key]] + return list(_FUNC_PATTERNS.values()) - patterns: list[re.Pattern] = [] - if eco_lower in ("go", "golang"): - patterns = [_FUNC_PATTERNS["go"]] - elif eco_lower in ("pypi", "python", "pip"): - patterns = [_FUNC_PATTERNS["python"]] - elif eco_lower in ("npm", "javascript", "node"): - patterns = [_FUNC_PATTERNS["javascript"]] - elif eco_lower in ("maven", "java"): - patterns = [_FUNC_PATTERNS["java"]] - elif eco_lower in ("conan", "c", "cpp", "c++"): - patterns = [_FUNC_PATTERNS["c"]] - else: - patterns = list(_FUNC_PATTERNS.values()) - - for line in patch_text.splitlines(): - context = None - if line.startswith("@@"): - parts = line.split("@@") - if len(parts) >= 3: - context = parts[2].strip() - elif line.startswith("+") and not line.startswith("+++"): - context = line[1:] - - if context is None: - continue - for pat in patterns: - m = pat.search(context) - if m: - name = next((g for g in m.groups() if g), None) - if (name and len(name) > 1 - and name.lower() not in _NOISE_FUNC_NAMES - and not name.startswith("Test") - and not name.startswith("test_") - and not _JAVA_TEST_RE.match(name)): - funcs.add(name) - break +def _match_func_name(text: str, patterns: list[re.Pattern], out: set[str]) -> None: + """Match a function name from text and add to out set if valid.""" + for pat in patterns: + m = pat.search(text) + if m: + name = next((g for g in m.groups() if g), None) + if (name and len(name) > 1 + and name.lower() not in _NOISE_FUNC_NAMES + and not name.startswith("Test") + and not name.startswith("test_") + and not _JAVA_TEST_RE.match(name)): + out.add(name) + break + +def extract_functions_from_parsed_patch(parsed_patch, ecosystem: str = "") -> set[str]: + """Extract function names from a ParsedPatch object. + + Checks added lines and @@ hunk section headers (which name the enclosing + function even when the signature itself isn't in the diff). + Skips test files. + """ + funcs: set[str] = set() + patterns = _get_ecosystem_patterns(ecosystem) + for pf in parsed_patch.files: + if TEST_FILE_RE.search(pf.target_path): + continue + for hunk in pf.hunks: + if hunk.section_header: + _match_func_name(hunk.section_header, patterns, funcs) + for line in hunk.added_lines: + _match_func_name(line, patterns, funcs) return funcs +def _append_enrichment_to_context(all_funcs: set[str], critical_context: list[str], source_label: str) -> None: + """Append extracted function names to critical_context.""" + critical_context.append( + f"Vulnerable functions (remediation patch hint): {', '.join(sorted(all_funcs))}" + ) + short_names = [f.rsplit(".", 1)[-1] for f in all_funcs if "." in f] + if short_names: + unique = list(dict.fromkeys(list(all_funcs) + short_names)) + critical_context.append(f"Search keywords: {', '.join(unique)}") + logger.info("Patch enrichment extracted functions from %s: %s", source_label, sorted(all_funcs)) + + async def enrich_vulnerable_functions_from_patch( cve_intel_list, critical_context: list[str], vulnerable_functions: set[str], ecosystem: str = "", + patch_result=None, ) -> None: """Extract vulnerable function names from remediation patch commits. - When GHSA/OSV have no vulnerable_functions, looks for GitHub commit URLs - in references, fetches the diff, and extracts function names from - hunk headers and modified function definitions. Test files are - skipped entirely to avoid injecting test-only names. + When GHSA/OSV have no vulnerable_functions, extracts function names from + patch diffs. Uses a pre-fetched patch_result when available (from the + cve_fetch_patches pipeline step), otherwise falls back to fetching via + fetch_patch_for_cve. Test files are skipped to avoid injecting test-only names. """ if vulnerable_functions: return - refs: list = [] - fix_refs: list = [] - for cve_intel in cve_intel_list: - if cve_intel.ghsa is not None: - ghsa_refs = getattr(cve_intel.ghsa, "references", None) or [] - for r in ghsa_refs: - if _is_fix_ref(r): - fix_refs.append(r) - else: - refs.append(r) - if cve_intel.nvd is not None: - refs.extend(cve_intel.nvd.references or []) - - all_refs = fix_refs + refs + from vuln_analysis.utils.web_patch_fetcher import WebPatchResult - commits: list[tuple[str, str, str]] = [] - seen: set[tuple[str, str, str]] = set() - for ref in all_refs: - url = _ref_to_url(ref) - m = _GITHUB_COMMIT_RE.search(url) - if m: - key = (m.group(1), m.group(2), m.group(3)) - if key not in seen: - seen.add(key) - commits.append(key) - if len(commits) >= _PATCH_MAX_COMMITS: - break + if isinstance(patch_result, dict): + patch_result = WebPatchResult.model_validate(patch_result) - if not commits: - return - - ghsa_api_key = os.environ.get("GHSA_API_KEY") - headers = {"Accept": "application/vnd.github+json"} - if ghsa_api_key: - headers["Authorization"] = f"Bearer {ghsa_api_key}" - else: - logger.warning("GHSA_API_KEY not set — GitHub API calls use unauthenticated rate limit (60 req/hr)") + if patch_result and patch_result.parsed_patch: + all_funcs = extract_functions_from_parsed_patch(patch_result.parsed_patch, ecosystem) + if all_funcs: + _append_enrichment_to_context(all_funcs, critical_context, f"pre-fetched patch ({patch_result.source})") + return try: import aiohttp - timeout = aiohttp.ClientTimeout(total=_GITHUB_API_TIMEOUT) - async with aiohttp.ClientSession(timeout=timeout) as session: - for owner, repo, sha in commits: - url = f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}" - async with session.get(url, headers=headers) as resp: - if resp.status != 200: - logger.warning( - "GitHub commit fetch for %s/%s/%s returned %d", - owner, repo, sha[:8], resp.status, - ) - continue - commit_data = await resp.json() - - all_funcs: set[str] = set() - for file_info in commit_data.get("files", []): - filename = file_info.get("filename", "") - if _TEST_FILE_RE.search(filename): - continue - patch = file_info.get("patch", "") - if patch: - all_funcs |= _extract_functions_from_patch(patch, ecosystem) - - if not all_funcs: - logger.info("Patch commit %s/%s@%s parsed but no functions extracted", owner, repo, sha[:8]) - - if all_funcs: - critical_context.append( - f"Vulnerable functions (remediation patch hint): {', '.join(sorted(all_funcs))}" - ) - short_names = [f.rsplit(".", 1)[-1] for f in all_funcs if "." in f] - if short_names: - unique = list(dict.fromkeys(list(all_funcs) + short_names)) - critical_context.append(f"Search keywords: {', '.join(unique)}") - logger.info( - "Patch enrichment extracted functions from %s/%s@%s: %s", - owner, repo, sha[:8], sorted(all_funcs), - ) - return + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + + merged_candidates: dict[str, list[str]] = {} + for cve_intel in cve_intel_list: + per_intel = extract_commit_url_candidates(cve_intel) + for source, urls in per_intel.items(): + merged_candidates.setdefault(source, []).extend(urls) + + vuln_id = cve_intel_list[0].vuln_id if cve_intel_list else "" + cve_description = None + for cve_intel in cve_intel_list: + if cve_intel.nvd and cve_intel.nvd.cve_description: + cve_description = cve_intel.nvd.cve_description + break + + async with aiohttp.ClientSession() as session: + result = await fetch_patch_for_cve( + session=session, + candidates=merged_candidates, + vuln_id=vuln_id, + cve_description=cve_description, + ) + + if result and result.parsed_patch: + all_funcs = extract_functions_from_parsed_patch(result.parsed_patch, ecosystem) + if all_funcs: + _append_enrichment_to_context(all_funcs, critical_context, f"fetched patch ({result.source})") + return + else: + logger.info("Patch fetched for %s but no functions extracted", vuln_id) except Exception: logger.warning("Patch-based function extraction failed", exc_info=True) diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 727830212..5191d2d49 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re import typing +from pathlib import Path from ordered_set import OrderedSet @@ -28,6 +30,10 @@ from vuln_analysis.data_models.output import JustificationOutput from vuln_analysis.data_models.output import CVSSOutput from vuln_analysis.functions.cve_generate_cvss import CVSS_VECTOR_STRING, CVSS_SCORE +from vuln_analysis.utils.web_patch_fetcher import WebPatchResult +from vuln_analysis.functions.code_agent_graph_defs import PatchFile +from vuln_analysis.utils.intel_utils import TEST_FILE_RE +from vuln_analysis.functions.cve_checker_report import infer_language_from_path, NON_CODE_LANGUAGES from vuln_analysis.data_models.state import AgentMorpheusEngineState from aiq.builder.builder import Builder @@ -93,12 +99,84 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE original_input=message) +_MAX_SNIPPET_LINES = 12 +_MAX_PATCH_FILES = 5 +_MD_LINK_RE = re.compile(r'([!\[\]()])') + + +def _build_full_pipeline_details_md(patch_result: WebPatchResult | dict | None) -> str | None: + """Build markdown details from a fetched patch result for the UI client.""" + if not patch_result: + return None + if isinstance(patch_result, dict): + patch_result = WebPatchResult.model_validate(patch_result) + if not patch_result.parsed_patch: + return None + + lines: list[str] = [] + + lines.append("## Vulnerability Fix Patch") + lines.append("") + + if patch_result.patch_url: + lines.append(f"**Patch source:** {_MD_LINK_RE.sub(r'\\\1', patch_result.patch_url)}") + lines.append("") + if patch_result.source: + lines.append(f"**Found via:** {_MD_LINK_RE.sub(r'\\\1', patch_result.source)}") + lines.append("") + if patch_result.commit_message: + first_line = patch_result.commit_message.strip().split("\n")[0][:200] + lines.append(f"**Commit message:** {_MD_LINK_RE.sub(r'\\\1', first_line)}") + lines.append("") + + parsed = patch_result.parsed_patch + code_pairs: list[tuple[PatchFile, str]] = [] + for pf in parsed.files: + path = pf.clean_target_path + lang = infer_language_from_path(path) + if lang and lang not in NON_CODE_LANGUAGES and not TEST_FILE_RE.search(path): + code_pairs.append((pf, lang)) + + for pf, lang in code_pairs[:_MAX_PATCH_FILES]: + target = pf.clean_target_path + lines.append(f"### `{target}`") + lines.append("") + + for hunk in pf.hunks: + if hunk.removed_lines: + lines.append("**Vulnerable code (removed):**") + code = "\n".join(hunk.removed_lines[:_MAX_SNIPPET_LINES]) + lines.append(f"```{lang}") + lines.append(code) + lines.append("```") + if len(hunk.removed_lines) > _MAX_SNIPPET_LINES: + lines.append(f"_... +{len(hunk.removed_lines) - _MAX_SNIPPET_LINES} more lines_") + lines.append("") + + if hunk.added_lines: + lines.append("**Fix code (added):**") + code = "\n".join(hunk.added_lines[:_MAX_SNIPPET_LINES]) + lines.append(f"```{lang}") + lines.append(code) + lines.append("```") + if len(hunk.added_lines) > _MAX_SNIPPET_LINES: + lines.append(f"_... +{len(hunk.added_lines) - _MAX_SNIPPET_LINES} more lines_") + lines.append("") + + if len(code_pairs) > _MAX_PATCH_FILES: + lines.append(f"_... +{len(code_pairs) - _MAX_PATCH_FILES} more files in patch_") + + result = "\n".join(lines).strip() + return result or None + + def parse_agent_morpheus_engine_output(vuln_id: str, checklist_results: list[dict[str, typing.Any]], summary: str, justification: dict[str, str], intel_score: int, - cvss: dict[str, str] | None) -> AgentMorpheusEngineOutput: + cvss: dict[str, str] | None, + patch_result: WebPatchResult | dict | None = None) -> AgentMorpheusEngineOutput: """ Parse the output fields for a single vulnerability into an AgentMorpheusEngineOutput object. """ @@ -115,17 +193,20 @@ def parse_agent_morpheus_engine_output(vuln_id: str, # Combine CVSS model outputs into a single CVSSOutput object if cvss: - cvss_output = CVSSOutput(vector_string=cvss[CVSS_VECTOR_STRING], + cvss_output = CVSSOutput(vector_string=cvss[CVSS_VECTOR_STRING], score=cvss[CVSS_SCORE]) else: cvss_output=None - + + details = _build_full_pipeline_details_md(patch_result) + return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=checklist_output, summary=summary, justification=justification_output, intel_score=intel_score, - cvss=cvss_output) + cvss=cvss_output, + details=details) def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -243,13 +324,16 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, for vuln_id in input_vuln_ids: if vuln_id in output_vuln_ids: + justification = result.justifications[vuln_id] + is_vulnerable = justification.get("affected_status") == "TRUE" output.append( parse_agent_morpheus_engine_output(vuln_id=vuln_id, checklist_results=result.checklist_results[vuln_id], summary=result.final_summaries[vuln_id], - justification=result.justifications[vuln_id], + justification=justification, intel_score=intel_map[vuln_id].intel_score, - cvss=result.cvss_results.get(vuln_id, None))) + cvss=result.cvss_results.get(vuln_id, None), + patch_result=result.patch_results.get(vuln_id) if is_vulnerable else None)) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) elif vuln_id in poor_quality_intel_vul: diff --git a/src/vuln_analysis/utils/osv_patch_retriever.py b/src/vuln_analysis/utils/osv_patch_retriever.py index 4eaaaf105..132cd62b5 100644 --- a/src/vuln_analysis/utils/osv_patch_retriever.py +++ b/src/vuln_analysis/utils/osv_patch_retriever.py @@ -144,6 +144,7 @@ def _parse_patch_content(patch_content: str, patch_filename: str) -> "ParsedPatc source_length=hunk.source_length, target_start=hunk.target_start, target_length=hunk.target_length, + section_header=hunk.section_header or "", context_lines=context, removed_lines=removed, added_lines=added, diff --git a/src/vuln_analysis/utils/tests/test_function_name_locator_go.py b/src/vuln_analysis/utils/tests/test_function_name_locator_go.py new file mode 100644 index 000000000..755c5a3af --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_function_name_locator_go.py @@ -0,0 +1,157 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from vuln_analysis.utils.function_name_locator import FunctionNameLocator + + +class TestFLGoPackageMatching: + def _make_locator(self, supported_packages): + """Build a FunctionNameLocator with a mocked Go retriever.""" + parser = GoLanguageFunctionsParser() + retriever = MagicMock() + retriever.language_parser = parser + retriever.ecosystem = Ecosystem.GO + retriever.supported_packages = supported_packages + retriever.documents_of_functions = [] + return FunctionNameLocator(retriever) + + def test_exact_match(self): + locator = self._make_locator(["github.com/golang-jwt/jwt/v5"]) + assert locator.search_in_third_party_packages("github.com/golang-jwt/jwt/v5") is True + + def test_fqdn_to_module_prefix_match(self): + """Sub-package path should resolve to its parent module.""" + locator = self._make_locator(["google.golang.org/protobuf"]) + assert locator.search_in_third_party_packages( + "google.golang.org/protobuf/encoding/protojson" + ) is True + + def test_fqdn_to_module_picks_longest_prefix(self): + """When multiple modules share a prefix, pick the longest match.""" + locator = self._make_locator([ + "google.golang.org/protobuf", + "google.golang.org/protobuf/encoding", + ]) + result = locator._resolve_go_fqdn_to_module( + "google.golang.org/protobuf/encoding/protojson" + ) + assert result == "google.golang.org/protobuf/encoding" + + def test_sibling_submodule_not_matched_by_prefix(self): + """_resolve_go_fqdn_to_module alone can't match siblings — only prefix matches.""" + locator = self._make_locator([ + "google.golang.org/protobuf/jsonpb", + "google.golang.org/grpc", + ]) + result = locator._resolve_go_fqdn_to_module( + "google.golang.org/protobuf/encoding/protojson" + ) + assert result is None + + def test_short_go_package_name_match(self): + """Short name lookup should find the full package.""" + locator = self._make_locator(["github.com/hashicorp/go-retryablehttp"]) + assert locator.search_in_third_party_packages("go-retryablehttp") is True + + def test_no_false_match(self): + locator = self._make_locator(["github.com/hashicorp/go-retryablehttp"]) + assert locator.search_in_third_party_packages("github.com/some/other/package") is False + + def test_is_same_package_substring(self): + """Go is_same_package uses substring matching: input IN tree_entry.""" + locator = self._make_locator(["github.com/golang-jwt/jwt/v5"]) + assert locator.search_in_third_party_packages("jwt") is True + assert locator.search_in_third_party_packages( + "github.com/golang-jwt/jwt/v5/extra/subpackage" + ) is True + + +class TestFLGoCommonRootAutoResolve: + """Tests for _find_go_common_root_auto_resolve — the sibling sub-module fix.""" + + def test_sibling_submodule_resolves_to_common_root(self): + """Core bug scenario: protobuf/encoding/protojson should resolve to + protobuf (the common root), NOT protobuf/jsonpb (the sibling).""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result == "google.golang.org/protobuf" + + def test_common_root_not_sibling(self): + """Returned value must be the common root, never the sibling match itself.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/hashicorp/vault/api/sub", + ["github.com/hashicorp/vault/sdk"], + ) + assert result == "github.com/hashicorp/vault" + assert result != "github.com/hashicorp/vault/sdk" + + def test_only_domain_in_common_no_match(self): + """Sharing only the domain (1 segment) should not resolve.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/grpc/codes", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result is None + + def test_different_github_orgs_no_match(self): + """Different GitHub orgs share only github.com (1 segment) — should not resolve.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/foo/bar/sub", + ["github.com/baz/bar/sub"], + ) + assert result is None + + def test_versioned_modules_common_root(self): + """jwt/v5/parser with jwt/v4 in tree should resolve to the unversioned root.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/golang-jwt/jwt/v5/parser", + ["github.com/golang-jwt/jwt/v4"], + ) + assert result == "github.com/golang-jwt/jwt" + + def test_multiple_siblings_picks_longest_root(self): + """When multiple close matches exist, pick the one with the longest common root.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + [ + "google.golang.org/protobuf/jsonpb", + "google.golang.org/protobuf/encoding/prototext", + ], + ) + # encoding/prototext shares 3 segments (google.golang.org/protobuf/encoding) + # jsonpb shares 2 segments (google.golang.org/protobuf) + assert result == "google.golang.org/protobuf/encoding" + + def test_empty_matches_returns_none(self): + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + [], + ) + assert result is None + + def test_case_insensitive_comparison(self): + """Comparison should be case-insensitive but return original case.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "Google.Golang.Org/Protobuf/Encoding/Protojson", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result == "google.golang.org/protobuf" + + def test_exact_package_in_matches(self): + """If the exact package is somehow in matches, all segments match.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/foo/bar", + ["github.com/foo/bar"], + ) + assert result == "github.com/foo/bar" + + def test_single_segment_package_no_match(self): + """Single-segment packages should never resolve (below 2-segment minimum).""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "protobuf", + ["protobuf-lite"], + ) + assert result is None diff --git a/src/vuln_analysis/utils/tests/test_intel_utils_exports.py b/src/vuln_analysis/utils/tests/test_intel_utils_exports.py new file mode 100644 index 000000000..f407a17f2 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_intel_utils_exports.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for intel_utils module.""" + +import vuln_analysis.utils.intel_utils as intel_utils + + +class TestTestFileRePublic: + """Tests for TEST_FILE_RE being public (fix 6).""" + + def test_exported_as_public(self): + assert hasattr(intel_utils, "TEST_FILE_RE") + assert not intel_utils.TEST_FILE_RE.pattern.startswith("_") + + def test_matches_go_test_file(self): + assert intel_utils.TEST_FILE_RE.search("pkg/handler_test.go") + + def test_matches_java_test_file(self): + assert intel_utils.TEST_FILE_RE.search("src/test/java/FooTest.java") + + def test_no_match_production_file(self): + assert not intel_utils.TEST_FILE_RE.search("src/main/java/Foo.java") + + def test_matches_python_test_file(self): + assert intel_utils.TEST_FILE_RE.search("tests/test_utils.py") + + def test_matches_js_spec_file(self): + assert intel_utils.TEST_FILE_RE.search("src/handler.spec.ts") + + +class TestDeadConstantsRemoved: + """Tests for dead constants removal (fix 8).""" + + def test_github_commit_re_removed(self): + assert not hasattr(intel_utils, "_GITHUB_COMMIT_RE") + + def test_github_api_timeout_removed(self): + assert not hasattr(intel_utils, "_GITHUB_API_TIMEOUT") + + def test_patch_max_commits_removed(self): + assert not hasattr(intel_utils, "_PATCH_MAX_COMMITS") \ No newline at end of file diff --git a/src/vuln_analysis/utils/tests/test_llm_engine_utils.py b/src/vuln_analysis/utils/tests/test_llm_engine_utils.py new file mode 100644 index 000000000..e32aad192 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_llm_engine_utils.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for llm_engine_utils module.""" + +import vuln_analysis.utils.llm_engine_utils as llm_engine_utils +from vuln_analysis.functions.code_agent_graph_defs import PatchHunk, PatchFile, ParsedPatch +from vuln_analysis.utils.web_patch_fetcher import WebPatchResult + + +class TestNonCodeExtensionsRemoved: + """Tests for _NON_CODE_EXTENSIONS removal (fix 5).""" + + def test_no_non_code_extensions_set(self): + assert not hasattr(llm_engine_utils, "_NON_CODE_EXTENSIONS") + + +class TestFilterBeforeSlice: + """Tests for filter-before-slice in _build_full_pipeline_details_md (fix 2).""" + + @staticmethod + def _make_patch_result(files: list[PatchFile]) -> WebPatchResult: + return WebPatchResult( + cve_id="CVE-2024-0001", + fixed_commit="abc123", + repo_url="https://github.com/example/repo", + patch_url="https://example.com/commit/abc.patch", + source="test", + parsed_patch=ParsedPatch(patch_filename="test.patch", files=files), + ) + + @staticmethod + def _make_file(path: str) -> PatchFile: + return PatchFile( + source_path=f"a/{path}", + target_path=f"b/{path}", + hunks=[PatchHunk( + source_start=1, source_length=1, target_start=1, target_length=1, + added_lines=["+ fix"], + )], + ) + + def test_non_code_files_filtered_before_limit(self): + """Files with unrecognized extensions should not count toward the 5-file limit.""" + non_code = [self._make_file(f"data/blob{i}.dat") for i in range(10)] + code = [self._make_file(f"src/fix{i}.java") for i in range(3)] + result = self._make_patch_result(non_code + code) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + for i in range(3): + assert f"src/fix{i}.java" in md + for i in range(10): + assert f"data/blob{i}.dat" not in md + + def test_markdown_and_yaml_files_filtered(self): + """Non-code files like .md and .yml should be excluded even though + infer_language_from_path returns a truthy language name for them.""" + files = [ + self._make_file("README.md"), + self._make_file("config.yml"), + self._make_file("data.json"), + self._make_file("src/Fix.java"), + ] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + assert "src/Fix.java" in md + assert "README.md" not in md + assert "config.yml" not in md + assert "data.json" not in md + + def test_test_files_filtered(self): + """Test files should be excluded from the details output.""" + files = [ + self._make_file("src/Foo.java"), + self._make_file("src/FooTest.java"), + self._make_file("tests/test_bar.py"), + ] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert "src/Foo.java" in md + assert "FooTest.java" not in md + assert "test_bar.py" not in md + + def test_all_non_code_returns_none(self): + """If all files have unrecognized extensions, result should be None.""" + files = [self._make_file("data.dat"), self._make_file("image.png")] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is None or "data.dat" not in md + + def test_overflow_count_uses_code_files_not_parsed_files(self): + """The 'more files' count should reflect code files, not total parsed files.""" + non_code = [self._make_file(f"docs/page{i}.md") for i in range(20)] + code = [self._make_file(f"src/fix{i}.java") for i in range(7)] + result = self._make_patch_result(non_code + code) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + assert "+2 more files" in md + assert "+22 more files" not in md \ No newline at end of file diff --git a/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py index 42f70dbaa..a9900cf8a 100644 --- a/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py +++ b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py @@ -18,6 +18,8 @@ import pytest from unittest.mock import MagicMock, patch +from vuln_analysis.functions.code_agent_graph_defs import PatchHunk, PatchFile, ParsedPatch +from vuln_analysis.utils.intel_utils import extract_functions_from_parsed_patch from vuln_analysis.utils.web_patch_fetcher import ( WebPatchFetcher, build_patch_url_from_repo, @@ -424,3 +426,132 @@ async def test_fetch_from_intel_refs_tries_multiple_urls(self, mock_session): assert result is mock_result assert mock_fetch.call_count == 2 + + +class TestPatchHunkSectionHeader: + """Tests for PatchHunk.section_header field (fix 1).""" + + def test_section_header_default_empty(self): + hunk = PatchHunk(source_start=1, source_length=5, target_start=1, target_length=5) + assert hunk.section_header == "" + + def test_section_header_populated(self): + hunk = PatchHunk( + source_start=10, source_length=5, target_start=10, target_length=5, + section_header="void processRequest(HttpServletRequest req)", + ) + assert hunk.section_header == "void processRequest(HttpServletRequest req)" + + def test_section_header_in_extract_functions(self): + """section_header containing a function signature should be picked up by extract_functions_from_parsed_patch.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/src/main.go", + target_path="b/src/main.go", + hunks=[PatchHunk( + source_start=10, source_length=3, target_start=10, target_length=3, + section_header="func handleRequest(w http.ResponseWriter, r *http.Request)", + added_lines=[" return nil"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="go") + assert "handleRequest" in funcs + + def test_section_header_only_no_added_lines(self): + """Function name should be extracted from section_header even when added_lines have no definitions.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/Foo.java", + target_path="b/Foo.java", + hunks=[PatchHunk( + source_start=50, source_length=2, target_start=50, target_length=2, + section_header="public String getVersion()", + added_lines=[" return VERSION;"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="java") + assert "getVersion" in funcs + + def test_section_header_skipped_for_test_files(self): + """Test files should be skipped even if section_header has a function.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/FooTest.java", + target_path="b/FooTest.java", + hunks=[PatchHunk( + source_start=10, source_length=2, target_start=10, target_length=2, + section_header="public void shouldParseInput()", + added_lines=[" assertEquals(1, 1);"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="java") + assert len(funcs) == 0 + + def test_section_header_empty_string_not_matched(self): + """Empty section_header should not contribute any function names.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/main.c", + target_path="b/main.c", + hunks=[PatchHunk( + source_start=1, source_length=1, target_start=1, target_length=1, + section_header="", + added_lines=["int x = 0;"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="c") + assert len(funcs) == 0 + + def test_python_section_header(self): + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/utils.py", + target_path="b/utils.py", + hunks=[PatchHunk( + source_start=20, source_length=3, target_start=20, target_length=3, + section_header="def validate_input(data):", + added_lines=[" if not data:"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="python") + assert "validate_input" in funcs + + +class TestCleanTargetPath: + """Tests for PatchFile.clean_target_path property (fix 3).""" + + def test_strip_b_prefix(self): + pf = PatchFile(source_path="a/foo.c", target_path="b/foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_strip_a_prefix(self): + pf = PatchFile(source_path="a/foo.c", target_path="a/foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_no_prefix(self): + pf = PatchFile(source_path="foo.c", target_path="foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_nested_path_b_prefix(self): + pf = PatchFile(source_path="a/src/main/Bar.java", target_path="b/src/main/Bar.java", hunks=[]) + assert pf.clean_target_path == "src/main/Bar.java" + + def test_b_in_path_not_prefix(self): + """A 'b/' that is part of the path (not a prefix) should not be stripped.""" + pf = PatchFile(source_path="a/lib/b/file.py", target_path="lib/b/file.py", hunks=[]) + assert pf.clean_target_path == "lib/b/file.py" + + def test_devnull_path(self): + """New files have /dev/null as source — target should still strip prefix.""" + pf = PatchFile(source_path="/dev/null", target_path="b/new_file.go", hunks=[], is_new_file=True) + assert pf.clean_target_path == "new_file.go" diff --git a/src/vuln_analysis/utils/web_patch_fetcher.py b/src/vuln_analysis/utils/web_patch_fetcher.py index cd0d2018b..320aeb71b 100644 --- a/src/vuln_analysis/utils/web_patch_fetcher.py +++ b/src/vuln_analysis/utils/web_patch_fetcher.py @@ -192,6 +192,7 @@ def _parse_patch_content(patch_content: str, patch_filename: str) -> ParsedPatch source_length=hunk.source_length, target_start=hunk.target_start, target_length=hunk.target_length, + section_header=hunk.section_header or "", context_lines=context, removed_lines=removed, added_lines=added, @@ -384,10 +385,8 @@ async def fetch_from_url( Returns: WebPatchResult on success, None on failure """ - # Check cache first cache_key = url.lower().rstrip("/") if cache_key in self._cache: - logger.debug("Cache hit for %s", url) return self._cache[cache_key] # Resolve URL to patch download URL @@ -1071,3 +1070,51 @@ def _build_patch_url(self, repo_url: str, commit_sha: str) -> str | None: if not patch_url: logger.debug("Unsupported repo URL: %s", repo_url) return patch_url + + +# --------------------------------------------------------------------------- +# Shared helper — used by both full pipeline and package checker paths +# --------------------------------------------------------------------------- + +async def fetch_patch_for_cve( + session: aiohttp.ClientSession, + candidates: dict[str, list[str]], + vuln_id: str, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, + upstream_version: str | None = None, + package_name: str | None = None, +) -> WebPatchResult | None: + """Try intel-ref URLs first, then fall back to OSV API. + + This is the shared fetch logic used by both the full pipeline + (cve_fetch_patches) and the package checker (upstream_search_preprocess). + + Args: + session: aiohttp session to reuse for all HTTP calls. + candidates: URL candidates from extract_commit_url_candidates(). + vuln_id: CVE identifier. + cve_description: Optional CVE description for Chromium CL disambiguation. + llm: Optional LLM for Chromium CL selection. + upstream_version: Optional version for OSV filtering. + package_name: Optional package name for OSV filtering. + + Returns: + WebPatchResult on success, None if no valid patch found. + """ + fetcher = WebPatchFetcher(session=session) + has_candidates = any(urls for urls in candidates.values()) + + if has_candidates: + result = await fetcher.fetch_from_intel_refs( + candidates, vuln_id, cve_description=cve_description, llm=llm, + ) + if result and result.parsed_patch: + return result + + client = OSVClient(session=session, patch_fetcher=fetcher) + result = await client.get_fix_patch(vuln_id, upstream_version, package_name) + if result and result.parsed_patch: + return result + + return None diff --git a/tests/test_intel_utils.py b/tests/test_intel_utils.py index a6a3d2f61..bb31ccee2 100644 --- a/tests/test_intel_utils.py +++ b/tests/test_intel_utils.py @@ -9,9 +9,10 @@ from vuln_analysis.utils.intel_utils import ( build_critical_context, _MAX_RHSA_CANDIDATES, - _extract_functions_from_patch, + extract_functions_from_parsed_patch, enrich_vulnerable_functions_from_patch, ) +from vuln_analysis.utils.web_patch_fetcher import ParsedPatch, PatchFile, PatchHunk class TestBuildCriticalContextRhsaCap: @@ -76,69 +77,81 @@ def test_context_note_still_shows_total_count(self): assert "50" in affected_notes[0] -class TestExtractFunctionsFromPatchNoiseFilter: - """Tests for test function filtering in _extract_functions_from_patch.""" - - JAVA_PATCH_WITH_TEST_METHODS = """\ -@@ -10,6 +10,30 @@ public class BeanUtilsTestCase { -+ public void testAllowAccessToClassPropertyFromBeanUtilsBean() { -+ // test body -+ } -+ public void testSuppressClassPropertyByDefaultFromPropertyUtilsBean() { -+ // test body -+ } -+ public String getName() { -+ return this.name; -+ } -+ public void setName(String name) { -+ this.name = name; -+ } -""" +class TestExtractFunctionsFromParsedPatchNoiseFilter: + """Tests for test function filtering in extract_functions_from_parsed_patch.""" + + def _make_parsed_patch(self, filename, added_lines): + """Build a ParsedPatch with a single file and hunk.""" + return ParsedPatch(patch_filename="test.patch", files=[ + PatchFile( + source_path="a/" + filename, + target_path="b/" + filename, + hunks=[PatchHunk( + source_start=1, source_length=0, + target_start=1, target_length=len(added_lines), + added_lines=added_lines, removed_lines=[], context_lines=[], + )], + ), + ]) + + JAVA_TEST_LINES = [ + " public void testAllowAccessToClassPropertyFromBeanUtilsBean() {", + " // test body", + " }", + " public void testSuppressClassPropertyByDefaultFromPropertyUtilsBean() {", + " // test body", + " }", + " public String getName() {", + " return this.name;", + " }", + " public void setName(String name) {", + " this.name = name;", + " }", + ] def test_java_testFoo_methods_filtered(self): """Java testFoo naming convention (lowercase test + uppercase letter) must be filtered.""" - funcs = _extract_functions_from_patch(self.JAVA_PATCH_WITH_TEST_METHODS, "java") + pp = self._make_parsed_patch("PropertyUtilsBean.java", self.JAVA_TEST_LINES) + funcs = extract_functions_from_parsed_patch(pp, "java") assert "testAllowAccessToClassPropertyFromBeanUtilsBean" not in funcs assert "testSuppressClassPropertyByDefaultFromPropertyUtilsBean" not in funcs def test_java_real_methods_kept(self): """Real methods (getName, setName) must not be filtered.""" - funcs = _extract_functions_from_patch(self.JAVA_PATCH_WITH_TEST_METHODS, "java") + pp = self._make_parsed_patch("PropertyUtilsBean.java", self.JAVA_TEST_LINES) + funcs = extract_functions_from_parsed_patch(pp, "java") assert "getName" in funcs assert "setName" in funcs def test_go_Test_prefix_filtered(self): """Go Test prefix (capital T) is filtered.""" - patch = """\ -@@ -5,0 +5,3 @@ -+func TestDoSAttack(t *testing.T) { -+func convert(input string) string { -""" - funcs = _extract_functions_from_patch(patch, "go") + pp = self._make_parsed_patch("handler.go", [ + "func TestDoSAttack(t *testing.T) {", + "func convert(input string) string {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "go") assert "TestDoSAttack" not in funcs assert "convert" in funcs def test_python_test_underscore_filtered(self): """Python test_ prefix is filtered.""" - patch = """\ -@@ -1,0 +1,4 @@ -+def test_parse_input(): -+def parse_input(data): -""" - funcs = _extract_functions_from_patch(patch, "python") + pp = self._make_parsed_patch("parser.py", [ + "def test_parse_input():", + "def parse_input(data):", + ]) + funcs = extract_functions_from_parsed_patch(pp, "python") assert "test_parse_input" not in funcs assert "parse_input" in funcs def test_java_testDoS_variants_filtered(self): """All testXxx variants are filtered regardless of suffix.""" - patch = """\ -@@ -1,0 +1,6 @@ -+ public void testCannotInjectEventHandler() { -+ public void testCannotUseJaxwsInputStreamToDeleteFile() { -+ public void testDoSAttackWithCollections() { -+ public void unmarshal() { -""" - funcs = _extract_functions_from_patch(patch, "java") + pp = self._make_parsed_patch("XStream.java", [ + " public void testCannotInjectEventHandler() {", + " public void testCannotUseJaxwsInputStreamToDeleteFile() {", + " public void testDoSAttackWithCollections() {", + " public void unmarshal() {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "java") assert "testCannotInjectEventHandler" not in funcs assert "testCannotUseJaxwsInputStreamToDeleteFile" not in funcs assert "testDoSAttackWithCollections" not in funcs @@ -146,73 +159,65 @@ def test_java_testDoS_variants_filtered(self): def test_noise_words_filtered(self): """Built-in noise words (main, init, setup, etc.) are filtered.""" - patch = """\ -@@ -1,0 +1,4 @@ -+func main() { -+func init() { -+func Setup() { -+func realFunc() { -""" - funcs = _extract_functions_from_patch(patch, "go") + pp = self._make_parsed_patch("main.go", [ + "func main() {", + "func init() {", + "func Setup() {", + "func realFunc() {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "go") assert "main" not in funcs assert "init" not in funcs assert "realFunc" in funcs class TestEnrichVulnerableFunctionsFromPatch: - """Patch-extracted functions populate vulnerable_functions; test files are skipped.""" - - def _make_mock_intel(self, commit_url): - from unittest.mock import MagicMock - mock_ghsa = MagicMock() - mock_ghsa.references = [{"type": "FIX", "url": commit_url}] - mock_cve_intel = MagicMock() - mock_cve_intel.ghsa = mock_ghsa - mock_cve_intel.nvd = None - return mock_cve_intel - - def _make_mock_session(self, commit_response): - from unittest.mock import AsyncMock, MagicMock - mock_resp = AsyncMock() - mock_resp.status = 200 - mock_resp.json = AsyncMock(return_value=commit_response) - mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) - mock_resp.__aexit__ = AsyncMock(return_value=False) - - mock_session = AsyncMock() - mock_session.get = MagicMock(return_value=mock_resp) - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=False) - return MagicMock(return_value=mock_session) + """Patch-extracted functions populate critical_context as hints; test files are skipped.""" + + def _make_patch_result(self, files, source="ghsa"): + """Build a WebPatchResult with the given ParsedPatch files.""" + from vuln_analysis.utils.web_patch_fetcher import WebPatchResult, ParsedPatch, PatchFile, PatchHunk + parsed_files = [] + for f in files: + hunks = [PatchHunk( + source_start=1, source_length=0, + target_start=1, target_length=len(f["added_lines"]), + added_lines=f["added_lines"], + removed_lines=f.get("removed_lines", []), + context_lines=[], + )] + parsed_files.append(PatchFile( + source_path="a/" + f["filename"], + target_path="b/" + f["filename"], + hunks=hunks, + )) + return WebPatchResult( + cve_id="CVE-2024-0000", + parsed_patch=ParsedPatch(patch_filename="test.patch", files=parsed_files), + source=source, + patch_url="https://example.com/patch", + fixed_commit="abc123", + repo_url="https://github.com/test/repo", + ) @pytest.mark.asyncio async def test_source_functions_added_to_critical_context_as_hints(self): """Functions from non-test source files are added to critical_context as hints.""" - from unittest.mock import patch - import aiohttp as real_aiohttp - critical_context: list[str] = [] vulnerable_functions: set[str] = set() - mock_cve_intel = self._make_mock_intel( - "https://github.com/apache/commons-beanutils/commit/bd20740d" + patch_result = self._make_patch_result([{ + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "added_lines": [ + " public Object getProperty(Object bean, String name) {", + " return null;", + " }", + ], + }]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "java", + patch_result=patch_result, ) - commit_response = { - "files": [{ - "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", - "patch": """\ -@@ -10,6 +10,10 @@ public class PropertyUtilsBean { -+ public Object getProperty(Object bean, String name) { -+ return null; -+ } -""" - }] - } - mock_client_session = self._make_mock_session(commit_response) - - with patch.object(real_aiohttp, "ClientSession", mock_client_session): - await enrich_vulnerable_functions_from_patch( - [mock_cve_intel], critical_context, vulnerable_functions, "java", - ) assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" patch_context = [c for c in critical_context if "remediation patch hint" in c] @@ -222,43 +227,32 @@ async def test_source_functions_added_to_critical_context_as_hints(self): @pytest.mark.asyncio async def test_test_files_skipped(self): """Files matching test patterns are skipped entirely.""" - from unittest.mock import patch - import aiohttp as real_aiohttp - critical_context: list[str] = [] vulnerable_functions: set[str] = set() - mock_cve_intel = self._make_mock_intel( - "https://github.com/apache/commons-beanutils/commit/bd20740d" + patch_result = self._make_patch_result([ + { + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "added_lines": [ + " public Object getProperty(Object bean, String name) {", + " return null;", + " }", + ], + }, + { + "filename": "src/test/java/org/apache/commons/beanutils/PropertyUtilsBeanTestCase.java", + "added_lines": [ + " public void testAllowAccessToClassProperty() {", + " }", + " public void testSuppressClassProperty() {", + " }", + ], + }, + ]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "java", + patch_result=patch_result, ) - commit_response = { - "files": [ - { - "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", - "patch": """\ -@@ -10,6 +10,10 @@ public class PropertyUtilsBean { -+ public Object getProperty(Object bean, String name) { -+ return null; -+ } -""" - }, - { - "filename": "src/test/java/org/apache/commons/beanutils/PropertyUtilsBeanTestCase.java", - "patch": """\ -@@ -10,6 +10,10 @@ public class PropertyUtilsBeanTestCase { -+ public void testAllowAccessToClassProperty() { -+ } -+ public void testSuppressClassProperty() { -+ } -""" - }, - ] - } - mock_client_session = self._make_mock_session(commit_response) - - with patch.object(real_aiohttp, "ClientSession", mock_client_session): - await enrich_vulnerable_functions_from_patch( - [mock_cve_intel], critical_context, vulnerable_functions, "java", - ) assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" patch_context = [c for c in critical_context if "remediation patch hint" in c] @@ -270,39 +264,28 @@ async def test_test_files_skipped(self): @pytest.mark.asyncio async def test_go_test_files_skipped(self): """Go _test.go files are skipped.""" - from unittest.mock import patch - import aiohttp as real_aiohttp - critical_context: list[str] = [] vulnerable_functions: set[str] = set() - mock_cve_intel = self._make_mock_intel( - "https://github.com/quic-go/quic-go/commit/abc12345" + patch_result = self._make_patch_result([ + { + "filename": "internal/ackhandler/sent_packet_handler.go", + "added_lines": [ + "func detectAndRemoveAckedPackets() {", + ], + }, + { + "filename": "internal/ackhandler/sent_packet_handler_test.go", + "added_lines": [ + "func TestDoSAttack(t *testing.T) {", + "func BenchmarkAckHandler(b *testing.B) {", + ], + }, + ]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "go", + patch_result=patch_result, ) - commit_response = { - "files": [ - { - "filename": "internal/ackhandler/sent_packet_handler.go", - "patch": """\ -@@ -5,0 +5,3 @@ -+func detectAndRemoveAckedPackets() { -""" - }, - { - "filename": "internal/ackhandler/sent_packet_handler_test.go", - "patch": """\ -@@ -5,0 +5,3 @@ -+func TestDoSAttack(t *testing.T) { -+func BenchmarkAckHandler(b *testing.B) { -""" - }, - ] - } - mock_client_session = self._make_mock_session(commit_response) - - with patch.object(real_aiohttp, "ClientSession", mock_client_session): - await enrich_vulnerable_functions_from_patch( - [mock_cve_intel], critical_context, vulnerable_functions, "go", - ) assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" patch_context = [c for c in critical_context if "remediation patch hint" in c] diff --git a/tests/test_process_steps.py b/tests/test_process_steps.py index 4c0777b3c..a19f174d3 100644 --- a/tests/test_process_steps.py +++ b/tests/test_process_steps.py @@ -68,7 +68,7 @@ async def test_cu_fallback_uses_reachability_graph(self, patch_externals, mock_g ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") mock_graph.ainvoke.assert_called_once() @@ -79,7 +79,7 @@ async def test_cu_fallback_uses_reachability_tracker(self, patch_externals, mock ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") call_args = mock_graph.ainvoke.call_args[0][0] tracker = call_args["rules_tracker"] @@ -96,7 +96,7 @@ async def test_direct_route_uses_correct_tracker(self, patch_externals): reach_graph = AsyncMock() agents = {"reachability": reach_graph, "code_understanding": cu_graph} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") call_args = cu_graph.ainvoke.call_args[0][0] tracker = call_args["rules_tracker"] @@ -110,7 +110,7 @@ async def test_reachability_route_uses_reachability_tracker(self, patch_external ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["Is func reachable?"], None) + await _process_steps(agents, MagicMock(), ["Is func reachable?"], None, vuln_id="CVE-test") call_args = mock_graph.ainvoke.call_args[0][0] tracker = call_args["rules_tracker"] @@ -127,7 +127,7 @@ async def test_initial_state_keys(self, patch_externals, mock_graph): ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["test question"], None) + await _process_steps(agents, MagicMock(), ["test question"], None, vuln_id="CVE-test") call_args = mock_graph.ainvoke.call_args[0][0] assert call_args["input"] == "test question" @@ -149,7 +149,7 @@ async def test_precomputed_intel_passed(self, patch_externals, mock_graph): ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["q"], None) + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") call_args = mock_graph.ainvoke.call_args[0][0] intel = call_args["precomputed_intel"] @@ -162,7 +162,7 @@ async def test_custom_max_iterations(self, patch_externals, mock_graph): ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["q"], None, max_iterations=25) + await _process_steps(agents, MagicMock(), ["q"], None, max_iterations=25, vuln_id="CVE-test") call_args = mock_graph.ainvoke.call_args[0][0] assert call_args["max_steps"] == 25 @@ -174,7 +174,7 @@ async def test_graph_config_recursion_limit(self, patch_externals, mock_graph): ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["q"], None) + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") config_arg = mock_graph.ainvoke.call_args[1]["config"] assert config_arg == {"recursion_limit": 50} @@ -191,7 +191,7 @@ async def test_multiple_steps_all_processed(self, patch_externals, mock_graph): agents = {"reachability": mock_graph} steps = ["q1", "q2", "q3"] - raw_results, questions = await _process_steps(agents, MagicMock(), steps, None) + raw_results, questions = await _process_steps(agents, MagicMock(), steps, None, vuln_id="CVE-test") assert len(raw_results) == 3 assert mock_graph.ainvoke.call_count == 3 @@ -218,7 +218,7 @@ async def slow_invoke(state, config=None): agents = {"reachability": mock_graph} semaphore = asyncio.Semaphore(2) - await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], semaphore) + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], semaphore, vuln_id="CVE-test") assert max_concurrent <= 2 @@ -243,7 +243,7 @@ async def slow_invoke(state, config=None): ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], None) + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], None, vuln_id="CVE-test") assert max_concurrent == 4 @@ -257,7 +257,7 @@ async def test_exception_in_step_returned_not_raised(self, patch_externals): ) agents = {"reachability": mock_graph} - raw_results, questions = await _process_steps(agents, MagicMock(), ["q1"], None) + raw_results, questions = await _process_steps(agents, MagicMock(), ["q1"], None, vuln_id="CVE-test") assert len(raw_results) == 1 assert isinstance(raw_results[0], RuntimeError) @@ -265,7 +265,7 @@ async def test_exception_in_step_returned_not_raised(self, patch_externals): @pytest.mark.asyncio async def test_empty_steps_returns_empty(self, patch_externals, mock_graph): agents = {"reachability": mock_graph} - raw_results, questions = await _process_steps(agents, MagicMock(), [], None) + raw_results, questions = await _process_steps(agents, MagicMock(), [], None, vuln_id="CVE-test") assert raw_results == [] assert questions == [] @@ -287,7 +287,7 @@ async def test_synthetic_question_injected_with_vulnerable_functions(self, patch ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) )} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") assert mock_graph.ainvoke.call_count == 1 call_args = mock_graph.ainvoke.call_args[0][0] @@ -306,7 +306,7 @@ async def test_synthetic_question_injected_without_vulnerable_functions(self, pa ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) )} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") assert mock_graph.ainvoke.call_count == 1 call_args = mock_graph.ainvoke.call_args[0][0] @@ -326,7 +326,7 @@ async def test_no_synthetic_question_without_candidate_packages(self, patch_exte cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) agents = {"reachability": mock_graph, "code_understanding": cu_graph} - await _process_steps(agents, MagicMock(), ["Is it configured?"], None) + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") mock_graph.ainvoke.assert_not_called() @@ -341,7 +341,7 @@ async def test_no_synthetic_when_reachability_already_routed(self, patch_externa ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["Is convert reachable?"], None) + await _process_steps(agents, MagicMock(), ["Is convert reachable?"], None, vuln_id="CVE-test") assert mock_graph.ainvoke.call_count == 1 call_args = mock_graph.ainvoke.call_args[0][0] @@ -358,7 +358,7 @@ async def test_span_logs_actual_type_on_fallback(self, patch_externals, mock_gra ) agents = {"reachability": mock_graph} - await _process_steps(agents, MagicMock(), ["q"], None) + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") call_args = patch_externals["tracer"].push_active_function.call_args assert "[reachability]" in call_args[1]["input_data"] @@ -372,7 +372,7 @@ async def test_span_logs_routed_type_when_available(self, patch_externals): cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) agents = {"reachability": MagicMock(), "code_understanding": cu_graph} - await _process_steps(agents, MagicMock(), ["q"], None) + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") call_args = patch_externals["tracer"].push_active_function.call_args assert "[code_understanding]" in call_args[1]["input_data"] From c56f416fccbdb1b2d02353fc98dd381864e2b507 Mon Sep 17 00:00:00 2001 From: Roni Hartuv Date: Sun, 14 Jun 2026 12:33:33 +0300 Subject: [PATCH 250/286] feat: add liveness probe to deployment (#249) --- kustomize/base/exploit_iq_service.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index b3ab3677a..ecb45e351 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -65,6 +65,10 @@ spec: - name: http protocol: TCP containerPort: 8080 + livenessProbe: + httpGet: + path: /health + port: http resources: limits: memory: "8Gi" From 0bb0dbb5b6100eddb4210f34f52f6d94fc03cfc1 Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 29 May 2026 17:36:13 -0400 Subject: [PATCH 251/286] add transitive dependency indexing support for python environment --- src/exploit_iq_commons/utils/dep_tree.py | 105 ++++++++++++++++-- .../utils/source_code_git_loader.py | 70 +++++++++++- 2 files changed, 161 insertions(+), 14 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index aa48755d7..f3a91a986 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -61,11 +61,20 @@ ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' TRANSITIVE_ENV_NAME = 'transitive_env' +INSTALLED_PACKAGES_FILE = 'installed_packages.txt' PYPROJECT_TOML = 'pyproject.toml' SETUP_PY = 'setup.py' +SETUP_CFG = 'setup.cfg' +PIPFILE = 'Pipfile' +UV_LOCK = 'uv.lock' +POETRY_LOCK = 'poetry.lock' README_MD = 'README.md' +# Manifest formats tried in priority order when requirements.txt is absent. +# Each entry is the filename; the install strategy is determined in _install_from_best_manifest. +_PYTHON_MANIFEST_FALLBACK_ORDER = [UV_LOCK, POETRY_LOCK, PYPROJECT_TOML, SETUP_PY, SETUP_CFG, PIPFILE] + _WALK_EXCLUDE_DIRS = frozenset({ ".venv", "venv", @@ -125,10 +134,9 @@ def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None: """ if os.path.isfile(git_repo_path / GOLANG_MANIFEST): return MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] - if ( - os.path.isfile(git_repo_path / PYTHON_MANIFEST) - or os.path.isfile(git_repo_path / PYPROJECT_TOML) - or os.path.isfile(git_repo_path / SETUP_PY) + if any( + os.path.isfile(git_repo_path / m) + for m in (PYTHON_MANIFEST, PYPROJECT_TOML, SETUP_PY, SETUP_CFG, UV_LOCK, POETRY_LOCK, PIPFILE) ): return MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] if os.path.isfile(git_repo_path / JS_MANIFEST): @@ -1597,18 +1605,82 @@ def _try_file(path: Path, extractor) -> str | None: def install_dependencies(self, manifest_path: Path): """Install Python dependencies for the given repository into a virtual environment. - Calls :meth:`determine_python_version` to select the interpreter; when a - version is found ``uv venv`` is invoked with ``--python ``, - otherwise ``uv`` selects the default interpreter. Each line of - ``requirements.txt`` is then installed via :meth:`install_dependency`. + Calls :meth:`determine_python_version` to select the interpreter. Creates + a ``transitive_env`` venv and installs packages from whichever manifest + format is present, trying formats in this priority order: + + 1. ``requirements.txt``: line-by-line install (original behaviour) + 2. ``uv.lock`` or ``poetry.lock``: ``uv export | uv pip install -r -`` + 3. ``pyproject.toml``, ``setup.py``, or ``setup.cfg``: ``uv pip install .`` + 4. ``Pipfile``: ``pipenv requirements | uv pip install -r -`` + + After installation, writes ``installed_packages.txt`` containing a + freeze-format snapshot of every package in the venv so that Code + Keyword Search can answer "is package X installed?" without source + traversal. Args: - manifest_path: Absolute path to the root of the cloned repository, - which is expected to contain a ``requirements.txt`` manifest. + manifest_path: Absolute path to the root of the cloned repository. """ - self._ensure_venv(manifest_path) + python_version = self.determine_python_version(str(manifest_path)) + if python_version: + logger.info("Creating transitive_env with Python %s using uv", python_version) + cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" + else: + logger.warning("Python version undetermined for %s; using uv default interpreter", manifest_path) + cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME}" + run_command(cmd) + + venv_python = f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python" site_packages = self._find_site_packages(manifest_path) - with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: + + installed_via = self._install_from_best_manifest(manifest_path, venv_python, site_packages) + if installed_via: + logger.info("Installed Python dependencies via %s", installed_via) + else: + logger.warning("No supported Python manifest found in %s; transitive_env will be empty", manifest_path) + + self._write_installed_packages(manifest_path) + + def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, + site_packages: Optional[Path]) -> Optional[str]: + """Try each Python manifest format in priority order; return the format name on success.""" + req_txt = manifest_path / PYTHON_MANIFEST + if req_txt.exists(): + self._install_from_requirements_txt(req_txt, manifest_path, site_packages) + return PYTHON_MANIFEST + + # Lock files: export to requirements format then pipe to uv pip install + for lock_file in (UV_LOCK, POETRY_LOCK): + if (manifest_path / lock_file).exists(): + res = run_command( + f"cd {manifest_path} && uv export --format requirements-txt --no-dev 2>/dev/null" + f" | uv pip install -r - --python {venv_python}" + ) + if res is not None: + return lock_file + + # Project manifests: uv pip install . resolves and installs all declared deps + for manifest_name in (PYPROJECT_TOML, SETUP_PY, SETUP_CFG): + if (manifest_path / manifest_name).exists(): + run_command(f"cd {manifest_path} && uv pip install . --python {venv_python}") + return manifest_name + + # Pipfile: requires pipenv; skip silently if not available + if (manifest_path / PIPFILE).exists(): + res = run_command( + f"cd {manifest_path} && pipenv requirements 2>/dev/null" + f" | uv pip install -r - --python {venv_python}" + ) + if res is not None: + return PIPFILE + + return None + + def _install_from_requirements_txt(self, req_txt: Path, manifest_path: Path, + site_packages: Optional[Path]) -> None: + """Install dependencies line-by-line from requirements.txt (original behaviour).""" + with open(req_txt, 'r') as manifest: for line in tqdm(manifest): if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): self.install_dependency(line, manifest_path) @@ -1616,6 +1688,15 @@ def install_dependencies(self, manifest_path: Path): package_name = re.split(r'[=>< \n]', line.strip())[0] self._fallback_if_stub_only(package_name, site_packages) + def _write_installed_packages(self, manifest_path: Path) -> None: + """Write a freeze-format snapshot of the venv to installed_packages.txt.""" + pip_freeze = run_command(f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip list --format=freeze") + if pip_freeze: + (manifest_path / INSTALLED_PACKAGES_FILE).write_text(pip_freeze) + logger.info("Wrote installed packages snapshot to %s/%s", manifest_path, INSTALLED_PACKAGES_FILE) + else: + logger.warning("Could not generate installed packages list for %s", manifest_path) + def install_dependency(self, dependency, repo_path): dependency = dependency.strip() valid_signs = ['==', '>=', '<=', '!='] diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index f6cca7386..30380dc1b 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -35,10 +35,26 @@ _credential_id_ctx, fetch_and_decrypt_credential, ) +from exploit_iq_commons.utils.dep_tree import INSTALLED_PACKAGES_FILE, TRANSITIVE_ENV_NAME from exploit_iq_commons.utils.transitive_code_searcher_tool import ( TransitiveCodeSearcher, ) +# Maximum number of .py files a site-packages package may contain before it +# is excluded from automatic indexing. +_SITE_PKG_MAX_PY_FILES: int = 150 + +# Directory-name suffixes/names to skip when scanning site-packages. +_SITE_PKG_SKIP_SUFFIXES: frozenset[str] = frozenset({".dist-info", ".egg-info"}) +_SITE_PKG_SKIP_DIRS: frozenset[str] = frozenset({ + "__pycache__", + "ansible_collections", # excluded: exceeds file-count threshold by a large margin + "tests", + "test", + "docs", + "doc", +}) + PathLike = typing.Union[str, os.PathLike] @@ -426,8 +442,18 @@ def yield_blobs(self) -> typing.Iterator[Blob]: for exc in self.exclude or {}: exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) - # Filter out files that are not in the repo - # include_files = include_files.intersection(all_files_in_repo) + # Always include installed_packages.txt when present so that Code + # Keyword Search can answer "is package X installed?" for transitive deps. + installed_pkg_file = base_path / INSTALLED_PACKAGES_FILE + if installed_pkg_file.is_file(): + include_files.add(INSTALLED_PACKAGES_FILE) + logger.debug("Including %s in document index", INSTALLED_PACKAGES_FILE) + + # Include Python source from site-packages so that CCA and Code Keyword + # Search can trace transitive call chains across package boundaries. + # Packages exceeding _SITE_PKG_MAX_PY_FILES .py files and known noisy + # directories are excluded to bound indexing cost. + self._add_site_packages_blobs(base_path, include_files) # Take the include files and remove the exclude files. final_files = include_files - exclude_files @@ -449,3 +475,43 @@ def yield_blobs(self) -> typing.Iterator[Blob]: logger.warning("Failed to read blob for '%s'. Ignoring this file. Error: %s", abs_file_path, e) else: logger.debug("Skipping path as it is a directory, not a file: '%s'", abs_file_path) + + @staticmethod + def _add_site_packages_blobs(base_path: Path, include_files: set[str]) -> None: + """Add Python source files from transitive_env site-packages to include_files. + + Only packages with at most ``_SITE_PKG_MAX_PY_FILES`` .py files are + indexed. Known heavy or noisy directories are skipped. + Files inside ``__pycache__`` sub-directories are always excluded. + """ + added_pkgs: list[str] = [] + skipped_pkgs: list[str] = [] + + for sp_dir in base_path.glob(f"{TRANSITIVE_ENV_NAME}/lib/*/site-packages"): + if not sp_dir.is_dir(): + continue + for pkg_dir in sp_dir.iterdir(): + if not pkg_dir.is_dir(): + continue + # Skip metadata directories and known noisy dirs + if any(pkg_dir.name.endswith(sfx) for sfx in _SITE_PKG_SKIP_SUFFIXES): + continue + if pkg_dir.name in _SITE_PKG_SKIP_DIRS: + continue + py_files = [ + f for f in pkg_dir.rglob("*.py") + if "__pycache__" not in f.parts + ] + if len(py_files) <= _SITE_PKG_MAX_PY_FILES: + for f in py_files: + include_files.add(str(f.relative_to(base_path))) + added_pkgs.append(pkg_dir.name) + else: + skipped_pkgs.append(f"{pkg_dir.name}({len(py_files)} files)") + + if added_pkgs: + logger.info("Indexed %d site-packages package(s) for transitive analysis: %s", + len(added_pkgs), ", ".join(added_pkgs)) + if skipped_pkgs: + logger.info("Skipped %d oversized site-packages package(s): %s", + len(skipped_pkgs), ", ".join(skipped_pkgs)) From ef81d7cfaf054d01dc29e4392d678d5b827f4ea7 Mon Sep 17 00:00:00 2001 From: etsien Date: Thu, 4 Jun 2026 13:31:06 -0400 Subject: [PATCH 252/286] fix uv venv dependency issue --- .tekton/on-pull-request.yaml | 9 ++++--- src/exploit_iq_commons/utils/dep_tree.py | 32 ++++++++++++++---------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index e15cb65f5..34c1a1380 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -204,7 +204,7 @@ spec: script: | #!/bin/bash set -eou pipefail - + print_banner() { echo echo "----------- ${1} -----------" @@ -219,10 +219,12 @@ spec: # Mark the workspace as safe before any tool (uv, git, etc.) touches it. git config --global --add safe.directory /workspace/source - # Install uv and Python 3.12 to shared PVC + # Install uv print_banner "Installing uv" curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.local/bin/env + source $HOME/.local/bin/env + # Download Python 3.12 to the shared PVC before creating the venv. + # UV_PYTHON_INSTALL_DIR alone does not trigger a download during uv venv. uv python install 3.12 print_banner "CREATING AND ACTIVATING TEST ENV" @@ -292,7 +294,6 @@ spec: # This is handled in the Makefile's lint-pr target and should be reverted after migration. make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME - print_banner "RUNNING UNIT TESTS" make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index f3a91a986..d69bc7a82 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1602,12 +1602,27 @@ def _try_file(path: Path, extractor) -> str | None: return None + def _ensure_venv(self, manifest_path: Path) -> str: + """Ensure transitive_env exists with a working python binary.""" + venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' + if Path(venv_python).exists(): + return venv_python + logger.warning("Venv python not found at %s — creating venv", venv_python) + python_version = self.determine_python_version(str(manifest_path)) + if not python_version: + import sys + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + logger.info("Python version undetermined; using current interpreter %s", python_version) + logger.info("Creating transitive_env with Python %s using uv", python_version) + run_command(f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}") + return venv_python + def install_dependencies(self, manifest_path: Path): """Install Python dependencies for the given repository into a virtual environment. - Calls :meth:`determine_python_version` to select the interpreter. Creates - a ``transitive_env`` venv and installs packages from whichever manifest - format is present, trying formats in this priority order: + Calls :meth:`_ensure_venv` to select the interpreter and create the venv. + Installs packages from whichever manifest format is present, trying formats + in this priority order: 1. ``requirements.txt``: line-by-line install (original behaviour) 2. ``uv.lock`` or ``poetry.lock``: ``uv export | uv pip install -r -`` @@ -1622,16 +1637,7 @@ def install_dependencies(self, manifest_path: Path): Args: manifest_path: Absolute path to the root of the cloned repository. """ - python_version = self.determine_python_version(str(manifest_path)) - if python_version: - logger.info("Creating transitive_env with Python %s using uv", python_version) - cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}" - else: - logger.warning("Python version undetermined for %s; using uv default interpreter", manifest_path) - cmd = f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME}" - run_command(cmd) - - venv_python = f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python" + venv_python = self._ensure_venv(manifest_path) site_packages = self._find_site_packages(manifest_path) installed_via = self._install_from_best_manifest(manifest_path, venv_python, site_packages) From 77113150426b32d33990f1ba65476c0129514978 Mon Sep 17 00:00:00 2001 From: etsien Date: Fri, 5 Jun 2026 21:19:20 -0400 Subject: [PATCH 253/286] bugfix --- src/vuln_analysis/utils/output_formatter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/utils/output_formatter.py b/src/vuln_analysis/utils/output_formatter.py index 6d3acf591..c01b9d9be 100644 --- a/src/vuln_analysis/utils/output_formatter.py +++ b/src/vuln_analysis/utils/output_formatter.py @@ -104,7 +104,7 @@ def _add_header(markdown_content, model_dict: AgentMorpheusOutput): """ input_image = model_dict.input.image # iterate over a list of dict objects, with vuln_id and ghsa being 2 keys in each element - for output in model_dict.output: + for output in model_dict.output.analysis: cve_id = output.vuln_id markdown_content[cve_id].append(f"# Vulnerability Analysis Report for {cve_id}") markdown_content[cve_id].append(f"> **Container Analyzed:** `{input_image.name}:{input_image.tag}`\n\n") @@ -279,7 +279,7 @@ def _add_table_of_contents(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id checklist = entry.checklist markdown_content[cve_id].append("### Checklist ") @@ -313,7 +313,7 @@ def _add_checklist_info(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id checklist = entry.checklist if checklist: @@ -424,7 +424,7 @@ def _add_vulnerability_analysis(markdown_content, model_dict: AgentMorpheusOutpu None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id summary = entry.summary justification = entry.justification From 16e58b368d8d6532a0cbe56510563e28529054d4 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 11 Jun 2026 08:14:05 +0300 Subject: [PATCH 254/286] chore: fix indetation in pipelinerun after merge conflicts Signed-off-by: Zvi Grinberg --- .tekton/on-pull-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 34c1a1380..fc1547d9d 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -222,7 +222,7 @@ spec: # Install uv print_banner "Installing uv" curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.local/bin/env + source $HOME/.local/bin/env # Download Python 3.12 to the shared PVC before creating the venv. # UV_PYTHON_INSTALL_DIR alone does not trigger a download during uv venv. uv python install 3.12 From 1699875d6729d85609586fc9e1a0a1058e6a8a95 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 11 Jun 2026 09:18:43 +0300 Subject: [PATCH 255/286] fix: prevents potential command injection attack vector through shell tests: remove obsolete python test case Signed-off-by: Zvi Grinberg --- src/exploit_iq_commons/utils/dep_tree.py | 17 ++++++++++------- tests/test_python_version_detection.py | 16 ---------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index d69bc7a82..c931f3706 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1659,11 +1659,14 @@ def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, # Lock files: export to requirements format then pipe to uv pip install for lock_file in (UV_LOCK, POETRY_LOCK): if (manifest_path / lock_file).exists(): - res = run_command( - f"cd {manifest_path} && uv export --format requirements-txt --no-dev 2>/dev/null" - f" | uv pip install -r - --python {venv_python}" - ) - if res is not None: + working_dir = manifest_path + create_requirements_command = ["uv","export","--format","requirements-txt","no-dev","2>/dev/null"] + requirements_txt_result = run_command(args=create_requirements_command, cwd=working_dir) + + if requirements_txt_result is not None: + run_command(args=["uv","pip","install","-r","-", "--python", venv_python], + input_data=requirements_txt_result) + return lock_file # Project manifests: uv pip install . resolves and installs all declared deps @@ -1674,11 +1677,11 @@ def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, # Pipfile: requires pipenv; skip silently if not available if (manifest_path / PIPFILE).exists(): - res = run_command( + requirements_txt_result = run_command( f"cd {manifest_path} && pipenv requirements 2>/dev/null" f" | uv pip install -r - --python {venv_python}" ) - if res is not None: + if requirements_txt_result is not None: return PIPFILE return None diff --git a/tests/test_python_version_detection.py b/tests/test_python_version_detection.py index 6032209ea..85f7dad24 100644 --- a/tests/test_python_version_detection.py +++ b/tests/test_python_version_detection.py @@ -271,19 +271,3 @@ def test_python2_project(self, builder, tmp_path): "setup.py": "from setuptools import setup\nsetup(python_requires='>=2.7,<3')\n", }) assert builder.determine_python_version(str(repo)) == "2.7" - - -class TestInstallDependenciesVersionGuard: - - def test_uses_python_flag_when_version_known(self, builder, tmp_path, monkeypatch): - (tmp_path / "requirements.txt").write_text("requests\n") - (tmp_path / "pyproject.toml").write_text("[project]\nrequires-python = \">=3.9\"\n") - calls = [] - monkeypatch.setattr( - "exploit_iq_commons.utils.dep_tree.run_command", - lambda args, cwd="", input_data=None: calls.extend(args) or "", - ) - builder.install_dependencies(tmp_path) - assert any("--python" in c for c in calls) and any("3.9" in c for c in calls) , f"calls: {calls}" -# def run_command(args: list[str], cwd: str | Path | None = None, input_data: str | None = None) -> str | None: - From 5243bc17b422fc824591f3b20b0282781694f8ba Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 11 Jun 2026 16:41:35 +0300 Subject: [PATCH 256/286] fix: tailor all python commands to the new run_command function' signature Signed-off-by: Zvi Grinberg --- src/exploit_iq_commons/utils/dep_tree.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index c931f3706..48bc355b8 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -1614,7 +1614,7 @@ def _ensure_venv(self, manifest_path: Path) -> str: python_version = f"{sys.version_info.major}.{sys.version_info.minor}" logger.info("Python version undetermined; using current interpreter %s", python_version) logger.info("Creating transitive_env with Python %s using uv", python_version) - run_command(f"cd {manifest_path} && uv venv {TRANSITIVE_ENV_NAME} --python {python_version}") + run_command(["uv", "venv" ,TRANSITIVE_ENV_NAME, "--python", python_version] ,cwd=manifest_path) return venv_python def install_dependencies(self, manifest_path: Path): @@ -1672,16 +1672,14 @@ def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, # Project manifests: uv pip install . resolves and installs all declared deps for manifest_name in (PYPROJECT_TOML, SETUP_PY, SETUP_CFG): if (manifest_path / manifest_name).exists(): - run_command(f"cd {manifest_path} && uv pip install . --python {venv_python}") + run_command([ "uv", "pip", "install", "." , "--python" "venv_python"] , cwd=manifest_path) return manifest_name # Pipfile: requires pipenv; skip silently if not available if (manifest_path / PIPFILE).exists(): - requirements_txt_result = run_command( - f"cd {manifest_path} && pipenv requirements 2>/dev/null" - f" | uv pip install -r - --python {venv_python}" - ) + requirements_txt_result = run_command(["pipenv","requirements","2>/dev/null"], cwd=manifest_path) if requirements_txt_result is not None: + run_command(["uv","pip","install","-r","-", "--python", venv_python],input_data=requirements_txt_result) return PIPFILE return None @@ -1699,7 +1697,8 @@ def _install_from_requirements_txt(self, req_txt: Path, manifest_path: Path, def _write_installed_packages(self, manifest_path: Path) -> None: """Write a freeze-format snapshot of the venv to installed_packages.txt.""" - pip_freeze = run_command(f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip list --format=freeze") + pip_full_path= f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip" + pip_freeze = run_command([pip_full_path,"list","--format=freeze"]) if pip_freeze: (manifest_path / INSTALLED_PACKAGES_FILE).write_text(pip_freeze) logger.info("Wrote installed packages snapshot to %s/%s", manifest_path, INSTALLED_PACKAGES_FILE) From f66b97d6adeb395aac6d8aaf72e39e8304de047b Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 16 Jun 2026 16:20:50 +0300 Subject: [PATCH 257/286] chore: adjust the liveness probe parameters to be tailored to a single cpu core python interpreter Signed-off-by: Zvi Grinberg --- kustomize/base/exploit_iq_service.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml index ecb45e351..2f99c7411 100644 --- a/kustomize/base/exploit_iq_service.yaml +++ b/kustomize/base/exploit_iq_service.yaml @@ -66,9 +66,19 @@ spec: protocol: TCP containerPort: 8080 livenessProbe: + failureThreshold: 24 + timeoutSeconds: 10 + periodSeconds: 20 httpGet: path: /health port: http +# If still doing problems under heavy cpu loads and heavy concurrent workloads , kindly uncomment the tcpSocket block below, +# and comment out the httpGet group as a temporary workaround. +# Upgrading the agent to python version 3.14 will solve this issue as it will +# be a real parallel multi-threading environment without GIL ( Global Interpreter Lock) restrictions. +# tcpSocket: +# port: 8080 + resources: limits: memory: "8Gi" From 469e21836a90541a683fc04c6ccac6b48772288c Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:22:24 +0300 Subject: [PATCH 258/286] Appeng 5342 - cve verification Adds a pre-analysis verification stage to check if the vulnerable package from CVE intel actually exists in the project's dependencies and whether the installed version falls within the vulnerable range. This enables early bypass with code_not_present when the package is not installed or the version is not affected, avoiding unnecessary LLM analysis. --- .tekton/on-cm-runner.yaml | 2 +- .tekton/on-pull-request.yaml | 2 +- ci/it/integration-tests-input.json | 51 +- kustomize/base/exploit-iq-config.yml | 21 +- kustomize/config-http-openai-local.yml | 19 +- .../data_models/dependencies.py | 16 + src/exploit_iq_commons/data_models/input.py | 6 +- src/exploit_iq_commons/utils/dep_tree.py | 174 ++-- .../utils/document_embedding.py | 126 ++- src/exploit_iq_commons/utils/git_utils.py | 49 + src/exploit_iq_commons/utils/java_utils.py | 121 +++ src/vuln_analysis/configs/config-http-nim.yml | 19 +- .../configs/config-http-openai.yml | 25 +- src/vuln_analysis/configs/config-tracing.yml | 19 +- src/vuln_analysis/configs/config.yml | 23 +- .../functions/cve_check_vuln_deps.py | 2 +- .../functions/cve_clone_and_deps.py | 158 +++ .../functions/cve_segmentation.py | 280 ++++++ .../functions/cve_verify_vuln_package.py | 938 ++++++++++++++++++ src/vuln_analysis/register.py | 138 ++- .../tools/configuration_scanner.py | 4 +- .../tools/tests/test_concurrency.py | 155 ++- .../tests/test_transitive_code_search.py | 28 + src/vuln_analysis/utils/intel_utils.py | 130 ++- src/vuln_analysis/utils/llm_engine_utils.py | 51 +- src/vuln_analysis/utils/lock_file_parsers.py | 243 +++++ src/vuln_analysis/utils/package_matchers.py | 697 +++++++++++++ src/vuln_analysis/utils/version_check.py | 688 +++++++++++++ tests/test_version_check.py | 560 +++++++++++ 29 files changed, 4520 insertions(+), 225 deletions(-) create mode 100644 src/vuln_analysis/functions/cve_clone_and_deps.py create mode 100644 src/vuln_analysis/functions/cve_segmentation.py create mode 100644 src/vuln_analysis/functions/cve_verify_vuln_package.py create mode 100644 src/vuln_analysis/utils/lock_file_parsers.py create mode 100644 src/vuln_analysis/utils/package_matchers.py create mode 100644 src/vuln_analysis/utils/version_check.py create mode 100644 tests/test_version_check.py diff --git a/.tekton/on-cm-runner.yaml b/.tekton/on-cm-runner.yaml index 17c42105d..439d6114e 100644 --- a/.tekton/on-cm-runner.yaml +++ b/.tekton/on-cm-runner.yaml @@ -322,7 +322,7 @@ spec: script: | #!/bin/bash set -e - pip install --no-cache-dir kafka-python pydantic requests + pip install --no-cache-dir kafka-python-ng pydantic requests echo "--- STARTING TRACE COLLECTOR ---" python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify echo "--- DEBUG: Current Directory ---" diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index fc1547d9d..8a27fb357 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -483,7 +483,7 @@ spec: script: | #!/bin/bash set -e - pip install --no-cache-dir kafka-python pydantic requests + pip install --no-cache-dir kafka-python-ng pydantic requests echo "--- STARTING TRACE COLLECTOR ---" python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify echo "--- DEBUG: Current Directory ---" diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json index 971cb50f6..215af15bb 100644 --- a/ci/it/integration-tests-input.json +++ b/ci/it/integration-tests-input.json @@ -2,6 +2,19 @@ "origin": "Agent Repository", "iterations": 1, "tests": [ + { + "language": "go", + "vuln_id": "CVE-2024-51744", + "git": { + "repo": "https://github.com/openshift/assisted-installer", + "ref": "ab9e2ade2f45890033a140b314ef87e367317b51" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], + "skip": false + }, { "language": "c", "vuln_id": "CVE-2025-1094", @@ -28,9 +41,9 @@ "ref": "f792cd138891dc1ead99fd089aa757fbca3aace9" }, "use_sbom": false, - "expected_label": "vulnerable", - "expected_result": "Exploitable", - "allowed_deviation_labels" : ["vulnerable"], + "expected_label": "false_positive", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["false_positive"], "skip": false }, { @@ -38,7 +51,7 @@ "vuln_id": "CVE-2024-49767", "git": { "repo": "https://github.com/TamarW0/example-repo", - "ref": "f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0" + "ref": "714b9fc466d1f17b032a6d40707ee5a143cfc1ab" }, "use_sbom": false, "expected_label": "vulnerable", @@ -46,19 +59,6 @@ "allowed_deviation_labels" : ["vulnerable"], "skip": false }, - { - "language": "go", - "vuln_id": "CVE-2024-51744", - "git": { - "repo": "https://github.com/openshift/assisted-installer", - "ref": "ab9e2ade2f45890033a140b314ef87e367317b51" - }, - "use_sbom": false, - "expected_label": "code_not_reachable", - "expected_result": "Not Exploitable", - "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], - "skip": false - }, { "language": "go", "vuln_id": "CVE-2024-28180", @@ -106,9 +106,22 @@ "ref": "aec80a8feef62f25ff8ff200b0d1ad2244b2a1e0" }, "use_sbom": false, - "expected_label": "code_not_reachable", + "expected_label": "false_positive", "expected_result": "Not Exploitable", - "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","false_positive"], + "skip": false + }, + { + "language": "javascript", + "vuln_id": "CVE-2019-10744", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "551e89592bf42c4d5d5dd82009abc7f5b4e0c601" + }, + "use_sbom": false, + "expected_label": "code_not_present", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","false_positive"], "skip": false } ] diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index abfa7d15b..249ee7471 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -30,15 +30,19 @@ general: project: cve_agent functions: - cve_generate_vdbs: - _type: cve_generate_vdbs - agent_name: cve_agent_executor # Used to determine which tools are enabled + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor embedder_name: nim_embedder base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git base_vdb_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}vdb base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index - base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle - base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -53,9 +57,6 @@ functions: cve_process_sbom: _type: cve_process_sbom - cve_check_vuln_deps : - _type: cve_check_vuln_deps - skip: true cve_checklist: _type: cve_checklist llm_name: checklist_llm @@ -273,11 +274,11 @@ embedders: workflow: _type: cve_agent - cve_generate_vdbs_name: cve_generate_vdbs + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation cve_fetch_intel_name: cve_fetch_intel cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom - cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index fa5b89769..695e48cc0 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -29,15 +29,19 @@ general: # api_key: ${LANGSMITH_API_KEY} functions: - cve_generate_vdbs: - _type: cve_generate_vdbs - agent_name: cve_agent_executor # Used to determine which tools are enabled + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor embedder_name: nim_embedder base_git_dir: .cache/am_cache/git base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index base_pickle_dir: .cache/am_cache/pickle - base_rpm_dir: .cache/am_cache/rpms ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel @@ -49,9 +53,6 @@ functions: endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom - cve_check_vuln_deps : - _type: cve_check_vuln_deps - skip: true cve_checklist: _type: cve_checklist llm_name: checklist_llm @@ -233,11 +234,11 @@ embedders: workflow: _type: cve_agent - cve_generate_vdbs_name: cve_generate_vdbs + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation cve_fetch_intel_name: cve_fetch_intel cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom - cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss diff --git a/src/exploit_iq_commons/data_models/dependencies.py b/src/exploit_iq_commons/data_models/dependencies.py index 1af1325f0..43bcda7b8 100644 --- a/src/exploit_iq_commons/data_models/dependencies.py +++ b/src/exploit_iq_commons/data_models/dependencies.py @@ -28,6 +28,19 @@ class DependencyPackage(BaseModel): relation: typing.Literal["SELF", "DIRECT", "INDIRECT"] | None = None +class CheckedNotVulnerablePackage(BaseModel): + """ + Information about a package that was checked against CVE intel but determined not vulnerable. + + - name: package name that was checked + - version: installed version of the package + - reason: LLM-generated explanation of why the package is not vulnerable + """ + name: str + version: str + reason: str + + class VulnerableSBOMPackage(BaseModel): """ Information about a vulnerable SBOM package and its related vulnerable dependency package. @@ -53,7 +66,10 @@ class VulnerableDependencies(BaseModel): the vulnerable package/version intel for the vuln_id. - vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are vulnerable for a given vuln_id. + - checked_not_vulnerable: list of CheckedNotVulnerablePackage objects, representing packages that were + checked but determined not vulnerable, along with the LLM reasoning. """ vuln_id: str vuln_package_intel_sources: list[str] vulnerable_sbom_packages: list[VulnerableSBOMPackage] + checked_not_vulnerable: list[CheckedNotVulnerablePackage] = [] diff --git a/src/exploit_iq_commons/data_models/input.py b/src/exploit_iq_commons/data_models/input.py index 77a325214..b15e2f09a 100644 --- a/src/exploit_iq_commons/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -37,6 +37,8 @@ from exploit_iq_commons.data_models.info import AgentMorpheusInfo from exploit_iq_commons.data_models.info import SBOMPackage +DEFAULT_FAILURE_REASON = "No failure reason provided" + class SourceDocumentsInfo(HashableModel): """ @@ -173,7 +175,7 @@ class ImageInfoInput(HashableModel): pipeline_mode: PipelineMode = PipelineMode.FULL_PIPELINE """ Controls which investigation path the pipeline takes after process_sbom: - - "full_pipeline": Full transitive analysis (check_vuln_deps -> llm_engine) + - "full_pipeline": Full transitive analysis (verify_vuln_package -> llm_engine) - "package_checker": Focused package vulnerability checker (package_checker -> checker_output) """ target_package: TargetPackage | None = None @@ -215,7 +217,7 @@ class AgentMorpheusInput(HashableModel): validation_alias=AliasChoices("credential_id", "credentialId"), ) code_index_success: bool | None = None - failure_reason: str | None = Field(default="No failure reason provided") + failure_reason: str | None = Field(default=DEFAULT_FAILURE_REASON) class AgentMorpheusEngineInput(BaseModel): diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 48bc355b8..530f99376 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -24,6 +24,7 @@ import subprocess import tarfile import tempfile +import threading import time import tomllib import urllib.error @@ -54,10 +55,26 @@ from exploit_iq_commons.logging.loggers_factory import LoggingFactory from exploit_iq_commons.utils.java_utils import add_missing_jar_string from exploit_iq_commons.utils.java_utils import is_maven_gav +from exploit_iq_commons.utils.java_utils import parse_depgraph_line from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager logger = LoggingFactory.get_agent_logger(__name__) +# Per-repo locking for Go operations that mutate go.mod/go.sum. +# Prevents concurrent 'go mod tidy' from corrupting shared cached repos. +_go_repo_locks: dict[str, threading.Lock] = {} +_go_repo_locks_lock = threading.Lock() + + +def _get_go_repo_lock(manifest_path) -> threading.Lock: + """Get or create a per-repo lock for serializing Go module operations.""" + key = str(manifest_path) + with _go_repo_locks_lock: + if key not in _go_repo_locks: + _go_repo_locks[key] = threading.Lock() + return _go_repo_locks[key] + + ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' TRANSITIVE_ENV_NAME = 'transitive_env' @@ -852,6 +869,82 @@ def get_go_mod_graph_tree(manifest_path) -> str: raise e return process_object.stdout + @staticmethod + def get_resolved_modules(manifest_path) -> dict[str, str]: + """ + Run 'go list -m all' to get MVS-resolved module versions. + + Unlike go.sum (which is a checksum file that may contain multiple versions), + this returns the actual versions selected by Go's Minimal Version Selection + algorithm that will be compiled into the binary. + + Args: + manifest_path: Path to the directory containing go.mod + + Returns: + dict mapping module paths to versions (without 'v' prefix) + + Raises: + RuntimeError: If 'go list -m all' fails + """ + env_with_gowork = {**os.environ, "GOWORK": "off"} + + # Serialize access to the same repo to prevent concurrent 'go mod tidy' + # from corrupting go.mod/go.sum in the shared cache. + repo_lock = _get_go_repo_lock(manifest_path) + with repo_lock: + try: + # First ensure go.mod/go.sum are consistent (required for repos with + # outdated or inconsistent module files in the cache) + tidy_result = subprocess.run( + ["go", "mod", "tidy"], + capture_output=True, + text=True, + cwd=manifest_path, + env=env_with_gowork, + timeout=90, + ) + tidy_failed = tidy_result.returncode != 0 + if tidy_failed: + logger.warning( + "go mod tidy failed at %s (exit %d): %s", + manifest_path, tidy_result.returncode, tidy_result.stderr.strip() + ) + + # Then get the resolved modules + # Use -mod=readonly if tidy succeeded (enforces consistency) + # Use -mod=mod if tidy failed (allows Go to update go.mod, bypasses vendor dir) + mod_flag = "-mod=mod" if tidy_failed else "-mod=readonly" + go_list_cmd = ["go", "list", mod_flag, "-m", "all"] + process_object = subprocess.run( + go_list_cmd, + capture_output=True, + text=True, + cwd=manifest_path, + env=env_with_gowork, + timeout=60, + ) + if process_object.returncode > 0: + raise RuntimeError( + f"'go list -m all' failed at {manifest_path}: {process_object.stderr}") + except subprocess.TimeoutExpired as e: + logger.error("Go command timed out at %s: %s exceeded %ds", manifest_path, e.cmd, e.timeout) + raise RuntimeError(f"Go command timed out at {manifest_path}: command exceeded {e.timeout}s") + except RuntimeError: + raise + except Exception as e: + logger.error(f"Failed to run 'go list -m all' at {manifest_path}: {repr(e)}") + raise + + packages: dict[str, str] = {} + for line in process_object.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2: + module, version = parts[0], parts[1] + packages[module] = version.lstrip('v') + + return packages + def download_go_mod_vendor(self, manifest_path): """ Run the 'go mod vendor' command, in order to download go' app @@ -1091,89 +1184,16 @@ def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[st Expected depgraph token shape (after indentation/branch prefix): groupId:artifactId:version:type:scope Example from your file: - org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile :contentReference[oaicite:3]{index=3} + org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile We return (depth, "groupId:artifactId:version") and ignore type/scope/optional marker. Also tolerates Maven log prefixes like "[INFO] " if they appear. """ - raw = (line or "").rstrip("\n") - if not raw.strip(): - return None, None - - # If Maven stdout and depgraph output got mixed, you may see mid-line "[INFO]" injection. - # Those lines are not safely recoverable as dependency tokens. - if "[INFO]" in raw and not raw.lstrip().startswith("[INFO]"): - return None, None - - s = raw.lstrip() - - # Strip Maven log prefix if present - if s.startswith("[INFO]"): - s = s[6:].lstrip() - - # Skip headers and build noise - if ( - not s - or "Dependency graph:" in s - or s.startswith(("---", "BUILD", "Reactor Summary", "Total time", "Finished at", "Scanning")) - or s.startswith("[") # other log levels like [WARNING], [ERROR], etc. - or ":" not in s - ): - return None, None - - # depgraph indentation blocks ("| " or " ") + optional "+- " or "\- " + rest - m = re.match(r"^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$", s) - if not m: - return None, None - - depth = (len(m.group("indent")) // 3) + (1 if m.group("branch") else 0) - rest = m.group("rest").strip() - - # First token up to whitespace or ')', optionally starting with '(' - m2 = re.match(r"^\(?([^\s\)]+)\)?", rest) - if not m2: + coord = parse_depgraph_line(line) + if coord is None: return None, None - - token = m2.group(1) # e.g. com.google.guava:guava:32.0.1-jre:jar:compile - parts = token.split(":") - - scopes = {"compile", "runtime", "test", "provided", "system", "import"} - if parts and parts[-1] in scopes: - parts = parts[:-1] - - if len(parts) < 3: - return None, None - - group, artifact = parts[0], parts[1] - - # depgraph text format puts version in position 2: - # group:artifact:version:type (scope already removed) - # We detect that by checking whether the last part is a packaging/type marker. - packaging = {"jar", "war", "pom", "bundle", "maven-plugin", "ear", "ejb", "rar", "zip", "test-jar"} - - def looks_like_version(v: str) -> bool: - return any(ch.isdigit() for ch in v) - - version: Optional[str] = None - - # depgraph: group:artifact:version:type - if len(parts) >= 4 and parts[-1] in packaging and looks_like_version(parts[2]): - version = parts[2] - # depgraph (rare): group:artifact:version:type:classifier - elif len(parts) >= 5 and parts[-2] in packaging and looks_like_version(parts[2]): - version = parts[2] - else: - # Fallback for other Maven-like formats where version is last - if looks_like_version(parts[-1]): - version = parts[-1] - elif looks_like_version(parts[2]): - version = parts[2] - else: - return None, None - - coord = f"{group}:{artifact}:{version}" - return depth, coord + return (coord.depth, coord.gav) class PythonDependencyTreeBuilder(DependencyTreeBuilder): DEP_SOURCE_DIR = TRANSITIVE_ENV_NAME diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index 642757b2f..ff0681710 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -44,7 +44,7 @@ from exploit_iq_commons.utils.java_segmenters_with_methods import JavaSegmenterWithMethods from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader -from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path, get_repo_path_with_ref from exploit_iq_commons.logging.loggers_factory import LoggingFactory from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended @@ -381,6 +381,9 @@ def _chunk_documents(self, documents: list[Document]) -> list[Document]: def get_repo_path(self, source_info: SourceDocumentsInfo): """ Returns the path to the git repository for a source document info. + + The path is ref-scoped to prevent conflicts when multiple threads work on + different refs of the same repository concurrently. Parameters ---------- @@ -390,13 +393,121 @@ def get_repo_path(self, source_info: SourceDocumentsInfo): Returns ------- Path - Returns the path to the git repository. + Returns the path to the git repository (ref-scoped). + + Example + ------- + For git_repo='https://github.com/org/repo' and ref='v1.0': + Returns: {git_directory}/https.github.com.org.repo_v1.0 + """ + return get_repo_path_with_ref(self._git_directory, source_info.git_repo, source_info.ref) + + def clone_and_install_dependencies(self, source_info: SourceDocumentsInfo) -> Path: """ - # Sanitize the git repo URL to create a valid filesystem path - # Remove protocol separators and path separators that could cause issues - # Example: 'https://github.com/RHEcosystemAppEng/vulnerability-analysis' -> 'https.github.com.RHEcosystemAppEng.vulnerability-analysis' - sanitized_repo_path = sanitize_git_url_for_path(source_info.git_repo) - return self._git_directory / PurePath(sanitized_repo_path) + Clone the repository and install dependencies without document parsing. + + This is the first phase of the split pipeline: clone repo, install deps, + return repo path. Document parsing/segmentation happens later via + collect_documents_from_cloned(). + + Thread-safe: Uses per-(git_repo, ref) lock to serialize operations. + + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info containing git_repo, ref, etc. + + Returns + ------- + Path + Path to the cloned repository (ref-scoped). + """ + repo_path = self.get_repo_path(source_info) + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + + with repo_lock: + loader = SourceCodeGitLoader( + repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude, + ) + loader.load_repo() + logger.info("clone_and_install_dependencies completed for '%s' ref='%s'", source_info.git_repo, source_info.ref) + + return repo_path + + def collect_documents_from_cloned(self, source_info: SourceDocumentsInfo) -> list[Document]: + """ + Collect and parse documents from an already-cloned repository. + + This is the second phase of the split pipeline: parse/segment files + from a repo that was previously cloned via clone_and_install_dependencies(). + + Thread-safe: Re-acquires per-(git_repo, ref) lock before working. + + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info to collect documents + + Returns + ------- + list[Document] + Returns a list of documents collected from the source document info. + """ + repo_path = self.get_repo_path(source_info) + cache_name = source_info.type if source_info.type != "code" else "" + + documents, documents_were_in_cache = retrieve_from_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents_name=cache_name, + ) + if documents_were_in_cache or len(documents) > 0: + return documents + + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + with repo_lock: + documents, documents_were_in_cache = retrieve_from_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents_name=cache_name, + ) + if documents_were_in_cache or len(documents) > 0: + return documents + + if not repo_path.exists(): + logger.warning( + "collect_documents_from_cloned called but repo not cloned: %s", + repo_path, + ) + return [] + + blob_loader = SourceCodeGitLoader( + repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude, + ) + blob_parser = ExtendedLanguageParser() + + loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + documents = loader.load() + + logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + save_to_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents, + documents_name=cache_name, + ) + return documents def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: """ @@ -418,7 +529,6 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: list[Document] Returns a list of documents collected from the source document info. """ - repo_path = self.get_repo_path(source_info) cache_name = source_info.type if source_info.type != "code" else "" documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, diff --git a/src/exploit_iq_commons/utils/git_utils.py b/src/exploit_iq_commons/utils/git_utils.py index 36c4a7ca8..54f477eba 100644 --- a/src/exploit_iq_commons/utils/git_utils.py +++ b/src/exploit_iq_commons/utils/git_utils.py @@ -27,6 +27,55 @@ def sanitize_git_url_for_path(git_url: str) -> str: return git_url.replace('//', '.').replace('/', '.').replace(':', '') +def sanitize_ref_for_path(ref: str) -> str: + """Sanitizes a git ref (branch, tag, commit) to create a valid filesystem path component. + + Git refs can contain characters that are problematic for filesystem paths: + - Forward slashes (e.g., 'feature/my-branch', 'refs/tags/v1.0') + - Colons (rare but possible) + + Uses reversible encoding: underscores are doubled first, then slashes/colons + become single underscores. This prevents collisions between refs like + 'feature/test' and 'feature_test'. + + Args: + ref: Git reference (branch name, tag name, or commit SHA) + + Returns: + Sanitized string safe for use in filesystem paths + + Example: + 'feature/my-branch' -> 'feature_my-branch' + 'feature_my-branch' -> 'feature__my-branch' + 'v1.0.0' -> 'v1.0.0' + 'refs/tags/v2.0' -> 'refs_tags_v2.0' + """ + return ref.replace('_', '__').replace('/', '_').replace(':', '_') + + +def get_repo_path_with_ref(base_dir: Path, git_url: str, ref: str) -> Path: + """Compute a ref-scoped directory path for a git repository. + + Creates a unique directory path for each (repo, ref) combination to prevent + conflicts when multiple threads work on different refs of the same repo. + + Args: + base_dir: Base directory for git repositories + git_url: Git repository URL + ref: Git reference (branch, tag, or commit SHA) + + Returns: + Path to the ref-scoped repository directory + + Example: + base_dir='/cache/git', git_url='https://github.com/org/repo', ref='v1.0' + -> '/cache/git/https.github.com.org.repo_v1.0' + """ + sanitized_url = sanitize_git_url_for_path(git_url) + sanitized_ref = sanitize_ref_for_path(ref) + return base_dir / f"{sanitized_url}_{sanitized_ref}" + + def get_repo_from_path(base_dir: str, git_repo: str = ".git") -> Repo: """ Utility function for getting GitPython `Repo` object representing a Git repository. diff --git a/src/exploit_iq_commons/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py index 1a1b37786..229bf7dab 100644 --- a/src/exploit_iq_commons/utils/java_utils.py +++ b/src/exploit_iq_commons/utils/java_utils.py @@ -73,6 +73,127 @@ re.X, ) +# Maven coordinate constants for depgraph parsing +MAVEN_SCOPES: frozenset[str] = frozenset({ + "compile", "runtime", "test", "provided", "system", "import" +}) + +MAVEN_PACKAGING_TYPES: frozenset[str] = frozenset({ + "jar", "war", "pom", "bundle", "maven-plugin", "ear", "ejb", "rar", "zip", "test-jar" +}) + +# Regex for parsing depgraph tree indentation +_DEPGRAPH_INDENT_RE = re.compile(r"^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$") +_DEPGRAPH_TOKEN_RE = re.compile(r"^\(?([^\s\)]+)\)?") + + +@dataclass(frozen=True, slots=True) +class DepgraphCoordinate: + """Parsed Maven coordinate from depgraph-maven-plugin output.""" + depth: int + group_id: str + artifact_id: str + version: str + + @property + def gav(self) -> str: + """Return groupId:artifactId:version string.""" + return f"{self.group_id}:{self.artifact_id}:{self.version}" + + +def parse_depgraph_line(line: str) -> Optional[DepgraphCoordinate]: + """ + Parse one line from depgraph-maven-plugin graphFormat=text output. + + Handles formats like: + - org.apache.activemq:artemis-server:2.28.0:jar:compile + - +- org.apache.activemq:artemis-core-client:2.28.0:jar:compile + - | +- io.netty:netty-buffer:4.1.86.Final:jar:compile + - (com.google.guava:guava:31.1-jre:jar:compile) + + Args: + line: A single line from dependency_tree.txt + + Returns: + DepgraphCoordinate with depth and (groupId, artifactId, version), + or None if line is not a valid dependency entry. + """ + raw = (line or "").rstrip("\n") + if not raw.strip(): + return None + + # Mid-line [INFO] injection cannot be safely recovered + if "[INFO]" in raw and not raw.lstrip().startswith("[INFO]"): + return None + + s = raw.lstrip() + + # Strip Maven log prefix if present + if s.startswith("[INFO]"): + s = s[6:].lstrip() + + # Skip headers and build noise + if ( + not s + or "Dependency graph:" in s + or s.startswith(("---", "BUILD", "Reactor Summary", "Total time", "Finished at", "Scanning")) + or s.startswith("[") + or ":" not in s + ): + return None + + # Parse depgraph indentation: "| " or " " blocks + optional "+- " or "\- " + m = _DEPGRAPH_INDENT_RE.match(s) + if not m: + return None + + indent_str = m.group("indent") or "" + depth = (len(indent_str) // 3) + (1 if m.group("branch") else 0) + rest = m.group("rest").strip() + + # Extract first token (coordinate), handling optional parentheses for omitted deps + m2 = _DEPGRAPH_TOKEN_RE.match(rest) + if not m2: + return None + + token = m2.group(1) + parts = token.split(":") + + # Strip scope if present at end + if parts and parts[-1] in MAVEN_SCOPES: + parts = parts[:-1] + + if len(parts) < 3: + return None + + group, artifact = parts[0], parts[1] + + def looks_like_version(v: str) -> bool: + return any(ch.isdigit() for ch in v) + + version: Optional[str] = None + + # depgraph: group:artifact:version:type + if len(parts) >= 4 and parts[-1] in MAVEN_PACKAGING_TYPES and looks_like_version(parts[2]): + version = parts[2] + # depgraph (rare): group:artifact:version:type:classifier (5 parts after scope removal) + elif len(parts) >= 5 and parts[-2] in MAVEN_PACKAGING_TYPES and looks_like_version(parts[2]): + version = parts[2] + # Standard 3-part: group:artifact:version + elif len(parts) == 3 and looks_like_version(parts[2]): + version = parts[2] + else: + # Fallback: try last part, then index 2 + if looks_like_version(parts[-1]): + version = parts[-1] + elif looks_like_version(parts[2]): + version = parts[2] + else: + return None + + return DepgraphCoordinate(depth=depth, group_id=group, artifact_id=artifact, version=version) + + # Matches real Java type headers like "class Foo", not ".class" _JAVA_TYPE_HEADER_RE = re.compile(r'(? AgentMorpheusEngineInput: if message.input.image.analysis_type == AnalysisType.SOURCE: logger.info("Source code analysis requested. Skipping vulnerable dependency check.") - message.info.vulnerable_dependencies = None + # Don't overwrite vulnerable_dependencies - it may have been set by verify_vuln_package return message image = f"{message.input.image.name}:{message.input.image.tag}" diff --git a/src/vuln_analysis/functions/cve_clone_and_deps.py b/src/vuln_analysis/functions/cve_clone_and_deps.py new file mode 100644 index 000000000..a7e2117ab --- /dev/null +++ b/src/vuln_analysis/functions/cve_clone_and_deps.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Early-phase node: clone repositories and install dependencies. + +This node runs early in the pipeline (before verify_vuln_package) to prepare +the codebase. Expensive VDB/indexing work is deferred to cve_segmentation, +which can be skipped if verify_vuln_package determines no CVE is vulnerable. +""" + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context +from exploit_iq_commons.utils.dep_tree import detect_ecosystem + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVECloneAndDepsConfig(FunctionBaseConfig, name="cve_clone_and_deps"): + """ + Configuration for the clone-and-deps node. + + This node clones source repositories and installs dependencies early in + the pipeline, before vulnerability verification. VDB/indexing is deferred. + """ + + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Directory for storing cloned git repositories.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="Directory for storing pickle cache files.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="Directory for storing RPM files.", + ) + ignore_errors: bool = Field( + default=False, + description="Whether to ignore errors during clone/install.", + ) + + +@register_function(config_type=CVECloneAndDepsConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def clone_and_deps(config: CVECloneAndDepsConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusInput + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + + embedder = DocumentEmbedding( + embedding=None, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """ + Clone repositories and install dependencies. + + This is the first phase of the split pipeline. VDB/indexing happens + later in segmentation (if the package turns out to be vulnerable). + """ + trace_id.set(message.scan.id) + source_infos = message.image.source_info + sbom_infos = message.image.sbom_info + failure_reason = "" + + try: + logger.debug( + "clone_and_deps: credential_id=%r scan_id=%s", + message.credential_id, + message.scan.id, + ) + + with credential_context(message.credential_id): + # Configure RPM manager for IMAGE analysis + if message.image.analysis_type == AnalysisType.IMAGE and isinstance( + sbom_infos, ManualSBOMInfoInput + ): + rpm_manager.sbom = sbom_infos.packages + image = f"{message.image.name}:{message.image.tag}" + rpm_manager.container_image = image + + # Clone and install dependencies for each source + code_sources = [si for si in source_infos if si.type == "code"] + for si in code_sources: + try: + embedder.clone_and_install_dependencies(si) + except Exception as e: + logger.warning( + "Error cloning/installing for source %s: %s", + si.git_repo, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + # Detect ecosystem from cloned repo manifests if not provided + if message.image.ecosystem is None and code_sources: + repo_path = embedder.get_repo_path(code_sources[0]) + detected = detect_ecosystem(repo_path) + if detected is not None: + message.image.ecosystem = detected + logger.info( + "Detected ecosystem '%s' from repo manifests", + detected.value, + ) + + except Exception as e: + failure_reason = f"Clone/install failed: {e}" + logger.error( + "clone_and_deps failed for image '%s': %s", + message.image.name, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + if failure_reason: + message.failure_reason = failure_reason + + info = AgentMorpheusInfo() + return AgentMorpheusEngineInput(input=message, info=info) + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusInput, + description="Clone repositories and install dependencies (early phase).", + ) diff --git a/src/vuln_analysis/functions/cve_segmentation.py b/src/vuln_analysis/functions/cve_segmentation.py new file mode 100644 index 000000000..7a1135d8b --- /dev/null +++ b/src/vuln_analysis/functions/cve_segmentation.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Late-phase node: document parsing, VDB creation, and code indexing. + +This node runs late in the pipeline (after verify_vuln_package) so it can +be skipped if no CVE is vulnerable. The repo should already be cloned by +cve_clone_and_deps. +""" + +import json +import os +import time +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.component_ref import FunctionRef +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVESegmentationConfig(FunctionBaseConfig, name="cve_segmentation"): + """ + Configuration for the segmentation node. + + This node builds VDBs and code indexes from already-cloned repositories. + It runs late in the pipeline after verify_vuln_package. + """ + + agent_name: FunctionRef = Field( + description="Name of the agent executor. Used to determine which tools are enabled.", + ) + embedder_name: str = Field(description="Name of embedder to use for VDB creation.") + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Directory for storing cloned git repositories.", + ) + base_vdb_dir: str = Field( + default=".cache/am_cache/vdb", + description="Directory for storing vector database files.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Directory for storing code index files.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="Directory for storing pickle cache files.", + ) + ignore_errors: bool = Field( + default=False, + description="Whether to ignore errors during segmentation.", + ) + ignore_code_embedding: bool = Field( + default=False, + description="Whether to skip building code vector database.", + ) + ignore_code_index: bool = Field( + default=False, + description="Whether to skip building code index.", + ) + + +class DocumentCollectionError(Exception): + """Raised when document parsing/collection fails in segmentation phase.""" + + +@register_function(config_type=CVESegmentationConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def segmentation(config: CVESegmentationConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import SourceDocumentsInfo + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache + from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig + from vuln_analysis.utils.full_text_search import FullTextSearch + + agent_config = builder.get_function_config(config.agent_name) + assert isinstance(agent_config, CVEAgentExecutorToolConfig) + + if ToolNames.CODE_SEMANTIC_SEARCH not in agent_config.tool_names: + logger.info("CODE_SEMANTIC_SEARCH not enabled, setting ignore_code_embedding=True") + config.ignore_code_embedding = True + + if ToolNames.CODE_KEYWORD_SEARCH not in agent_config.tool_names: + logger.info("CODE_KEYWORD_SEARCH not enabled, setting ignore_code_index=True") + config.ignore_code_index = True + + embedding = await builder.get_embedder( + embedder_name=config.embedder_name, + wrapper_type=LLMFrameworkEnum.LANGCHAIN, + ) + + cache_std = StandardLibraryCache.get_instance() + cache_std.set_cache_directory(config.base_pickle_dir) + + embedder = DocumentEmbedding( + embedding=embedding, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + def _create_code_index( + source_infos: list[SourceDocumentsInfo], + embedder: DocumentEmbedding, + output_path: Path, + ) -> bool: + logger.info( + "Collecting documents for code index. Source Infos: %s", + json.dumps([x.model_dump(mode="json") for x in source_infos]), + ) + + output_path = Path(output_path) + if output_path.exists(): + logger.warning("Code index already exists and will be overwritten: %s", output_path) + + documents = [] + for si in source_infos: + try: + documents.extend(embedder.collect_documents_from_cloned(si)) + except Exception as e: + logger.warning("Error collecting documents for %s: %s", si, e, exc_info=True) + raise DocumentCollectionError(e) from e + + if len(documents) == 0: + logger.warning("No documents collected, skipping code index for: %s", source_infos) + return False + + chunked_documents = embedder._chunk_documents(documents) + + logger.info( + "Creating code index. Doc count: %d, Chunks: %d, Location: %s", + len(documents), + len(chunked_documents), + output_path, + ) + + full_text_search = FullTextSearch(cache_path=str(output_path)) + indexing_start_time = time.time() + full_text_search.add_documents_from_langchain_chunks(chunked_documents) + logger.info( + "Completed code indexing in %.2f seconds for '%s'", + time.time() - indexing_start_time, + output_path, + ) + return True + + def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: + code_sources = [si for si in source_infos if si.type == "code"] + if not code_sources: + return None + + index_embedder = DocumentEmbedding( + embedding=None, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + code_index_path = FullTextSearch.get_index_directory( + base_path=config.base_code_index_dir, + hash_value=index_embedder.hash_source_documents_info(code_sources), + ) + + if not code_index_path.exists() or os.environ.get("MORPHEUS_ALWAYS_REBUILD_VDB", "0") == "1": + documents_exist = _create_code_index(code_sources, index_embedder, code_index_path) + if not documents_exist: + return None + else: + logger.info("Cache hit on code index: %s", code_index_path) + + return code_index_path + + async def _arun(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """ + Build VDBs and code indexes from already-cloned repositories. + + This is the second phase of the split pipeline. Repos should already + be cloned by clone_and_deps. + """ + trace_id.set(state.input.scan.id) + message = state.input + source_infos = message.image.source_info + base_image = message.image.name + + vdb_code_path = None + vdb_doc_path = None + code_index_path = None + failure_reason = "" + + try: + logger.debug( + "segmentation: credential_id=%r scan_id=%s", + message.credential_id, + message.scan.id, + ) + + with credential_context(message.credential_id): + vdb_code_path, vdb_doc_path = embedder.build_vdbs( + source_infos, + config.ignore_code_embedding, + ) + + if vdb_code_path is None and not config.ignore_code_embedding: + logger.warning( + "Failed to generate code VDB for image '%s'", + base_image, + ) + + if vdb_doc_path is None: + logger.warning( + "Failed to generate documentation VDB for image '%s'", + base_image, + ) + + if not config.ignore_code_index: + code_index_path = _build_code_index(source_infos) + if code_index_path is None: + logger.warning( + "Failed to generate code index for image '%s'", + base_image, + ) + + except Exception as e: + failure_reason = f"Segmentation failed: {e}" + logger.error( + "segmentation failed for image '%s': %s", + base_image, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + vdb_code_path = str(vdb_code_path) if vdb_code_path else None + vdb_doc_path = str(vdb_doc_path) if vdb_doc_path else None + code_index_path = str(code_index_path) if code_index_path else None + code_index_success = code_index_path is not None + + if not code_index_success: + message.failure_reason = failure_reason + + message.code_index_success = code_index_success + + state.info.vdb = AgentMorpheusInfo.VdbPaths( + code_vdb_path=vdb_code_path, + doc_vdb_path=vdb_doc_path, + code_index_path=code_index_path, + ) + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Build VDBs and code indexes from cloned repositories (late phase).", + ) diff --git a/src/vuln_analysis/functions/cve_verify_vuln_package.py b/src/vuln_analysis/functions/cve_verify_vuln_package.py new file mode 100644 index 000000000..5270790c0 --- /dev/null +++ b/src/vuln_analysis/functions/cve_verify_vuln_package.py @@ -0,0 +1,938 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Verify vulnerable package presence in source code dependencies before LLM analysis. + +This node runs for SOURCE analysis in FULL_PIPELINE mode. It checks if the vulnerable +package (from CVE intel) actually exists in the project's dependency lock files, +and uses LLM to verify if the installed version falls within the vulnerable range. +""" + +from pathlib import Path +from typing import Any + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import BaseModel, Field + +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.dependencies import ( + CheckedNotVulnerablePackage, + DependencyPackage, + VulnerableDependencies, + VulnerableSBOMPackage, +) +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from nat.builder.context import Context +from exploit_iq_commons.utils.dep_tree import Ecosystem +from vuln_analysis.utils.intel_utils import extract_vuln_packages_from_intel + +logger = LoggingFactory.get_agent_logger(__name__) + +INTEL_SOURCE_RHSA = "rhsa" + +# Canonical mapping between Ecosystem enum and system_name strings. +# Single source of truth - derive inverse mapping programmatically. +ECOSYSTEM_TO_SYSTEM_NAME: dict[Ecosystem, str] = { + Ecosystem.GO: "go", + Ecosystem.PYTHON: "pypi", + Ecosystem.JAVASCRIPT: "npm", + Ecosystem.JAVA: "maven", + Ecosystem.C_CPP: "c", +} +SYSTEM_NAME_TO_ECOSYSTEM: dict[str, Ecosystem] = {v: k for k, v in ECOSYSTEM_TO_SYSTEM_NAME.items()} + +# Language ecosystems use module paths (e.g., "go.etcd.io/etcd", "requests"). +# RHSA package_state entries contain RPM package names (e.g., "etcd", "python3-requests") +# which don't map to language module paths and cause false matches. +LANGUAGE_ECOSYSTEM_SYSTEM_NAMES: frozenset[str] = frozenset({"go", "pypi", "npm", "maven"}) + +# Python version extraction patterns for different file formats. +# pyproject.toml: requires-python = ">=3.8" or "~=3.8" or "^3.8" +# Version specifiers (>=, ~=, ^, etc.) appear AFTER the opening quote. +PYPROJECT_PYTHON_VERSION_RE = r'requires-python\s*=\s*["\'][><=~^!]*\s*(\d+\.\d+(?:\.\d+)?)' +# .python-version: bare version like "3.12.0" or "3.12" (no prefix required) +PYTHON_VERSION_FILE_RE = r'^(\d+\.\d+(?:\.\d+)?)' + + +def _is_language_ecosystem(system_name: str) -> bool: + """Check if ecosystem uses language module paths rather than RPM names. + + Language ecosystems (go, python, java, javascript) should not use RHSA + package_state entries, which contain RPM package names that don't map + to language module paths. + + Args: + system_name: The ecosystem identifier (e.g., 'go', 'pypi', 'maven', 'npm') + + Returns: + True if this is a language ecosystem + """ + return system_name in LANGUAGE_ECOSYSTEM_SYSTEM_NAMES + + +def _extract_toolchain_version(repo_path: Path, ecosystem: Ecosystem) -> str | None: + """Extract the language toolchain version from project configuration files. + + Args: + repo_path: Path to the repository root + ecosystem: The ecosystem enum (GO, PYTHON, JAVA, JAVASCRIPT) + + Returns: + The toolchain version string, or None if not found + """ + import re + + if ecosystem == Ecosystem.GO: + go_mod = repo_path / "go.mod" + if go_mod.exists(): + content = go_mod.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"^go\s+(\d+\.\d+(?:\.\d+)?)", content, re.MULTILINE) + if match: + return match.group(1) + + elif ecosystem == Ecosystem.PYTHON: + # Try pyproject.toml first (requires-python = ">=3.8" format) + pyproject = repo_path / "pyproject.toml" + if pyproject.exists(): + content = pyproject.read_text(encoding="utf-8", errors="ignore") + match = re.search(PYPROJECT_PYTHON_VERSION_RE, content, re.IGNORECASE) + if match: + return match.group(1) + logger.debug("pyproject.toml exists but no requires-python found") + + # Fall back to .python-version (bare version like "3.12.0") + pyver_file = repo_path / ".python-version" + if pyver_file.exists(): + content = pyver_file.read_text(encoding="utf-8", errors="ignore").strip() + match = re.match(PYTHON_VERSION_FILE_RE, content) + if match: + return match.group(1) + logger.debug(".python-version exists but no valid version found: %s", content[:50]) + + elif ecosystem == Ecosystem.JAVASCRIPT: + package_json = repo_path / "package.json" + if package_json.exists(): + import json + try: + data = json.loads(package_json.read_text(encoding="utf-8")) + engines = data.get("engines", {}) + node_ver = engines.get("node", "") + match = re.search(r"(\d+\.\d+(?:\.\d+)?)", node_ver) + if match: + return match.group(1) + except json.JSONDecodeError: + pass + + elif ecosystem == Ecosystem.JAVA: + pom_xml = repo_path / "pom.xml" + if pom_xml.exists(): + content = pom_xml.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"(\d+(?:\.\d+)?)", content) + if match: + return match.group(1) + match = re.search(r"(\d+(?:\.\d+)?)", content) + if match: + return match.group(1) + + return None + + +def _get_tracer(): + try: + return Context.get() + except Exception: + return None + + +def _record_version_check_trace( + *, + cve_id: str, + package: str, + installed_version: str, + ecosystem: str, + intel_source: str | None, + compare_tech: str, + is_vulnerable: bool, + reason: str, + classification_path: str | None = None, + hard_reason: str | None = None, + gate_reason: str | None = None, + error: str | None = None, +) -> None: + """Emit a trace span for a single version comparison decision.""" + tracer = _get_tracer() + if not tracer: + return + + input_data = { + "cve_id": cve_id, + "package": package, + "installed_version": installed_version, + "ecosystem": ecosystem, + "intel_source": intel_source, + } + output: dict[str, Any] = { + "compare_tech": compare_tech, + "is_vulnerable": is_vulnerable, + "reason": reason, + } + if classification_path is not None: + output["classification_path"] = classification_path + if hard_reason is not None: + output["hard_reason"] = hard_reason + if gate_reason is not None: + output["gate_reason"] = gate_reason + if error is not None: + output["error"] = error + + with tracer.push_active_function( + "verify_vuln_package_version_check", + input_data=input_data, + ) as span: + span.set_output(output) + + +def _record_stdlib_check_trace( + *, + cve_id: str, + ecosystem: str, + toolchain_version: str, + is_vulnerable: bool, + affected_component: str, + reason: str, + error: str | None = None, +) -> None: + """Emit a trace span for a stdlib/toolchain vulnerability check.""" + tracer = _get_tracer() + if not tracer: + return + + input_data = { + "cve_id": cve_id, + "ecosystem": ecosystem, + "toolchain_version": toolchain_version, + "check_type": "stdlib_fallback", + } + output: dict[str, Any] = { + "is_vulnerable": is_vulnerable, + "affected_component": affected_component, + "reason": reason, + } + if error is not None: + output["error"] = error + + with tracer.push_active_function( + "verify_vuln_package_stdlib_check", + input_data=input_data, + ) as span: + span.set_output(output) + + +# ============================================================ +# LLM Structured Output Models +# ============================================================ + +class VersionComparisonResult(BaseModel): + """Structured output for LLM version comparison.""" + is_vulnerable: bool = Field( + description="True if the installed version falls within the vulnerable range" + ) + reason: str = Field( + description="Brief explanation of the version comparison result" + ) + + +class PackageMatchResult(BaseModel): + """Structured output for LLM package name matching.""" + is_match: bool = Field( + description="True if the SBOM package is the same or a variant of the CVE package" + ) + confidence: str = Field( + description="Confidence level: high, medium, or low" + ) + reason: str = Field( + description="Brief explanation of why the packages match or don't match" + ) + + +# ============================================================ +# CVE Processing Configuration +# ============================================================ + +from dataclasses import dataclass +from typing import Callable, Awaitable + + +@dataclass +class CveProcessingConfig: + """Configuration for CVE processing loop behavior. + + This dataclass parameterizes the differences between IMAGE and SOURCE + analysis paths, allowing a single shared processing loop. + """ + system_name: str + filter_fix_state: bool = False + use_llm_fallback: bool = False + package_matcher: Callable[[str, str, str], Awaitable[tuple[bool, str, str]]] | None = None + stdlib_fallback_checker: Callable[[CveIntel, str, str], Awaitable[tuple[bool, str, str]]] | None = None + toolchain_version: str | None = None + + +# ============================================================ +# LLM Prompts +# ============================================================ + +PACKAGE_MATCH_PROMPT = """You are a Linux/RPM package naming expert. Determine if an SBOM package matches a CVE-affected package name. + +CVE/Intel package name: {intel_package} +SBOM package name: {sbom_package} +SBOM package version: {sbom_version} + +Rules for package matching: +1. RPM packages often have suffixes like -libs, -devel, -common, -utils that indicate variants of the same base package +2. The CVE typically affects the core library, which means all variants (e.g., openssl, openssl-libs, openssl-devel) are potentially affected +3. Consider that "openssl" CVE affects "openssl-libs" (YES match) +4. Consider that "libxml2" CVE affects "libxml2-devel" (YES match) +5. Be careful: "curl" is different from "libcurl" (usually same project but may need to check) +6. Watch for false positives: "python3" is not the same as "python3-pip" + +Determine if the SBOM package is the same software (or a direct variant) as the CVE package.""" + + +# ============================================================ +# Version Comparison Prompt +# ============================================================ + +HARD_VERSION_COMPARISON_PROMPT = """You are a version comparison expert. Determine if an installed package version is vulnerable. + +CVE ID: {cve_id} +NVD/CVE description (primary ground truth): +{description} + +Package: {package} +Installed version: {installed_version} +Gate reason (why deterministic comparison was skipped): {gate_reason} + +Structured intel (reference only — may be wrong stream or incomplete): +- Start (inclusive): {version_start_incl} +- Start (exclusive): {version_start_excl} +- End (inclusive): {version_end_incl} +- End (exclusive): {version_end_excl} +- GHSA vulnerable range: {vulnerable_range} +- First patched version: {first_patched} +- RHSA fix_state: {fix_state} + +Rules: +1. Prefer the CVE description for multi-branch fixes (e.g. PostgreSQL 13.x → 13.19). +2. Do not compare across distro streams (e.g. el7 fix vs el8 install). +3. Letter-suffix upstream versions (e.g. OpenSSL 1.1.1j vs 1.1.1k) require description context. +4. RPM NEVRA/module strings may embed upstream version separately from distro tags. +5. If description clearly shows the installed version is at or past the fix, answer not vulnerable. + +Respond with whether the installed version is vulnerable and explain your reasoning.""" + + +class CVEVerifyVulnPackageConfig(FunctionBaseConfig, name="cve_verify_vuln_package"): + """ + Defines a function that verifies vulnerable package presence in source dependencies. + + For SOURCE analysis, parses lock files (go.sum, etc.) and checks if the CVE's + vulnerable package exists and if the installed version is vulnerable using LLM + comparison. + """ + skip: bool = Field( + default=False, + description="Whether to skip vulnerable package verification" + ) + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Base directory for git repos" + ) + llm_name: str = Field( + default="checklist_llm", + description="Name of the LLM to use for version comparison" + ) + + +@register_function(config_type=CVEVerifyVulnPackageConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_verify_vuln_package(config: CVEVerifyVulnPackageConfig, builder: Builder): + # pylint: disable=unused-argument + from functools import partial + + from langchain_core.messages import HumanMessage + + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.package_matchers import get_package_matcher, PackageMatch + from vuln_analysis.utils.version_check import ( + StdlibVulnerabilityResult, + check_stdlib_vulnerability as _check_stdlib_vulnerability, + ) + + # Get LLM for version comparison, package matching, and stdlib analysis + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + version_comparison_llm = llm.with_structured_output(VersionComparisonResult) + package_match_llm = llm.with_structured_output(PackageMatchResult) + stdlib_vulnerability_llm = llm.with_structured_output(StdlibVulnerabilityResult) + + async def check_version_vulnerable( + package: str, + installed_version: str, + version_info: dict[str, Any], + cve_intel: CveIntel, + ecosystem: str, + ) -> tuple[bool, str]: + """ + Route version check through gate: vendor shortcut, univers, or LLM. + + Returns: + Tuple of (is_vulnerable, reason) + """ + from vuln_analysis.utils.version_check import ( + VersionCheckPath, + classify_version_check, + deterministic_version_check, + get_cve_description, + ) + + description = get_cve_description(cve_intel) + classification = classify_version_check( + installed_version=installed_version, + version_info=version_info, + description=description or None, + ecosystem=ecosystem, + package_name=package, + intel=cve_intel, + ) + trace_kwargs = { + "cve_id": cve_intel.vuln_id, + "package": package, + "installed_version": installed_version, + "ecosystem": ecosystem, + "intel_source": version_info.get("source"), + "classification_path": classification.path.value, + "hard_reason": ( + classification.hard_reason.value + if classification.hard_reason is not None + else None + ), + } + + if classification.path == VersionCheckPath.VENDOR_NOT_AFFECTED: + reason = "RHSA fix_state: package not affected on this distro" + _record_version_check_trace( + compare_tech="vendor_not_affected", + is_vulnerable=False, + reason=reason, + **trace_kwargs, + ) + return False, reason + + if classification.path == VersionCheckPath.EASY: + try: + is_vulnerable, reason = deterministic_version_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + logger.info( + " Easy path verdict: vulnerable=%s, reason=%s", + is_vulnerable, + reason, + ) + _record_version_check_trace( + compare_tech="univers", + is_vulnerable=is_vulnerable, + reason=reason, + **trace_kwargs, + ) + return is_vulnerable, reason + except Exception as e: + logger.warning( + "Deterministic version comparison failed for %s: %s", + package, + e, + exc_info=True, + ) + + gate_reason = ( + classification.hard_reason.value + if classification.hard_reason is not None + else "unknown" + ) + prompt = HARD_VERSION_COMPARISON_PROMPT.format( + cve_id=cve_intel.vuln_id, + description=description or "N/A", + package=package, + installed_version=installed_version, + gate_reason=gate_reason, + version_start_incl=version_info.get("version_start_incl") or "N/A", + version_start_excl=version_info.get("version_start_excl") or "N/A", + version_end_incl=version_info.get("version_end_incl") or "N/A", + version_end_excl=version_info.get("version_end_excl") or "N/A", + vulnerable_range=version_info.get("vulnerable_range") or "N/A", + first_patched=version_info.get("first_patched") or "N/A", + fix_state=version_info.get("fix_state") or "N/A", + ) + + try: + result: VersionComparisonResult = await version_comparison_llm.ainvoke( + [HumanMessage(content=prompt)] + ) + logger.info( + " Hard path verdict: vulnerable=%s, reason=%s", + result.is_vulnerable, + result.reason, + ) + _record_version_check_trace( + compare_tech="llm", + is_vulnerable=result.is_vulnerable, + reason=result.reason, + gate_reason=gate_reason, + **trace_kwargs, + ) + return result.is_vulnerable, result.reason + except Exception as e: + logger.warning( + "LLM version comparison failed for %s: %s", + package, + e, + exc_info=True, + ) + reason = f"LLM comparison failed, assuming vulnerable: {e}" + _record_version_check_trace( + compare_tech="llm", + is_vulnerable=True, + reason=reason, + gate_reason=gate_reason, + error=str(e), + **trace_kwargs, + ) + return True, reason + + async def check_package_match( + intel_package: str, + sbom_package: str, + sbom_version: str + ) -> tuple[bool, str, str]: + """ + Use LLM to check if an SBOM package matches a CVE-affected package name. + + This is used as a fallback when static matching rules don't find a match. + + Returns: + Tuple of (is_match, confidence, reason) + """ + prompt = PACKAGE_MATCH_PROMPT.format( + intel_package=intel_package, + sbom_package=sbom_package, + sbom_version=sbom_version, + ) + + try: + result: PackageMatchResult = await package_match_llm.ainvoke( + [HumanMessage(content=prompt)] + ) + return result.is_match, result.confidence, result.reason + except Exception as e: + logger.warning("LLM package match failed for %s vs %s: %s", + intel_package, sbom_package, e) + return False, "error", f"LLM match failed: {e}" + + async def _invoke_stdlib_llm(prompt: str) -> StdlibVulnerabilityResult: + """Adapter bridging LangChain LLM to StdlibLLMChecker signature.""" + return await stdlib_vulnerability_llm.ainvoke([HumanMessage(content=prompt)]) + + async def _process_cve_intel_loop( + intel_list: list[CveIntel], + matcher, + dependencies: dict[str, str], + version_checker, + proc_config: CveProcessingConfig, + ) -> list[VulnerableDependencies]: + """ + Shared CVE processing loop for both IMAGE and SOURCE analysis paths. + + This function consolidates the duplicated logic for: + 1. Extracting vulnerable packages from intel + 2. Matching against dependencies (with optional LLM fallback) + 3. Checking version vulnerability (with has_version_info guard) + 4. Building VulnerableDependencies result + + Args: + intel_list: List of CveIntel objects to process + matcher: Ecosystem-specific package matcher + dependencies: Dict mapping package names to versions + version_checker: Async function to check version vulnerability + proc_config: Configuration for processing behavior + + Returns: + List of VulnerableDependencies for each CVE + """ + vulnerable_dependencies: list[VulnerableDependencies] = [] + + # Check once if this is a language ecosystem. + # Language ecosystems should skip RHSA package_state entries (RPM names). + is_language_ecosystem = _is_language_ecosystem(proc_config.system_name) + + for intel in intel_list: + vuln_packages = extract_vuln_packages_from_intel(intel) + logger.info("CVE %s: %d vulnerable package(s) in intel", + intel.vuln_id, len(vuln_packages)) + + vuln_sbom_packages: list[VulnerableSBOMPackage] = [] + checked_not_vulnerable: list[CheckedNotVulnerablePackage] = [] + intel_sources: list[str] = [vp["source"] for vp in vuln_packages] + + found_vulnerable = False + for vp in vuln_packages: + if found_vulnerable: + break + + pkg_name = vp["package"] + + # Skip ALL RHSA entries for language ecosystems. + # RHSA contains RPM package names (e.g., "etcd", "golang") which don't + # map to language module paths (e.g., "go.etcd.io/etcd"). Language + # ecosystems should rely on GHSA (proper module paths) and NVD instead. + if vp["source"] == INTEL_SOURCE_RHSA and is_language_ecosystem: + logger.info( + " SKIP [%s]: %s - RHSA not applicable for %s ecosystem", + vp["source"], pkg_name, proc_config.system_name + ) + continue + + # Optionally filter by fix_state (IMAGE path uses this) + if proc_config.filter_fix_state: + fix_state = vp.get("fix_state") + if fix_state and fix_state.lower() in ("not affected", "will not fix", "out of support scope"): + logger.info(" SKIP [%s]: %s fix_state=%s", + vp["source"], pkg_name, fix_state) + continue + + # Use ecosystem-specific matcher to find packages + matches: list[PackageMatch] = matcher.find_package(pkg_name, dependencies) + + # Optionally use LLM fallback when static matching fails (IMAGE path uses this) + if not matches and proc_config.use_llm_fallback and proc_config.package_matcher: + logger.info(" Static match failed for %s, trying LLM fallback...", pkg_name) + candidates = [] + pkg_lower = pkg_name.lower() + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + if (pkg_lower in dep_lower or dep_lower in pkg_lower or + any(part in dep_lower for part in pkg_lower.split("-") if len(part) > 3)): + candidates.append((dep_name, version)) + if len(candidates) >= 5: + break + + for dep_name, version in candidates: + is_match, confidence, reason = await proc_config.package_matcher( + intel_package=pkg_name, + sbom_package=dep_name, + sbom_version=version + ) + if is_match and confidence in ("high", "medium"): + logger.info(" LLM MATCH [%s]: %s @ %s (confidence=%s)", + vp["source"], dep_name, version, confidence) + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=pkg_name, + )) + + if not matches: + logger.info(" NO MATCH [%s]: %s not found in dependencies", + vp["source"], pkg_name) + continue + + # Process each matched package + for match in matches: + logger.info(" MATCH [%s]: %s @ %s", + vp["source"], match.dep_name, match.installed_version) + + is_vulnerable, reason = await version_checker( + package=match.dep_name or pkg_name, + installed_version=match.installed_version or "unknown", + version_info=vp, + cve_intel=intel, + ecosystem=proc_config.system_name, + ) + + if is_vulnerable: + vuln_sbom_packages.append(VulnerableSBOMPackage( + name=match.dep_name or pkg_name, + version=match.installed_version or "unknown", + vulnerable_dependency_package=DependencyPackage( + system=proc_config.system_name, + name=match.dep_name or pkg_name, + version=match.installed_version, + relation="SELF" + ) + )) + found_vulnerable = True + break + else: + checked_not_vulnerable.append(CheckedNotVulnerablePackage( + name=match.dep_name or pkg_name, + version=match.installed_version or "unknown", + reason=reason + )) + + # Stdlib/toolchain fallback: when no packages matched and it's a language ecosystem + no_packages_found = not vuln_sbom_packages and not checked_not_vulnerable + if ( + no_packages_found + and is_language_ecosystem + and proc_config.stdlib_fallback_checker is not None + ): + logger.info( + "CVE %s: No packages matched, checking stdlib/toolchain vulnerability", + intel.vuln_id, + ) + toolchain_ver = proc_config.toolchain_version or "unknown" + is_vulnerable, component, reason = await proc_config.stdlib_fallback_checker( + intel, proc_config.system_name, toolchain_ver + ) + + # Record trace for stdlib vulnerability check + _record_stdlib_check_trace( + cve_id=intel.vuln_id, + ecosystem=proc_config.system_name, + toolchain_version=toolchain_ver, + is_vulnerable=is_vulnerable, + affected_component=component, + reason=reason, + ) + + if is_vulnerable: + vuln_sbom_packages.append(VulnerableSBOMPackage( + name=f"stdlib:{component}", + version=toolchain_ver, + vulnerable_dependency_package=DependencyPackage( + system=proc_config.system_name, + name=f"stdlib:{component}", + version=toolchain_ver, + relation="SELF" + ) + )) + intel_sources.append("llm_stdlib_check") + else: + checked_not_vulnerable.append(CheckedNotVulnerablePackage( + name=f"stdlib:{component}", + version=toolchain_ver, + reason=reason + )) + intel_sources.append("llm_stdlib_check") + + # Create VulnerableDependencies entry for this CVE + vulnerable_dependencies.append(VulnerableDependencies( + vuln_id=intel.vuln_id, + vuln_package_intel_sources=list(set(intel_sources)), + vulnerable_sbom_packages=vuln_sbom_packages, + checked_not_vulnerable=checked_not_vulnerable + )) + + # Log summary for this CVE + if vuln_sbom_packages: + logger.info("CVE %s: %d vulnerable package(s) confirmed", + intel.vuln_id, len(vuln_sbom_packages)) + else: + logger.info("CVE %s: No vulnerable packages confirmed", + intel.vuln_id) + + return vulnerable_dependencies + + async def _handle_cpp_image_analysis( + message: AgentMorpheusEngineInput, + version_checker, + package_matcher, + ) -> AgentMorpheusEngineInput: + """ + Handle C/C++ container IMAGE analysis using SBOM matching and LLM version checking. + + This function: + 1. Gets SBOM packages from RPMDependencyManager + 2. Uses shared _process_cve_intel_loop with IMAGE-specific config + 3. Builds VulnerableDependencies result + """ + from vuln_analysis.utils.package_matchers import CppPackageMatcher + + logger.info("C/C++ IMAGE analysis: using SBOM + LLM version checking") + + # Get SBOM dependencies + matcher = CppPackageMatcher() + dependencies = matcher.get_sbom_dependencies() + logger.info("SBOM contains %d packages", len(dependencies)) + + if not dependencies: + logger.warning("SBOM is empty. Cannot verify vulnerable packages.") + logger.info("=== verify_vuln_package END (empty SBOM) ===") + return message + + # Check if we have intel to process + if not message.info.intel: + logger.warning("No intel available for CVE matching.") + logger.info("=== verify_vuln_package END (no intel) ===") + return message + + # Use shared processing loop with IMAGE-specific config + proc_config = CveProcessingConfig( + system_name="rpm", + filter_fix_state=True, + use_llm_fallback=True, + package_matcher=package_matcher, + ) + vulnerable_dependencies = await _process_cve_intel_loop( + message.info.intel, matcher, dependencies, + version_checker, proc_config + ) + + # Update message with vulnerable_dependencies + message.info.vulnerable_dependencies = vulnerable_dependencies + + logger.info("=== verify_vuln_package END (C/C++ IMAGE - %d CVEs processed) ===", + len(vulnerable_dependencies)) + return message + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + from vuln_analysis.utils.package_matchers import CppPackageMatcher + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + + trace_id.set(message.input.scan.id) + + logger.info("=== verify_vuln_package START ===") + logger.info("analysis_type: %s", message.input.image.analysis_type) + logger.info("ecosystem: %s", message.input.image.ecosystem) + logger.info("vulns count: %d", len(message.input.scan.vulns)) + logger.info("intel count: %d", len(message.info.intel) if message.info.intel else 0) + + if config.skip: + logger.info("config.skip is True. Skipping verify_vuln_package.") + logger.info("=== verify_vuln_package END (passthrough) ===") + return message + + # Get ecosystem + ecosystem = message.input.image.ecosystem + + # Route based on analysis type and ecosystem + if message.input.image.analysis_type == AnalysisType.IMAGE and ecosystem == Ecosystem.C_CPP: + # C/C++ container analysis: use SBOM + LLM version checking + return await _handle_cpp_image_analysis( + message, check_version_vulnerable, check_package_match + ) + elif message.input.image.analysis_type == AnalysisType.SOURCE: + # SOURCE analysis: use lock files (Go/Python/Java) + pass # Continue to existing lock file logic below + else: + # Other analysis types: passthrough + logger.info("Non-SOURCE/non-C++ IMAGE analysis. Skipping.") + logger.info("=== verify_vuln_package END (passthrough) ===") + return message + + # --- SOURCE analysis logic (Go/Python/Java with lock files) --- + + if ecosystem is None: + logger.warning("Ecosystem not set. Cannot parse lock files.") + logger.info("=== verify_vuln_package END (no ecosystem) ===") + return message + + # Get package matcher for ecosystem + try: + matcher = get_package_matcher(ecosystem) + logger.info("Using %s for ecosystem %s", type(matcher).__name__, ecosystem) + except NotImplementedError as e: + logger.info("Ecosystem %s not yet supported: %s", ecosystem, e) + logger.info("=== verify_vuln_package END (unsupported ecosystem) ===") + return message + + # Get repo path from source_info + source_info = message.input.image.source_info + if not source_info: + logger.warning("No source_info available. Cannot locate repo.") + logger.info("=== verify_vuln_package END (no source_info) ===") + return message + + # Use first source_info entry for repo path + first_source = source_info[0] + repo_path = get_repo_path_with_ref( + Path(config.base_git_dir), first_source.git_repo, first_source.ref + ) + + logger.info("Repo path: %s", repo_path) + + # Parse dependencies using ecosystem-specific matcher + try: + dependencies = matcher.parse_dependencies( + repo_path, git_repo=first_source.git_repo, ref=first_source.ref + ) + logger.info("Parsed %d packages from lock file", len(dependencies)) + except FileNotFoundError as e: + logger.warning("Lock file not found: %s", e) + logger.info("=== verify_vuln_package END (no lock file) ===") + return message + except NotImplementedError as e: + logger.warning("Parser not implemented: %s", e) + logger.info("=== verify_vuln_package END (parser not implemented) ===") + return message + + # For each CVE, extract vulnerable packages and check against dependencies + if not message.info.intel: + logger.warning("No intel available for CVE matching.") + logger.info("=== verify_vuln_package END (no intel) ===") + return message + + # Map ecosystem to system name for DependencyPackage + system_name = ECOSYSTEM_TO_SYSTEM_NAME.get(ecosystem, ecosystem.value) + + # Extract toolchain version for stdlib fallback + toolchain_version = _extract_toolchain_version(repo_path, ecosystem) + if toolchain_version: + logger.info("Detected toolchain version: %s", toolchain_version) + + # Use shared processing loop with SOURCE-specific config + # Bind LLM adapter to stdlib checker (dependency injection) + stdlib_checker_with_llm = partial(_check_stdlib_vulnerability, llm_checker=_invoke_stdlib_llm) + proc_config = CveProcessingConfig( + system_name=system_name, + filter_fix_state=False, + use_llm_fallback=False, + stdlib_fallback_checker=stdlib_checker_with_llm, + toolchain_version=toolchain_version, + ) + vulnerable_dependencies = await _process_cve_intel_loop( + message.info.intel, matcher, dependencies, + check_version_vulnerable, proc_config + ) + + # Update message with vulnerable_dependencies + message.info.vulnerable_dependencies = vulnerable_dependencies + + logger.info("=== verify_vuln_package END (SOURCE - %d CVEs processed) ===", + len(vulnerable_dependencies)) + return message + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Verifies vulnerable package presence in source dependencies before LLM analysis." + ) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index d76430e77..eaeef6039 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -27,22 +27,24 @@ from exploit_iq_commons.data_models.checker_status import PackageCheckerStatus, PACKAGE_CHECKER_STATUS_DESCRIPTIONS from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from exploit_iq_commons.data_models.input import AgentMorpheusInput +from exploit_iq_commons.data_models.input import DEFAULT_FAILURE_REASON from exploit_iq_commons.data_models.info import AgentMorpheusInfo from vuln_analysis.data_models.output import AgentMorpheusEngineOutput, AgentMorpheusOutput, JustificationOutput, OutputPayload from vuln_analysis.data_models.state import AgentMorpheusEngineState # pylint: disable=unused-import from vuln_analysis.functions import cve_agent -from vuln_analysis.functions import cve_check_vuln_deps from vuln_analysis.functions import cve_checklist from vuln_analysis.functions import cve_fetch_intel from vuln_analysis.functions import cve_file_output -from vuln_analysis.functions import cve_generate_vdbs +from vuln_analysis.functions import cve_clone_and_deps +from vuln_analysis.functions import cve_segmentation from vuln_analysis.functions import cve_http_output from vuln_analysis.functions import cve_justify from vuln_analysis.functions import cve_package_code_agent from vuln_analysis.functions import cve_checker_segmentation from vuln_analysis.functions import cve_source_acquisition from vuln_analysis.functions import cve_process_sbom +from vuln_analysis.functions import cve_verify_vuln_package from vuln_analysis.functions import cve_summarize from vuln_analysis.functions import cve_checker_report from vuln_analysis.functions import cve_build_agent @@ -72,11 +74,19 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): """ Defines the workflow function for determining the impact of a documented CVEs on a specific project or container. """ - cve_generate_vdbs_name: str = Field(description="Function name to generate vector databases") + cve_clone_and_deps_name: str = Field( + description="Function name for early clone and dependency installation.", + ) + cve_segmentation_name: str = Field( + description="Function name for late segmentation (VDB and code index building).", + ) cve_fetch_intel_name: str = Field(description="Function name to fetch intel") cve_calculate_intel_score_name: str = Field(description="Function name to calculate intel score") cve_process_sbom_name: str = Field(description="Function name to process SBOMs") - cve_check_vuln_deps_name: str = Field(description="Function name to check vulnerable dependencies") + cve_verify_vuln_package_name: str | None = Field( + default=None, + description="Function name to verify vulnerable package presence in source dependencies" + ) cve_checklist_name: str = Field(description="Function name to generate checklist") cve_agent_executor_name: str = Field(description="Function name to run CVE agent on checklist") cve_summarize_name: str = Field(description="Function name to generate summary") @@ -123,11 +133,13 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): from langgraph.graph import StateGraph # Access functions that will be used in this workflow - cve_generate_vdbs_fn = builder.get_function(name=config.cve_generate_vdbs_name) cve_fetch_intel_fn = builder.get_function(name=config.cve_fetch_intel_name) cve_calculate_intel_score_fn = builder.get_function(name=config.cve_calculate_intel_score_name) cve_process_sbom_fn = builder.get_function(name=config.cve_process_sbom_name) - cve_check_vuln_deps_fn = builder.get_function(name=config.cve_check_vuln_deps_name) + cve_verify_vuln_package_fn = ( + builder.get_function(name=config.cve_verify_vuln_package_name) + if config.cve_verify_vuln_package_name else None + ) cve_checklist_fn = builder.get_function(name=config.cve_checklist_name) cve_agent_executor_fn = builder.get_function(name=config.cve_agent_executor_name) cve_summary_fn = builder.get_function(name=config.cve_summarize_name) @@ -159,6 +171,8 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): builder.get_function(name=config.cve_fetch_patches_name) if config.cve_fetch_patches_name else None ) + cve_clone_and_deps_fn = builder.get_function(name=config.cve_clone_and_deps_name) + cve_segmentation_fn = builder.get_function(name=config.cve_segmentation_name) # Define langgraph node functions @catch_pipeline_errors_async @@ -167,16 +181,6 @@ async def add_start_time_node(state: AgentMorpheusInput) -> AgentMorpheusInput: state.scan.started_at = datetime.now(timezone.utc).isoformat() return state - @catch_pipeline_errors_async - async def generate_vdbs_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: - """Generates VDBs based on CVE input""" - - engine_input = await cve_generate_vdbs_fn.ainvoke(state.model_dump()) - result = engine_input.model_dump() - result["code_index_success"] = engine_input.input.code_index_success - result["failure_reason"] = engine_input.input.failure_reason - return result - @catch_pipeline_errors_async async def fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Fetch intel for CVE input""" @@ -197,10 +201,59 @@ async def process_sbom_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEng return await cve_process_sbom_fn.ainvoke(state.model_dump()) @catch_pipeline_errors_async - async def check_vuln_deps_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - """Check for vulnerable dependencies""" + async def verify_vuln_package_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Verify vulnerable package presence in source dependencies""" + if cve_verify_vuln_package_fn: + return await cve_verify_vuln_package_fn.ainvoke(state.model_dump()) + return state - return await cve_check_vuln_deps_fn.ainvoke(state.model_dump()) + # --- Split pipeline nodes (clone_and_deps + segmentation) --- + + @catch_pipeline_errors_async + async def clone_and_deps_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """Clone repositories and install dependencies (early phase).""" + return await cve_clone_and_deps_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async + async def segmentation_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Build VDBs and code indexes from cloned repos (late phase).""" + result = await cve_segmentation_fn.ainvoke(state.model_dump()) + result_dict = result.model_dump() + result_dict["code_index_success"] = result.input.code_index_success + result_dict["failure_reason"] = result.input.failure_reason + return result_dict + + def route_after_verify_vuln_package(state: AgentMorpheusEngineInput) -> str: + """Route to segmentation if any CVE is vulnerable, else skip to llm_engine.""" + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + return "segmentation" + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + return "segmentation" if any_vulnerable else "llm_engine" + + def route_after_clone_and_deps(state: AgentMorpheusEngineInput | AgentMorpheusInput) -> str: + """Route to fetch_intel on success, or failure if clone failed. + + Note: state may be AgentMorpheusEngineInput (has .input.failure_reason) or + AgentMorpheusInput (has .failure_reason directly) depending on LangGraph's + state propagation behavior. + """ + if isinstance(state, AgentMorpheusEngineInput): + failure_reason = state.input.failure_reason + else: + failure_reason = state.failure_reason + + if failure_reason and failure_reason != DEFAULT_FAILURE_REASON: + return "failure" + return "fetch_intel" + + async def failure_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Handles pipeline failure (e.g., clone/install failed).""" + return AgentMorpheusOutput( + input=state.input, + info=state.info, + output=OutputPayload(analysis=[], vex=None), + ) async def checklist_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: """Generates a checklist based on CVE input""" @@ -266,21 +319,6 @@ async def checker_calculate_intel_score_node(state: AgentMorpheusEngineInput) -> """Calculate intel score for CVE input (package checker path).""" return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) - async def check_vdbs_success(state: AgentMorpheusInput) -> str: - """Checks if the VDBs were successfully generated""" - if state.code_index_success: - return "fetch_intel" - else: - return "failure" - - async def failure_node(state: AgentMorpheusInput) -> AgentMorpheusOutput: - """Handles the failure of the pipeline""" - from exploit_iq_commons.data_models.info import AgentMorpheusInfo - from vuln_analysis.data_models.output import OutputPayload - return AgentMorpheusOutput(input=state, info=AgentMorpheusInfo(), output=OutputPayload(analysis=[], vex=None)) - - - @catch_pipeline_errors_async async def source_acquisition_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Acquires source code for the target package (source containers, git fallback).""" @@ -409,7 +447,7 @@ def route_after_add_start_time(state: AgentMorpheusInput): """Route to full pipeline or package checker based on pipeline_mode.""" if state.image.pipeline_mode == PipelineMode.PACKAGE_CHECKER: return "checker_init_state" - return "generate_vdbs" + return "clone_and_deps" # build llm engine subgraph subgraph_builder = StateGraph(AgentMorpheusEngineState) @@ -449,11 +487,12 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): # build parent graph graph_builder = StateGraph(AgentMorpheusOutput, input=AgentMorpheusInput) graph_builder.add_node("add_start_time", add_start_time_node) - graph_builder.add_node("generate_vdbs", generate_vdbs_node) graph_builder.add_node("fetch_intel", fetch_intel_node) graph_builder.add_node("calculate_intel_score_node", calculate_intel_score_node) graph_builder.add_node("process_sbom", process_sbom_node) - graph_builder.add_node("check_vuln_deps", check_vuln_deps_node) + graph_builder.add_node("verify_vuln_package", verify_vuln_package_node) + graph_builder.add_node("clone_and_deps", clone_and_deps_node) + graph_builder.add_node("segmentation", segmentation_node) graph_builder.add_node("llm_engine", call_llm_engine_subgraph_node) graph_builder.add_node("add_completed_time", add_completed_time_node) graph_builder.add_node("output_results", output_results_node) @@ -475,18 +514,33 @@ async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): "add_start_time", route_after_add_start_time, { - "generate_vdbs": "generate_vdbs", + "clone_and_deps": "clone_and_deps", "checker_init_state": "checker_init_state", }, ) - graph_builder.add_conditional_edges("generate_vdbs", check_vdbs_success,{"fetch_intel": "fetch_intel", "failure": "failure"}) + # Split pipeline: clone_and_deps -> [check success] -> fetch_intel -> ... -> verify_vuln_package -> [conditional] -> segmentation -> llm_engine + graph_builder.add_conditional_edges( + "clone_and_deps", + route_after_clone_and_deps, + { + "fetch_intel": "fetch_intel", + "failure": "failure", + }, + ) graph_builder.add_edge("failure", "add_completed_time") - #graph_builder.add_edge("generate_vdbs", "fetch_intel") graph_builder.add_edge("fetch_intel", "calculate_intel_score_node") graph_builder.add_edge("calculate_intel_score_node", "process_sbom") - graph_builder.add_edge("process_sbom", "check_vuln_deps") - graph_builder.add_edge("check_vuln_deps", "llm_engine") + graph_builder.add_edge("process_sbom", "verify_vuln_package") + graph_builder.add_conditional_edges( + "verify_vuln_package", + route_after_verify_vuln_package, + { + "segmentation": "segmentation", + "llm_engine": "llm_engine", + }, + ) + graph_builder.add_edge("segmentation", "llm_engine") graph_builder.add_edge("llm_engine", "add_completed_time") # Package checker path diff --git a/src/vuln_analysis/tools/configuration_scanner.py b/src/vuln_analysis/tools/configuration_scanner.py index 0ed1ada34..a9801d450 100644 --- a/src/vuln_analysis/tools/configuration_scanner.py +++ b/src/vuln_analysis/tools/configuration_scanner.py @@ -27,7 +27,7 @@ from pydantic import Field from exploit_iq_commons.logging.loggers_factory import LoggingFactory -from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY from vuln_analysis.utils.error_handling_decorator import catch_tool_errors from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope, format_app_dep_output @@ -188,7 +188,7 @@ async def _arun(query: str) -> str: for si in source_infos: if not hasattr(si, "git_repo"): continue - repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(si.git_repo) + repo_path = get_repo_path_with_ref(Path(DEFAULT_GIT_DIRECTORY), si.git_repo, si.ref) if not repo_path.is_dir(): continue diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py index f2009ef44..4d0df58c5 100644 --- a/src/vuln_analysis/tools/tests/test_concurrency.py +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -464,4 +464,157 @@ async def test_nonjava_caches_under_repo_key(): await _build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey", _DEFAULT_THRESHOLD) assert repo_key in _searcher_cache, "Non-Java should cache under repo_key" - assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" \ No newline at end of file + assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" + + +# --------------------------------------------------------------------------- +# Split Clone/Segmentation Pipeline Tests +# --------------------------------------------------------------------------- + +class TestRefScopedCloning: + """Tests for ref-scoped directory paths and concurrent cloning.""" + + def test_different_refs_produce_different_paths(self): + """Same repo with different refs should have different directory paths.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + + path_v1 = get_repo_path_with_ref(base_dir, git_url, "v1.0") + path_v2 = get_repo_path_with_ref(base_dir, git_url, "v2.0") + path_main = get_repo_path_with_ref(base_dir, git_url, "main") + + assert path_v1 != path_v2, "Different refs should produce different paths" + assert path_v1 != path_main, "Different refs should produce different paths" + assert path_v2 != path_main, "Different refs should produce different paths" + + assert "v1.0" in str(path_v1) or "v1_0" in str(path_v1) + assert "v2.0" in str(path_v2) or "v2_0" in str(path_v2) + + def test_same_ref_produces_same_path(self): + """Same repo and ref should always produce the same path.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + ref = "v1.0" + + path1 = get_repo_path_with_ref(base_dir, git_url, ref) + path2 = get_repo_path_with_ref(base_dir, git_url, ref) + + assert path1 == path2, "Same repo+ref should always produce same path" + + def test_ref_with_slashes_is_sanitized(self): + """Refs with slashes (e.g., refs/tags/v1.0) should be sanitized for filesystem.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + ref = "refs/tags/v1.0" + + path = get_repo_path_with_ref(base_dir, git_url, ref) + + assert "/" not in path.name or path.name.count("/") == 0, \ + "Ref slashes should be sanitized in directory name" + + +class TestCloneAndDepsLocking: + """Tests for per-repo locking in DocumentEmbedding.""" + + def test_get_repo_lock_returns_same_lock_for_same_key(self): + """Same (git_repo, ref) should return the same lock instance.""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock1 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "main") + lock2 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "main") + + assert lock1 is lock2, "Same repo+ref should return same lock" + + def test_get_repo_lock_returns_different_locks_for_different_refs(self): + """Different refs should get different locks (can clone concurrently).""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock_v1 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "v1.0") + lock_v2 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "v2.0") + + assert lock_v1 is not lock_v2, "Different refs should get different locks" + + def test_get_repo_lock_returns_different_locks_for_different_repos(self): + """Different repos should get different locks.""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock_a = DocumentEmbedding._get_repo_lock("https://github.com/a/repo", "main") + lock_b = DocumentEmbedding._get_repo_lock("https://github.com/b/repo", "main") + + assert lock_a is not lock_b, "Different repos should get different locks" + + +class TestVulnerabilityRouting: + """Tests for conditional routing based on vulnerability status.""" + + def _make_vuln_dep(self, has_vulns: bool): + """Create a mock vulnerable dependency.""" + mock = MagicMock() + mock.vulnerable_sbom_packages = ["pkg-1.0"] if has_vulns else [] + return mock + + def _make_engine_input(self, vuln_deps): + """Create a mock AgentMorpheusEngineInput with vulnerable_dependencies.""" + mock = MagicMock() + mock.info.vulnerable_dependencies = vuln_deps + return mock + + def test_route_to_segmentation_when_any_vulnerable(self): + """Should route to segmentation if any CVE has vulnerable packages.""" + state = self._make_engine_input([ + self._make_vuln_dep(has_vulns=False), + self._make_vuln_dep(has_vulns=True), + self._make_vuln_dep(has_vulns=False), + ]) + + vuln_deps = state.info.vulnerable_dependencies + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + + assert any_vulnerable is True, "Should detect at least one vulnerable" + + def test_route_to_llm_engine_when_none_vulnerable(self): + """Should skip segmentation if no CVE has vulnerable packages.""" + state = self._make_engine_input([ + self._make_vuln_dep(has_vulns=False), + self._make_vuln_dep(has_vulns=False), + ]) + + vuln_deps = state.info.vulnerable_dependencies + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + + assert any_vulnerable is False, "Should detect no vulnerables" + + def test_route_to_segmentation_when_vuln_deps_is_none(self): + """Should route to segmentation when vulnerable_dependencies is None (unknown state).""" + state = self._make_engine_input(None) + + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + route = "segmentation" + else: + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + route = "segmentation" if any_vulnerable else "llm_engine" + + assert route == "segmentation", "None vuln_deps should route to segmentation" + + def test_route_to_llm_engine_when_empty_vuln_deps(self): + """Should skip segmentation when vulnerable_dependencies is empty list.""" + state = self._make_engine_input([]) + + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + route = "segmentation" + else: + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + route = "segmentation" if any_vulnerable else "llm_engine" + + assert route == "llm_engine", "Empty vuln_deps should skip segmentation" diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index 48276726e..b6b5180c1 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -20,8 +20,36 @@ from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder +from exploit_iq_commons.utils.document_embedding import DocumentEmbedding +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path, get_repo_path_with_ref from pathlib import Path + +@pytest.fixture(autouse=True) +def patch_repo_path_with_fallback(): + """ + Patch get_repo_path to fall back to old cache format for existing CI caches. + + New format: {base_dir}/{sanitized_url}_{sanitized_ref} + Old format: {base_dir}/{sanitized_url} + """ + def get_repo_path_with_fallback(self, source_info): + new_path = get_repo_path_with_ref( + self._git_directory, source_info.git_repo, source_info.ref + ) + if new_path.exists(): + return new_path + + old_path = self._git_directory / sanitize_git_url_for_path(source_info.git_repo) + if old_path.exists(): + return old_path + + return new_path + + with patch.object(DocumentEmbedding, 'get_repo_path', get_repo_path_with_fallback): + yield + + transitive_code_search_runner_coroutine = None diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index c88e06c7e..7be93359f 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -21,9 +21,12 @@ from pydpkg import Dpkg from pydpkg.exceptions import DpkgVersionError -from exploit_iq_commons.data_models.cve_intel import CveIntelNvd,CveIntel +from typing import Any + +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd, CveIntel +from vuln_analysis.utils.package_identifier import _RPM_NEVRA_RE from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY -from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -393,7 +396,7 @@ def validate_go_vendor_packages(source_info, candidate_packages): code_si = next((si for si in source_info if si.type == "code"), None) if code_si is None: return candidate_packages, [] - repo_path = Path(DEFAULT_GIT_DIRECTORY) / sanitize_git_url_for_path(code_si.git_repo) + repo_path = get_repo_path_with_ref(Path(DEFAULT_GIT_DIRECTORY), code_si.git_repo, code_si.ref) vendor_path = repo_path / "vendor" if not vendor_path.is_dir(): return candidate_packages, [] @@ -696,3 +699,124 @@ def filter_context_to_package(critical_context: list[str], selected: str, all_ca entry = re.sub(r"\s+", " ", entry).strip() filtered.append(entry) return filtered + + +def extract_vuln_packages_from_intel(cve_intel: CveIntel) -> list[dict[str, Any]]: + """ + Extract vulnerable package names and version ranges from intel sources. + + Args: + cve_intel: CveIntel object containing NVD, GHSA, and RHSA data + + Returns: + List of dicts with package info: + { + "source": "nvd" | "ghsa" | "rhsa", + "package": str, + "ecosystem": str | None, + "version_start_incl": str | None, + "version_start_excl": str | None, + "version_end_incl": str | None, + "version_end_excl": str | None, + "vulnerable_range": str | None, # GHSA format like "< 2.9.2" + "first_patched": str | None, + "fix_state": str | None, # RHSA fix_state ("affected", "not affected", etc.) + } + """ + packages: list[dict[str, Any]] = [] + + # Extract from NVD configurations + if cve_intel.nvd and cve_intel.nvd.configurations: + for config in cve_intel.nvd.configurations: + packages.append({ + "source": "nvd", + "package": config.package, + "ecosystem": config.system, + "version_start_incl": config.versionStartIncluding, + "version_start_excl": config.versionStartExcluding, + "version_end_incl": config.versionEndIncluding, + "version_end_excl": config.versionEndExcluding, + "vulnerable_range": None, + "first_patched": None, + "fix_state": None, + }) + + # Extract from GHSA vulnerabilities + if cve_intel.ghsa and cve_intel.ghsa.vulnerabilities: + for vuln in cve_intel.ghsa.vulnerabilities: + # Handle both dict and object formats + if isinstance(vuln, dict): + pkg_info = vuln.get("package", {}) + pkg_name = pkg_info.get("name") if isinstance(pkg_info, dict) else None + pkg_ecosystem = pkg_info.get("ecosystem") if isinstance(pkg_info, dict) else None + vuln_range = vuln.get("vulnerable_version_range") + first_patched = vuln.get("first_patched_version") + else: + pkg_info = getattr(vuln, "package", {}) + pkg_name = pkg_info.get("name") if isinstance(pkg_info, dict) else None + pkg_ecosystem = pkg_info.get("ecosystem") if isinstance(pkg_info, dict) else None + vuln_range = getattr(vuln, "vulnerable_version_range", None) + first_patched = getattr(vuln, "first_patched_version", None) + + if pkg_name: + packages.append({ + "source": "ghsa", + "package": pkg_name, + "ecosystem": pkg_ecosystem, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + "vulnerable_range": vuln_range, + "first_patched": first_patched, + "fix_state": None, + }) + + # Extract from RHSA package_state (affected packages with fix status) + if cve_intel.rhsa and cve_intel.rhsa.package_state: + for ps in cve_intel.rhsa.package_state: + if ps.package_name: + packages.append({ + "source": "rhsa", + "package": ps.package_name, + "ecosystem": "rpm", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + "vulnerable_range": None, + "first_patched": None, + "fix_state": ps.fix_state, + }) + + # Extract from RHSA affected_release (fixed versions) + if cve_intel.rhsa and hasattr(cve_intel.rhsa, 'affected_release') and cve_intel.rhsa.affected_release: + for entry in cve_intel.rhsa.affected_release: + pkg_str = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if pkg_str: + # Parse NEVRA format: name-[epoch:]version-release.arch + m = _RPM_NEVRA_RE.match(pkg_str) + if m: + name = m.group(1) + version = m.group(3) + release_arch = m.group(4) + release = release_arch.rsplit(".", 1)[0] if "." in release_arch else release_arch + ver_rel = f"{version}-{release}" + else: + name = pkg_str + ver_rel = None + + packages.append({ + "source": "rhsa", + "package": name, + "ecosystem": "rpm", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": ver_rel, # Fixed version = end of vulnerable range + "vulnerable_range": f"< {ver_rel}" if ver_rel else None, + "first_patched": ver_rel, + "fix_state": None, + }) + + return packages diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 5191d2d49..41eaf1899 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -20,7 +20,7 @@ from ordered_set import OrderedSet from exploit_iq_commons.data_models.common import AnalysisType -from exploit_iq_commons.data_models.dependencies import VulnerableDependencies +from exploit_iq_commons.data_models.dependencies import CheckedNotVulnerablePackage, VulnerableDependencies from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from exploit_iq_commons.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusEngineOutput @@ -59,9 +59,9 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE # Scan through the VDC output for CVE's to run through the agent if message.info.vulnerable_dependencies is None: - vdc_run_agent = [True for _ in am_input.scan.vulns] + cve_version_checker_run_agent = [True for _ in am_input.scan.vulns] else: - vdc_run_agent = [ + cve_version_checker_run_agent = [ len(v.vulnerable_sbom_packages) > 0 or len(v.vuln_package_intel_sources) == 0 for v in message.info.vulnerable_dependencies ] @@ -70,7 +70,7 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] # Create the filter list by combining the VDC and intel criteria - run_agent = [all(tup) for tup in zip(vdc_run_agent, sufficient_intel_run_agent)] + run_agent = [all(tup) for tup in zip(cve_version_checker_run_agent, sufficient_intel_run_agent)] vuln_ids = [vuln.vuln_id for vuln in am_input.scan.vulns] vulns_for_agent = [] @@ -92,9 +92,10 @@ def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusE logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(vulns_for_agent)) - return AgentMorpheusEngineState(code_vdb_path=message.info.vdb.code_vdb_path, - doc_vdb_path=message.info.vdb.doc_vdb_path, - code_index_path=message.info.vdb.code_index_path, + vdb = message.info.vdb + return AgentMorpheusEngineState(code_vdb_path=vdb.code_vdb_path if vdb else None, + doc_vdb_path=vdb.doc_vdb_path if vdb else None, + code_index_path=vdb.code_index_path if vdb else None, cve_intel=filtered_intel, original_input=message) @@ -231,20 +232,37 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: cvss=cvss) -def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: - summary = ("The VulnerableDependencyChecker did not find any vulnerable packages " - "or dependencies in the SBOM.") +def build_no_vuln_packages_output( + vuln_id: str, + checked_not_vulnerable: list[CheckedNotVulnerablePackage] | None = None +) -> AgentMorpheusEngineOutput: + if checked_not_vulnerable: + package_details = "; ".join( + f"{pkg.name}@{pkg.version}: {pkg.reason}" for pkg in checked_not_vulnerable + ) + summary = (f"Packages were checked but determined not vulnerable. " + f"Details: {package_details}") + justification_reason = (f"The following packages were checked and found not vulnerable: " + f"{package_details}") + response_text = (f"The VulnerableDependencyChecker found matching packages but the LLM " + f"determined they are not vulnerable. Details: {package_details}") + else: + summary = ("The VulnerableDependencyChecker did not find any vulnerable packages " + "or dependencies in the SBOM.") + justification_reason = "No vulnerable packages or dependencies were detected in the SBOM." + response_text = ("The VulnerableDependencyChecker did not find any vulnerable packages " + "or dependencies in the SBOM and so the agent was bypassed.") + justification = JustificationOutput(label="false_positive", - reason="No vulnerable packages or dependencies were detected in the SBOM.", + reason=justification_reason, status="FALSE") cvss = None - + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[ ChecklistItemOutput(input="Agent bypassed: no vulnerable packages detected. Checklist not generated.", - response=("The VulnerableDependencyChecker did not find any vulnerable packages " - "or dependencies in the SBOM and so the agent was bypassed."), + response=response_text, intermediate_steps=None) ], summary=summary, @@ -300,11 +318,13 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, if vulnerable_dependencies is None: vdc_skipped_vulns = [] + vdc_map: dict[str, VulnerableDependencies] = {} else: vdc_skipped_vulns = [ v.vuln_id for v in vulnerable_dependencies if len(v.vulnerable_sbom_packages) == 0 and len(v.vuln_package_intel_sources) > 0 ] + vdc_map = {v.vuln_id: v for v in vulnerable_dependencies} deficient_intel = [i.vuln_id for i in message.info.intel if not i.has_sufficient_intel_for_agent] @@ -339,7 +359,8 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, elif vuln_id in poor_quality_intel_vul: output.append(build_low_intel_score_output(vuln_id, poor_quality_intel_vul[vuln_id])) elif vuln_id in vdc_skipped_vulns: - output.append(build_no_vuln_packages_output(vuln_id)) + checked_not_vulnerable = vdc_map[vuln_id].checked_not_vulnerable if vuln_id in vdc_map else None + output.append(build_no_vuln_packages_output(vuln_id, checked_not_vulnerable)) else: assert False, "CVE has vulnerable dependencies but there is no workflow output." diff --git a/src/vuln_analysis/utils/lock_file_parsers.py b/src/vuln_analysis/utils/lock_file_parsers.py new file mode 100644 index 000000000..6f3941cd1 --- /dev/null +++ b/src/vuln_analysis/utils/lock_file_parsers.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Lock file parsers for extracting dependency {package: version} mappings. + +Each ecosystem has its own parser function that can be imported directly. +""" + +import os +import re +import subprocess +from pathlib import Path + +from exploit_iq_commons.utils.dep_tree import TRANSITIVE_ENV_NAME, run_command +from exploit_iq_commons.utils.java_utils import parse_depgraph_line +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + + +def _pep503_normalize(name: str) -> str: + """Normalize package name per PEP 503. + + Converts all runs of dashes, underscores, and dots to a single dash, + and lowercases the result. This ensures equivalent package names + compare equal (e.g., zope.interface == zope-interface == zope_interface). + """ + return re.sub(r'[-_.]', '-', name.lower()) + +logger = LoggingFactory.get_agent_logger(__name__) + + +# ============================================================ +# IMPLEMENTED PARSERS +# ============================================================ + +def parse_go_modules(repo_path: Path) -> dict[str, str]: + """ + Get resolved Go module versions using 'go list -m all'. + + This runs Go's Minimal Version Selection (MVS) algorithm to determine + the actual versions that will be compiled into the binary, rather than + parsing go.sum which may contain superseded versions. + + Delegates to GoDependencyTreeBuilder to avoid duplicating Go command + execution logic. + + Args: + repo_path: Path to the repository root containing go.mod + + Returns: + dict mapping module paths to versions (without 'v' prefix) + + Raises: + RuntimeError: If 'go list -m all' fails + """ + from exploit_iq_commons.utils.dep_tree import GoDependencyTreeBuilder + + packages = GoDependencyTreeBuilder.get_resolved_modules(repo_path) + logger.debug("Parsed %d packages from 'go list -m all'", len(packages)) + return packages + + +def parse_python_venv(repo_path: Path) -> dict[str, str]: + """ + Parse installed Python dependencies from venv using uv pip freeze. + + Extracts package names and versions from an installed virtual environment + by running `uv pip freeze`. Package names are normalized per PEP 503 + (dashes, underscores, and dots all become dashes, lowercased) for + consistent matching. + + Args: + repo_path: Path to repository containing the transitive_env venv + + Returns: + dict mapping normalized package names to versions + + Raises: + FileNotFoundError: If venv doesn't exist + """ + venv_path = repo_path / TRANSITIVE_ENV_NAME + if not venv_path.exists(): + raise FileNotFoundError(f"Virtual environment not found at {venv_path}") + + venv_python = venv_path / "bin" / "python" + cmd = ['uv', 'pip', 'freeze', '--python', str(venv_python)] + output = run_command(cmd) + + packages: dict[str, str] = {} + if not output: + logger.warning("uv pip freeze returned empty output for %s", venv_path) + return packages + + for line in output.splitlines(): + line = line.strip() + if not line: + continue + + # Skip editable installs (-e) + if line.startswith('-e'): + logger.debug("Skipping editable install: %s", line) + continue + + # Skip direct URL references (package @ url) + if ' @ ' in line: + logger.debug("Skipping URL reference: %s", line) + continue + + # Parse package==version format + if '==' in line: + name, version = line.split('==', 1) + normalized_name = _pep503_normalize(name) + packages[normalized_name] = version + + logger.debug("Parsed %d packages from Python venv", len(packages)) + return packages + + +# ============================================================ +# JAVA ECOSYSTEM +# ============================================================ + + +def parse_dependency_tree_txt(repo_path: Path) -> dict[str, str]: + """ + Parse dependency_tree.txt generated by depgraph-maven-plugin. + + The file is generated by JavaDependencyTreeBuilder.build_tree() using: + mvn com.github.ferstl:depgraph-maven-plugin:aggregate -DgraphFormat=text + + Format: groupId:artifactId:version:type:scope (indented tree structure) + Example: org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile + + Args: + repo_path: Path to directory containing dependency_tree.txt + + Returns: + dict mapping groupId:artifactId to version + + Raises: + FileNotFoundError: If dependency_tree.txt doesn't exist + """ + import re + + tree_file = repo_path / "dependency_tree.txt" + if not tree_file.exists(): + raise FileNotFoundError(f"dependency_tree.txt not found at {tree_file}") + + packages: dict[str, str] = {} + + for line in tree_file.read_text().splitlines(): + coord = _extract_maven_coordinate(line) + if coord: + group, artifact, version = coord + ga_key = f"{group}:{artifact}" + if ga_key not in packages: + packages[ga_key] = version + + logger.debug("Parsed %d packages from dependency_tree.txt", len(packages)) + return packages + + +def _extract_maven_coordinate(line: str) -> tuple[str, str, str] | None: + """ + Extract (groupId, artifactId, version) from a depgraph-maven-plugin line. + + Handles formats like: + - org.apache.activemq:artemis-server:2.28.0:jar:compile + - +- org.apache.activemq:artemis-core-client:2.28.0:jar:compile + - | +- io.netty:netty-buffer:4.1.86.Final:jar:compile + - (com.google.guava:guava:31.1-jre:jar:compile) + + Args: + line: A single line from dependency_tree.txt + + Returns: + Tuple of (groupId, artifactId, version) or None if not parseable + """ + coord = parse_depgraph_line(line) + if coord is None: + return None + return (coord.group_id, coord.artifact_id, coord.version) + + +def _generate_dependency_tree_txt(repo_path: Path) -> bool: + """ + Generate dependency_tree.txt using depgraph-maven-plugin. + + Uses the same command as JavaDependencyTreeBuilder.build_tree() but without + the targetIncludes filter (we want all dependencies for verification). + + Args: + repo_path: Path to the repository root containing pom.xml + + Returns: + True if successful, False otherwise. + """ + mvn_cmd = "./mvnw" if (repo_path / "mvnw").exists() else "mvn" + output_file = repo_path / "dependency_tree.txt" + + settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH") + + cmd = [ + mvn_cmd, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DoutputFileName=dependency_tree.txt", + "-DclasspathScope=runtime", + ] + + if settings_path: + cmd.insert(2, "-s") + cmd.insert(3, settings_path) + + try: + result = subprocess.run( + cmd, + cwd=repo_path, + capture_output=True, + timeout=300, + ) + return result.returncode == 0 and output_file.exists() + except (subprocess.TimeoutExpired, FileNotFoundError) as e: + logger.warning("depgraph-maven-plugin failed: %s", e) + return False + + diff --git a/src/vuln_analysis/utils/package_matchers.py b/src/vuln_analysis/utils/package_matchers.py new file mode 100644 index 000000000..245c78277 --- /dev/null +++ b/src/vuln_analysis/utils/package_matchers.py @@ -0,0 +1,697 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Ecosystem-specific package matchers for verifying vulnerable package presence. + +Each ecosystem has different naming conventions and lock file formats. +This module provides a unified interface for matching vulnerable packages +from CVE intel against project dependencies. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +import json +import os +import shutil +import subprocess +import tempfile + +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.lock_file_parsers import _pep503_normalize + +logger = LoggingFactory.get_agent_logger(__name__) + + +@dataclass +class PackageMatch: + """Result of matching a vulnerable package against dependencies.""" + found: bool + dep_name: str | None = None + installed_version: str | None = None + intel_package: str | None = None + source: str | None = None + version_info: dict[str, Any] = field(default_factory=dict) + + +class EcosystemPackageMatcher(ABC): + """Abstract base for ecosystem-specific package matching.""" + + @property + @abstractmethod + def ecosystem(self) -> Ecosystem: + """Return the ecosystem this matcher handles.""" + pass + + @abstractmethod + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse lock file to extract {package: version} mapping. + + Args: + repo_path: Path to the repository root + git_repo: Optional git repository URL (for locking) + ref: Optional git ref (for locking) + + Returns: + dict mapping package names to versions + + Raises: + FileNotFoundError: If lock file doesn't exist + """ + pass + + @abstractmethod + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against dependencies. + + Handles ecosystem-specific naming differences (e.g., Go module paths, + Python underscore/dash normalization, Maven groupId:artifactId). + + Args: + intel_package: Package name from CVE intel + dependencies: Dict of {package: version} from lock file + + Returns: + List of PackageMatch objects (empty if no match found) + """ + pass + + +# ============================================================ +# GO ECOSYSTEM +# ============================================================ + +class GoPackageMatcher(EcosystemPackageMatcher): + """ + Go-specific package matcher. + + Handles Go module paths like: + - golang.org/x/net + - github.com/user/repo + - k8s.io/api + + Intel may have full module path or just the package name. + """ + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.GO + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + from vuln_analysis.utils.lock_file_parsers import parse_go_modules + return parse_go_modules(repo_path) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + # Exact match first + if intel_package in dependencies: + return [PackageMatch( + found=True, + dep_name=intel_package, + installed_version=dependencies[intel_package], + intel_package=intel_package, + )] + + intel_lower = intel_package.lower() + + # Go modules: check if intel_package is a suffix of any dependency + # e.g., intel: "golang.org/x/net" should match dep: "golang.org/x/net" + # or intel: "net" might need to match "golang.org/x/net" + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + + # Check exact case-insensitive match + if dep_lower == intel_lower: + return [PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )] + + # Check if intel package is a suffix (for short names) + # e.g., "x/net" matches "golang.org/x/net" + if dep_lower.endswith(f"/{intel_lower}"): + return [PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )] + + return [] + + +# ============================================================ +# PYTHON ECOSYSTEM +# ============================================================ + +class PythonPackageMatcher(EcosystemPackageMatcher): + """ + Python-specific package matcher. + + Handles PyPI naming conventions per PEP 503: + - Dashes, underscores, and dots are all equivalent + (e.g., zope.interface == zope-interface == zope_interface) + - Case insensitivity + + Dependencies are parsed from an installed virtual environment using + `uv pip freeze`. Package names are normalized per PEP 503 for consistent + matching. + """ + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.PYTHON + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + from vuln_analysis.utils.lock_file_parsers import parse_python_venv + return parse_python_venv(repo_path) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + normalized = _pep503_normalize(intel_package) + + if normalized in dependencies: + return [PackageMatch( + found=True, + dep_name=normalized, + installed_version=dependencies[normalized], + intel_package=intel_package, + )] + + return [] + + +# ============================================================ +# JAVASCRIPT ECOSYSTEM +# ============================================================ + +class JavaScriptPackageMatcher(EcosystemPackageMatcher): + """ + JavaScript-specific package matcher. + + Handles npm naming conventions: + - Scoped packages (@angular/core, @types/node) + - Case insensitivity + - Multiple versions of the same package in the dependency tree + + Dependencies are parsed using `npm ls --json --all` which returns the + full installed dependency tree. Multiple versions of the same package + are tracked internally and returned as separate PackageMatch results. + """ + + MAX_MATCHES = 10 + + def __init__(self): + self._all_versions: dict[str, list[str]] = {} + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.JAVASCRIPT + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse installed npm dependencies using npm ls --json --all. + + Args: + repo_path: Path to the repository root containing node_modules + git_repo: Optional git repository URL (unused) + ref: Optional git ref (unused) + + Returns: + dict mapping package names to versions (first-seen version) + + Raises: + FileNotFoundError: If node_modules doesn't exist + """ + node_modules = repo_path / "node_modules" + if not node_modules.exists(): + raise FileNotFoundError(f"node_modules not found at {repo_path}") + + npm_tree = self._get_npm_tree(repo_path) + if npm_tree is None: + return {} + + first_seen: dict[str, str] = {} + self._all_versions = {} + self._flatten_npm_tree(npm_tree, first_seen, self._all_versions) + + logger.debug("Parsed %d packages from npm ls", len(first_seen)) + return first_seen + + def _get_npm_tree(self, repo_path: Path) -> dict | None: + """ + Run npm ls --json --all and return parsed JSON. + + Returns None if npm ls fails or returns invalid JSON. + """ + try: + result = subprocess.run( + ["npm", "ls", "--json", "--all", "--ignore-scripts"], + capture_output=True, + text=True, + cwd=repo_path, + timeout=60, + ) + except subprocess.TimeoutExpired: + logger.error("npm ls timed out after 60s for %s", repo_path) + return None + + if not result.stdout: + logger.warning("npm ls returned empty output for %s", repo_path) + return None + + try: + return json.loads(result.stdout) + except json.JSONDecodeError as e: + logger.warning("Failed to parse npm ls output: %s", e) + return None + + def _flatten_npm_tree( + self, + node: dict, + first_seen: dict[str, str], + all_versions: dict[str, list[str]] + ): + """ + Recursively flatten npm ls JSON tree to extract package versions. + + Populates both first_seen (for interface compatibility) and + all_versions (for multi-version tracking). + + Args: + node: Current node in the npm tree + first_seen: Dict to store first-seen version of each package + all_versions: Dict to store all versions of each package + """ + dependencies = node.get("dependencies", {}) + + for dep_name, dep_info in dependencies.items(): + if not isinstance(dep_info, dict): + continue + + version = dep_info.get("version", "unknown") + + if dep_name not in first_seen: + first_seen[dep_name] = version + + if dep_name not in all_versions: + all_versions[dep_name] = [] + + if version not in all_versions[dep_name]: + all_versions[dep_name].append(version) + + self._flatten_npm_tree(dep_info, first_seen, all_versions) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against npm dependencies. + + Uses cascading match strategy: + 1. Exact match (case-insensitive) + 2. Scoped suffix match: "core" matches "@angular/core" + + Returns PackageMatch for all versions of matching packages + tracked in _all_versions. + """ + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + matched_packages: set[str] = set() + + for dep_name in dependencies.keys(): + dep_lower = dep_name.lower() + + # Strategy 1: Exact match (case-insensitive) + if dep_lower == intel_lower: + matched_packages.add(dep_name) + continue + + # Strategy 2: Scoped suffix match + # e.g., intel "core" matches "@angular/core" + if dep_lower.endswith(f"/{intel_lower}"): + matched_packages.add(dep_name) + continue + + for dep_name in matched_packages: + versions = self._all_versions.get(dep_name, [dependencies.get(dep_name, "unknown")]) + for version in versions: + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.info( + "Intel '%s' matched %d package versions: %s", + intel_package, + len(matches), + ", ".join(f"{m.dep_name}@{m.installed_version}" for m in matches) + ) + + return matches + + +# ============================================================ +# JAVA ECOSYSTEM +# ============================================================ + + +class JavaPackageMatcher(EcosystemPackageMatcher): + """ + Java-specific package matcher. + + Handles Maven coordinates in two formats: + - Full groupId:artifactId from GHSA (e.g., org.apache.logging.log4j:log4j-core) + - Artifact-only names from NVD (e.g., log4j, artemis) + + Uses cascading match strategy: + 1. Exact groupId:artifactId match (if intel contains ":") + 2. Artifact prefix match (for NVD artifact-only names) + + Returns up to MAX_MATCHES results with warning if multiple found. + + Concurrency: Uses lock+copy pattern for dependency_tree.txt to prevent + race conditions where concurrent threads might read a partially-written file. + Reuses DocumentEmbedding._get_repo_lock() for consistent per-repo locking. + """ + + MAX_MATCHES = 5 + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.JAVA + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse Java dependencies using lock+copy pattern for thread safety. + + The dependency_tree.txt file is generated by maven and read by multiple + threads. To prevent race conditions: + 1. Acquire per-repo lock (via DocumentEmbedding._get_repo_lock) + 2. Generate file if missing + 3. Copy to temp file while holding lock + 4. Release lock + 5. Parse from temp file (thread-safe) + 6. Clean up temp file + + Args: + repo_path: Path to the repository root + git_repo: Git repository URL (required for locking) + ref: Git ref (required for locking) + """ + from vuln_analysis.utils.lock_file_parsers import ( + parse_dependency_tree_txt, + _generate_dependency_tree_txt, + ) + + # If git_repo/ref not provided, fall back to unlocked behavior + if not git_repo or not ref: + logger.warning("git_repo/ref not provided to JavaPackageMatcher; skipping lock") + return self._parse_unlocked(repo_path) + + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + repo_lock = DocumentEmbedding._get_repo_lock(git_repo, ref) + temp_dir: str | None = None + + with repo_lock: + tree_file = repo_path / "dependency_tree.txt" + + if not tree_file.exists(): + logger.info("dependency_tree.txt not found, generating via depgraph-maven-plugin") + if not _generate_dependency_tree_txt(repo_path): + logger.warning("Could not resolve Java dependencies for %s", repo_path) + return {} + + if not tree_file.exists(): + logger.warning("dependency_tree.txt still not found after generation for %s", repo_path) + return {} + + # Copy to temp directory while holding the lock to get a consistent snapshot + try: + temp_dir = tempfile.mkdtemp(prefix="java_deps_") + shutil.copy(tree_file, Path(temp_dir) / "dependency_tree.txt") + logger.debug("Copied dependency_tree.txt to temp dir: %s", temp_dir) + except OSError as e: + logger.warning("Failed to copy dependency_tree.txt to temp: %s", e) + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + return {} + + # Parse from temp directory outside the lock (thread-safe, each thread has its own copy) + try: + return parse_dependency_tree_txt(Path(temp_dir)) + except FileNotFoundError: + logger.warning("Temp dependency_tree.txt disappeared: %s", temp_dir) + return {} + finally: + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + + def _parse_unlocked(self, repo_path: Path) -> dict[str, str]: + """Fallback parsing without locking (original behavior).""" + from vuln_analysis.utils.lock_file_parsers import ( + parse_dependency_tree_txt, + _generate_dependency_tree_txt, + ) + + try: + return parse_dependency_tree_txt(repo_path) + except FileNotFoundError: + pass + + logger.info("dependency_tree.txt not found, generating via depgraph-maven-plugin") + if _generate_dependency_tree_txt(repo_path): + return parse_dependency_tree_txt(repo_path) + + logger.warning("Could not resolve Java dependencies for %s", repo_path) + return {} + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + + # Strategy 1: Exact groupId:artifactId match (for GHSA data) + if ":" in intel_package: + # Exact case-sensitive match + if intel_package in dependencies: + return [PackageMatch( + found=True, + dep_name=intel_package, + installed_version=dependencies[intel_package], + intel_package=intel_package, + )] + + # Case-insensitive fallback + for ga, version in dependencies.items(): + if ga.lower() == intel_lower: + return [PackageMatch( + found=True, + dep_name=ga, + installed_version=version, + intel_package=intel_package, + )] + + # No GA match found + return [] + + # Strategy 2: Artifact prefix match (for NVD artifact-only data) + for ga, version in dependencies.items(): + # Extract artifactId from groupId:artifactId + artifact = ga.split(":")[-1] + if artifact.lower().startswith(intel_lower): + matches.append(PackageMatch( + found=True, + dep_name=ga, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.warning( + "Intel '%s' matched %d artifacts: %s", + intel_package, + len(matches), + ", ".join(m.dep_name or "" for m in matches) + ) + + return matches + + +# ============================================================ +# C/C++ ECOSYSTEM +# ============================================================ + +class CppPackageMatcher(EcosystemPackageMatcher): + """ + C/C++ package matcher for container SBOM-based analysis. + + Handles RPM package naming conventions: + - Exact match (case-insensitive): "openssl" matches "openssl" + - Prefix match: "openssl" matches "openssl-libs", "openssl-devel" + - Library suffix variations: handles -libs, -devel, -common suffixes + + For container analysis, dependencies come from SBOM (not lock files). + """ + + MAX_MATCHES = 10 + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.C_CPP + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + For C/C++ container analysis, dependencies come from SBOM not repo. + + This method is called when repo_path is provided (SOURCE analysis), + but for IMAGE analysis, use get_sbom_dependencies() instead. + + Args: + repo_path: Path to the repository root (unused for C/C++) + git_repo: Optional git repository URL (unused) + ref: Optional git ref (unused) + """ + return self.get_sbom_dependencies() + + def get_sbom_dependencies(self) -> dict[str, str]: + """Get dependencies from the SBOM via RPMDependencyManager.""" + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + sbom = RPMDependencyManager.get_instance().sbom + return {pkg.name: pkg.version for pkg in sbom} + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against SBOM dependencies. + + Uses cascading match strategy: + 1. Exact match (case-insensitive) + 2. Prefix match: intel "openssl" matches "openssl-libs" + 3. Base name match: handles packages where intel has full name + + Returns multiple matches when appropriate (e.g., openssl matches + openssl, openssl-libs, openssl-devel). + """ + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + + # Match strategies: + # 1. Exact match (case-insensitive) + # 2. Prefix match - intel "openssl" matches "openssl-libs" + # 3. Reverse prefix - intel "libxml2-devel" matches dep "libxml2" + if (dep_lower == intel_lower or + dep_lower.startswith(intel_lower + "-") or + intel_lower.startswith(dep_lower + "-")): + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.info( + "Intel '%s' matched %d packages: %s", + intel_package, + len(matches), + ", ".join(m.dep_name or "" for m in matches) + ) + + return matches + + +# ============================================================ +# DISPATCHER +# ============================================================ + +PACKAGE_MATCHERS: dict[Ecosystem, type[EcosystemPackageMatcher]] = { + Ecosystem.GO: GoPackageMatcher, + Ecosystem.PYTHON: PythonPackageMatcher, + Ecosystem.JAVASCRIPT: JavaScriptPackageMatcher, + Ecosystem.JAVA: JavaPackageMatcher, + Ecosystem.C_CPP: CppPackageMatcher, +} + + +def get_package_matcher(ecosystem: Ecosystem) -> EcosystemPackageMatcher: + """ + Get the appropriate package matcher for an ecosystem. + + Args: + ecosystem: The ecosystem to get a matcher for + + Returns: + An instance of the appropriate EcosystemPackageMatcher + + Raises: + NotImplementedError: If no matcher is available for the ecosystem + """ + matcher_cls = PACKAGE_MATCHERS.get(ecosystem) + if matcher_cls is None: + raise NotImplementedError(f"No package matcher available for ecosystem: {ecosystem}") + return matcher_cls() diff --git a/src/vuln_analysis/utils/version_check.py b/src/vuln_analysis/utils/version_check.py new file mode 100644 index 000000000..11536bca4 --- /dev/null +++ b/src/vuln_analysis/utils/version_check.py @@ -0,0 +1,688 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Version vulnerability gate: route to univers (easy) or LLM (hard).""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Awaitable, Callable, Literal + +from pydantic import BaseModel, Field + +from univers import versions + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.package_identifier import ( + _extract_dist_tag, + _interpret_fix_state, + _match_package_state_for_distro, +) + +logger = LoggingFactory.get_agent_logger(__name__) + +ECOSYSTEM_RPM = "rpm" +INTEL_VALUE_NOT_APPLICABLE = "N/A" + +_LETTER_SUFFIX_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+[a-z]", re.IGNORECASE) +_PLAIN_SEMVER_RE = re.compile(r"^\d+\.\d+") +_MODULE_NEVRA_MARKER = ".module+" +_MULTI_BRANCH_DESCRIPTION_RE = re.compile( + r"before\s+.+,\s+.+\s+and\s+", + re.IGNORECASE, +) + +KEY_FIRST_PATCHED = "first_patched" +KEY_VULNERABLE_RANGE = "vulnerable_range" +KEY_VERSION_START_INCL = "version_start_incl" +KEY_VERSION_START_EXCL = "version_start_excl" +KEY_VERSION_END_INCL = "version_end_incl" +KEY_VERSION_END_EXCL = "version_end_excl" +KEY_FIX_STATE = "fix_state" + +NVD_RANGE_KEYS = ( + KEY_VERSION_START_EXCL, + KEY_VERSION_END_EXCL, + KEY_VERSION_START_INCL, + KEY_VERSION_END_INCL, +) + +# ============================================================ +# Stdlib/Toolchain Vulnerability Prompt (LLM Hard Path) +# ============================================================ + +STDLIB_VULNERABILITY_PROMPT = """You are a security expert analyzing whether a CVE affects a project's language toolchain/stdlib. + +CVE ID: {cve_id} +Ecosystem: {ecosystem} + +CVE Description: +{description} + +Project's toolchain version: {toolchain_version} +(This is the language version from go.mod, pyproject.toml, package.json, or pom.xml) + +Context: +- This CVE had no matching packages in structured vulnerability databases (GHSA/NVD configurations were empty) +- This suggests it may be a stdlib/toolchain vulnerability affecting the language runtime itself +- Common examples: Go crypto/x509, net/http; Python ssl, urllib; Java security providers + +Analysis Tasks: +1. Identify if this CVE affects a stdlib/runtime component (not a third-party library) +2. Determine which versions of the toolchain are affected based on the description +3. Compare the project's toolchain version against the affected range + +Rules: +1. Pay close attention to statements like "affects only Go 1.24 release candidates" or "fixed in Python 3.11.4" +2. If the CVE only affects pre-release/RC versions and the project uses a stable release, it's NOT vulnerable +3. If the project's toolchain version is at or past the fix version, it's NOT vulnerable +4. If you cannot determine the affected versions from the description, assume vulnerable (fail safe) + +Respond with: +- is_vulnerable: whether the project's toolchain version is affected +- affected_component: the stdlib component (e.g., "crypto/x509", "net/http", "ssl") +- reason: clear explanation of your analysis""" + +ComparisonMode = Literal["first_patched", "nvd_range", "none", "ambiguous"] + + +class VersionCheckPath(str, Enum): + VENDOR_NOT_AFFECTED = "vendor_not_affected" + EASY = "easy" + HARD = "hard" + + +class HardPathReason(str, Enum): + LETTER_SUFFIX_VERSION = "letter_suffix_version" + MODULE_NEVRA = "module_nevra" + MULTI_BRANCH_DESCRIPTION = "multi_branch_description" + CROSS_DIST_INTEL = "cross_dist_intel" + AXIS_MISMATCH = "axis_mismatch" + PARSE_ERROR = "parse_error" + COMPARATOR_DISAGREEMENT = "comparator_disagreement" + NO_STRUCTURED_BOUNDS = "no_structured_bounds" + NON_SEMVER_INSTALLED = "non_semver_installed" + AMBIGUOUS_INTEL = "ambiguous_intel" + + +@dataclass(frozen=True) +class VersionCheckClassification: + path: VersionCheckPath + hard_reason: HardPathReason | None = None + + +class StdlibVulnerabilityResult(BaseModel): + """Structured output for LLM stdlib/toolchain vulnerability analysis. + + Used when no specific packages match from intel (GHSA/NVD empty) but the CVE + may affect the language runtime/stdlib (e.g., Go crypto/x509, Python ssl). + """ + + is_vulnerable: bool = Field( + description="True if the project's toolchain version is affected by this CVE" + ) + affected_component: str = Field( + description="The stdlib/runtime component affected (e.g., 'crypto/x509', 'net/http')" + ) + reason: str = Field( + description="Explanation of why the toolchain is or isn't vulnerable" + ) + + +# Type alias for the LLM checker callable (dependency injection) +StdlibLLMChecker = Callable[ + [str], Awaitable[StdlibVulnerabilityResult] +] + + +def get_cve_description(intel: CveIntel) -> str: + """Collect the best available CVE description text for hard-path LLM prompts. + + Combines the primary description (NVD > GHSA > RHSA details) with the RHSA + statement field when available. The RHSA statement often contains critical + context about affected versions/configurations (e.g., "This vulnerability + affects only the Go 1.24 release candidates"). + """ + parts: list[str] = [] + + if intel.nvd and intel.nvd.cve_description: + parts.append(intel.nvd.cve_description) + elif intel.ghsa: + if intel.ghsa.description: + parts.append(intel.ghsa.description) + elif intel.ghsa.summary: + parts.append(intel.ghsa.summary) + elif intel.rhsa and intel.rhsa.details: + parts.append(" ".join(intel.rhsa.details)) + + if intel.rhsa and intel.rhsa.statement: + parts.append(f"RHSA Statement: {intel.rhsa.statement}") + + return "\n\n".join(parts) + + +async def check_stdlib_vulnerability( + cve_intel: CveIntel, + ecosystem: str, + toolchain_version: str, + llm_checker: StdlibLLMChecker, +) -> tuple[bool, str, str]: + """Check if a stdlib/toolchain CVE affects the project using LLM analysis. + + This is the LLM hard-path for stdlib vulnerabilities. Called when no packages + matched from structured intel (GHSA/NVD empty), suggesting this may be a + language runtime vulnerability (e.g., Go crypto/x509, Python ssl). + + Args: + cve_intel: The CVE intel object containing description and metadata + ecosystem: The ecosystem identifier (e.g., 'go', 'pypi', 'npm', 'maven') + toolchain_version: The project's language version (e.g., '1.21' for Go) + llm_checker: Async callable that invokes the LLM with structured output. + Signature: (prompt: str) -> StdlibVulnerabilityResult + + Returns: + Tuple of (is_vulnerable, affected_component, reason) + """ + description = get_cve_description(cve_intel) + prompt = STDLIB_VULNERABILITY_PROMPT.format( + cve_id=cve_intel.vuln_id, + ecosystem=ecosystem, + description=description or "No description available", + toolchain_version=toolchain_version or "unknown", + ) + + try: + result = await llm_checker(prompt) + logger.info( + "Stdlib vulnerability check for %s: vulnerable=%s, component=%s", + cve_intel.vuln_id, + result.is_vulnerable, + result.affected_component, + ) + return result.is_vulnerable, result.affected_component, result.reason + + except Exception as e: + logger.error( + "LLM stdlib vulnerability check failed for %s", + cve_intel.vuln_id, + exc_info=True, + ) + return True, "unknown", f"LLM check failed, assuming vulnerable: {e}" + + +def extract_dist_from_installed(installed_version: str) -> str | None: + """Extract elN dist tag from an installed version or NVR string.""" + return _extract_dist_tag(installed_version) + + +def classify_version_check( + *, + installed_version: str, + version_info: dict[str, Any], + description: str | None, + ecosystem: str, + package_name: str, + intel: CveIntel | None = None, +) -> VersionCheckClassification: + """Decide which version-check path to use. Does not compare versions.""" + if not installed_version or installed_version == "unknown": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + vendor_classification = _classify_vendor_fix_state( + version_info=version_info, + package_name=package_name, + installed_version=installed_version, + intel=intel, + ) + if vendor_classification is not None: + return vendor_classification + + if _description_has_multi_branch(description): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.MULTI_BRANCH_DESCRIPTION, + ) + + if ecosystem == ECOSYSTEM_RPM: + if _installed_version_is_non_perfect(installed_version, ecosystem): + return _hard_for_installed(installed_version, ecosystem) + + comparison_mode = _comparison_mode(version_info) + if comparison_mode == "none": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + if comparison_mode == "ambiguous": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AMBIGUOUS_INTEL, + ) + + if comparison_mode == "first_patched": + return _classify_first_patched_match( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + return _classify_nvd_range_match( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + +def deterministic_version_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + """Compare versions with univers. Caller must have classified as EASY first.""" + comparison_mode = _comparison_mode(version_info) + if comparison_mode == "first_patched": + return _deterministic_first_patched_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + if comparison_mode == "nvd_range": + return _deterministic_nvd_range_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + msg = "deterministic_version_check called without a supported comparison mode" + logger.error(msg) + raise ValueError(msg) + + +def _deterministic_first_patched_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + if not first_patched: + raise ValueError("first_patched comparison requested but value is missing") + + version_func = _version_func_for_ecosystem(ecosystem) + installed = version_func(installed_version) + reference = version_func(first_patched) + is_vulnerable = installed < reference + return is_vulnerable, ( + f"Installed {installed_version} " + f"{'<' if is_vulnerable else '>='} " + f"first_patched {first_patched}" + ) + + +def _deterministic_nvd_range_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + version_range = [_intel_value(version_info.get(key)) for key in NVD_RANGE_KEYS] + in_range = _check_version_in_range(installed_version, version_range, ecosystem) + return in_range, ( + f"Installed {installed_version} " + f"{'in' if in_range else 'outside'} " + f"NVD range" + ) + + +def _classify_vendor_fix_state( + *, + version_info: dict[str, Any], + package_name: str, + installed_version: str, + intel: CveIntel | None, +) -> VersionCheckClassification | None: + fix_state = version_info.get(KEY_FIX_STATE) + if fix_state and _interpret_fix_state(fix_state) == EnumIdentifyResult.NO: + return VersionCheckClassification(VersionCheckPath.VENDOR_NOT_AFFECTED) + + if intel is None or intel.rhsa is None or not intel.rhsa.package_state: + return None + + target_dist = extract_dist_from_installed(installed_version) + matched = _match_package_state_for_distro( + intel.rhsa.package_state, + package_name, + target_dist, + ) + if matched is None: + return None + + if _interpret_fix_state(matched.fix_state) == EnumIdentifyResult.NO: + return VersionCheckClassification(VersionCheckPath.VENDOR_NOT_AFFECTED) + + return None + + +def _classify_first_patched_match( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> VersionCheckClassification: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + if not first_patched: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + if ecosystem == ECOSYSTEM_RPM: + if _bound_is_non_perfect(first_patched, ecosystem): + return _hard_for_bound(first_patched, ecosystem) + rpm_hard = _classify_rpm_perfect_match(installed_version, first_patched) + if rpm_hard is not None: + return rpm_hard + + if not _versions_parse_cleanly(installed_version, first_patched, ecosystem): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + + return VersionCheckClassification(VersionCheckPath.EASY) + + +def _classify_nvd_range_match( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> VersionCheckClassification: + bound_values = [ + _intel_value(version_info.get(key)) + for key in NVD_RANGE_KEYS + if _intel_value(version_info.get(key)) + ] + if not bound_values: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + if ecosystem == ECOSYSTEM_RPM: + for bound in bound_values: + if _bound_is_non_perfect(bound, ecosystem): + return _hard_for_bound(bound, ecosystem) + for bound in bound_values: + rpm_hard = _classify_rpm_perfect_match(installed_version, bound) + if rpm_hard is not None: + return rpm_hard + + for bound in bound_values: + if not _versions_parse_cleanly(installed_version, bound, ecosystem): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + + return VersionCheckClassification(VersionCheckPath.EASY) + + +def _comparison_mode(version_info: dict[str, Any]) -> ComparisonMode: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + vulnerable_range = _intel_value(version_info.get(KEY_VULNERABLE_RANGE)) + has_nvd_range = any( + _intel_value(version_info.get(key)) is not None + for key in NVD_RANGE_KEYS + ) + + if vulnerable_range: + return "ambiguous" + + if first_patched and has_nvd_range: + return "ambiguous" + + if first_patched: + return "first_patched" + + if has_nvd_range: + return "nvd_range" + + return "none" + + +def _description_has_multi_branch(description: str | None) -> bool: + if not description: + return False + return bool(_MULTI_BRANCH_DESCRIPTION_RE.search(description)) + + +def _installed_version_is_non_perfect(installed_version: str, ecosystem: str) -> bool: + if _MODULE_NEVRA_MARKER in installed_version: + return True + + upstream = _extract_upstream_component(installed_version, ecosystem) + if upstream is None: + return True + if _has_letter_suffix(upstream): + return True + if ecosystem != ECOSYSTEM_RPM and not _looks_like_plain_semver(upstream): + return True + return False + + +def _hard_for_installed(installed_version: str, ecosystem: str) -> VersionCheckClassification: + if _MODULE_NEVRA_MARKER in installed_version: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.MODULE_NEVRA, + ) + + upstream = _extract_upstream_component(installed_version, ecosystem) + if upstream and _has_letter_suffix(upstream): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.LETTER_SUFFIX_VERSION, + ) + + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NON_SEMVER_INSTALLED, + ) + + +def _bound_is_non_perfect(bound: str, ecosystem: str) -> bool: + if _has_letter_suffix(bound): + return True + + if ecosystem != ECOSYSTEM_RPM: + return not _looks_like_plain_semver(bound.lstrip("vV")) + + upstream = _extract_upstream_component(bound, ECOSYSTEM_RPM) + if upstream and _has_letter_suffix(upstream): + return True + + if _extract_dist_tag(bound) is not None: + return False + + return "-" in bound + + +def _hard_for_bound(bound: str, ecosystem: str) -> VersionCheckClassification: + if _has_letter_suffix(bound) or ( + ecosystem == ECOSYSTEM_RPM + and _extract_upstream_component(bound, ECOSYSTEM_RPM) + and _has_letter_suffix(_extract_upstream_component(bound, ECOSYSTEM_RPM) or "") + ): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.LETTER_SUFFIX_VERSION, + ) + + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AXIS_MISMATCH, + ) + + +def _classify_rpm_perfect_match( + installed_version: str, + bound: str, +) -> VersionCheckClassification | None: + installed_dist = extract_dist_from_installed(installed_version) + bound_dist = _extract_dist_tag(bound) + + if bound_dist is not None: + if installed_dist is None or installed_dist != bound_dist: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.CROSS_DIST_INTEL, + ) + if not _versions_parse_cleanly(installed_version, bound, ECOSYSTEM_RPM): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + if not _rpm_comparators_agree(installed_version, bound): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.COMPARATOR_DISAGREEMENT, + ) + return None + + if installed_dist is not None: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AXIS_MISMATCH, + ) + + return None + + +def _intel_value(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + if not text or text == INTEL_VALUE_NOT_APPLICABLE: + return None + return text + + +def _has_letter_suffix(version: str) -> bool: + return bool(_LETTER_SUFFIX_VERSION_RE.match(version.strip())) + + +def _looks_like_plain_semver(version: str) -> bool: + return bool(_PLAIN_SEMVER_RE.match(version.strip())) + + +def _extract_upstream_component(version: str, ecosystem: str) -> str | None: + if ecosystem != ECOSYSTEM_RPM: + return version.lstrip("vV").split("+")[0].split("-")[0] or None + + if _MODULE_NEVRA_MARKER in version: + return version.split("-", 1)[0] or None + + if "-" not in version: + return version + + return version.split("-", 1)[0] or None + + +_ECOSYSTEM_VERSION_MAPPING = { + ECOSYSTEM_RPM: versions.RpmVersion, + "pypi": versions.PypiVersion, + "npm": versions.SemverVersion, + "go": versions.GolangVersion, + "maven": versions.MavenVersion, +} + + +def _version_func_for_ecosystem(ecosystem: str): + return _ECOSYSTEM_VERSION_MAPPING.get(ecosystem, versions.GenericVersion) + + +def _versions_parse_cleanly(installed: str, bound: str, ecosystem: str) -> bool: + version_func = _version_func_for_ecosystem(ecosystem) + try: + version_func(installed) + version_func(bound) + except versions.InvalidVersion: + logger.debug( + "Version parse failed for installed=%r bound=%r ecosystem=%s", + installed, + bound, + ecosystem, + exc_info=True, + ) + return False + return True + + +def _rpm_comparators_agree(installed: str, bound: str) -> bool: + try: + generic_less = versions.GenericVersion(installed) < versions.GenericVersion(bound) + rpm_less = versions.RpmVersion(installed) < versions.RpmVersion(bound) + except versions.InvalidVersion: + logger.debug( + "RPM comparator probe failed for installed=%r bound=%r", + installed, + bound, + exc_info=True, + ) + return False + return generic_less == rpm_less + + +def _check_version_in_range( + version_to_check: str, + version_range: list[str | None], + ecosystem: str, +) -> bool: + """Range check mirroring package_identifier._check_version_in_range.""" + ver_start_excl, ver_end_excl, ver_start_incl, ver_end_incl = version_range + version_func = _version_func_for_ecosystem(ecosystem) + + try: + vtc = version_func(version_to_check) + vsi = version_func(ver_start_incl) if ver_start_incl else None + vse = version_func(ver_start_excl) if ver_start_excl else None + vei = version_func(ver_end_incl) if ver_end_incl else None + vee = version_func(ver_end_excl) if ver_end_excl else None + except versions.InvalidVersion: + logger.debug( + "Range parse failed for version_to_check=%r", + version_to_check, + exc_info=True, + ) + raise + + if vsi and not (vsi <= vtc): + return False + if vse and not (vse < vtc): + return False + if vei and not (vtc <= vei): + return False + if vee and not (vtc < vee): + return False + return True diff --git a/tests/test_version_check.py b/tests/test_version_check.py new file mode 100644 index 000000000..2bfd431ee --- /dev/null +++ b/tests/test_version_check.py @@ -0,0 +1,560 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for version_check gate and deterministic comparison.""" + +import pytest + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelNvd, CveIntelRhsa + +from vuln_analysis.utils.version_check import ( + HardPathReason, + StdlibVulnerabilityResult, + VersionCheckPath, + check_stdlib_vulnerability, + classify_version_check, + deterministic_version_check, + get_cve_description, +) + + +POSTGRES_MULTI_BRANCH_DESCRIPTION = ( + "Versions before PostgreSQL 17.3, 16.7, 15.11, 14.16, and 13.19 are affected." +) + + +class TestClassifyVersionCheck: + def test_lodash_ghsa_first_patched_is_easy(self): + version_info = { + "first_patched": "4.17.21", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="4.17.20", + version_info=version_info, + description=None, + ecosystem="npm", + package_name="lodash", + ) + assert result.path == VersionCheckPath.EASY + + def test_postgresql_module_nevra_is_hard(self): + version_info = { + "first_patched": "9.2.24-9.el7_9", + "vulnerable_range": "< 9.2.24-9.el7_9", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "9.2.24-9.el7_9", + } + result = classify_version_check( + installed_version="13.14-1.module+el8.9.0+21288+3d364c44", + version_info=version_info, + description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ecosystem="rpm", + package_name="postgresql", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason in { + HardPathReason.MODULE_NEVRA, + HardPathReason.MULTI_BRANCH_DESCRIPTION, + } + + def test_openssl_letter_suffix_is_hard(self): + version_info = { + "first_patched": "1.1.1k", + "vulnerable_range": None, + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.1.1k", + } + result = classify_version_check( + installed_version="1.1.1j-2.el8", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="openssl-libs", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason == HardPathReason.LETTER_SUFFIX_VERSION + + def test_cross_dist_rhsa_is_hard(self): + version_info = { + "first_patched": "9.2.24-9.el7_9", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="13.14-1.el8", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="postgresql", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason == HardPathReason.CROSS_DIST_INTEL + + def test_rhsa_not_affected_is_vendor_shortcut(self): + intel = CveIntel( + vuln_id="CVE-2016-8687", + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + version_info = { + "fix_state": None, + "first_patched": None, + "vulnerable_range": None, + "version_start_incl": "3.2.1", + "version_start_excl": None, + "version_end_incl": "3.2.1", + "version_end_excl": None, + } + result = classify_version_check( + installed_version="3.1.2-14.el7_9.1", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="libarchive", + intel=intel, + ) + assert result.path == VersionCheckPath.VENDOR_NOT_AFFECTED + + def test_nvd_range_clearly_outside_is_easy(self): + version_info = { + "first_patched": None, + "vulnerable_range": None, + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.5.0", + } + result = classify_version_check( + installed_version="2.0.0", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_same_dist_rpm_nvr_is_easy(self): + version_info = { + "first_patched": "7.76.1-31.el9", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="7.76.1-26.el9", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="curl", + ) + assert result.path == VersionCheckPath.EASY + + def test_pypi_prerelease_version_is_easy(self): + """PyPI pre-release versions (PEP 440) should use EASY path with PypiVersion.""" + version_info = { + "first_patched": "1.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0a1", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_pypi_post_release_version_is_easy(self): + """PyPI post-release versions (PEP 440) should use EASY path.""" + version_info = { + "first_patched": "2.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.post1", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_npm_prerelease_version_is_easy(self): + """npm semver pre-release versions should use EASY path with SemverVersion.""" + version_info = { + "first_patched": "1.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0-beta.1", + version_info=version_info, + description=None, + ecosystem="npm", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_go_version_is_easy(self): + """Go versions should use EASY path with GolangVersion.""" + version_info = { + "first_patched": "v1.21.5", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="v1.21.0", + version_info=version_info, + description=None, + ecosystem="go", + package_name="somemodule", + ) + assert result.path == VersionCheckPath.EASY + + def test_go_pseudo_version_is_easy(self): + """Go pseudo versions (pre-release commits) should use EASY path with GolangVersion.""" + version_info = { + "first_patched": "v1.5.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="v1.4.1-0.20260512091500-f9e8d7c6b5a4", + version_info=version_info, + description=None, + ecosystem="go", + package_name="somemodule", + ) + assert result.path == VersionCheckPath.EASY + + def test_maven_version_is_easy(self): + """Maven versions should use EASY path with MavenVersion.""" + version_info = { + "first_patched": "2.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0-SNAPSHOT", + version_info=version_info, + description=None, + ecosystem="maven", + package_name="com.example:artifact", + ) + assert result.path == VersionCheckPath.EASY + + +class TestDeterministicVersionCheck: + def test_lodash_is_vulnerable(self): + version_info = {"first_patched": "4.17.21"} + is_vulnerable, reason = deterministic_version_check( + installed_version="4.17.20", + version_info=version_info, + ecosystem="npm", + ) + assert is_vulnerable is True + assert "4.17.21" in reason + + def test_lodash_is_not_vulnerable_at_fix(self): + version_info = {"first_patched": "4.17.21"} + is_vulnerable, _reason = deterministic_version_check( + installed_version="4.17.21", + version_info=version_info, + ecosystem="npm", + ) + assert is_vulnerable is False + + def test_nvd_range_outside_is_not_vulnerable(self): + version_info = { + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.5.0", + } + is_vulnerable, reason = deterministic_version_check( + installed_version="2.0.0", + version_info=version_info, + ecosystem="pypi", + ) + assert is_vulnerable is False + assert "outside" in reason + + +class TestGetCveDescription: + def test_prefers_nvd_description(self): + intel = CveIntel( + vuln_id="CVE-2025-1094", + nvd=CveIntelNvd( + cve_id="CVE-2025-1094", + cve_description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ), + ghsa=CveIntelGhsa( + ghsa_id="GHSA-xxxx-yyyy-zzzz", + summary="summary only", + ), + ) + assert get_cve_description(intel) == POSTGRES_MULTI_BRANCH_DESCRIPTION + + def test_appends_rhsa_statement_to_nvd_description(self): + """RHSA statement should be appended to NVD description when available.""" + rhsa_statement = "This vulnerability affects only the Go 1.24 release candidates." + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability description.", + ), + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "Go crypto/x509 vulnerability description." in result + assert f"RHSA Statement: {rhsa_statement}" in result + assert "\n\n" in result + + def test_appends_rhsa_statement_to_ghsa_description(self): + """RHSA statement should be appended to GHSA description when NVD is absent.""" + rhsa_statement = "Only affects specific configurations." + intel = CveIntel( + vuln_id="CVE-2025-12345", + ghsa=CveIntelGhsa( + ghsa_id="GHSA-abcd-efgh-ijkl", + description="GHSA description text.", + ), + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "GHSA description text." in result + assert f"RHSA Statement: {rhsa_statement}" in result + + def test_rhsa_statement_only_when_no_other_description(self): + """RHSA statement alone should be returned when no other description exists.""" + rhsa_statement = "Package not affected in RHEL 9." + intel = CveIntel( + vuln_id="CVE-2025-99999", + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert result == f"RHSA Statement: {rhsa_statement}" + + def test_no_rhsa_statement_preserves_existing_behavior(self): + """When RHSA statement is None, existing behavior is preserved.""" + intel = CveIntel( + vuln_id="CVE-2025-1094", + nvd=CveIntelNvd( + cve_id="CVE-2025-1094", + cve_description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ), + rhsa=CveIntelRhsa( + details=["Some RHSA details."], + statement=None, + ), + ) + result = get_cve_description(intel) + assert result == POSTGRES_MULTI_BRANCH_DESCRIPTION + assert "RHSA Statement" not in result + + def test_rhsa_details_with_statement(self): + """RHSA details as primary description with statement appended.""" + rhsa_statement = "Upstream fix available in version 2.0." + intel = CveIntel( + vuln_id="CVE-2025-11111", + rhsa=CveIntelRhsa( + details=["First detail.", "Second detail."], + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "First detail. Second detail." in result + assert f"RHSA Statement: {rhsa_statement}" in result + + +class TestCheckStdlibVulnerability: + """Tests for check_stdlib_vulnerability() LLM fallback function. + + This function is used when no packages matched from structured intel (GHSA/NVD empty), + suggesting a stdlib/toolchain vulnerability (e.g., Go crypto/x509, Python ssl). + """ + + @pytest.mark.asyncio + async def test_returns_vulnerable_when_llm_says_vulnerable(self): + """LLM determines project's toolchain is vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability in parsing.", + ), + rhsa=CveIntelRhsa( + statement="This vulnerability affects Go 1.24 release candidates and Go 1.23.x before 1.23.5.", + ), + ) + + async def mock_llm_checker(prompt: str) -> StdlibVulnerabilityResult: + return StdlibVulnerabilityResult( + is_vulnerable=True, + affected_component="crypto/x509", + reason="Project uses Go 1.23.3 which is before the fix in 1.23.5", + ) + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.23.3", + llm_checker=mock_llm_checker, + ) + + assert is_vulnerable is True + assert component == "crypto/x509" + assert "1.23.5" in reason + + @pytest.mark.asyncio + async def test_returns_not_vulnerable_when_llm_says_not_vulnerable(self): + """LLM determines project's toolchain is NOT vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability.", + ), + rhsa=CveIntelRhsa( + statement="This vulnerability affects only the Go 1.24 release candidates.", + ), + ) + + async def mock_llm_checker(prompt: str) -> StdlibVulnerabilityResult: + return StdlibVulnerabilityResult( + is_vulnerable=False, + affected_component="crypto/x509", + reason="CVE only affects Go 1.24 RC, project uses stable Go 1.21", + ) + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.21", + llm_checker=mock_llm_checker, + ) + + assert is_vulnerable is False + assert component == "crypto/x509" + assert "1.24 RC" in reason + + @pytest.mark.asyncio + async def test_returns_fail_safe_when_llm_fails(self): + """LLM failure returns fail-safe: assume vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability.", + ), + ) + + async def mock_llm_checker_raises(prompt: str) -> StdlibVulnerabilityResult: + raise RuntimeError("LLM service unavailable") + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.21", + llm_checker=mock_llm_checker_raises, + ) + + assert is_vulnerable is True + assert component == "unknown" + assert "LLM check failed" in reason + assert "LLM service unavailable" in reason + + @pytest.mark.asyncio + async def test_prompt_includes_cve_id_and_toolchain_version(self): + """Verify prompt contains key context for LLM analysis.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Test description.", + ), + ) + + captured_prompt: str = "" + + async def mock_llm_checker_capture(prompt: str) -> StdlibVulnerabilityResult: + nonlocal captured_prompt + captured_prompt = prompt + return StdlibVulnerabilityResult( + is_vulnerable=False, + affected_component="test", + reason="test reason", + ) + + await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="pypi", + toolchain_version="3.11.4", + llm_checker=mock_llm_checker_capture, + ) + + assert "CVE-2025-22865" in captured_prompt + assert "pypi" in captured_prompt + assert "3.11.4" in captured_prompt + assert "Test description." in captured_prompt From 5de8cfa2cbd8f32eb24515ae0d1eba3aa797972e Mon Sep 17 00:00:00 2001 From: Roni Hartuv Date: Thu, 18 Jun 2026 08:47:26 +0300 Subject: [PATCH 259/286] feat: add cache invalidation automation (#254) --- .tekton/cache-cleaner.yaml | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .tekton/cache-cleaner.yaml diff --git a/.tekton/cache-cleaner.yaml b/.tekton/cache-cleaner.yaml new file mode 100644 index 000000000..2b456e9d4 --- /dev/null +++ b/.tekton/cache-cleaner.yaml @@ -0,0 +1,65 @@ +--- +# RBAC: The Tekton ServiceAccount (usually `pipeline`) needs `edit` (or a custom +# Role) in target application namespaces to create/delete Jobs and mount PVCs +# (e.g., exploit-iq-testings, exploit-iq-prod). +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: generic-cache-cleaner +spec: + description: >- + Purges application cache from a PVC by spawning a short-lived Job in the + target namespace, waiting for completion, and removing the Job. + params: + - name: pvc-name + type: string + description: Name of the PersistentVolumeClaim holding the cache. + - name: namespace + type: string + description: Target namespace where the PVC lives (e.g., exploit-iq-testings). + steps: + - name: create-cache-cleaner-job + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + script: | + #!/bin/sh + set -eu + + oc apply -f - < Date: Thu, 18 Jun 2026 16:19:17 +0300 Subject: [PATCH 260/286] Security: regenerate Maven wrapper before execution in build_tree and lock_file_parsers (#256) * Security: regenerate Maven wrapper before execution in build_tree and lock_file_parsers - Extract resolve_mvn_command() and _extract_mvnw_maven_version() as module-level functions in dep_tree.py - Add mvnw security check to JavaDependencyTreeBuilder.build_tree() (previously used ./mvnw without validation) - Add mvnw security check to _generate_dependency_tree_txt() in lock_file_parsers.py (previously used ./mvnw without validation) * Small change following review Signed-off-by: Theodor Mihalache --- src/exploit_iq_commons/utils/dep_tree.py | 86 ++++++++++++-------- src/vuln_analysis/utils/lock_file_parsers.py | 4 +- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py index 530f99376..e98ed0dc2 100644 --- a/src/exploit_iq_commons/utils/dep_tree.py +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -965,9 +965,58 @@ def extract_package_name(self, package_name: str) -> str: return package_name +_MAVEN_VERSION_RE = re.compile(r'^\d+\.\d+\.\d+$') + + +def _extract_mvnw_maven_version(manifest_path: Path) -> str | None: + """Extract and validate the Maven version from the wrapper properties file.""" + props = manifest_path / ".mvn" / "wrapper" / "maven-wrapper.properties" + if not props.is_file(): + return None + try: + for line in props.read_text().splitlines(): + if line.startswith("distributionUrl="): + match = re.search(r'apache-maven-(\d+\.\d+\.\d+)', line) + if match and _MAVEN_VERSION_RE.match(match.group(1)): + return match.group(1) + except OSError: + pass + return None + + +def resolve_mvn_command(manifest_path: Path) -> str: + """Return the Maven command to use for *manifest_path*. + + If a ``mvnw`` wrapper exists, regenerate it from a trusted ``mvn`` + binary (using the version declared in ``maven-wrapper.properties``) + to avoid executing an untrusted script. Falls back to ``"mvn"`` + when regeneration fails or the version cannot be determined. + """ + if not (Path(manifest_path) / "mvnw").is_file(): + return "mvn" + version = _extract_mvnw_maven_version(manifest_path) + if not version: + logger.warning("Could not extract valid Maven version from wrapper properties, using system mvn") + return "mvn" + logger.info("Regenerating Maven wrapper with version %s", version) + result = subprocess.run( + ["mvn", "wrapper:wrapper", f"-Dmaven={version}"], + cwd=manifest_path, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return "./mvnw" + logger.warning( + "Failed to regenerate Maven wrapper (exit %d): %s", + result.returncode, + result.stderr or result.stdout, + ) + return "mvn" + + class JavaDependencyTreeBuilder(DependencyTreeBuilder): DEP_SOURCE_DIR = "dependencies-sources" - _MAVEN_VERSION_RE = re.compile(r'^\d+\.\d+\.\d+$') def __init__(self, query: str): self._query = query @@ -979,39 +1028,11 @@ def __check_file_exists(self, dir_path: str | Path, filename: str) -> bool: p = Path(dir_path) / filename return p.is_file() - @staticmethod - def _extract_mvnw_maven_version(manifest_path: Path) -> str | None: - """Extract and validate the Maven version from the wrapper properties file.""" - props = manifest_path / ".mvn" / "wrapper" / "maven-wrapper.properties" - if not props.is_file(): - return None - try: - for line in props.read_text().splitlines(): - if line.startswith("distributionUrl="): - match = re.search(r'apache-maven-(\d+\.\d+\.\d+)', line) - if match and JavaDependencyTreeBuilder._MAVEN_VERSION_RE.match(match.group(1)): - return match.group(1) - except OSError: - pass - return None - def install_dependencies(self, manifest_path: Path): - mvn_command = "mvn" + mvn_command = resolve_mvn_command(manifest_path) settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml') source_path = self.DEP_SOURCE_DIR - if self.__check_file_exists(manifest_path, "mvnw"): - version = self._extract_mvnw_maven_version(manifest_path) - if version: - logger.info("Regenerating Maven wrapper with version %s", version) - result = subprocess.run(["mvn", "wrapper:wrapper", f"-Dmaven={version}"], cwd=manifest_path) - if result.returncode == 0: - mvn_command = "./mvnw" - else: - logger.warning("Failed to regenerate Maven wrapper, falling back to system mvn") - else: - logger.warning("Could not extract valid Maven version from wrapper properties, using system mvn") - process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) @@ -1059,14 +1080,11 @@ def install_dependencies(self, manifest_path: Path): logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: - mvn_command = "mvn" + mvn_command = resolve_mvn_command(manifest_path) settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml") dependency_file = manifest_path / "dependency_tree.txt" package_name = self._query.split(",")[0] - if self.__check_file_exists(manifest_path, "mvnw"): - mvn_command = "./mvnw" - if is_maven_gav(package_name): subprocess.run( [ diff --git a/src/vuln_analysis/utils/lock_file_parsers.py b/src/vuln_analysis/utils/lock_file_parsers.py index 6f3941cd1..60166c26f 100644 --- a/src/vuln_analysis/utils/lock_file_parsers.py +++ b/src/vuln_analysis/utils/lock_file_parsers.py @@ -24,7 +24,7 @@ import subprocess from pathlib import Path -from exploit_iq_commons.utils.dep_tree import TRANSITIVE_ENV_NAME, run_command +from exploit_iq_commons.utils.dep_tree import TRANSITIVE_ENV_NAME, resolve_mvn_command, run_command from exploit_iq_commons.utils.java_utils import parse_depgraph_line from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -207,7 +207,7 @@ def _generate_dependency_tree_txt(repo_path: Path) -> bool: Returns: True if successful, False otherwise. """ - mvn_cmd = "./mvnw" if (repo_path / "mvnw").exists() else "mvn" + mvn_cmd = resolve_mvn_command(repo_path) output_file = repo_path / "dependency_tree.txt" settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH") From 545adf32e41dfe5f250bd85c78899e262a81e1be Mon Sep 17 00:00:00 2001 From: Gal Netanel Date: Sun, 21 Jun 2026 09:53:30 +0300 Subject: [PATCH 261/286] Appeng 5400 return client error when language was not detected (#250) In addition, in case credential is not provided and when cloning for git repo result with user/pass prompt, fail the operation * PPENG-5400: Fail scan request as soon as echosystem fail to be detected for the the git repo * Handle error none exist of private repository when expect to be public repository by the code * add unit tests for public source code clone --------- Co-authored-by: Gal Netanel --- .../utils/source_code_git_loader.py | 19 ++++++++++-- .../functions/cve_generate_vdbs.py | 29 +++++++++++++++---- .../functions/cve_http_output.py | 2 +- .../tests/test_source_code_git_loader.py | 15 ++++++++++ 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 30380dc1b..7ae72a728 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -201,8 +201,23 @@ def load_repo(self): repo = self._clone_with_credential(credential_id) credential_clone = True else: - # Create a shallow clone of the repository without checking out - repo = Repo.clone_from(self.clone_url, self.repo_path, depth=1, no_checkout=True) + # Create a shallow clone of the repository without checking out. + # GIT_TERMINAL_PROMPT=0: never block on username/password prompts. + + clone_env = {"GIT_TERMINAL_PROMPT": "0"} + try: + repo = Repo.clone_from( + self.clone_url, + self.repo_path, + depth=1, + no_checkout=True, + env=clone_env, + ) + except GitCommandError as e: + logger.error("Failed to clone repository '%s': %s", self.clone_url, e) + raise ValueError( + f"Failed to clone repository '{self.clone_url}'" + ) from e # depth=1 implies --single-branch, which narrows the fetch # refspec to only the default branch. Widen it so that # fetching any branch creates a remote-tracking ref and diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index df3e9c542..354631eea 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -37,6 +37,13 @@ class GitCloneError(Exception): """Raised when cloning or updating a repo for document collection fails.""" + +class EcosystemNotDetectedError(Exception): + """Raised when no ecosystem can be detected from cloned repo manifests.""" + + def __init__(self, message: str = "No ecosystem detected from repo manifests") -> None: + super().__init__(message) + class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): """ Defines a function that generates a vector database from code repositories and documentation. @@ -119,7 +126,9 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume documents.extend(embedder.collect_documents(si)) except Exception as e: logger.warning("Error collecting documents for source info %s: %s", si, e) - raise GitCloneError(e) + logger.error("Failed to clone repository %s--%s", si.git_repo, e) + raise GitCloneError(f"Failed to clone repository {si.git_repo}") + # Warn and return early if no documents were collected if len(documents) == 0: @@ -189,6 +198,7 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: vdb_code_path = None vdb_doc_path = None code_index_path = None + ecosystem_detected = True base_image = message.image.name source_infos = message.image.source_info @@ -239,6 +249,18 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: if detected is not None: message.image.ecosystem = detected logger.info("Detected ecosystem '%s' from repo manifests", detected.value) + else: + ecosystem_detected = False + raise EcosystemNotDetectedError() + + except EcosystemNotDetectedError: + failure_reason = "No ecosystem detected from repo manifests" + logger.error( + "No ecosystem detected from repo manifests for image '%s', source code info: %s", + base_image, + source_infos, + ) + logger.warning("Ignoring VDB generation error and continuing with missing VDBs.") except Exception as e: failure_reason = f"Failure to build VDB , Error: {e}" @@ -248,15 +270,12 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: e, exc_info=True) - if not config.ignore_errors: - raise - logger.warning("Ignoring VDB generation error and continuing with missing VDBs.") vdb_code_path = str(vdb_code_path) if vdb_code_path is not None else None vdb_doc_path = str(vdb_doc_path) if vdb_doc_path is not None else None code_index_path = str(code_index_path) if code_index_path is not None else None - code_index_success = True if code_index_path is not None else False + code_index_success = True if code_index_path is not None and ecosystem_detected else False if not code_index_success: message.failure_reason = failure_reason diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index ceb2539bf..792b0f107 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -128,7 +128,7 @@ def _build_output_payload( report = FailureReport( scan_id=message.input.scan.id, error_type="processing-error", - error_message=f"Failed to clone repository {repo_url}--{message.input.failure_reason}", + error_message=message.input.failure_reason if message.input.failure_reason else "Unknown failure reason", ) logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") return OutputPayload(json=report.model_dump_json(by_alias=True), url=failure_url, skip_mlops=True) diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py index 0896fa677..2858753f0 100644 --- a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -1,5 +1,6 @@ import pytest import shutil + from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader @@ -24,6 +25,20 @@ def test_https_to_ssh_url(https_url, expected_ssh): assert SourceCodeGitLoader._https_to_ssh_url(https_url) == expected_ssh + +def test_public_clone_fails_instead_of_prompting(tmp_path): + """Unauthenticated clone of a missing/private repo must fail fast, not hang on a prompt.""" + clone_url = "https://github.com/example/missing" + repo_path = tmp_path / "repo" + + loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=clone_url, ref="main") + + with pytest.raises(ValueError, match=r"Failed to clone repository .+missing"): + loader.load_repo() + + assert not repo_path.exists() or not (repo_path / ".git").exists() + + def test_load_repo_branch_checkout(tmp_path): """Branch names must be checkable after a shallow clone (APPENG-4896).""" repo_url = "https://github.com/medik8s/node-remediation-console" From b2205fa8408e0a20ca36a136647f439a8e87026e Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Sun, 21 Jun 2026 15:42:32 +0300 Subject: [PATCH 262/286] A few fixes (#257) - Add dedicated ServiceAccount, Role, and RoleBinding for MCP server (anyuid SCC) - Switch MCP server image to quay.io/ecosystem-appeng/ registry - Add runAsNonRoot security context and EXPLOITIQ_READ_ONLY/REDACT_RESPONSES env vars - Update kustomization image reference to match new registry - Upgrade git loader fetch failure logs from debug to warning for better troubleshooting - Deduplicate checked_not_vulnerable packages using seen_not_vulnerable set - Improve justification output: group packages by shared reason, show package names without version * Address PR #257 review feedback: - Restore git error details in warning logs for fetch failure diagnostics - Include package version in justification output to disambiguate duplicates Signed-off-by: Theodor Mihalache --- kustomize/base/exploitiq_mcp_server.yaml | 46 ++++++++++++++++++- kustomize/base/kustomization.yaml | 2 +- .../utils/source_code_git_loader.py | 4 +- .../functions/cve_verify_vuln_package.py | 14 ++++-- src/vuln_analysis/utils/llm_engine_utils.py | 31 +++++++++---- src/vuln_analysis/utils/version_check.py | 4 +- 6 files changed, 81 insertions(+), 20 deletions(-) diff --git a/kustomize/base/exploitiq_mcp_server.yaml b/kustomize/base/exploitiq_mcp_server.yaml index 1e48a81fc..5b48f5213 100644 --- a/kustomize/base/exploitiq_mcp_server.yaml +++ b/kustomize/base/exploitiq_mcp_server.yaml @@ -1,3 +1,39 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploitiq-mcp-server-sa + labels: + app: exploit-iq + component: exploitiq-mcp-server +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exploitiq-mcp-server-anyuid-scc + labels: + app: exploit-iq + component: exploitiq-mcp-server +rules: + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["anyuid"] + verbs: ["use"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exploitiq-mcp-server-anyuid-scc + labels: + app: exploit-iq + component: exploitiq-mcp-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exploitiq-mcp-server-anyuid-scc +subjects: + - kind: ServiceAccount + name: exploitiq-mcp-server-sa +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -19,11 +55,13 @@ spec: app: exploit-iq component: exploitiq-mcp-server spec: - serviceAccountName: exploit-iq-client-sa + serviceAccountName: exploitiq-mcp-server-sa imagePullSecrets: [] containers: - name: exploitiq-mcp-server - image: quay.io/exploit-iq/exploitiq-mcp-server:latest + securityContext: + runAsNonRoot: true + image: quay.io/ecosystem-appeng/exploitiq-mcp-server:latest imagePullPolicy: Always ports: - name: http @@ -40,6 +78,10 @@ spec: value: "3000" - name: NODE_EXTRA_CA_CERTS value: /etc/tls/tls.crt + - name: EXPLOITIQ_READ_ONLY + value: "true" + - name: EXPLOITIQ_REDACT_RESPONSES + value: "true" - name: OPENSHIFT_DOMAIN valueFrom: secretKeyRef: diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index af75cb844..16abd4d8e 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -100,5 +100,5 @@ images: - name: quay.io/ecosystem-appeng/agent-morpheus-client newTag: latest - - name: quay.io/exploit-iq/exploitiq-mcp-server + - name: quay.io/ecosystem-appeng/exploitiq-mcp-server newTag: latest diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index 7ae72a728..edc1bd6a5 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -298,7 +298,7 @@ def _do_fetch(self, repo: Repo) -> None: logger.info("Fetched ref '%s' as tag", self.ref) return except GitCommandError as e: - logger.debug("Tag fetch failed for '%s': %s", self.ref, e) + logger.warning("Ref '%s' is not a tag, retrying as branch/commit SHA: %s", self.ref, e) # Not a tag — try as a branch or commit SHA try: @@ -306,7 +306,7 @@ def _do_fetch(self, repo: Repo) -> None: logger.info("Fetched ref '%s' as branch/commit", self.ref) return except GitCommandError as e: - logger.debug("Branch/commit fetch failed for '%s': %s", self.ref, e) + logger.warning("Ref '%s' not found on remote, checking if it exists locally: %s", self.ref, e) # Neither fetch worked — check if ref already exists locally try: diff --git a/src/vuln_analysis/functions/cve_verify_vuln_package.py b/src/vuln_analysis/functions/cve_verify_vuln_package.py index 5270790c0..87a4c5dd5 100644 --- a/src/vuln_analysis/functions/cve_verify_vuln_package.py +++ b/src/vuln_analysis/functions/cve_verify_vuln_package.py @@ -600,6 +600,7 @@ async def _process_cve_intel_loop( vuln_sbom_packages: list[VulnerableSBOMPackage] = [] checked_not_vulnerable: list[CheckedNotVulnerablePackage] = [] + seen_not_vulnerable: set[tuple[str, str]] = set() intel_sources: list[str] = [vp["source"] for vp in vuln_packages] found_vulnerable = False @@ -692,11 +693,14 @@ async def _process_cve_intel_loop( found_vulnerable = True break else: - checked_not_vulnerable.append(CheckedNotVulnerablePackage( - name=match.dep_name or pkg_name, - version=match.installed_version or "unknown", - reason=reason - )) + pkg_key = (match.dep_name or pkg_name, match.installed_version or "unknown") + if pkg_key not in seen_not_vulnerable: + seen_not_vulnerable.add(pkg_key) + checked_not_vulnerable.append(CheckedNotVulnerablePackage( + name=pkg_key[0], + version=pkg_key[1], + reason=reason + )) # Stdlib/toolchain fallback: when no packages matched and it's a language ecosystem no_packages_found = not vuln_sbom_packages and not checked_not_vulnerable diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 41eaf1899..1857bdd89 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -237,15 +237,30 @@ def build_no_vuln_packages_output( checked_not_vulnerable: list[CheckedNotVulnerablePackage] | None = None ) -> AgentMorpheusEngineOutput: if checked_not_vulnerable: - package_details = "; ".join( - f"{pkg.name}@{pkg.version}: {pkg.reason}" for pkg in checked_not_vulnerable + n = len(checked_not_vulnerable) + reasons = {pkg.reason for pkg in checked_not_vulnerable} + package_lines = "\n".join( + f"- {pkg.name} ({pkg.version})" for pkg in checked_not_vulnerable ) - summary = (f"Packages were checked but determined not vulnerable. " - f"Details: {package_details}") - justification_reason = (f"The following packages were checked and found not vulnerable: " - f"{package_details}") - response_text = (f"The VulnerableDependencyChecker found matching packages but the LLM " - f"determined they are not vulnerable. Details: {package_details}") + if len(reasons) == 1: + shared_reason = next(iter(reasons)) + summary = f"Not vulnerable — {shared_reason}." + justification_reason = (f"Not vulnerable — {shared_reason}. {n} package(s)\n" + f" checked:\n" + f"{package_lines}") + response_text = (f"Version check completed — {n} package(s) matched but none are vulnerable. " + f"Reason: {shared_reason}.\n\n" + f"{package_lines}") + else: + package_lines_with_reason = "\n".join( + f"- {pkg.name} ({pkg.version}) — {pkg.reason}" + for pkg in checked_not_vulnerable + ) + summary = f"Not vulnerable — {n} package(s) checked." + justification_reason = (f"Not vulnerable. {n} package(s) checked:\n\n" + f"{package_lines_with_reason}") + response_text = (f"Version check completed — {n} package(s) matched but none are vulnerable.\n\n" + f"{package_lines_with_reason}") else: summary = ("The VulnerableDependencyChecker did not find any vulnerable packages " "or dependencies in the SBOM.") diff --git a/src/vuln_analysis/utils/version_check.py b/src/vuln_analysis/utils/version_check.py index 11536bca4..818c768fe 100644 --- a/src/vuln_analysis/utils/version_check.py +++ b/src/vuln_analysis/utils/version_check.py @@ -333,7 +333,7 @@ def _deterministic_first_patched_check( reference = version_func(first_patched) is_vulnerable = installed < reference return is_vulnerable, ( - f"Installed {installed_version} " + f"Installed version {installed_version} " f"{'<' if is_vulnerable else '>='} " f"first_patched {first_patched}" ) @@ -348,7 +348,7 @@ def _deterministic_nvd_range_check( version_range = [_intel_value(version_info.get(key)) for key in NVD_RANGE_KEYS] in_range = _check_version_in_range(installed_version, version_range, ecosystem) return in_range, ( - f"Installed {installed_version} " + f"Installed version {installed_version} " f"{'in' if in_range else 'outside'} " f"NVD range" ) From e64b5cc046e20e549b9a9b66f860633e4d40c6d9 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Mon, 22 Jun 2026 11:48:33 +0300 Subject: [PATCH 263/286] feat(nginx): replace docker.io nginx with Red Hat hardened image --- kustomize/base/nginx.yaml | 71 ++++++++++++++----- .../overlays/remote-nim-all/nginx-patch.yaml | 4 +- .../nginx-patch.yaml | 4 +- kustomize/overlays/tests/nginx-patch.yaml | 4 +- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index 865347357..c4bbd0521 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -20,21 +20,36 @@ spec: component: nginx-cache spec: imagePullSecrets: [] - containers: - - name: nginx - image: docker.io/nginxinc/nginx-unprivileged:1.27 - imagePullPolicy: IfNotPresent - ports: - - name: http - protocol: TCP - containerPort: 8080 - resources: - limits: - memory: "256Mi" - cpu: "250m" - requests: - memory: "100Mi" - cpu: "100m" + initContainers: + - name: render-nginx-config + image: registry.redhat.io/ubi9/python-312@sha256:55ddb67dcfc6c7ec072fce58753910b290c2903b10cd71d0d64c631d44ef4ab0 + command: ["python3", "-c"] + args: + - | + import os, sys, string + from pathlib import Path + + required = [ + "SERPAPI_API_KEY", "NVIDIA_API_KEY", "OPENAI_API_KEY", + "NGC_API_KEY", "NGC_ORG_ID", "NGINX_UPSTREAM_NVAI", + "NGINX_UPSTREAM_NIM_LLM", "NGINX_UPSTREAM_NIM_EMBED", + "NGINX_UPSTREAM_OPENAI", + ] + missing = [v for v in required if v not in os.environ] + if missing: + print(f"missing required variables: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + def render_dir(src_dir, dst_dir): + src, dst = Path(src_dir), Path(dst_dir) + dst.mkdir(parents=True, exist_ok=True) + for tmpl in src.glob("*.conf.template"): + out = dst / tmpl.stem + out.write_text(string.Template(tmpl.read_text()).safe_substitute(os.environ)) + print(f"rendered: {tmpl} -> {out}") + + render_dir("/templates/routes", "/generated/routes") + render_dir("/templates/variables", "/generated/variables") env: - name: SERPAPI_API_KEY valueFrom: @@ -46,7 +61,6 @@ spec: secretKeyRef: key: nvidia_api_key name: exploit-iq-secret - - name: OPENAI_API_KEY valueFrom: secretKeyRef: @@ -66,12 +80,33 @@ spec: value: https://integrate.api.nvidia.com volumeMounts: - name: routes - mountPath: /etc/nginx/templates/routes + mountPath: /templates/routes + readOnly: true - name: variables - mountPath: /etc/nginx/templates/variables + mountPath: /templates/variables + readOnly: true + - name: extra-config + mountPath: /generated + containers: + - name: nginx + image: registry.access.redhat.com/hi/nginx:1.30.3 + imagePullPolicy: IfNotPresent + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "100Mi" + cpu: "100m" + volumeMounts: - name: config mountPath: /etc/nginx/nginx.conf subPath: nginx.conf + readOnly: true - name: extra-config mountPath: /etc/nginx/conf.d - name: cache diff --git a/kustomize/overlays/remote-nim-all/nginx-patch.yaml b/kustomize/overlays/remote-nim-all/nginx-patch.yaml index 4afc4aeb2..e7fb543c6 100644 --- a/kustomize/overlays/remote-nim-all/nginx-patch.yaml +++ b/kustomize/overlays/remote-nim-all/nginx-patch.yaml @@ -5,8 +5,8 @@ metadata: spec: template: spec: - containers: - - name: nginx + initContainers: + - name: render-nginx-config env: - name: NGINX_UPSTREAM_NIM_LLM value: https://integrate.api.nvidia.com diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml index a814d02f3..b3d598543 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml @@ -5,8 +5,8 @@ metadata: spec: template: spec: - containers: - - name: nginx + initContainers: + - name: render-nginx-config env: - name: NGINX_UPSTREAM_NIM_LLM value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 diff --git a/kustomize/overlays/tests/nginx-patch.yaml b/kustomize/overlays/tests/nginx-patch.yaml index f8cd9dcad..c4aae48ee 100644 --- a/kustomize/overlays/tests/nginx-patch.yaml +++ b/kustomize/overlays/tests/nginx-patch.yaml @@ -5,8 +5,8 @@ metadata: spec: template: spec: - containers: - - name: nginx + initContainers: + - name: render-nginx-config env: - name: NGINX_UPSTREAM_NIM_EMBED value: http://localhost:8000 From 0166003ff6511fa64e3615ef29d86325d9bf88c8 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 23 Jun 2026 14:55:38 +0300 Subject: [PATCH 264/286] docs(kustomize): rewrite deployment guide per Red Hat style --- kustomize/README.md | 634 ++++++++++++++++++++++++++++---------------- 1 file changed, 405 insertions(+), 229 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 1f9be713b..ec0d06f52 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -15,129 +15,71 @@ See the License for the specific language governing permissions and limitations under the License. --> -# Procedure to Run and Deploy +# Deploying Exploit Intelligence on OpenShift Container Platform -## Install and Run Locally +## Prerequisites -One can run ExploitIQ on his local machine ( No GPU dependency is required!), for the purpose of testing, debugging and troubleshooting problems: +Ensure that the following tools are installed and available on your system path: -1. Install the lightweight [uv package manager](https://docs.astral.sh/uv/getting-started/installation). -2. Ensure Python 3.12 is installed for your operating system. -```shell - uv python install 3.12 -``` -3. Navigate to the project root, then create and activate a virtual environment using `uv`. - -```shell -# Make sure you're on the repo's root directory. -cd $(git rev-parse --show-toplevel) -# Create and activate the virtual environment -rm -rf .venv || true -uv venv --python 3.12 --no-cache -source .venv/bin/activate -``` - -4. **Install Dependencies**: Once the environment is active, install the required packages. - -```shell -uv sync --no-cache -``` - -5. **Set Environment Variables**: Define the following environment variables. - -```shell -export CHECKLIST_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export CODE_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export CVE_AGENT_EXECUTOR_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export DOC_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export JUSTIFY_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export NVIDIA_API_BASE=http://YOUR_SELF_HOSTED_OPENAI_LLM_ADDRESSS/v1 -export PYTHONUNBUFFERED=1 -export GOPROXY=https://proxy.golang.org,direct -export SERPAPI_API_KEY=YOUR_SERPAPI_KEY -export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 -export REGISTRY_REDHAT_USERNAME="your_username" -export REGISTRY_REDHAT_PASSWORD="your_password_or_token" -``` - -6. **Run the Application**: After a successful installation, run the application. -```shell -nat --log-level debug serve --config_file=src/vuln_analysis/configs/config-http-openai.yml --host 0.0.0.0 --port 26466 -``` -7. In another terminal, in the same venv python env, start Arize phoenix service to enable tracing for local deployment: -```shell -phoenix serve -``` +- [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) — OpenShift CLI, logged in to your target cluster +- [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) — version 5 or later +- `openssl` — for generating secrets and verifying certificates -### Prerequisites -One need to install the following tools in order to run the agent locally without problems, all tools binaries expected -to be on the system path: -- [Java JDK >=21](https://www.oracle.com/il-en/java/technologies/downloads/#java21) -- [Maven Package manager](https://maven.apache.org/install.html) -- [Golang](https://go.dev/doc/install) - -### Container Source Download Configuration +You also need credentials for the following external services. Refer to the main [README](../README.md#obtain-api-keys) for instructions on obtaining them: -For C/C++ projects, you can enable container source download to extract RPM dependencies from Red Hat container registries. This feature uses skopeo to download container source layers and automatically extracts RPM packages, excluding the main application RPMs to focus on dependencies only. -**Required Environment Variables:** -# Enable container source download mode (default behavior ) -export USE_CONTAINER_SOURCES=true +- NVIDIA API key +- SerpAPI key +- GitHub Advisory Database (GHSA) API key +- Red Hat registry credentials (`registry.redhat.io`) -**How it works:** -1. Downloads container source layers using skopeo -2. Extracts RPM packages from the downloaded layers -3. Filters out main application RPMs (e.g., `postgresql-*` for PostgreSQL containers) -4. Copies dependency RPMs to the standard RPM cache directory +> [!NOTE] +> All commands in this guide assume that your current working directory is `kustomize/`. +--- -**Prerequisites:** -- `skopeo` must be installed on the system -- Valid Red Hat registry credentials -- Network access to `registry.redhat.io` +## Preparing Your Deployment +Complete the following steps before running any deployment command. -## Deploy And Run On OCP +### Step 1. Create API Credentials -1. Create a `base/secrets.env` file containing the API keys for external services `ExploitIQ` might use. Not all keys are mandatory. Refer to the main [README](../README.md#obtain-api-keys) for details on how to create the Red Hat credentials and other API keys. +Create the `base/secrets.env` file with your API keys. You do not need all keys for every deployment variant. ```shell cat > base/secrets.env << EOF -nvd_api_key=you_api_key -serpapi_api_key=your_api_key -nvidia_api_key=your_api_key -ghsa_api_key=your_api_key -registry_redhat_username=your_registry_username -registry_redhat_password=your_registry_pass_token - +nvd_api_key= +serpapi_api_key= +nvidia_api_key= +ghsa_api_key= +registry_redhat_username= +registry_redhat_password= EOF ``` -2. If a namespace does not exist, create one: +### Step 2. Create a Project Namespace + +If a namespace does not exist, create one: ```shell -export YOUR_NAMESPACE_NAME=yourNamespaceNameHere +export YOUR_NAMESPACE_NAME= oc new-project $YOUR_NAMESPACE_NAME ``` -3. Create a `base/argilla/feedback_secret.env` file containing the credentials for the Argilla feedback service: -```shell -cat > base/argilla/feedback_secret.env << EOF -argilla_username=your_argilla_username -argilla_password=your_argilla_password -argilla_api_key=your_argilla_api_key +### Step 3. Create an Image Pull Secret -EOF -``` -4. Create an image pull secret to authorize pulling the `ExploitIQ` and `Argilla` container images: +Create the secret that authorizes pulling Exploit Intelligence and Argilla container images: ```shell -oc create secret generic exploit-iq-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +oc create secret generic exploit-iq-pull-secret \ + --from-file=.dockerconfigjson= \ + --type=kubernetes.io/dockerconfigjson ``` -5. Edit the `image-registry-credentials.env` and provide image registry credentials required by product scanning to access and pull component images. +### Step 4. Configure Image Registry Credentials + +Create the `base/image-registry-credentials.env` file with credentials for the registries that product scanning uses to pull component images. The `auth` value is the base64 encoding of `:`. ```shell -# Example: add your desired registries and credentials. "auth" is base64 of ":". cat > base/image-registry-credentials.env << 'EOF' { "auths": { @@ -149,26 +91,66 @@ cat > base/image-registry-credentials.env << 'EOF' EOF ``` ->[!IMPORTANT] ->This secret is essential for product scanning to authenticate and pull component images. If you skip this step, kustomize will still deploy, but authenticated pulls will not work until you provide real credentials. +> [!IMPORTANT] +> Product scanning requires valid registry credentials to pull component images. If you skip this step, the deployment succeeds but authenticated image pulls fail. -6. Create the `oauth-secret.env` file containing the `client-secret` and `openshift-domain` values required by the [ExploitIQ Client](./base/exploit_iq_client.yaml) configuration. +### Step 5. Configure Argilla Feedback Credentials + +Create the `base/argilla/feedback_secret.env` file with credentials for the Argilla feedback service: -If openshift resource of kind `OAuthClient` named `exploit-iq-client` exists, just get the secret from there: ```shell -export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +cat > base/argilla/feedback_secret.env << EOF +argilla_username= +argilla_password= +argilla_api_key= +EOF +``` + +### Step 6. Configure OAuth Authentication + +Exploit Intelligence uses OpenShift OAuth for user authentication. Select the option that matches your situation. + +#### Option A: Create a New OAuthClient + +Generate a new client secret and write the credentials file: + +```shell +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) +export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) + +cat > base/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +session-secret=$OAUTH_SESSION_SECRET +openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +EOF ``` -Otherwise, Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret of your own: + +> [!IMPORTANT] +> Save the value of `$OAUTH_CLIENT_SECRET`. You need it when you create the `OAuthClient` resource after deployment. + +Use this option when deploying Exploit Intelligence for the first time. + +#### Option B: Reuse an Existing OAuthClient + +Retrieve the secret from the existing `OAuthClient` resource and verify that it is at least 32 characters: ```shell -export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" +export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +echo -n "$OAUTH_CLIENT_SECRET" | wc -c ``` +If the result is less than 32, rotate the secret before continuing: + ```shell -export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) +oc patch oauthclient exploit-iq-client -p "{\"secret\":\"$OAUTH_CLIENT_SECRET\"}" ``` +Then write the credentials file: + ```shell +export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) + cat > base/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET session-secret=$OAUTH_SESSION_SECRET @@ -176,7 +158,7 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -7. Create MongoDB credentials: +### Step 7. Create Database Credentials ```shell cat > base/mongodb-credentials.env << EOF @@ -187,128 +169,172 @@ exploit-iq-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) EOF ``` -8. Update `ExploitIQ` configuration file with the correct callback URL for the client service. +### Step 8. Set the Application Callback URL ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` -### Configuring Git SSL Certificate Authority for Custom CAs +### Step 9. Configure a Custom Git Server CA -If your Git server uses a certificate that is signed by a custom Certificate Authority (CA), you must provide the CA certificate bundle to enable ExploitIQ to verify the Git server identity. +Complete this step if your Git server uses a certificate signed by a custom Certificate Authority (CA). > [!IMPORTANT] -> If you need to access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`, you must complete this procedure. - -#### Procedure +> Complete this step if you access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`. -1. Create the certificate directory: +**1.** Create the certificate directory: ```shell -mkdir -p kustomize/base/ca-certs +mkdir -p base/ca-certs ``` -2. Obtain your CA certificates. +**2.** Obtain your CA certificates. For Red Hat internal Git repositories: ```shell -# Download the Red Hat Root CA -curl -o kustomize/base/ca-certs/internal-root-ca.pem \ +curl -o base/ca-certs/internal-root-ca.pem \ https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem -# Download the Red Hat internal Intermediate CA -curl -o kustomize/base/ca-certs/rhcs-intermediate-ca.crt \ +curl -o base/ca-certs/rhcs-intermediate-ca.crt \ https://certs.corp.redhat.com/chains/rhcs-ca-chain-2022-self-signed.crt ``` For other custom CAs: ```shell -# Copy your custom CA certificate files to the directory -cp /path/to/your-custom-ca.pem kustomize/base/ca-certs/ +cp /path/to/your-custom-ca.pem base/ca-certs/ ``` -3. Create the CA bundle by concatenating all certificates: +**3.** Create the CA bundle: ```shell -cat kustomize/base/ca-certs/*.{pem,crt} > kustomize/base/ca-certs/ca-bundle.crt +cat base/ca-certs/*.{pem,crt} > base/ca-certs/ca-bundle.crt ``` -4. Verify that the bundle contains your certificates: +**4.** Verify the bundle: ```shell -openssl crl2pkcs7 -nocrl -certfile kustomize/base/ca-certs/ca-bundle.crt | \ +openssl crl2pkcs7 -nocrl -certfile base/ca-certs/ca-bundle.crt | \ openssl pkcs7 -print_certs -noout ``` ->[!IMPORTANT] -You should only run one of the steps 9,10 or 11, depending on if you want to run the service with a self hosted LLM, self hosted LLM with MLOps or Nvidia remote NIM. -9. To deploy `ExploitIQ` with a self-hosted LLM , run: +### Step 10. Configure the OAuth Endpoint CA -```shell -# Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM -oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +Complete this step if your OCP ingress router uses a CA that is not trusted by default. -``` +> [!IMPORTANT] +> If you see the error `PKIX path building failed` in the `exploit-iq-client` pod logs after deployment, complete this step and redeploy. -10. To deploy `ExploitIQ` with a self-hosted LLM and MLOps, run: +**1.** Create the certificate directory: ```shell -# Patch overlay kustomization yaml with deployment namespace value (Grafana and Tempo) -sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomization.yaml -sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/tempo/kustomization.yaml +mkdir -p base/oidc-ca ``` +**2.** Fetch the CA chain from the OAuth endpoint: + ```shell -# replace EXPLOIT_IQ_GRAFANA_SA_TOKEN with ExploitIQ Grafana SA Token from bitwarden vault (1 year expiration date) -oc create secret generic grafana-bearer-token --from-literal=token='EXPLOIT_IQ_GRAFANA_SA_TOKEN' +openssl s_client \ + -connect oauth-openshift.apps.$(oc get dns cluster -o jsonpath='{.spec.baseDomain}'):443 \ + -showcerts 2>/dev/null | awk '/BEGIN CERT/,/END CERT/' \ + > base/oidc-ca/ca-bundle.crt ``` +Alternatively, fetch the CA bundle from the cluster: + ```shell -# Deploy ExploitIQ with self hosted llama3.1-70b-4bit LLM and MLOps -oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME +kubectl get cm default-ingress-cert -n openshift-config-managed \ + -o jsonpath='{.data.ca-bundle\.crt}' > base/oidc-ca/ca-bundle.crt +``` +**3.** Verify the bundle: + +```shell +openssl crl2pkcs7 -nocrl -certfile base/oidc-ca/ca-bundle.crt | \ + openssl pkcs7 -print_certs -noout ``` -#### ⚠️ Note: Grafana Resources May Fail to Deploy +--- + +## Selecting a Deployment Variant + +Exploit Intelligence supports the following deployment variants. Run only one deployment command in the next section. + +| Variant | Overlay | LLM | Use When | +|---|---|---|---| +| Self-Hosted LLM | `self-hosted-llama3.1-70b-4bit` | On-cluster GPU node | You have GPU capacity in the cluster | +| Self-Hosted LLM + MLOps | `mlops` | On-cluster GPU node | You also need Grafana and Tempo observability | +| Remote NIM | `remote-nim-all` | NVIDIA-hosted NIM | You use NVIDIA-hosted inference | + +--- -When applying the Grafana overlay, some Grafana resources **may fail to be created**. This is usually because the **Grafana Operator CSV has not yet reached the `Succeeded` phase**. +## Deploying Exploit Intelligence -**How to check** +### Deploy with a Self-Hosted LLM -Check the Grafana Operator CSV status: +```shell +oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +### Deploy with a Self-Hosted LLM and MLOps Observability + +Patch the overlay with your namespace: ```shell -oc get csv -n $YOUR_NAMESPACE_NAME | grep grafana-operator +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomization.yaml +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/tempo/kustomization.yaml ``` -Look at the PHASE column — it should show: -```text -Succeeded +Create the Grafana token secret. Retrieve the token value from the Bitwarden vault entry **Exploit Intelligence Grafana SA Token**: + +```shell +oc create secret generic grafana-bearer-token \ + --from-literal=token='' ``` -**Redeploy Grafana resources** +Deploy: -Once the CSV is in Succeeded state, you can safely redeploy the Grafana overlay filtered to Grafana CRs (all grafana CRs are labeled with managed-by: grafana-operator): ```shell -oc kustomize overlays/mlops \ - | yq e 'select(.metadata.labels.managed-by == "grafana-operator")' - \ - | oc replace -f - -n $YOUR_NAMESPACE_NAME +oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` +> [!WARNING] +> Grafana custom resources may fail to create if the Grafana Operator has not yet reached the `Succeeded` phase. Check the operator status: +> +> ```shell +> oc get csv -n $YOUR_NAMESPACE_NAME | grep grafana-operator +> ``` +> +> When the PHASE column shows `Succeeded`, redeploy the Grafana resources: +> +> ```shell +> oc kustomize overlays/mlops \ +> | yq e 'select(.metadata.labels.managed-by == "grafana-operator")' - \ +> | oc replace -f - -n $YOUR_NAMESPACE_NAME +> ``` + +### Deploy with Remote NIM -10. Alternatively, to deploy `ExploitIQ` with a fully remote nim LLM, run: ```shell -# Deploy ExploitIQ with remote nim llama-3.1-70b-16bit LLM -oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME +oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` ->[!WARNING] -Without completing the following step with the correct secret from step 6, authentication and logging into the UI App will fail! -11. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 6) and generated route -```bash +--- + +## Configuring OpenShift OAuth + +> [!WARNING] +> Complete this step before attempting to log in to the Exploit Intelligence UI. Authentication fails if the `OAuthClient` resource is not configured correctly. + +After the deployment completes and the `exploit-iq-client` route is available, configure the OpenShift OAuth client. + +### Create the OAuthClient Resource + +Create the `OAuthClient` resource using the secret from [Step 6, Option A](#option-a-create-a-new-oauthclient): + +```shell oc create -f - < base/oauth-secrets.env << EOF -client-secret=$OAUTH_CLIENT_SECRET -openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') -EOF + +oc patch oauthclient exploit-iq-client \ + -p '{"redirectURIs":["'$HTTP_ROUTE'","'$HTTPS_ROUTE'"]}' ``` -12. **(Optional) Enable OAuth for the ExploitIQ MCP Server.** If you want MCP clients (Claude Code, Cursor, etc.) to authenticate via OpenShift OAuth, create an `OAuthClient` CR for the MCP server: +--- + +## Post-Deployment Configuration + +### Enable OAuth for the MCP Server -```bash +This step is optional. Complete it if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. Configure an `OAuthClient` resource for the MCP server. + +If the resource does not exist, create it: + +```shell oc create -f - < | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -To restrict MCP access to specific OpenShift groups, edit the `EXPLOITIQ_ALLOWED_GROUPS` value in `base/exploitiq_mcp_server.yaml` (defaults to `exploitiq-users`): -```yaml -- name: EXPLOITIQ_ALLOWED_GROUPS - value: "exploitiq-users,security-team" +To restrict MCP access to specific OpenShift groups, set the `EXPLOITIQ_ALLOWED_GROUPS` environment variable in `base/exploitiq_mcp_server.yaml`. The default value is `exploitiq-users`. To allow any authenticated user, remove the environment variable entirely. + +### Configure the Swagger UI Endpoint + +This step is optional and applies to non-production environments only. Configure the Swagger UI endpoint for API testing: + +```shell +oc set env deployment -l component=exploit-iq-client \ + QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') +``` + +--- + +## Uninstalling Exploit Intelligence + +Set your deployment variant and run one of the following commands: + +```shell +export DEPLOYMENT_VARIANT_NAME=remote-nim-all +# export DEPLOYMENT_VARIANT_NAME=self-hosted-llama3.1-70b-4bit +# export DEPLOYMENT_VARIANT_NAME=mlops +``` + +To delete all resources but preserve data in PersistentVolumeClaims: + +```shell +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=persistent -f - +``` + +To delete all resources including persistent data: + +```shell +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - +``` + +--- + +## Running Exploit Intelligence Locally + +You can run Exploit Intelligence on a local machine without GPU hardware, for development, debugging, and troubleshooting. + +### Prerequisites + +Install the following tools. All binaries must be available on your system path: + +- [uv package manager](https://docs.astral.sh/uv/getting-started/installation) +- Python 3.12 +- [Java JDK 21 or later](https://www.oracle.com/il-en/java/technologies/downloads/#java21) +- [Maven](https://maven.apache.org/install.html) +- [Go](https://go.dev/doc/install) + +### Procedure + +**1.** Navigate to the repository root and create a Python virtual environment: + +```shell +cd $(git rev-parse --show-toplevel) +rm -rf .venv || true +uv venv --python 3.12 --no-cache +source .venv/bin/activate +``` + +**2.** Install dependencies: + +```shell +uv sync --no-cache +``` + +**3.** Set the required environment variables: + +```shell +export CHECKLIST_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CODE_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CVE_AGENT_EXECUTOR_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export DOC_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export JUSTIFY_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export NVIDIA_API_BASE=http:///v1 +export PYTHONUNBUFFERED=1 +export GOPROXY=https://proxy.golang.org,direct +export SERPAPI_API_KEY= +export REGISTRY_REDHAT_USERNAME="" +export REGISTRY_REDHAT_PASSWORD="" +``` + +**4.** Start the application: + +```shell +nat --log-level debug serve \ + --config_file=src/vuln_analysis/configs/config-http-openai.yml \ + --host 0.0.0.0 \ + --port 26466 ``` -When set, group membership is checked during the OAuth callback — unauthorized users see a 403 error in the browser before a token is issued, instead of a misleading "Authentication successful" followed by a connection failure. A second group check runs as Express middleware on every `/mcp` request as defense-in-depth. To allow any authenticated user, remove the env var entirely. -13. On Non Production environments, the Swagger-UI endpoint is enabled, and can be configured for testing on environment -that way: +**5.** In a separate terminal with the virtual environment active, start the Arize Phoenix tracing service: + ```shell -oc set env deployment -l component=exploit-iq-client QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') +phoenix serve ``` -14. To Uninstall the ExploitIQ System, kindly run the following command, after setting the Deployment variant environment variable, depending on your deployment variant of choice: +### Enabling Container Source Download + +This step is optional and applies to C/C++ projects only. You can enable container source download to extract RPM dependencies from Red Hat container registries. The feature uses `skopeo` to download container source layers and extract RPM packages, excluding main application RPMs. + +**Prerequisites:** + +- `skopeo` installed and available on your system path +- Valid Red Hat registry credentials +- Network access to `registry.redhat.io` + +Set the following environment variable before starting the application: ```shell -DEPLOYMENT_VARIANT_NAME=remote-nim-all -#DEPLOYMENT_VARIANT_NAME=self-hosted-llama3.1-70b-4bit -# Delete all resources but keep all data saved in PVCs -kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=persistent -f - -# Or, Delete Everything -kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - +export USE_CONTAINER_SOURCES=true ``` -### Deploy Test overlay variant (Rapid deployment) -1. Download and install [GnuPG](https://www.gnupg.org/download/) and [sops](https://github.com/getsops/sops/releases) -2. Create new namespace/project: + +--- + +## Deploying the Test Variant + +The test variant uses encrypted secret files that require GnuPG and SOPS to decrypt. + +### Prerequisites + +- [GnuPG](https://www.gnupg.org/download/) +- [SOPS](https://github.com/getsops/sops/releases) +- The private decryption key from the Bitwarden vault entry **Exploit Intelligence Tests Deployment Variant Private Key for Decryption** + +### Procedure + +**1.** Create a project namespace: + ```shell export PROJECT_NAME=exploit-test oc new-project $PROJECT_NAME ``` -3. Take private key from vault ( `ExploitIQ Tests Deployment Variant Private Key for Decryption.`) and import it to GPG: + +**2.** Import the private decryption key: + ```shell gpg --import /path/to/sec-decryption.key ``` -4. Decrypt all secret files: + +**3.** Navigate to the test overlay directory and decrypt all secret files: + ```shell cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests mkdir -p secrets -sops -d exploit-iq-ips.secret > secrets/exploit-iq-ips.json -sops -d google-sheets-secrets-enc.yaml > secrets/google-sheets-secrets.yaml -sops -d integration-tests-secrets-enc.yaml > secrets/integration-tests-secrets.yaml -sops -d mongodb-credentials.env2 > secrets/mongodb-credentials.env -sops -d oauth-secrets.env2 > secrets/oauth-secrets.env -sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml -sops -d secrets.env2 > secrets/secrets.env -sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml -sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml -sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml -sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env + +sops -d exploit-iq-ips.secret > secrets/exploit-iq-ips.json +sops -d google-sheets-secrets-enc.yaml > secrets/google-sheets-secrets.yaml +sops -d integration-tests-secrets-enc.yaml > secrets/integration-tests-secrets.yaml +sops -d mongodb-credentials.env2 > secrets/mongodb-credentials.env +sops -d oauth-secrets.env2 > secrets/oauth-secrets.env +sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml +sops -d secrets.env2 > secrets/secrets.env +sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml +sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml +sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml +sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env ``` -5. Override any secret that you need in the decrypted files, if not needed, you can continue to next step. +**4.** Override any decrypted secret values as needed, then continue to the next step. + +**5.** Configure the OAuth client secret. ->[!WARNING] -Without completing the following step with the correct secret from step 6, authentication and logging into the UI App will fail! +> [!WARNING] +> Authentication fails if you do not complete this step with the correct OAuth secret before deployment. +If an `OAuthClient` named `exploit-iq-client` already exists on the cluster, retrieve its secret: -6. If openshift resource of kind `OAuthClient` named `exploit-iq-client` exists, just get the secret from there: ```shell export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') -echo oauthClientSecret=$OAUTH_CLIENT_SECRET ``` -Otherwise, Replace `some-long-secret-used-by-the-oauth-client` with a more secure, unique secret of your own: + +Otherwise, generate a new secret (minimum 32 characters): ```shell -export OAUTH_CLIENT_SECRET="some-long-secret-used-by-the-oauth-client" +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) ``` -Eventually, Run the following: + +Write the credentials file: + ```shell cat > secrets/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET @@ -436,7 +589,8 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -7. Update `ExploitIQ` configuration file with the correct callback URL for the client service +**6.** Set the application callback URL: + ```shell cd $(git rev-parse --show-toplevel)/kustomize export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" @@ -444,15 +598,17 @@ find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEH cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests ``` -8. Now deploy to the cluster the exploitIQ system ( minus agent) with all resources: +**7.** Deploy the test overlay: + ```shell kustomize build . | oc apply -f - ``` +**8.** Configure the `OAuthClient` resource. -9. If it doesn't already exist, create the `OAuthClient` Custom Resource using the secret (from step 6) and generated route +If the resource does not exist, create it: -```bash +```shell oc create -f - < \ + exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models ``` -12. Remove untracked decrypted secrets files +**11.** Remove the decrypted secret files: + ```shell -rm -rf secrets/ +rm -rf secrets/ ``` -13. Tear down: +### Tearing Down the Test Environment + ```shell helm delete exploit-iq-tests - oc delete project $(oc project --short -q) ``` -14. Need to install on cluster [Openshift pipelines operator](https://docs.redhat.com/en/documentation/red_hat_openshift_pipelines/1.19/html/installing_and_configuring/installing-pipelines)If need to install the [exploit-iq-pac](https://github.com/apps/exploit-iq-pac/) PAC (pipeline as code) github application on a new cluster , you need to make sure to configure it according to the [PAC github application docs](https://pipelinesascode.com/docs/install/github_apps/#configure-pipelines-as-code-on-your-cluster-to-access-the-github-app). -In this case, you need to supply to the secret in the documentation github application private key generated in the github app settings, and webhook secret defined and set it in the github application settings. +### Installing Pipelines as Code + +If you need to install the OpenShift Pipelines Operator on a new cluster, refer to the [OpenShift Pipelines installation documentation](https://docs.redhat.com/en/documentation/red_hat_openshift_pipelines/1.19/html/installing_and_configuring/installing-pipelines). + +To configure the [Exploit Intelligence PAC GitHub application](https://github.com/apps/exploit-iq-pac/) on a new cluster, follow the [PAC GitHub application configuration guide](https://pipelinesascode.com/docs/install/github_apps/#configure-pipelines-as-code-on-your-cluster-to-access-the-github-app). You need the GitHub application private key and the webhook secret from the application settings. From 56c6a6a2ceafd896000bb33ca42d09c42155d1cf Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 23 Jun 2026 15:45:55 +0300 Subject: [PATCH 265/286] docs(kustomize): apply Red Hat style fixes and clarify credential requirements --- kustomize/README.md | 65 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index ec0d06f52..7b0b7db5a 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -19,17 +19,17 @@ limitations under the License. ## Prerequisites -Ensure that the following tools are installed and available on your system path: +Install the following tools and verify that all binaries are available on your system path: - [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) — OpenShift CLI, logged in to your target cluster - [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) — version 5 or later - `openssl` — for generating secrets and verifying certificates -You also need credentials for the following external services. Refer to the main [README](../README.md#obtain-api-keys) for instructions on obtaining them: +Obtain credentials for the following services before you begin. For instructions on creating each credential, refer to the main [README](../README.md#obtain-api-keys): -- NVIDIA API key - SerpAPI key - GitHub Advisory Database (GHSA) API key +- NVIDIA API key (required for Remote NIM deployments; a placeholder value is sufficient for self-hosted variants) - Red Hat registry credentials (`registry.redhat.io`) > [!NOTE] @@ -43,14 +43,21 @@ Complete the following steps before running any deployment command. ### Step 1. Create API Credentials -Create the `base/secrets.env` file with your API keys. You do not need all keys for every deployment variant. +Create the `base/secrets.env` file with your API keys. The following table shows which keys are required for each deployment variant: + +| Key | Self-Hosted LLM | Remote NIM | Description | +| --- | --- | --- | --- | +| `serpapi_api_key` | Required | Required | Web search for patch intelligence | +| `ghsa_api_key` | Required | Required | GitHub token for advisory lookups and repository scanning | +| `nvidia_api_key` | Placeholder | Required | A placeholder value is sufficient for self-hosted variants | +| `registry_redhat_username` | Required | Required | Red Hat registry credentials for container image scanning | +| `registry_redhat_password` | Required | Required | Red Hat registry credentials for container image scanning | ```shell cat > base/secrets.env << EOF -nvd_api_key= serpapi_api_key= -nvidia_api_key= ghsa_api_key= +nvidia_api_key= registry_redhat_username= registry_redhat_password= EOF @@ -58,8 +65,6 @@ EOF ### Step 2. Create a Project Namespace -If a namespace does not exist, create one: - ```shell export YOUR_NAMESPACE_NAME= oc new-project $YOUR_NAMESPACE_NAME @@ -67,7 +72,7 @@ oc new-project $YOUR_NAMESPACE_NAME ### Step 3. Create an Image Pull Secret -Create the secret that authorizes pulling Exploit Intelligence and Argilla container images: +Create the secret that grants permission to pull Exploit Intelligence and Argilla container images: ```shell oc create secret generic exploit-iq-pull-secret \ @@ -112,7 +117,7 @@ Exploit Intelligence uses OpenShift OAuth for user authentication. Select the op #### Option A: Create a New OAuthClient -Generate a new client secret and write the credentials file: +Use this option when deploying Exploit Intelligence for the first time. Generate a new client secret and write the credentials file: ```shell export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) @@ -128,8 +133,6 @@ EOF > [!IMPORTANT] > Save the value of `$OAUTH_CLIENT_SECRET`. You need it when you create the `OAuthClient` resource after deployment. -Use this option when deploying Exploit Intelligence for the first time. - #### Option B: Reuse an Existing OAuthClient Retrieve the secret from the existing `OAuthClient` resource and verify that it is at least 32 characters: @@ -146,7 +149,7 @@ export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) oc patch oauthclient exploit-iq-client -p "{\"secret\":\"$OAUTH_CLIENT_SECRET\"}" ``` -Then write the credentials file: +Write the credentials file: ```shell export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) @@ -242,7 +245,7 @@ openssl s_client \ > base/oidc-ca/ca-bundle.crt ``` -Alternatively, fetch the CA bundle from the cluster: +Or fetch the CA bundle from the cluster: ```shell kubectl get cm default-ingress-cert -n openshift-config-managed \ @@ -263,9 +266,9 @@ openssl crl2pkcs7 -nocrl -certfile base/oidc-ca/ca-bundle.crt | \ Exploit Intelligence supports the following deployment variants. Run only one deployment command in the next section. | Variant | Overlay | LLM | Use When | -|---|---|---|---| +| --- | --- | --- | --- | | Self-Hosted LLM | `self-hosted-llama3.1-70b-4bit` | On-cluster GPU node | You have GPU capacity in the cluster | -| Self-Hosted LLM + MLOps | `mlops` | On-cluster GPU node | You also need Grafana and Tempo observability | +| Self-Hosted LLM + MLOps | `mlops` | On-cluster GPU node | You need Grafana and Tempo observability | | Remote NIM | `remote-nim-all` | NVIDIA-hosted NIM | You use NVIDIA-hosted inference | --- @@ -367,9 +370,9 @@ oc patch oauthclient exploit-iq-client \ ### Enable OAuth for the MCP Server -This step is optional. Complete it if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. Configure an `OAuthClient` resource for the MCP server. +Complete this step if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. -If the resource does not exist, create it: +If the `OAuthClient` resource does not exist, create it: ```shell oc create -f - < Date: Tue, 23 Jun 2026 15:56:56 +0300 Subject: [PATCH 266/286] Rpm Commit Discovery Pipeline This PR adds Commit Discovery Pipeline - a 4-phase enhancement that extracts fix commit information from upstream repositories when no downstream patch is available. This enables better vulnerability analysis by discovering the actual fix code from advisory references. Key Features OSIDB Intel Integration - New intel source for internal Red Hat users Reference Mining - Extract commit hints from advisory URLs Repository Resolution - Map package names to upstream git repositories Git Commit Search - Clone repos and find fix commits using extracted hints --- .tekton/on-pull-request.yaml | 2 + ci/it/integration-tests-input.json | 85 ++ kustomize/base/exploit-iq-config.yml | 1 + kustomize/config-http-openai-local.yml | 1 + pyproject.toml | 1 + .../data/package_repo_mapping.json | 314 ++++++ .../data_models/checker_status.py | 40 +- .../data_models/cve_intel.py | 50 + src/vuln_analysis/configs/config-http-nim.yml | 1 + .../configs/config-http-openai.yml | 1 + src/vuln_analysis/configs/config-tracing.yml | 1 + src/vuln_analysis/configs/config.yml | 1 + .../functions/code_agent_graph_defs.py | 359 ++++++- .../functions/cve_build_agent.py | 78 +- .../functions/cve_checker_report.py | 52 +- .../functions/cve_fetch_intel.py | 10 +- .../functions/cve_package_code_agent.py | 673 ++++++++++++- .../functions/react_internals.py | 34 + src/vuln_analysis/tools/source_grep.py | 8 + src/vuln_analysis/tools/source_inspector.py | 12 +- .../utils/clients/osidb_client.py | 168 ++++ .../utils/git_commit_searcher.py | 952 ++++++++++++++++++ src/vuln_analysis/utils/git_repo_manager.py | 424 ++++++++ src/vuln_analysis/utils/intel_retriever.py | 40 +- src/vuln_analysis/utils/intel_utils.py | 187 ++++ src/vuln_analysis/utils/package_identifier.py | 96 +- src/vuln_analysis/utils/reference_fetcher.py | 193 ++++ src/vuln_analysis/utils/reference_parser.py | 575 +++++++++++ src/vuln_analysis/utils/repo_resolver.py | 512 ++++++++++ .../utils/rpm_checker_prompts.py | 219 +++- tests/test_git_commit_searcher.py | 462 +++++++++ tests/test_package_identifier.py | 8 + tests/test_repo_resolver.py | 331 ++++++ 33 files changed, 5742 insertions(+), 149 deletions(-) create mode 100644 src/exploit_iq_commons/data/package_repo_mapping.json create mode 100644 src/vuln_analysis/utils/clients/osidb_client.py create mode 100644 src/vuln_analysis/utils/git_commit_searcher.py create mode 100644 src/vuln_analysis/utils/git_repo_manager.py create mode 100644 src/vuln_analysis/utils/reference_fetcher.py create mode 100644 src/vuln_analysis/utils/reference_parser.py create mode 100644 src/vuln_analysis/utils/repo_resolver.py create mode 100644 tests/test_git_commit_searcher.py create mode 100644 tests/test_repo_resolver.py diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index 8a27fb357..e72258311 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -392,6 +392,8 @@ spec: value: cve_agent - name: OTEL_TRACES_ENDPOINT value: http://localhost:4318/v1/traces + - name: RPM_USER_TYPE + value: external script: | #!/bin/sh diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json index 215af15bb..99238da9a 100644 --- a/ci/it/integration-tests-input.json +++ b/ci/it/integration-tests-input.json @@ -123,6 +123,91 @@ "expected_result": "Not Exploitable", "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","false_positive"], "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2026-5121" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": true + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-2511" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "openssl", + "version": "3.5.1", + "release": "7.el9_7", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": true + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-2511" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "openssl", + "version": "3.2.1", + "release": "1.fc40", + "arch": "x86_64" + } + }, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels": ["vulnerable"], + "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-48957" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.7.2", + "release": "4.fc40", + "arch": "x86_64" + } + }, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels": ["vulnerable"], + "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-48957" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.7.2", + "release": "7.fc40", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": false } ] } diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 249ee7471..819084623 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -46,6 +46,7 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} retry_on_client_errors: false intel_plugin_config: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml index 695e48cc0..99cd4c809 100644 --- a/kustomize/config-http-openai-local.yml +++ b/kustomize/config-http-openai-local.yml @@ -45,6 +45,7 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} retry_on_client_errors: false intel_plugin_config: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin diff --git a/pyproject.toml b/pyproject.toml index c3041e8be..4a11258c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dynamic = ["version"] dependencies = [ "exploit-iq-commons", "aiohttp-client-cache==0.11", + "beautifulsoup4>=4.12.0", "aioresponses==0.7.6", "nvidia-nat[langchain,profiling,telemetry]>=1.2.0rc8,<1.3.0", "aiofiles", diff --git a/src/exploit_iq_commons/data/package_repo_mapping.json b/src/exploit_iq_commons/data/package_repo_mapping.json new file mode 100644 index 000000000..511537e56 --- /dev/null +++ b/src/exploit_iq_commons/data/package_repo_mapping.json @@ -0,0 +1,314 @@ +{ + "_meta": { + "description": "Package name to upstream git repository mapping for RPM/C/C++ packages", + "version": "1.0.0", + "last_updated": "2026-06-17", + "usage": "PRIMARY resolution strategy for REPO_RESOLUTION_NODE" + }, + "packages": { + "httpd": "https://github.com/apache/httpd", + "curl": "https://github.com/curl/curl", + "openssl": "https://github.com/openssl/openssl", + "nginx": "https://github.com/nginx/nginx", + "postgresql": "https://github.com/postgres/postgres", + "mysql": "https://github.com/mysql/mysql-server", + "sqlite": "https://github.com/sqlite/sqlite", + "redis": "https://github.com/redis/redis", + "memcached": "https://github.com/memcached/memcached", + "openssh": "https://github.com/openssh/openssh-portable", + "bind": "https://gitlab.isc.org/isc-projects/bind9", + "linux": "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git", + "glibc": "https://sourceware.org/git/glibc.git", + "gcc": "https://gcc.gnu.org/git/gcc.git", + "gnutls": "https://gitlab.com/gnutls/gnutls", + "libxml2": "https://gitlab.gnome.org/GNOME/libxml2", + "libxslt": "https://gitlab.gnome.org/GNOME/libxslt", + "python": "https://github.com/python/cpython", + "php": "https://github.com/php/php-src", + "ruby": "https://github.com/ruby/ruby", + "nodejs": "https://github.com/nodejs/node", + "golang": "https://github.com/golang/go", + "rust": "https://github.com/rust-lang/rust", + "git": "https://github.com/git/git", + "vim": "https://github.com/vim/vim", + "tmux": "https://github.com/tmux/tmux", + "bash": "https://git.savannah.gnu.org/git/bash.git", + "coreutils": "https://git.savannah.gnu.org/git/coreutils.git", + "systemd": "https://github.com/systemd/systemd", + "dbus": "https://gitlab.freedesktop.org/dbus/dbus", + "cups": "https://github.com/OpenPrinting/cups", + "samba": "https://git.samba.org/samba.git", + "openvpn": "https://github.com/OpenVPN/openvpn", + "strongswan": "https://github.com/strongswan/strongswan", + "iptables": "https://git.netfilter.org/iptables", + "nftables": "https://git.netfilter.org/nftables", + "wireshark": "https://gitlab.com/wireshark/wireshark", + "tcpdump": "https://github.com/the-tcpdump-group/tcpdump", + "libpcap": "https://github.com/the-tcpdump-group/libpcap", + "zlib": "https://github.com/madler/zlib", + "xz": "https://github.com/tukaani-project/xz", + "bzip2": "https://sourceware.org/git/bzip2.git", + "lz4": "https://github.com/lz4/lz4", + "zstd": "https://github.com/facebook/zstd", + "libpng": "https://github.com/pnggroup/libpng", + "libjpeg-turbo": "https://github.com/libjpeg-turbo/libjpeg-turbo", + "imagemagick": "https://github.com/ImageMagick/ImageMagick", + "ffmpeg": "https://git.ffmpeg.org/ffmpeg.git", + "vlc": "https://code.videolan.org/videolan/vlc", + "haproxy": "https://github.com/haproxy/haproxy", + "traefik": "https://github.com/traefik/traefik", + "envoy": "https://github.com/envoyproxy/envoy", + "kubernetes": "https://github.com/kubernetes/kubernetes", + "docker": "https://github.com/moby/moby", + "containerd": "https://github.com/containerd/containerd", + "runc": "https://github.com/opencontainers/runc", + "podman": "https://github.com/containers/podman", + "buildah": "https://github.com/containers/buildah", + "skopeo": "https://github.com/containers/skopeo", + "cri-o": "https://github.com/cri-o/cri-o", + "etcd": "https://github.com/etcd-io/etcd", + "consul": "https://github.com/hashicorp/consul", + "vault": "https://github.com/hashicorp/vault", + "terraform": "https://github.com/hashicorp/terraform", + "ansible": "https://github.com/ansible/ansible", + "puppet": "https://github.com/puppetlabs/puppet", + "chef": "https://github.com/chef/chef", + "salt": "https://github.com/saltstack/salt", + "jenkins": "https://github.com/jenkinsci/jenkins", + "grafana": "https://github.com/grafana/grafana", + "prometheus": "https://github.com/prometheus/prometheus", + "elasticsearch": "https://github.com/elastic/elasticsearch", + "logstash": "https://github.com/elastic/logstash", + "kibana": "https://github.com/elastic/kibana", + "kafka": "https://github.com/apache/kafka", + "zookeeper": "https://github.com/apache/zookeeper", + "cassandra": "https://github.com/apache/cassandra", + "mongodb": "https://github.com/mongodb/mongo", + "mariadb": "https://github.com/MariaDB/server", + "tomcat": "https://github.com/apache/tomcat", + "jetty": "https://github.com/jetty/jetty.project", + "spring-boot": "https://github.com/spring-projects/spring-boot", + "spring-framework": "https://github.com/spring-projects/spring-framework", + "struts": "https://github.com/apache/struts", + "log4j": "https://github.com/apache/logging-log4j2", + "jackson": "https://github.com/FasterXML/jackson-databind", + "guava": "https://github.com/google/guava", + "netty": "https://github.com/netty/netty", + "grpc": "https://github.com/grpc/grpc", + "protobuf": "https://github.com/protocolbuffers/protobuf", + "flatbuffers": "https://github.com/google/flatbuffers", + "capnproto": "https://github.com/capnproto/capnproto", + "thrift": "https://github.com/apache/thrift", + "avro": "https://github.com/apache/avro", + "arrow": "https://github.com/apache/arrow", + "parquet": "https://github.com/apache/parquet-java", + "spark": "https://github.com/apache/spark", + "flink": "https://github.com/apache/flink", + "hadoop": "https://github.com/apache/hadoop", + "hive": "https://github.com/apache/hive", + "presto": "https://github.com/prestodb/presto", + "trino": "https://github.com/trinodb/trino", + "airflow": "https://github.com/apache/airflow", + "celery": "https://github.com/celery/celery", + "rabbitmq": "https://github.com/rabbitmq/rabbitmq-server", + "activemq": "https://github.com/apache/activemq", + "pulsar": "https://github.com/apache/pulsar", + "nats": "https://github.com/nats-io/nats-server", + "zeromq": "https://github.com/zeromq/libzmq", + "expat": "https://github.com/libexpat/libexpat", + "freetype": "https://gitlab.freedesktop.org/freetype/freetype", + "fontconfig": "https://gitlab.freedesktop.org/fontconfig/fontconfig", + "cairo": "https://gitlab.freedesktop.org/cairo/cairo", + "pango": "https://gitlab.gnome.org/GNOME/pango", + "gdk-pixbuf": "https://gitlab.gnome.org/GNOME/gdk-pixbuf", + "gtk": "https://gitlab.gnome.org/GNOME/gtk", + "glib": "https://gitlab.gnome.org/GNOME/glib", + "libsoup": "https://gitlab.gnome.org/GNOME/libsoup", + "webkit": "https://github.com/nickthecoder/nickthecoder-website", + "webkitgtk": "https://github.com/nickthecoder/nickthecoder-website", + "chromium": "https://chromium.googlesource.com/chromium/src.git", + "firefox": "https://github.com/nickthecoder/nickthecoder-website", + "libtiff": "https://gitlab.com/libtiff/libtiff", + "giflib": "https://sourceforge.net/projects/giflib", + "libwebp": "https://chromium.googlesource.com/webm/libwebp", + "openjpeg": "https://github.com/uclouvain/openjpeg", + "libvpx": "https://chromium.googlesource.com/webm/libvpx", + "x264": "https://code.videolan.org/videolan/x264", + "x265": "https://bitbucket.org/multicoreware/x265_git", + "dav1d": "https://code.videolan.org/videolan/dav1d", + "aom": "https://aomedia.googlesource.com/aom", + "opus": "https://gitlab.xiph.org/xiph/opus", + "flac": "https://github.com/xiph/flac", + "vorbis": "https://gitlab.xiph.org/xiph/vorbis", + "ogg": "https://gitlab.xiph.org/xiph/ogg", + "libsndfile": "https://github.com/libsndfile/libsndfile", + "pulseaudio": "https://gitlab.freedesktop.org/pulseaudio/pulseaudio", + "pipewire": "https://gitlab.freedesktop.org/pipewire/pipewire", + "alsa-lib": "https://github.com/alsa-project/alsa-lib", + "openldap": "https://git.openldap.org/openldap/openldap", + "cyrus-sasl": "https://github.com/cyrusimap/cyrus-sasl", + "krb5": "https://github.com/krb5/krb5", + "nss": "https://github.com/nickthecoder/nickthecoder-website", + "nspr": "https://github.com/nickthecoder/nickthecoder-website", + "libssh": "https://git.libssh.org/projects/libssh.git", + "libssh2": "https://github.com/libssh2/libssh2", + "libevent": "https://github.com/libevent/libevent", + "libuv": "https://github.com/libuv/libuv", + "libev": "https://github.com/enki/libev", + "boost": "https://github.com/boostorg/boost", + "poco": "https://github.com/pocoproject/poco", + "cpprestsdk": "https://github.com/microsoft/cpprestsdk", + "libcurl": "https://github.com/curl/curl", + "wget": "https://git.savannah.gnu.org/git/wget.git", + "rsync": "https://github.com/WayneD/rsync", + "sudo": "https://github.com/sudo-project/sudo", + "polkit": "https://gitlab.freedesktop.org/polkit/polkit", + "shadow": "https://github.com/shadow-maint/shadow", + "pam": "https://github.com/linux-pam/linux-pam", + "util-linux": "https://github.com/util-linux/util-linux", + "e2fsprogs": "https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git", + "xfsprogs": "https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git", + "btrfs-progs": "https://github.com/kdave/btrfs-progs", + "lvm2": "https://sourceware.org/git/lvm2.git", + "mdadm": "https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git", + "parted": "https://git.savannah.gnu.org/git/parted.git", + "grub": "https://git.savannah.gnu.org/git/grub.git", + "dracut": "https://github.com/dracutdevs/dracut", + "systemd-boot": "https://github.com/systemd/systemd", + "dhcp": "https://gitlab.isc.org/isc-projects/dhcp", + "dnsmasq": "https://thekelleys.org.uk/gitweb/?p=dnsmasq.git", + "unbound": "https://github.com/NLnetLabs/unbound", + "nsd": "https://github.com/NLnetLabs/nsd", + "postfix": "https://github.com/vdukhovni/postfix", + "dovecot": "https://github.com/dovecot/core", + "exim": "https://git.exim.org/exim.git", + "sendmail": "https://github.com/nickthecoder/nickthecoder-website", + "spamassassin": "https://github.com/apache/spamassassin", + "clamav": "https://github.com/Cisco-Talos/clamav", + "fail2ban": "https://github.com/fail2ban/fail2ban", + "iptables-nft": "https://git.netfilter.org/iptables", + "nmap": "https://github.com/nmap/nmap", + "socat": "https://repo.or.cz/socat.git", + "netcat": "https://sourceforge.net/projects/netcat", + "iproute2": "https://git.kernel.org/pub/scm/network/iproute2/iproute2.git", + "ethtool": "https://git.kernel.org/pub/scm/network/ethtool/ethtool.git", + "bridge-utils": "https://git.kernel.org/pub/scm/network/bridge/bridge-utils.git", + "iw": "https://git.kernel.org/pub/scm/linux/kernel/git/jberg/iw.git", + "wpa_supplicant": "https://w1.fi/cgit/hostap", + "hostapd": "https://w1.fi/cgit/hostap", + "NetworkManager": "https://gitlab.freedesktop.org/NetworkManager/NetworkManager", + "connman": "https://git.kernel.org/pub/scm/network/connman/connman.git", + "avahi": "https://github.com/avahi/avahi", + "bluez": "https://github.com/bluez/bluez", + "qemu": "https://gitlab.com/qemu-project/qemu", + "libvirt": "https://gitlab.com/libvirt/libvirt", + "virt-manager": "https://github.com/virt-manager/virt-manager", + "lxc": "https://github.com/lxc/lxc", + "lxd": "https://github.com/canonical/lxd", + "firecracker": "https://github.com/firecracker-microvm/firecracker", + "kata-containers": "https://github.com/kata-containers/kata-containers", + "crun": "https://github.com/containers/crun", + "bubblewrap": "https://github.com/containers/bubblewrap", + "flatpak": "https://github.com/flatpak/flatpak", + "ostree": "https://github.com/ostreedev/ostree", + "rpm": "https://github.com/rpm-software-management/rpm", + "dnf": "https://github.com/rpm-software-management/dnf", + "yum": "https://github.com/rpm-software-management/yum", + "libsolv": "https://github.com/openSUSE/libsolv", + "librepo": "https://github.com/rpm-software-management/librepo", + "createrepo": "https://github.com/rpm-software-management/createrepo_c", + "dpkg": "https://git.dpkg.org/git/dpkg/dpkg.git", + "apt": "https://salsa.debian.org/apt-team/apt", + "pacman": "https://gitlab.archlinux.org/pacman/pacman", + "zypper": "https://github.com/openSUSE/zypper", + "snapd": "https://github.com/snapcore/snapd" + }, + "aliases": { + "apache": "httpd", + "apache2": "httpd", + "apache-httpd": "httpd", + "postgres": "postgresql", + "pgsql": "postgresql", + "cpython": "python", + "python3": "python", + "node": "nodejs", + "go": "golang", + "moby": "docker", + "mongo": "mongodb", + "mysql-server": "mysql", + "mariadb-server": "mariadb", + "kernel": "linux", + "linux-kernel": "linux", + "openssh-server": "openssh", + "openssh-client": "openssh", + "ssh": "openssh", + "sshd": "openssh", + "named": "bind", + "bind9": "bind", + "nginx-core": "nginx", + "libcurl4": "curl", + "curl-minimal": "curl", + "openssl-libs": "openssl", + "libssl": "openssl", + "libopenssl": "openssl", + "gnutls-utils": "gnutls", + "libgnutls": "gnutls", + "vim-minimal": "vim", + "vim-enhanced": "vim", + "neovim": "vim", + "ntp": "chrony", + "ntpd": "chrony", + "libxml2-dev": "libxml2", + "libxml2-devel": "libxml2", + "libxslt-dev": "libxslt", + "libxslt-devel": "libxslt", + "php-fpm": "php", + "php-cli": "php", + "postgresql-server": "postgresql", + "postgresql-client": "postgresql", + "mysql-client": "mysql", + "redis-server": "redis", + "redis-cli": "redis", + "memcached-devel": "memcached", + "sqlite3": "sqlite", + "libsqlite3": "sqlite", + "systemd-libs": "systemd", + "libsystemd": "systemd", + "cups-libs": "cups", + "libcups": "cups", + "samba-client": "samba", + "samba-common": "samba", + "samba-libs": "samba", + "libsmbclient": "samba", + "qemu-kvm": "qemu", + "qemu-system": "qemu", + "qemu-img": "qemu", + "libvirt-daemon": "libvirt", + "libvirt-client": "libvirt", + "docker-ce": "docker", + "docker-engine": "docker", + "containerd.io": "containerd", + "podman-docker": "podman", + "zlib-devel": "zlib", + "zlib1g": "zlib", + "xz-libs": "xz", + "liblzma": "xz", + "bzip2-libs": "bzip2", + "libbz2": "bzip2", + "zstd-devel": "zstd", + "libzstd": "zstd", + "libpng-devel": "libpng", + "libpng16": "libpng", + "libjpeg": "libjpeg-turbo", + "jpeg": "libjpeg-turbo", + "turbojpeg": "libjpeg-turbo", + "ImageMagick-libs": "imagemagick", + "libMagick": "imagemagick", + "ffmpeg-libs": "ffmpeg", + "libavcodec": "ffmpeg", + "libavformat": "ffmpeg", + "libavutil": "ffmpeg", + "vlc-libs": "vlc", + "libvlc": "vlc" + } +} diff --git a/src/exploit_iq_commons/data_models/checker_status.py b/src/exploit_iq_commons/data_models/checker_status.py index bdfbbb8e5..4047d02ea 100644 --- a/src/exploit_iq_commons/data_models/checker_status.py +++ b/src/exploit_iq_commons/data_models/checker_status.py @@ -130,6 +130,11 @@ class VulnerabilityIntel(BaseModel): default_factory=list, description="Recommended grep patterns ordered by specificity (most specific first)" ) + component_names: list[str] = Field( + default_factory=list, + description="Module or component names explicitly mentioned in CVE intel (e.g., mod_http2, libxml2_sax). " + "Only extract if explicitly named in advisory/description - do NOT infer or guess." + ) affected_bitness: Literal["32-bit", "64-bit", "both"] = Field( default="both", description="Which bitness is affected: 32-bit only, 64-bit only, or both (default)" @@ -155,6 +160,24 @@ class VulnerabilityIntel(BaseModel): description="Vendor-provided mitigations from RHSA or other intel sources (e.g., compiler flags, config changes)" ) + # Version range context from Identify phase + affected_version_range: str = Field( + default="", + description="Affected version range from NVD configurations (e.g., '< 2.4.68')" + ) + fixed_version: str = Field( + default="", + description="First fixed version from NVD intel (e.g., '2.4.68')" + ) + target_version_in_vulnerable_range: bool | None = Field( + default=None, + description="True if target package version is within the vulnerable range (from Identify phase)" + ) + target_package_version: str = Field( + default="", + description="Actual version of the target package being analyzed (e.g., '2.4.63')" + ) + def format_for_prompt(self) -> str: """Format VulnerabilityIntel for injection into L1 agent runtime prompt. @@ -182,6 +205,8 @@ def format_for_prompt(self) -> str: lines.append(f" - {p}") if self.search_keywords: lines.append(f"SEARCH_KEYWORDS: {', '.join(self.search_keywords)}") + if self.component_names: + lines.append(f"COMPONENT_NAMES: {', '.join(self.component_names)}") if self.root_cause: lines.append(f"ROOT_CAUSE: {self.root_cause}") if self.affected_bitness and self.affected_bitness != "both": @@ -190,6 +215,13 @@ def format_for_prompt(self) -> str: lines.append(f"AFFECTED_ARCHITECTURES: {', '.join(self.affected_architectures)}") if self.known_mitigations: lines.append(f"KNOWN_MITIGATIONS: {self.known_mitigations}") + # Version context - target version first for clear comparison + if self.target_package_version: + lines.append(f"TARGET_PACKAGE_VERSION: {self.target_package_version}") + if self.affected_version_range: + lines.append(f"AFFECTED_VERSION_RANGE: {self.affected_version_range}") + if self.fixed_version: + lines.append(f"FIXED_VERSION: {self.fixed_version}") return "\n".join(lines) @@ -197,11 +229,15 @@ class L1InvestigationResult(BaseModel): """Intermediate result from L1 investigation, input to L2 or report generation.""" downstream_report: dict[str, Any] | None = Field( default=None, - description="Serialized DownstreamSearchReport from L1 investigation", + description="DownstreamSearchReport from L1 investigation (as dict)", ) upstream_report: dict[str, Any] | None = Field( default=None, - description="Serialized UpstreamSearchReport from L1 investigation", + description="UpstreamSearchReport from L1 investigation (as dict)", + ) + git_search_report: dict[str, Any] | None = Field( + default=None, + description="GitSearchReport from git commit search (as dict)", ) l1_agent_answer: str | None = Field( default=None, diff --git a/src/exploit_iq_commons/data_models/cve_intel.py b/src/exploit_iq_commons/data_models/cve_intel.py index 33f92aecb..6d573c297 100644 --- a/src/exploit_iq_commons/data_models/cve_intel.py +++ b/src/exploit_iq_commons/data_models/cve_intel.py @@ -223,6 +223,55 @@ def description_fields(self): return [] +class CveIntelOsidb(IntelSource): + """ + Information about an OSIDB (Open Source Information Database) flaw entry. + Available only for internal Red Hat users on VPN. + """ + model_config = ConfigDict(extra="allow") + + class UpstreamPurl(BaseModel): + model_config = ConfigDict(extra="allow") + name: str | None = None + purl: str | None = None + ecosystem: str | None = None + + class Affect(BaseModel): + model_config = ConfigDict(extra="allow") + ps_module: str | None = None + ps_product: str | None = None + ps_component: str | None = None + ps_update_stream: str | None = None + resolution: str | None = None + affectedness: str | None = None + purl: str | None = None + + class CvssScore(BaseModel): + model_config = ConfigDict(extra="allow") + issuer: str | None = None + score: float | None = None + vector: str | None = None + cvss_version: str | None = None + + cve_id: str | None = None + impact: str | None = None + title: str | None = None + comment_zero: str | None = None + cve_description: str | None = None + statement: str | None = None + cwe_id: str | None = None + source: str | None = None + mitigation: str | None = None + upstream_purls: list[UpstreamPurl] = [] + affects: list[Affect] = [] + cvss_scores: list[CvssScore] = [] + references: list[str] = [] + + @property + def description_fields(self): + return ['comment_zero', 'cve_description', 'statement'] + + class IntelPluginData(BaseModel): label: str description: str @@ -243,6 +292,7 @@ class CveIntel(BaseModel): rhsa: CveIntelRhsa | None = None ubuntu: CveIntelUbuntu | None = None epss: CveIntelEpss | None = None + osidb: CveIntelOsidb | None = None plugin_data: list[IntelPluginData] = [] intel_score: int = 0 diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml index 0a62be3cb..23a87e85e 100644 --- a/src/vuln_analysis/configs/config-http-nim.yml +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -39,6 +39,7 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} intel_plugin_config: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml index ca40357b8..08a5b20e3 100644 --- a/src/vuln_analysis/configs/config-http-openai.yml +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -46,6 +46,7 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} intel_plugin_config: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 2b1a68575..c464451f2 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -50,6 +50,7 @@ functions: ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} intel_plugin_config: plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin plugin_config: diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index f5e42f9a0..f93258a84 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -38,6 +38,7 @@ functions: base_pickle_dir: .cache/am_cache/pickle cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} cve_process_sbom: _type: cve_process_sbom cve_verify_vuln_package: diff --git a/src/vuln_analysis/functions/code_agent_graph_defs.py b/src/vuln_analysis/functions/code_agent_graph_defs.py index f655ac375..c1278f49a 100644 --- a/src/vuln_analysis/functions/code_agent_graph_defs.py +++ b/src/vuln_analysis/functions/code_agent_graph_defs.py @@ -86,6 +86,14 @@ class CodeAgentState(MessagesState): observation: NotRequired[Observation | None] vulnerability_intel: NotRequired["VulnerabilityIntel | None"] arch_mismatch_reason: NotRequired[str | None] + # Reference Mining state (populated when no patch found via existing flows) + reference_hints: NotRequired["ReferenceHints | None"] + advisory_urls_remaining: NotRequired[list[str]] + advisory_urls_processed: NotRequired[list[str]] + # Repository Resolution state (Phase 2: package → upstream git repo) + repo_resolution: NotRequired["RepoResolution | None"] + # Git Search state (Phase 3: commit discovery in upstream repo) + git_search_report: NotRequired["GitSearchReport | None"] # --------------------------------------------------------------------------- @@ -235,6 +243,268 @@ class ReflectionBase(BaseModel): description="True if results are good enough to proceed.") +# --------------------------------------------------------------------------- +# Reference Mining schemas +# --------------------------------------------------------------------------- + +HINT_STRENGTH_STRONG = "strong" +HINT_STRENGTH_MEDIUM = "medium" +HINT_STRENGTH_WEAK = "weak" +HINT_STRENGTH_NONE = "none" + + +class ReferenceHints(BaseModel): + """Extracted hints from advisory references for commit discovery. + + These hints are extracted by LLM from advisory pages (mailing lists, + vendor advisories, security bulletins) when no direct patch/commit + URL is available. Used to guide git log searches and grep patterns. + """ + + revision_hint: str | None = Field( + default=None, + description="Single SVN revision (r1935008) or git commit hash for THIS CVE's fix", + ) + function_hints: list[str] = Field( + default_factory=list, + description="Vulnerable or patched function names", + ) + file_hints: list[str] = Field( + default_factory=list, + description="Source file paths (.c, .h, .py)", + ) + branch_hints: list[str] = Field( + default_factory=list, + description="Branch names where fix was applied", + ) + version_hints: list[str] = Field( + default_factory=list, + description="Version numbers containing the fix", + ) + sources_processed: list[str] = Field( + default_factory=list, + description="Advisory URLs that were processed", + ) + hint_strength: str = Field( + default=HINT_STRENGTH_NONE, + description="Overall strength of extracted hints: strong, medium, weak, or none", + ) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="LLM confidence in extraction quality (0.0-1.0)", + ) + + +class AdvisoryContent(BaseModel): + """Fetched advisory page content for LLM extraction. + + Represents the result of fetching a single advisory URL (mailing list + post, vendor security bulletin, etc.) for hint extraction. + """ + + url: str = Field(description="The advisory URL that was fetched") + source_type: str = Field( + description="Classification of the source: openwall, mailing_list, vendor_advisory, etc.", + ) + raw_text: str = Field( + default="", + description="Extracted text content from the advisory page", + ) + fetch_success: bool = Field( + default=True, + description="Whether the HTTP fetch succeeded", + ) + error_message: str | None = Field( + default=None, + description="Error message if fetch_success is False", + ) + + +# --------------------------------------------------------------------------- +# Repository Resolution schema (Phase 2: package → upstream git repo) +# --------------------------------------------------------------------------- + +RESOLUTION_METHOD_OSIDB_PURL = "osidb_purl" +RESOLUTION_METHOD_CURATED_MAPPING = "curated_mapping" +RESOLUTION_METHOD_PURL_PARSE = "purl_parse" +RESOLUTION_METHOD_NOT_FOUND = "not_found" + + +# --------------------------------------------------------------------------- +# Git Search schema (Phase 3: commit discovery in upstream repo) +# --------------------------------------------------------------------------- + +SEARCH_METHOD_REVISION = "revision" +SEARCH_METHOD_FUNCTION_MESSAGE = "function_message" +SEARCH_METHOD_FUNCTION_PICKAXE = "function_pickaxe" + +CONFIDENCE_REVISION_DIRECT = 0.99 +CONFIDENCE_REVISION_BRANCH = 0.95 +CONFIDENCE_REVISION_FALLBACK = 0.85 +CONFIDENCE_FUNCTION_MESSAGE = 0.75 +CONFIDENCE_FUNCTION_PICKAXE = 0.70 +CONFIDENCE_THRESHOLD_MIN = 0.50 + + +class CommitSearchResult(BaseModel): + """Result from a single search strategy. + + Represents a candidate commit found during git search, with metadata + about how it was discovered and confidence in the match. + """ + + commit_hash: str = Field( + description="Full 40-character git commit hash", + ) + commit_hash_short: str = Field( + description="Short 7-character commit hash for display", + ) + commit_message: str = Field( + description="Full commit message", + ) + author: str = Field( + description="Commit author name and email", + ) + date: str = Field( + description="Commit date in ISO format", + ) + files_changed: list[str] = Field( + default_factory=list, + description="List of files modified in commit", + ) + + search_method: Literal[ + "revision", + "function_message", + "function_pickaxe", + ] = Field(description="Strategy that found this commit") + confidence: float = Field( + ge=0.0, + le=1.0, + description="Confidence score for this result", + ) + match_details: str = Field( + default="", + description="What matched (e.g., 'grep matched r1935008')", + ) + + +class GitSearchReport(BaseModel): + """Complete report from git search node. + + Aggregates results from all search strategies with execution metadata. + The best_result is converted to ParsedPatch format for L1 agent consumption. + """ + + repo_url: str = Field(description="Repository URL that was searched") + repo_path: str | None = Field( + default=None, + description="Local path to cloned repository", + ) + branches_searched: list[str] = Field( + default_factory=list, + description="Branches that were fetched and searched", + ) + + best_result: CommitSearchResult | None = Field( + default=None, + description="Highest confidence result", + ) + all_results: list[CommitSearchResult] = Field( + default_factory=list, + description="All candidate commits found (ranked by confidence)", + ) + + commit_diff: str | None = Field( + default=None, + description="Full diff of the best result commit", + ) + parsed_patch: ParsedPatch | None = Field( + default=None, + description="Converted to ParsedPatch format for L1 agent", + ) + + search_success: bool = Field( + default=False, + description="True if at least one commit was found", + ) + strategies_tried: list[str] = Field( + default_factory=list, + description="Search strategies that were attempted", + ) + strategies_succeeded: list[str] = Field( + default_factory=list, + description="Search strategies that found results", + ) + error_messages: list[str] = Field( + default_factory=list, + description="Errors encountered during search", + ) + + clone_time_ms: int = Field(default=0, description="Time to clone/update repo") + search_time_ms: int = Field(default=0, description="Time to search") + total_time_ms: int = Field(default=0, description="Total node execution time") + + +class RepoResolution(BaseModel): + """Result of repository resolution for a package. + + Maps package name → upstream git repository URL using multiple strategies: + 1. OSIDB upstream_purls (highest confidence) + 2. Curated package → repo mapping (PRIMARY for C/C++) + 3. PURL parsing from any intel source + + Only public repositories are supported. If resolution fails, GIT_SEARCH_NODE + is skipped and the flow proceeds directly to L1_AGENT_NODE. + """ + + # Resolution result + repo_url: str | None = Field( + default=None, + description="Upstream git repository URL (clone URL). Only public repos supported.", + ) + repo_platform: str | None = Field( + default=None, + description="Hosting platform: github, gitlab, kernel.org, bitbucket, etc.", + ) + + # Resolution metadata + resolution_method: Literal[ + "osidb_purl", + "curated_mapping", + "purl_parse", + "not_found", + ] = Field(default=RESOLUTION_METHOD_NOT_FOUND) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in resolution: 0.95 (OSIDB), 0.90 (mapping), 0.85 (alias), 0.80 (PURL)", + ) + is_alias: bool = Field( + default=False, + description="True if resolved via alias in curated mapping (confidence 0.85 vs 0.90)", + ) + + # Alternative repos (if multiple found) + alternative_repos: list[str] = Field( + default_factory=list, + description="Other candidate repo URLs found during resolution", + ) + + # Tracking + strategies_tried: list[str] = Field( + default_factory=list, + description="List of resolution strategies attempted", + ) + error_messages: list[str] = Field( + default_factory=list, + description="Errors encountered during resolution", + ) + + # --------------------------------------------------------------------------- # Code Agent Report schema # --------------------------------------------------------------------------- @@ -992,18 +1262,27 @@ def _cap_snippets_by_type( return out -def _extract_downstream_patch_code_snippets( - downstream_report: DownstreamSearchReport | None, +def _extract_snippets_from_patch( + parsed_patch: ParsedPatch | None, + source: str, ) -> list[CodeSnippet]: - """Extract vulnerable/fix snippets from the downstream parsed patch only. + """Extract vulnerable/fix snippets from a parsed patch. For purely additive patches (no removed lines), shows context lines as "vulnerable" since they represent the code lacking the fix. + + Args: + parsed_patch: The parsed patch to extract snippets from. + source: The source identifier for the snippets (e.g., "downstream_patch", + "upstream_patch", "git_search"). + + Returns: + List of CodeSnippet objects extracted from the patch. """ - if not downstream_report or not downstream_report.parsed_patch: + if not parsed_patch: return [] snippets: list[CodeSnippet] = [] - for pf in downstream_report.parsed_patch.files: + for pf in parsed_patch.files: for hunk in pf.hunks: if hunk.removed_lines: snippets.append(CodeSnippet( @@ -1011,7 +1290,7 @@ def _extract_downstream_patch_code_snippets( line_number=hunk.source_start, code="\n".join(hunk.removed_lines[:10]), snippet_type="vulnerable", - source="downstream_patch", + source=source, )) elif hunk.context_lines and hunk.added_lines: snippets.append(CodeSnippet( @@ -1019,7 +1298,7 @@ def _extract_downstream_patch_code_snippets( line_number=hunk.source_start, code="\n".join(hunk.context_lines[:10]), snippet_type="vulnerable", - source="downstream_patch", + source=source, )) if hunk.added_lines: snippets.append(CodeSnippet( @@ -1027,50 +1306,35 @@ def _extract_downstream_patch_code_snippets( line_number=hunk.target_start, code="\n".join(hunk.added_lines[:10]), snippet_type="fix", - source="downstream_patch", + source=source, )) return snippets +def _extract_downstream_patch_code_snippets( + downstream_report: DownstreamSearchReport | None, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from the downstream parsed patch.""" + patch = downstream_report.parsed_patch if downstream_report else None + return _extract_snippets_from_patch(patch, "downstream_patch") + + +def _extract_git_search_code_snippets( + git_search_report: GitSearchReport | None, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from the git search parsed patch.""" + patch = git_search_report.parsed_patch if git_search_report else None + return _extract_snippets_from_patch(patch, "git_search") + + def _extract_code_snippets( downstream_report: DownstreamSearchReport | None, upstream_report: UpstreamSearchReport | None, ) -> list[CodeSnippet]: - """Extract code snippets from parsed patches. - - For purely additive patches (no removed lines), shows context lines - as "vulnerable" since they represent the code lacking the fix. - """ - snippets: list[CodeSnippet] = _extract_downstream_patch_code_snippets(downstream_report) - - if upstream_report and upstream_report.fixed_parsed_patch: - for pf in upstream_report.fixed_parsed_patch.files: - for hunk in pf.hunks: - if hunk.removed_lines: - snippets.append(CodeSnippet( - file_path=pf.clean_target_path, - line_number=hunk.source_start, - code="\n".join(hunk.removed_lines[:10]), - snippet_type="vulnerable", - source="upstream_patch", - )) - elif hunk.context_lines and hunk.added_lines: - snippets.append(CodeSnippet( - file_path=pf.clean_target_path, - line_number=hunk.source_start, - code="\n".join(hunk.context_lines[:10]), - snippet_type="vulnerable", - source="upstream_patch", - )) - if hunk.added_lines: - snippets.append(CodeSnippet( - file_path=pf.clean_target_path, - line_number=hunk.target_start, - code="\n".join(hunk.added_lines[:10]), - snippet_type="fix", - source="upstream_patch", - )) - + """Extract code snippets from downstream and upstream parsed patches.""" + snippets = _extract_downstream_patch_code_snippets(downstream_report) + upstream_patch = upstream_report.fixed_parsed_patch if upstream_report else None + snippets.extend(_extract_snippets_from_patch(upstream_patch, "upstream_patch")) return snippets @@ -1128,6 +1392,7 @@ async def generate_code_agent_report( descriptions: list[tuple[str, str]], downstream_report: DownstreamSearchReport | None, upstream_report: UpstreamSearchReport | None, + git_search_report: GitSearchReport | None = None, l1_agent_answer: str | None, tracer, policy_context: str = "", @@ -1153,6 +1418,9 @@ async def generate_code_agent_report( Output of downstream search (may be None). upstream_report: Output of upstream search (may be None). + git_search_report: + Output of git search via reference mining (may be None). Used as + third-priority source for code snippets after downstream and upstream. l1_agent_answer: Final answer from the L1 ReAct agent (may be None). tracer: @@ -1263,6 +1531,11 @@ async def generate_code_agent_report( ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) report.code_snippets = _cap_snippets_by_type(ranked) snippet_source = "upstream_patch" + elif git_search_report and git_search_report.parsed_patch: + raw = _extract_git_search_code_snippets(git_search_report) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "git_search" elif not report.code_snippets: report.code_snippets = _extract_code_snippets(downstream_report, upstream_report) diff --git a/src/vuln_analysis/functions/cve_build_agent.py b/src/vuln_analysis/functions/cve_build_agent.py index bb4e8b458..dfca3dce4 100644 --- a/src/vuln_analysis/functions/cve_build_agent.py +++ b/src/vuln_analysis/functions/cve_build_agent.py @@ -41,7 +41,15 @@ from nat.builder.context import Context from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output, invoke_comprehension +from vuln_analysis.functions.react_internals import ( + CheckerThought, + CodeFindings, + Observation, + FORCED_FINISH_PROMPT, + check_empty_output, + invoke_comprehension, + CheckerSearchTracker, +) from vuln_analysis.functions.build_agent_graph_defs import ( BuildAgentState, @@ -185,6 +193,9 @@ async def create_graph_build_agent( investigation_stack: list[L2InvestigationPhase] = [] investigation_stack.append(L2InvestigationPhase.HARDENING) investigation_stack.append(L2InvestigationPhase.CONFIGURATION) + + search_tracker = CheckerSearchTracker() + # ------------------------------------------------------------------------- # L2 graph nodes # ------------------------------------------------------------------------- @@ -340,7 +351,7 @@ async def thought_node(state: BuildAgentState) -> dict: logger.info("thought_node: starting step %d", step_num) runtime_prompt = state.get("runtime_prompt") - _messages = [SystemMessage(content=runtime_prompt)] + state["messages"] # Reserved for LLM call + _messages = [SystemMessage(content=runtime_prompt)] + state["messages"] with tracer.push_active_function("thought_node", input_data={}) as span: obs = state.get("observation", None) @@ -351,12 +362,41 @@ async def thought_node(state: BuildAgentState) -> dict: findings_context = "\n".join(f"- {f}" for f in recent_findings) context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" _messages.append(SystemMessage(content=context_block)) - + response: CheckerThought = await thought_llm.ainvoke(_messages) - + + duplicate_warning = None + if response.mode == "act" and response.actions: + tool_name = response.actions.tool + query = response.actions.query or "" + duplicate_warning = search_tracker.check_before_execute(tool_name, query) + + if duplicate_warning: + logger.warning( + "thought_node: duplicate search detected - tool=%s, query=%s", + tool_name, + query, + ) + _messages.append(HumanMessage(content=duplicate_warning)) + response = await thought_llm.ainvoke(_messages) + + if response.mode == "act" and response.actions: + retry_warning = search_tracker.check_before_execute( + response.actions.tool, + response.actions.query or "", + ) + if retry_warning: + logger.warning("thought_node: duplicate persists after re-prompt, forcing finish") + response = CheckerThought( + thought="Duplicate search loop detected - concluding with available evidence.", + mode="finish", + actions=None, + final_answer="Unable to gather additional evidence. Conclude based on KNOWLEDGE.", + ) + if response.mode == "finish": ai_message = AIMessage(content=response.final_answer or "Analysis complete.") - else: + elif response.actions: tool_name = response.actions.tool arguments = response.actions.query tool_call_id = str(uuid.uuid4()) @@ -364,11 +404,22 @@ async def thought_node(state: BuildAgentState) -> dict: content=response.thought, tool_calls=[{"name": tool_name, "args": {"query": arguments}, "id": tool_call_id}] ) + else: + logger.warning("thought_node: mode='act' but actions is None, forcing finish") + response = CheckerThought( + thought=response.thought or "No tool action provided", + mode="finish", + actions=None, + final_answer=response.thought or "Analysis complete.", + ) + ai_message = AIMessage(content=response.final_answer) + span.set_output({ "thought": response.thought, "mode": response.mode, "actions": response.actions, "final_answer": response.final_answer, + "duplicate_warning": duplicate_warning, }) return { @@ -387,6 +438,11 @@ async def observation_node(state: BuildAgentState) -> dict: return { "messages": [AIMessage(content="No thought found")], } + if not last_thought.actions: + logger.warning("observation_node: thought has no actions (mode=%s), skipping", last_thought.mode) + return { + "messages": [AIMessage(content="No tool action to process")], + } last_thought_text = last_thought.thought tool_used = last_thought.actions.tool tool_input_detail = last_thought.actions.query @@ -609,6 +665,7 @@ async def investigation_phase_node(state: BuildAgentState) -> dict: # since build_architecture detection is no longer performed. # Normal path: proceed to hardening phase (non-kernel packages) + search_tracker.reset() runtime_prompt = await build_runtime_prompt(preprocess_data) messages = state["messages"] prune_messages = [] @@ -717,22 +774,15 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: logger.error("build_agent: L2CompileVerdict is none should never happen") return message hardening_verdict = result.get("L2HardeningVerdict") or None - harvest_report = result.get("harvest_report") or BuildHarvestReport() hardening_reason = None hardening_flags = [] if compile_verdict.compilation_status == "not_compiled": hardening_relevant = False l2_override_verdict = "not_vulnerable" elif hardening_verdict is None: - # Hardening was skipped - could be kernel package or architecture mismatch + # Hardening skipped (kernel, no build log/binary, etc.) — defer to L1/code agent hardening_relevant = False - if harvest_report.is_kernel: - # Kernel package: hardening skipped because RHEL kernels always have it - # Code IS compiled, so defer to L1 verdict (no L2 override) - l2_override_verdict = None - else: - # Architecture mismatch case: compiled but not exploitable on this arch - l2_override_verdict = "not_vulnerable" + l2_override_verdict = None else: hardening_relevant = True hardening_reason = hardening_verdict.reasoning diff --git a/src/vuln_analysis/functions/cve_checker_report.py b/src/vuln_analysis/functions/cve_checker_report.py index 805e8d5c2..130d34d2c 100644 --- a/src/vuln_analysis/functions/cve_checker_report.py +++ b/src/vuln_analysis/functions/cve_checker_report.py @@ -54,6 +54,7 @@ CodeAgentReport, CodeSnippet, DownstreamSearchReport, + GitSearchReport, ParsedPatch, UpstreamSearchReport, generate_code_agent_report, @@ -274,6 +275,16 @@ def _summarize_parsed_patch_lines(parsed_patch: ParsedPatch | None) -> list[str] return lines +def _build_commit_url(repo_url: str, commit_hash: str) -> str | None: + """Build a viewable commit URL from repo URL and commit hash.""" + from vuln_analysis.utils.web_patch_fetcher import build_patch_url_from_repo + + patch_url = build_patch_url_from_repo(repo_url, commit_hash) + if patch_url: + return patch_url.removesuffix(".patch") + return None + + def _format_target_fix_search_md(blocks: "ReportBlocks") -> str: lines: list[str] = [] lines.extend(_md_section_header(f"CVE fix search on target package ({_target_nvr(blocks)})")) @@ -385,6 +396,32 @@ def _format_external_fix_clues_md(blocks: "ReportBlocks") -> str: if upstream.reason_code_fixed_by_rebase.strip(): clue_lines.append(f" - {upstream.reason_code_fixed_by_rebase.strip()}") + git = blocks.git_search_report + if git and git.search_success and git.best_result: + result = git.best_result + clue_lines.append("- **Git commit search:**") + clue_lines.append(f" - **Repository:** `{git.repo_url}`") + # debug information + #clue_lines.append(f" - **Commit:** `{result.commit_hash_short}`") + + commit_url = _build_commit_url(git.repo_url, result.commit_hash) + if commit_url: + clue_lines.append(f" - **Commit URL:** {commit_url}") + + # debug information + #clue_lines.append(f" - **Search method:** {result.search_method}") + #clue_lines.append(f" - **Confidence:** {result.confidence:.0%}") + + if result.commit_message: + msg_preview = result.commit_message[:100] + if len(result.commit_message) > 100: + msg_preview += "..." + clue_lines.append(f" - **Message:** {msg_preview}") + + if git.parsed_patch: + patch_lines = _summarize_parsed_patch_lines(git.parsed_patch) + clue_lines.extend(f" {line}" for line in patch_lines) + if clue_lines: lines.extend(clue_lines) else: @@ -526,8 +563,9 @@ class ReportBlocks: vulnerable_snippets: list[CodeSnippet] fix_snippets: list[CodeSnippet] - # Investigation context (upstream / L1 / L2) + # Investigation context (upstream / L1 / L2 / git search) upstream_report: UpstreamSearchReport | None = None + git_search_report: GitSearchReport | None = None l1_agent_answer: str | None = None investigation_mode: str | None = None arch_gate_reason: str | None = None @@ -738,6 +776,7 @@ def _build_report_blocks( l1_result: L1InvestigationResult | None = None, l2_result: L2BuildResult | None = None, identify_result=None, + git_search_report: GitSearchReport | None = None, ) -> ReportBlocks: """Extract and format all report data into blocks.""" target_package = message.input.image.target_package @@ -806,6 +845,7 @@ def _build_report_blocks( vulnerable_snippets=vulnerable_snippets, fix_snippets=fix_snippets, upstream_report=upstream_report, + git_search_report=git_search_report, l1_agent_answer=l1_agent_answer, investigation_mode=investigation_mode, arch_gate_reason=arch_gate_reason, @@ -970,6 +1010,11 @@ def _build_analysis( - details: investigation narrative markdown (target fix search through build check) """ ctx = message.info.checker_context if message.info else None + + git_search_report: GitSearchReport | None = None + if l1_result and l1_result.git_search_report: + git_search_report = GitSearchReport.model_validate(l1_result.git_search_report) + blocks = _build_report_blocks( message, code_agent_report, @@ -979,6 +1024,7 @@ def _build_analysis( l1_result=l1_result, l2_result=l2_result, identify_result=ctx.identify_result if ctx else None, + git_search_report=git_search_report, ) label = blocks.justification_label @@ -1068,11 +1114,14 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusOutput: downstream_report: DownstreamSearchReport | None = None upstream_report: UpstreamSearchReport | None = None + git_search_report: GitSearchReport | None = None if l1_result.downstream_report: downstream_report = DownstreamSearchReport.model_validate(l1_result.downstream_report) if l1_result.upstream_report: upstream_report = UpstreamSearchReport.model_validate(l1_result.upstream_report) + if l1_result.git_search_report: + git_search_report = GitSearchReport.model_validate(l1_result.git_search_report) vuln_id = message.input.scan.vulns[0].vuln_id target_package = message.input.image.target_package @@ -1109,6 +1158,7 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusOutput: descriptions=descriptions, downstream_report=downstream_report, upstream_report=upstream_report, + git_search_report=git_search_report, l1_agent_answer=l1_result.l1_agent_answer, tracer=tracer, policy_context=policy_context, diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index 26bb52c39..b21b9238a 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -37,6 +37,10 @@ class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): """ retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") intel_plugin_config: PluginConfig = Field(description="Intel Plugin configuration") + rpm_user_type: str = Field( + default="external", + description="User profile: 'internal' (Red Hat VPN) enables OSIDB intel, 'external' disables it." + ) @register_function(config_type=CVEFetchIntelConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -49,8 +53,10 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: async def _inner(): async with aiohttp.ClientSession() as session: trace_id.set(message.input.scan.id) - intel_retriever = IntelRetriever(session=session, retry_on_client_errors=config.retry_on_client_errors, - plugins_config=[config.intel_plugin_config]) + intel_retriever = IntelRetriever(session=session, + retry_on_client_errors=config.retry_on_client_errors, + plugins_config=[config.intel_plugin_config], + rpm_user_type=config.rpm_user_type) intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py index a9721848d..251a0d17a 100644 --- a/src/vuln_analysis/functions/cve_package_code_agent.py +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -25,7 +25,7 @@ from pydantic import Field from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id -from exploit_iq_commons.data_models.checker_status import L1InvestigationResult +from exploit_iq_commons.data_models.checker_status import L1InvestigationResult, EnumIdentifyResult from langgraph.graph import StateGraph, START, END from langgraph.prebuilt import ToolNode @@ -44,6 +44,10 @@ VulnerabilityIntel, format_patch_data_for_intel, get_relevant_hunks, + ReferenceHints, + AdvisoryContent, + GitSearchReport, + CONFIDENCE_THRESHOLD_MIN, ) from vuln_analysis.utils.rpm_checker_prompts import ( L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, @@ -61,11 +65,13 @@ VULNERABILITY_INTEL_EXTRACTION_PROMPT, ) from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile -from vuln_analysis.utils.package_identifier import _extract_dist_tag +from vuln_analysis.utils.package_identifier import _extract_dist_tag, extract_nvd_version_info from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output, invoke_comprehension -from vuln_analysis.utils.intel_utils import extract_commit_url_candidates +from vuln_analysis.utils.intel_utils import extract_commit_url_candidates, extract_advisory_urls from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer +from vuln_analysis.utils.reference_fetcher import ReferenceFetcher +from vuln_analysis.utils.reference_parser import ReflectiveReferenceParser, ParserConfig from vuln_analysis.utils.token_utils import truncate_tool_output from vuln_analysis.runtime_context import ctx_state @@ -73,6 +79,7 @@ import uuid import tiktoken +import aiohttp # NEVRA = Name-Epoch-Version-Release-Architecture, the standard RPM package naming format. _RPM_NEVRA_RE = re.compile(r"^(.+?)-(?:(\d+):)?(\d\S*?)-(\S+)$") @@ -159,6 +166,30 @@ def _build_tool_strategy(tool_names: list[str]) -> str: return "\n".join(strategies) if strategies else "Use available tools to search for vulnerable and fixed code patterns." +_DISCOVERED_FILE_PATTERN = re.compile(r' in ([./\w-]+\.(?:c|h|cpp|hpp|cc|cxx|hxx))(?::\d+)?') + + +def _extract_discovered_files_from_memory(memory: list[str]) -> set[str]: + """Extract file basenames from L1 observation memory entries. + + Parses entries like: + - "VULNERABLE_CODE_FOUND: mod_proxy_ftp in ./httpd-2.4/.../mod_proxy_ftp.c" + - "CVE_CODE_FOUND: function in /path/to/file.h:123" + + Returns set of unique file basenames for build log matching (e.g., 'mod_proxy_ftp.c'). + """ + file_basenames = set() + + for entry in memory: + match = _DISCOVERED_FILE_PATTERN.search(entry) + if match: + full_path = match.group(1) + basename = Path(full_path).name + file_basenames.add(basename) + + return file_basenames + + # --------------------------------------------------------------------------- # Policy context formatting for L1 reports (Feedback-2 gap coverage) # --------------------------------------------------------------------------- @@ -240,6 +271,53 @@ def _format_policy_context_for_l1_report( return "\n".join(lines) +def _merge_hints(existing: ReferenceHints, new: ReferenceHints) -> ReferenceHints: + """Merge two ReferenceHints objects, keeping first revision and deduplicating lists.""" + return ReferenceHints( + revision_hint=existing.revision_hint if existing.revision_hint else new.revision_hint, + function_hints=list(set(existing.function_hints + new.function_hints)), + file_hints=list(set(existing.file_hints + new.file_hints)), + branch_hints=list(set(existing.branch_hints + new.branch_hints)), + version_hints=list(set(existing.version_hints + new.version_hints)), + sources_processed=existing.sources_processed + new.sources_processed, + hint_strength=new.hint_strength if new.confidence > existing.confidence else existing.hint_strength, + confidence=max(existing.confidence, new.confidence), + ) + + +def merge_reference_hints_to_intel( + intel: VulnerabilityIntel, + hints: ReferenceHints, +) -> VulnerabilityIntel: + """Merge extracted reference hints into VulnerabilityIntel. + + Reference mining hints are lower confidence than patch-derived intel, + so they are appended (not prepended) to existing lists. + + Args: + intel: Existing VulnerabilityIntel (may have data from patch analysis) + hints: ReferenceHints from reference mining + + Returns: + Updated VulnerabilityIntel with merged hints + """ + existing_funcs = set(intel.vulnerable_functions) + new_funcs = [f for f in hints.function_hints if f not in existing_funcs] + intel.vulnerable_functions = intel.vulnerable_functions + new_funcs + + existing_files = set(intel.affected_files) + new_files = [f for f in hints.file_hints if f not in existing_files] + intel.affected_files = intel.affected_files + new_files + + if hints.revision_hint and hints.revision_hint not in intel.search_keywords: + intel.search_keywords = intel.search_keywords + [hints.revision_hint] + + if not intel.fixed_version and hints.version_hints: + intel.fixed_version = hints.version_hints[0] + + return intel + + class CVEPackageCodeAgentConfig(FunctionBaseConfig, name="cve_package_code_agent"): """ Level 1 Package Code Agent. Investigates each CVE using extracted source @@ -271,6 +349,64 @@ class CVEPackageCodeAgentConfig(FunctionBaseConfig, name="cve_package_code_agent max_iterations: int = Field(default=10, description="The maximum number of iterations for the agent.") context_window_token_limit: int = Field(default=5000, description="Token limit for context window before pruning old messages.") + # Reference Mining configuration + reference_mining_enabled: bool = Field( + default=True, + description="Enable reference mining when no upstream patch is found", + ) + reference_mining_max_urls: int = Field( + default=3, + description="Maximum number of advisory URLs to process", + ) + reference_mining_fetch_timeout: int = Field( + default=30, + description="HTTP timeout per URL in seconds", + ) + reference_mining_chunk_overlap: int = Field( + default=200, + description="Token overlap between chunks to preserve context", + ) + reference_mining_prompt_overhead: int = Field( + default=2500, + description="Tokens reserved for system prompt + CVE context + output", + ) + reference_mining_max_content_bytes: int = Field( + default=500_000, + description="Maximum advisory content to fetch per URL (bytes)", + ) + + # Repository Resolution configuration + repo_resolution_validate_urls: bool = Field( + default=True, + description="Validate repo URLs are accessible before returning", + ) + + # Git Search configuration + git_search_enabled: bool = Field( + default=True, + description="Enable git commit search node", + ) + git_search_cache_dir: str = Field( + default=".cache/am_cache/checker/upstream_repos", + description="Directory to cache cloned repositories", + ) + git_search_clone_depth: int = Field( + default=1000, + description="Default depth for shallow clones", + ) + git_search_fetch_timeout_seconds: int = Field( + default=120, + description="Timeout for git clone/fetch operations", + ) + git_search_min_confidence: float = Field( + default=0.50, + description="Minimum confidence to use git search result", + ) + git_search_max_diff_size: int = Field( + default=50000, + description="Maximum diff size in characters", + ) + async def create_graph_code_agent(config: CVEPackageCodeAgentConfig, builder: Builder, state: AgentMorpheusEngineInput, tracer): # Node name constants @@ -280,6 +416,9 @@ async def create_graph_code_agent(config: CVEPackageCodeAgentConfig, builder: Bu OBSERVATION_NODE = "observation_node" DOWNSTREAM_SEARCH_NODE = "downstream_search" GATHER_MORE_INFO_NODE = "gather_more_info" + REFERENCE_MINING_NODE = "reference_mining" + REPO_RESOLUTION_NODE = "repo_resolution_node" + GIT_SEARCH_NODE = "git_search_node" L1_AGENT_NODE = "L1_agent" llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) @@ -344,13 +483,22 @@ def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observati brew_downloader = None descriptions: list[tuple[str, str]] = [] - if aIntel.ghsa: - cve_text = aIntel.ghsa.description or aIntel.ghsa.summary or "" - if cve_text: - descriptions.append(("ghsa", cve_text)) - if aIntel.ubuntu and aIntel.ubuntu.description: - descriptions.append(("ubuntu", aIntel.ubuntu.description)) - + + if aIntel.osidb: + if aIntel.osidb.statement: + descriptions.append(("osidb_statement", aIntel.osidb.statement)) + if aIntel.osidb.cve_description: + descriptions.append(("osidb", aIntel.osidb.cve_description)) + if aIntel.osidb.comment_zero and not aIntel.osidb.cve_description: + descriptions.append(("osidb_vendor", aIntel.osidb.comment_zero)) + else: + if aIntel.ghsa: + cve_text = aIntel.ghsa.description or aIntel.ghsa.summary or "" + if cve_text: + descriptions.append(("ghsa", cve_text)) + if aIntel.ubuntu and aIntel.ubuntu.description: + descriptions.append(("ubuntu", aIntel.ubuntu.description)) + cve_description = "\n".join(f"[{src}] {txt}" for src, txt in descriptions) @@ -358,23 +506,34 @@ async def L1_agent(state: CodeAgentState) -> dict: logger.info("L1_agent: starting") downstream_report = state.get("downstream_report") upstream_report = state.get("upstream_report") + git_search_report = state.get("git_search_report") # Extract potential commit URLs from intel references for analysis commit_url_candidates = extract_commit_url_candidates(aIntel) with tracer.push_active_function("Initial_Intelligence_Gathering", input_data={"commit_url_candidates": commit_url_candidates}) as span: - + if downstream_report and downstream_report.is_patch_file_available: parsed_patch = downstream_report.parsed_patch patch_data = format_patch_data_for_intel(parsed_patch) elif upstream_report and upstream_report.is_fixed_srpm_is_needed: parsed_patch = upstream_report.fixed_parsed_patch patch_data = format_patch_data_for_intel(parsed_patch) + elif git_search_report and git_search_report.parsed_patch: + parsed_patch = git_search_report.parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + logger.info( + "L1_agent: Using discovered patch from git search (commit=%s, confidence=%.2f)", + git_search_report.best_result.commit_hash_short if git_search_report.best_result else "unknown", + git_search_report.best_result.confidence if git_search_report.best_result else 0, + ) else: parsed_patch = None patch_data = "" - # Extract vendor mitigations from intel (RHSA, etc.) + # Extract vendor mitigations from intel (OSIDB preferred, fallback to RHSA) vendor_mitigations = "" - if aIntel and aIntel.rhsa: + if aIntel.osidb and aIntel.osidb.mitigation: + vendor_mitigations = aIntel.osidb.mitigation + elif aIntel.rhsa: mitigation = getattr(aIntel.rhsa, 'mitigation', None) if mitigation: mit_text = mitigation.get('value', '') if isinstance(mitigation, dict) else str(mitigation) @@ -394,6 +553,12 @@ async def L1_agent(state: CodeAgentState) -> dict: vulnerability_intel = VulnerabilityIntelSanitizer(parsed_patch).apply( vulnerability_intel ) + # Prepend component_names to search_keywords for file discovery prioritization + # This ensures L1 Agent sees module names (e.g., "mod_http2") as first search patterns + if vulnerability_intel.component_names: + vulnerability_intel.search_keywords = ( + vulnerability_intel.component_names + vulnerability_intel.search_keywords + ) # Preserve vendor mitigations in the intel object if not already extracted if vendor_mitigations and not vulnerability_intel.known_mitigations: vulnerability_intel.known_mitigations = vendor_mitigations @@ -403,6 +568,37 @@ async def L1_agent(state: CodeAgentState) -> dict: vulnerability_intel.is_patch_applied_in_build = downstream_report.is_patch_applied_in_build vulnerability_intel.patch_file_name = downstream_report.patch_file_name or "" + # Populate version range context from Identify phase + if ctx and ctx.identify_result: + identify_result = ctx.identify_result + if identify_result.is_target_package_affected == EnumIdentifyResult.YES: + vulnerability_intel.target_version_in_vulnerable_range = True + elif identify_result.is_target_package_affected == EnumIdentifyResult.NO: + vulnerability_intel.target_version_in_vulnerable_range = False + + affected_range, fixed_ver = extract_nvd_version_info(aIntel, target_package.name) + if affected_range: + vulnerability_intel.affected_version_range = affected_range + if fixed_ver: + vulnerability_intel.fixed_version = fixed_ver + + # Always set target version for LLM reasoning (not dependent on identify phase) + vulnerability_intel.target_package_version = target_package.version + + # Merge reference hints if available from reference mining + reference_hints = state.get("reference_hints") + if reference_hints: + vulnerability_intel = merge_reference_hints_to_intel( + intel=vulnerability_intel, + hints=reference_hints, + ) + logger.info( + "L1_agent: Merged reference hints - %d functions, %d files, revision=%s", + len(reference_hints.function_hints), + len(reference_hints.file_hints), + reference_hints.revision_hint, + ) + # Early architecture check - skip further investigation if CVE doesn't apply target_arch = target_package.arch target_bitness = ARCH_TO_BITNESS.get(target_arch) @@ -511,6 +707,24 @@ async def L1_agent(state: CodeAgentState) -> dict: "mode": "upstream_patch_verification", "patch_filename": upstream_report.fixed_srpm_file_name, }) + # use case 4: Fix commit discovered via git search + elif git_search_report and git_search_report.parsed_patch: + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "git_search_patch_verification", + "commit_hash": git_search_report.best_result.commit_hash_short if git_search_report.best_result else "", + "confidence": git_search_report.best_result.confidence if git_search_report.best_result else 0, + "search_method": git_search_report.best_result.search_method if git_search_report.best_result else "", + }) else: # Use case 4: Default prompt - no patch context, use VulnerabilityIntel from CVE description runtime_prompt = L1_AGENT_PROMPT_TEMPLATE_NO_PATCH.format( @@ -601,6 +815,399 @@ async def gather_more_info(state: CodeAgentState) -> dict: "upstream_report": report, } + async def should_continue_after_gather(state: CodeAgentState) -> str: + """Route after gather_more_info: to reference mining if no upstream patch found.""" + if not config.reference_mining_enabled: + return L1_AGENT_NODE + + upstream_report = state.get("upstream_report") + + has_upstream_patch = ( + upstream_report and + (upstream_report.fixed_parsed_patch or upstream_report.osv_result) + ) + + if has_upstream_patch: + logger.info("should_continue_after_gather: upstream patch found, proceeding to L1_agent") + return L1_AGENT_NODE + else: + logger.info("should_continue_after_gather: no upstream patch, trying reference mining") + return REFERENCE_MINING_NODE + + async def reference_mining_node(state: CodeAgentState) -> dict: + """Extract hints from advisory references when no patch is found. + + Uses LLM with reflection pattern to: + 1. Extract hints from advisory content + 2. Decide if more sources need to be fetched + + Returns: + State update with reference_hints + """ + logger.info("reference_mining_node: starting for %s", vuln_id) + + with tracer.push_active_function("reference_mining", input_data={}) as span: + # Extract advisory URLs from intel + advisory_urls = extract_advisory_urls(aIntel) + if not advisory_urls: + logger.info("reference_mining_node: no advisory URLs found") + span.set_output({"status": "no_advisory_urls"}) + return { + "messages": [AIMessage(content="No advisory URLs available for reference mining")], + "reference_hints": None, + "advisory_urls_remaining": [], + "advisory_urls_processed": [], + } + + logger.info("reference_mining_node: found %d advisory URLs", len(advisory_urls)) + + # Initialize parser and fetcher + max_chunk_tokens = config.context_window_token_limit - config.reference_mining_prompt_overhead + parser_config = ParserConfig( + max_chunk_tokens=max_chunk_tokens, + chunk_overlap_tokens=config.reference_mining_chunk_overlap, + ) + parser = ReflectiveReferenceParser(llm=llm, config=parser_config) + fetcher = ReferenceFetcher( + timeout_seconds=config.reference_mining_fetch_timeout, + max_content_length=config.reference_mining_max_content_bytes, + ) + + # Initialize state + hints = ReferenceHints() + processed: list[str] = [] + remaining = list(advisory_urls) + max_refs = config.reference_mining_max_urls + + async with aiohttp.ClientSession() as session: + while remaining and len(processed) < max_refs: + # Pop highest priority URL (sorted by priority) + url, source_type, priority = remaining.pop(0) + processed.append(url) + + logger.info( + "reference_mining_node: processing URL %d/%d: %s", + len(processed), max_refs, url + ) + + # Fetch advisory content + content: AdvisoryContent = await fetcher.fetch_advisory( + session=session, + url=url, + source_type=source_type, + ) + + if not content.fetch_success: + logger.warning( + "reference_mining_node: failed to fetch %s: %s", + url, content.error_message + ) + continue + + if not content.raw_text.strip(): + logger.warning("reference_mining_node: empty content from %s", url) + continue + + # Extract hints from content + with tracer.push_active_function("extract_advisory_hints", input_data={ + "url": url, + "source_type": source_type, + "content_length": len(content.raw_text), + }) as extract_span: + extraction = await parser.extract_hints( + advisory_content=content.raw_text, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=target_package.name, + advisory_url=url, + source_type=source_type, + ) + + extract_span.set_output({ + "revision_hint": extraction.revision_hint, + "function_hints": extraction.function_hints[:5], + "file_hints": extraction.file_hints[:5], + }) + + # Merge hints BEFORE reflection so reflection sees accumulated state + hints = _merge_hints(hints, extraction) + + # Reflect on ACCUMULATED hints to assess overall strength + reflection = await parser.reflect_on_extraction( + extraction=hints, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=target_package.name, + current_url=url, + remaining_urls=remaining, + ) + + hints.hint_strength = reflection.hint_strength + hints.confidence = max(hints.confidence, reflection.confidence) + + # Control flow decision + if not reflection.needs_more_sources: + logger.info( + "reference_mining_node: hints sufficient (strength=%s, confidence=%.2f)", + reflection.hint_strength, reflection.confidence + ) + break + + # Update sources processed + hints.sources_processed = processed + + span.set_output({ + "urls_processed": len(processed), + "hint_strength": hints.hint_strength, + "confidence": hints.confidence, + "revision_hint": hints.revision_hint, + "function_hints": len(hints.function_hints), + "file_hints": len(hints.file_hints), + }) + + logger.info( + "reference_mining_node: completed - revision=%s, %d functions, %d files", + hints.revision_hint, + len(hints.function_hints), + len(hints.file_hints), + ) + + return { + "messages": [AIMessage(content="Reference mining completed")], + "reference_hints": hints if (hints.revision_hint or hints.function_hints or hints.file_hints) else None, + "advisory_urls_remaining": [u[0] for u in remaining], + "advisory_urls_processed": processed, + } + + def should_run_repo_resolution(state: CodeAgentState) -> str: + """Decide if repo resolution should run after reference mining. + + Run repo resolution when reference mining produced usable hints. + + Note: By the time we reach here, we already know there's no upstream + patch (that's why we went to reference_mining in the first place). + + Returns: + Node name to route to. + """ + reference_hints = state.get("reference_hints") + + # Skip if no hints to search for + if not reference_hints: + logger.debug("should_run_repo_resolution: no reference hints") + return L1_AGENT_NODE + + # Check hint strength + hint_strength = getattr(reference_hints, "hint_strength", "none") + if hint_strength == "none": + logger.debug("should_run_repo_resolution: hint strength is none") + return L1_AGENT_NODE + + logger.info("should_run_repo_resolution: routing to repo_resolution") + return REPO_RESOLUTION_NODE + + async def repo_resolution_node(state: CodeAgentState) -> dict: + """Resolve package name to upstream git repository URL. + + Uses multiple strategies in priority order: + 1. OSIDB upstream_purls (confidence 0.95) + 2. Curated package → repo mapping (confidence 0.90, PRIMARY) + 3. PURL parsing from any intel source (confidence 0.80) + + Returns: + State update with repo_resolution + """ + logger.info("repo_resolution_node: starting for %s", target_package.name) + + with tracer.push_active_function("repo_resolution", input_data={}) as span: + from vuln_analysis.utils.repo_resolver import RepoResolver + + resolver = RepoResolver() + resolution = resolver.resolve( + package_name=target_package.name, + intel=aIntel, + ) + + span.set_output({ + "repo_url": resolution.repo_url, + "resolution_method": resolution.resolution_method, + "confidence": resolution.confidence, + "is_alias": resolution.is_alias, + "strategies_tried": resolution.strategies_tried, + }) + + if resolution.repo_url: + logger.info( + "repo_resolution_node: resolved %s → %s (method=%s, confidence=%.2f)", + target_package.name, + resolution.repo_url, + resolution.resolution_method, + resolution.confidence, + ) + msg_content = f"Resolved repository: {resolution.repo_url}" + else: + logger.info( + "repo_resolution_node: no repo found for %s", + target_package.name, + ) + msg_content = f"Could not resolve repository for {target_package.name}" + + return { + "messages": [AIMessage(content=msg_content)], + "repo_resolution": resolution, + } + + def should_run_git_search(state: CodeAgentState) -> str: + """Decide if git search should run after repo resolution. + + Run git search when: + 1. Git search is enabled in config + 2. Repo URL was resolved successfully + 3. Reference hints exist (something to search for) + """ + if not config.git_search_enabled: + logger.debug("should_run_git_search: disabled in config") + return L1_AGENT_NODE + + repo_resolution = state.get("repo_resolution") + reference_hints = state.get("reference_hints") + + if not repo_resolution or not repo_resolution.repo_url: + logger.info("should_run_git_search: no repo_url, skipping to L1") + return L1_AGENT_NODE + + if not reference_hints: + logger.info("should_run_git_search: no hints, skipping to L1") + return L1_AGENT_NODE + + has_useful_hints = ( + reference_hints.revision_hint or + reference_hints.function_hints + ) + + if not has_useful_hints: + logger.info("should_run_git_search: no useful hints, skipping to L1") + return L1_AGENT_NODE + + logger.info("should_run_git_search: routing to git_search") + return GIT_SEARCH_NODE + + async def git_search_node(state: CodeAgentState) -> dict: + """Search git repository for CVE fix commit. + + Requires: + - state.repo_resolution.repo_url (from REPO_RESOLUTION_NODE) + - state.reference_hints (from REFERENCE_MINING_NODE) + + Returns: + State update with git_search_report + """ + import time + from vuln_analysis.utils.git_repo_manager import GitRepoManager + from vuln_analysis.utils.git_commit_searcher import GitCommitSearcher + + start_time = time.time() + + repo_resolution = state.get("repo_resolution") + reference_hints = state.get("reference_hints") + + if not repo_resolution or not repo_resolution.repo_url: + logger.warning("git_search_node: No repo_url available, skipping") + return {"git_search_report": None} + + logger.info("git_search_node: searching %s", repo_resolution.repo_url) + + with tracer.push_active_function("git_search", input_data={ + "repo_url": repo_resolution.repo_url, + "revision_hint": reference_hints.revision_hint if reference_hints else None, + "has_function_hints": bool(reference_hints and reference_hints.function_hints), + }) as span: + try: + repo_manager = GitRepoManager( + cache_dir=config.git_search_cache_dir, + default_depth=config.git_search_clone_depth, + fetch_timeout_seconds=config.git_search_fetch_timeout_seconds, + ) + searcher = GitCommitSearcher(repo_manager, tracer=tracer) + + repo_path, is_fresh = await repo_manager.clone_or_update( + repo_resolution.repo_url + ) + clone_time = int((time.time() - start_time) * 1000) + + search_start = time.time() + report = await searcher.search( + repo_path=repo_path, + repo_url=repo_resolution.repo_url, + hints=reference_hints, + llm=llm, + package_name=target_package.name, + cve_description=cve_description, + ) + search_time = int((time.time() - search_start) * 1000) + + if report.best_result and report.best_result.confidence >= config.git_search_min_confidence: + parsed_patch = await searcher.commit_to_parsed_patch( + repo_path=repo_path, + commit_hash=report.best_result.commit_hash, + max_diff_size=config.git_search_max_diff_size, + repo_url=repo_resolution.repo_url, + ) + report.parsed_patch = parsed_patch + report.commit_diff = await searcher.get_commit_diff( + repo_path=repo_path, + commit_hash=report.best_result.commit_hash, + max_diff_size=config.git_search_max_diff_size, + ) + + report.clone_time_ms = clone_time + report.search_time_ms = search_time + report.total_time_ms = int((time.time() - start_time) * 1000) + + span.set_output({ + "search_success": report.search_success, + "strategies_tried": report.strategies_tried, + "strategies_succeeded": report.strategies_succeeded, + "best_result_method": report.best_result.search_method if report.best_result else None, + "best_result_confidence": report.best_result.confidence if report.best_result else 0, + "total_results": len(report.all_results), + "clone_time_ms": report.clone_time_ms, + "search_time_ms": report.search_time_ms, + "cache_hit": not is_fresh, + }) + + if report.best_result: + msg_content = ( + f"Found fix commit: {report.best_result.commit_hash_short} " + f"(confidence={report.best_result.confidence:.2f}, method={report.best_result.search_method})" + ) + logger.info( + "git_search_node: Found commit %s with confidence %.2f", + report.best_result.commit_hash_short, + report.best_result.confidence, + ) + else: + msg_content = "Git search did not find a fix commit" + logger.info("git_search_node: No commit found") + + return { + "messages": [AIMessage(content=msg_content)], + "git_search_report": report, + } + + except Exception as e: + logger.exception("git_search_node: Error during search") + error_report = GitSearchReport( + repo_url=repo_resolution.repo_url, + search_success=False, + error_messages=[str(e)], + ) + span.set_output({"error": str(e)}) + return { + "messages": [AIMessage(content=f"Git search failed: {e}")], + "git_search_report": error_report, + } + async def thought_node(state: CodeAgentState) -> dict: """Generate next thought/action using the LLM.""" step_num = state.get("step", 0) @@ -696,6 +1303,11 @@ async def observation_node(state: CodeAgentState) -> dict: return { "messages": [AIMessage(content="No thought found")], } + if not last_thought.actions: + logger.warning("observation_node: thought has no actions (mode=%s), skipping", last_thought.mode) + return { + "messages": [AIMessage(content="No tool action to process")], + } last_thought_text = last_thought.thought tool_used = last_thought.actions.tool tool_input_detail = last_thought.actions.query @@ -825,6 +1437,9 @@ def should_continue_after_intel(state: CodeAgentState) -> str: flow.add_node(DOWNSTREAM_SEARCH_NODE, downstream_search) flow.add_node(GATHER_MORE_INFO_NODE, gather_more_info) + flow.add_node(REFERENCE_MINING_NODE, reference_mining_node) + flow.add_node(REPO_RESOLUTION_NODE, repo_resolution_node) + flow.add_node(GIT_SEARCH_NODE, git_search_node) flow.add_node(L1_AGENT_NODE, L1_agent) flow.add_node(THOUGHT_NODE, thought_node) flow.add_node(TOOL_NODE, tools_node) @@ -836,7 +1451,23 @@ def should_continue_after_intel(state: CodeAgentState) -> str: L1_AGENT_NODE: L1_AGENT_NODE, GATHER_MORE_INFO_NODE: GATHER_MORE_INFO_NODE, }) - flow.add_edge(GATHER_MORE_INFO_NODE, L1_AGENT_NODE) + # Route from gather_more_info: to reference mining if no upstream patch, else directly to L1 + flow.add_conditional_edges(GATHER_MORE_INFO_NODE, should_continue_after_gather, { + L1_AGENT_NODE: L1_AGENT_NODE, + REFERENCE_MINING_NODE: REFERENCE_MINING_NODE, + }) + # Route from reference mining: to repo resolution if hints found, else directly to L1 + flow.add_conditional_edges(REFERENCE_MINING_NODE, should_run_repo_resolution, { + L1_AGENT_NODE: L1_AGENT_NODE, + REPO_RESOLUTION_NODE: REPO_RESOLUTION_NODE, + }) + # Route from repo resolution: to git search if enabled and hints available, else directly to L1 + flow.add_conditional_edges(REPO_RESOLUTION_NODE, should_run_git_search, { + L1_AGENT_NODE: L1_AGENT_NODE, + GIT_SEARCH_NODE: GIT_SEARCH_NODE, + }) + # Git search always leads to L1 agent + flow.add_edge(GIT_SEARCH_NODE, L1_AGENT_NODE) flow.add_conditional_edges(L1_AGENT_NODE, should_continue_after_intel, { END: END, THOUGHT_NODE: THOUGHT_NODE, @@ -895,6 +1526,7 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: l1_result = L1InvestigationResult( downstream_report=None, upstream_report=None, + git_search_report=None, l1_agent_answer=arch_mismatch_reason, vulnerability_intel=result.get("vulnerability_intel"), preliminary_verdict="not_present", @@ -926,11 +1558,24 @@ async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: downstream_report: DownstreamSearchReport | None = result.get("downstream_report") upstream_report: UpstreamSearchReport | None = result.get("upstream_report") + git_search_report: GitSearchReport | None = result.get("git_search_report") vulnerability_intel: VulnerabilityIntel | None = result.get("vulnerability_intel") + observation = result.get("observation") + if observation and vulnerability_intel: + discovered_files = _extract_discovered_files_from_memory(observation.memory or []) + if discovered_files: + existing_files = set(vulnerability_intel.affected_files or []) + vulnerability_intel.affected_files = list(existing_files | discovered_files) + logger.debug( + "package_code_agent: Updated affected_files with L1 discoveries: %s", + discovered_files, + ) + l1_result = L1InvestigationResult( downstream_report=downstream_report.model_dump() if downstream_report else None, upstream_report=upstream_report.model_dump() if upstream_report else None, + git_search_report=git_search_report.model_dump() if git_search_report else None, l1_agent_answer=final_answer, vulnerability_intel=vulnerability_intel, preliminary_verdict=preliminary_verdict, diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 15386f61e..22300de74 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -276,6 +276,40 @@ def check_thought_behavior(self, action: str, action_input: str, output) -> tupl return False, "" +class CheckerSearchTracker(BaseRulesTracker): + """Lightweight search tracker for L2 Build Agent. + + Prevents duplicate tool calls by checking BEFORE execution. Inherits + duplicate detection logic from BaseRulesTracker but provides a simpler + interface for the "check-then-execute" pattern needed when context + pruning causes the LLM to forget previous searches. + """ + + def check_before_execute(self, tool: str, query: str) -> str | None: + """Check if this search was already performed BEFORE executing. + + Args: + tool: Tool name (e.g., 'Source Grep') + query: Search query (e.g., 'logs:mod_proxy_ftp.c') + + Returns: + None if this is a new search (automatically recorded). + Warning message string if this is a duplicate search. + """ + if self._rule_duplicate_call(tool, query): + return ( + f"DUPLICATE SEARCH DETECTED: You already called '{tool}' with '{query}'. " + f"Check KNOWLEDGE for the previous result. " + f"Use a DIFFERENT query or tool, or finish with available evidence." + ) + self.add_action(tool, query, output=None) + return None + + def reset(self): + """Clear search history. Call when switching investigation phases.""" + self.action_history.clear() + + class ReachabilityRulesTracker(BaseRulesTracker): """Behavioral rules for the reachability sub-agent.""" def __init__(self): diff --git a/src/vuln_analysis/tools/source_grep.py b/src/vuln_analysis/tools/source_grep.py index 38b25cf98..b549df58f 100644 --- a/src/vuln_analysis/tools/source_grep.py +++ b/src/vuln_analysis/tools/source_grep.py @@ -168,6 +168,14 @@ async def _arun(query: str) -> str: source_key = checker_context.source_key pattern, file_glob, target, word_boundary = _parse_query(query) + # For logs target, normalize path patterns to just filename + # LLMs often pass full paths like "modules/proxy/mod_proxy_ftp.c" from AFFECTED_FILES + # but build logs only reference filenames like "mod_proxy_ftp.c" + if target == "logs" and isinstance(pattern, str) and "/" in pattern: + original_pattern = pattern + pattern = pattern.rsplit("/", 1)[-1] + logger.debug("logs target: normalized pattern '%s' -> '%s'", original_pattern, pattern) + # For logs target, use arch-specific subdirectory if target == "logs": target_package = workflow_state.original_input.input.image.target_package diff --git a/src/vuln_analysis/tools/source_inspector.py b/src/vuln_analysis/tools/source_inspector.py index 2df030882..34ed3d39d 100644 --- a/src/vuln_analysis/tools/source_inspector.py +++ b/src/vuln_analysis/tools/source_inspector.py @@ -164,8 +164,18 @@ async def grep_native( cmd.append("-i") if word_boundary: cmd.append("-w") + + # Determine if searching a specific file (not a wildcard pattern) + # Specific file searches get asymmetric context (-B 2 -A 6) to see full code structures + # Wildcard searches use symmetric context (-C) to limit output volume + is_specific_file = file_glob and "*" not in file_glob + if context_lines > 0: - cmd.extend(["-C", str(context_lines)]) + if is_specific_file: + # More context after match to see full code block structure + cmd.extend(["-B", "2", "-A", "6"]) + else: + cmd.extend(["-C", str(context_lines)]) # If file_glob contains a path (e.g., "lib/connect.c"), split into: # - path_filter: directory portion for post-filtering results (e.g., "lib/") diff --git a/src/vuln_analysis/utils/clients/osidb_client.py b/src/vuln_analysis/utils/clients/osidb_client.py new file mode 100644 index 000000000..a2c728cba --- /dev/null +++ b/src/vuln_analysis/utils/clients/osidb_client.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import aiohttp + +from exploit_iq_commons.data_models.cve_intel import CveIntelOsidb +from vuln_analysis.utils.clients.intel_client import IntelClient +from vuln_analysis.utils.url_utils import url_join + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +OSIDB_DEFAULT_BASE_URL = "https://osidb.prodsec.redhat.com/osidb/api/v2" + + +class OsidbClient(IntelClient): + """ + Async client for OSIDB (Open Source Information Database) API. + Requires Red Hat VPN connectivity. + """ + + def __init__(self, + *, + base_url: str | None = None, + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): + + super().__init__(session=session, + base_url=base_url or os.environ.get('OSIDB_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) + + @classmethod + def default_base_url(cls) -> str: + return OSIDB_DEFAULT_BASE_URL + + async def get_intel(self, cve_id: str) -> CveIntelOsidb | None: + """ + Fetch OSIDB flaw data for a given CVE ID. + + Parameters + ---------- + cve_id : str + The CVE identifier (e.g., "CVE-2026-29170") + + Returns + ------- + CveIntelOsidb | None + Parsed OSIDB intel or None if not found + """ + intel_dict = await self._fetch_flaw(cve_id) + if not intel_dict: + return None + + return self._parse_response(intel_dict) + + async def _fetch_flaw(self, cve_id: str) -> dict | None: + """Fetch raw flaw data from OSIDB API.""" + try: + response = await self.request( + method='GET', + url=url_join(self.base_url, "flaws", cve_id), + log_on_error=False + ) + return response + + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.debug("No OSIDB flaw entry found for %s", cve_id) + return None + logger.error("OSIDB API error for %s: %s", cve_id, e, exc_info=True) + return None + + except aiohttp.ClientError as e: + logger.warning("OSIDB connection error for %s (VPN required): %s", cve_id, e) + return None + + def _parse_response(self, data: dict) -> CveIntelOsidb: + """Parse OSIDB API response into CveIntelOsidb model.""" + upstream_purls = self._extract_upstream_purls(data) + affects = self._extract_affects(data) + cvss_scores = self._extract_cvss_scores(data) + references = self._extract_references(data) + + return CveIntelOsidb( + cve_id=data.get("cve_id"), + impact=data.get("impact"), + title=data.get("title"), + comment_zero=data.get("comment_zero"), + cve_description=data.get("cve_description"), + statement=data.get("statement"), + cwe_id=data.get("cwe_id"), + source=data.get("source"), + mitigation=data.get("mitigation"), + upstream_purls=upstream_purls, + affects=affects, + cvss_scores=cvss_scores, + references=references + ) + + def _extract_upstream_purls(self, data: dict) -> list[dict]: + """Extract upstream purl information from nested upstream_data structure.""" + upstream_purls = [] + + for upstream_entry in data.get("upstream_data", []): + for purl_info in upstream_entry.get("upstream_purls", []): + upstream_purls.append({ + "name": purl_info.get("name"), + "purl": purl_info.get("purl"), + "ecosystem": purl_info.get("ecosystem") + }) + + return upstream_purls + + def _extract_affects(self, data: dict) -> list[dict]: + """Extract affects information.""" + return [ + { + "ps_module": affect.get("ps_module"), + "ps_product": affect.get("ps_product"), + "ps_component": affect.get("ps_component"), + "ps_update_stream": affect.get("ps_update_stream"), + "resolution": affect.get("resolution"), + "affectedness": affect.get("affectedness"), + "purl": affect.get("purl") + } + for affect in data.get("affects", []) + ] + + def _extract_cvss_scores(self, data: dict) -> list[dict]: + """Extract CVSS scores from multiple issuers.""" + return [ + { + "issuer": score.get("issuer"), + "score": score.get("score"), + "vector": score.get("vector"), + "cvss_version": score.get("cvss_version") + } + for score in data.get("cvss_scores", []) + ] + + def _extract_references(self, data: dict) -> list[str]: + """Extract reference URLs.""" + return [ + ref.get("url") + for ref in data.get("references", []) + if ref.get("url") + ] diff --git a/src/vuln_analysis/utils/git_commit_searcher.py b/src/vuln_analysis/utils/git_commit_searcher.py new file mode 100644 index 000000000..994f6727b --- /dev/null +++ b/src/vuln_analysis/utils/git_commit_searcher.py @@ -0,0 +1,952 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Git Commit Searcher for CVE fix discovery. + +Searches git repositories for CVE fix commits using multiple strategies: +1. Revision search (SVN/git revision lookup) - highest confidence +2. Function search (function name in commit messages or code) + +Results are ranked by confidence and security keyword matching. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING + +import aiohttp + +from langchain_core.messages import SystemMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + CONFIDENCE_FUNCTION_MESSAGE, + CONFIDENCE_FUNCTION_PICKAXE, + CONFIDENCE_REVISION_BRANCH, + CONFIDENCE_REVISION_DIRECT, + CONFIDENCE_REVISION_FALLBACK, + SEARCH_METHOD_FUNCTION_MESSAGE, + SEARCH_METHOD_FUNCTION_PICKAXE, + SEARCH_METHOD_REVISION, + CommitSearchResult, + GitSearchReport, + ParsedPatch, + ReferenceHints, +) +from vuln_analysis.utils.web_patch_fetcher import ( + WebPatchFetcher, + _parse_patch_content, + build_patch_url_from_repo, +) +from vuln_analysis.utils.git_repo_manager import ( + GIT_HASH_LENGTH_SHORT, + GitCommandError, + GitRepoManager, +) + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MAX_RESULTS_PER_STRATEGY = 5 +DATE_RANGE_MONTHS = 12 + +GIT_HASH_FULL_PATTERN = re.compile(r"^[a-f0-9]{40}$") +GIT_HASH_SHORT_PATTERN = re.compile(r"^[a-f0-9]{7,12}$") +SVN_REVISION_PATTERN = re.compile(r"^r?(\d{5,8})$") + +GIT_LOG_FORMAT = "%H\x1f%h\x1f%s\x1f%an <%ae>\x1f%aI" +GIT_LOG_SEPARATOR = "\x1f" # ASCII Unit Separator - won't appear in commit messages + + +class _NoOpSpan: + """No-op context manager when tracer is not available.""" + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def set_output(self, data): + pass + + +SECURITY_KEYWORDS = [ + ("fix", 0.05), + ("security", 0.10), + ("vulnerab", 0.10), + ("overflow", 0.08), + ("injection", 0.08), + ("sanitiz", 0.05), + ("validat", 0.05), + ("CVE", 0.05), + ("crash", 0.03), + ("denial", 0.05), + ("remote", 0.05), + ("arbitrary", 0.08), +] + +MERGE_COMMIT_PENALTY = 0.10 +MAX_CONFIDENCE = 1.0 + + +# --------------------------------------------------------------------------- +# Branch Selection (LLM-assisted) +# --------------------------------------------------------------------------- + +BRANCH_SELECTION_PROMPT = """You are selecting which git branch(es) to search for a CVE fix commit. + +## Context + +Package: {package_name} +CVE Description: {cve_description} +Fixed Version (from advisory): {version_hints} + +## Available Branches in Repository + +{available_branches} + +## Task + +Select the branch(es) most likely to contain the fix commit for this CVE. + +Consider: +1. Version hints - "2.4.68" likely means branch "2.4.x" or "release/2.4" +2. Stable/release branches are more likely than development branches +3. If multiple versions affected, select the primary stable branch + +## Output + +Return JSON with selected_branches (at most 2) and reasoning. +""" + + +class BranchSelection(BaseModel): + """LLM output for branch selection.""" + + selected_branches: list[str] = Field( + description="Branch names to search (at most 2)", + ) + reasoning: str = Field( + description="Why these branches were selected", + ) + + +# --------------------------------------------------------------------------- +# GitCommitSearcher +# --------------------------------------------------------------------------- + + +class GitCommitSearcher: + """Search git repositories for CVE fix commits. + + Implements multiple search strategies ordered by confidence: + 1. Revision search (direct hash or SVN revision grep) + 2. Function search (commit message or pickaxe) + """ + + def __init__( + self, + repo_manager: GitRepoManager, + max_results_per_strategy: int = MAX_RESULTS_PER_STRATEGY, + tracer=None, + ): + self.repo_manager = repo_manager + self.max_results = max_results_per_strategy + self._tracer = tracer + + def _trace(self, name: str, input_data=None): + """Create a tracing span if tracer is available, otherwise return a no-op context manager.""" + if self._tracer: + return self._tracer.push_active_function(name, input_data=input_data or {}) + return _NoOpSpan() + + async def search( + self, + repo_path: Path, + repo_url: str, + hints: ReferenceHints | None, + cve_date: str | None = None, + branches: list[str] | None = None, + llm: "BaseChatModel | None" = None, + package_name: str | None = None, + cve_description: str | None = None, + ) -> GitSearchReport: + """Execute all search strategies and return best results. + + Args: + repo_path: Path to cloned repository + repo_url: Original repository URL + hints: Extracted hints from reference mining + cve_date: CVE disclosure date for date filtering + branches: Specific branches to search (from hints) + llm: LLM for branch resolution (optional) + package_name: Package name for branch resolution context + cve_description: CVE description for branch resolution context + + Returns: + GitSearchReport with ranked results + """ + report = GitSearchReport( + repo_url=repo_url, + repo_path=str(repo_path), + ) + + if not hints: + logger.warning("search: No hints provided, cannot search") + report.error_messages.append("No hints provided for search") + return report + + resolved_branches = await self._resolve_branches( + repo_path=repo_path, + repo_url=repo_url, + hints=hints, + branches=branches, + llm=llm, + package_name=package_name, + cve_description=cve_description, + ) + report.branches_searched = resolved_branches + + all_results: list[CommitSearchResult] = [] + + if hints.revision_hint: + report.strategies_tried.append(SEARCH_METHOD_REVISION) + result = await self._search_by_revision( + repo_path=repo_path, + revision_hint=hints.revision_hint, + branches=resolved_branches, + repo_url=repo_url, + ) + if result: + report.strategies_succeeded.append(SEARCH_METHOD_REVISION) + all_results.append(result) + + if hints.function_hints: + report.strategies_tried.append(SEARCH_METHOD_FUNCTION_MESSAGE) + report.strategies_tried.append(SEARCH_METHOD_FUNCTION_PICKAXE) + + results = await self._search_by_functions( + repo_path=repo_path, + function_hints=hints.function_hints, + cve_date=cve_date, + ) + if results: + for r in results: + if r.search_method == SEARCH_METHOD_FUNCTION_MESSAGE: + if SEARCH_METHOD_FUNCTION_MESSAGE not in report.strategies_succeeded: + report.strategies_succeeded.append(SEARCH_METHOD_FUNCTION_MESSAGE) + elif r.search_method == SEARCH_METHOD_FUNCTION_PICKAXE: + if SEARCH_METHOD_FUNCTION_PICKAXE not in report.strategies_succeeded: + report.strategies_succeeded.append(SEARCH_METHOD_FUNCTION_PICKAXE) + all_results.extend(results) + + ranked_results = self._rank_results( + results=all_results, + file_hints=hints.file_hints, + function_hints=hints.function_hints, + ) + report.all_results = ranked_results[: self.max_results] + + if ranked_results: + report.best_result = ranked_results[0] + report.search_success = True + logger.info( + "search: Found %d results, best: %s (confidence=%.2f, method=%s)", + len(ranked_results), + report.best_result.commit_hash_short, + report.best_result.confidence, + report.best_result.search_method, + ) + else: + logger.info("search: No commits found") + + return report + + async def _resolve_branches( + self, + repo_path: Path, + repo_url: str, + hints: ReferenceHints, + branches: list[str] | None, + llm: "BaseChatModel | None", + package_name: str | None, + cve_description: str | None, + ) -> list[str]: + """Resolve which branches to search. + + Priority: + 1. Explicit branches parameter + 2. branch_hints from Reference Mining + 3. LLM-selected branches (if LLM provided) + 4. Default branches (main, master, trunk) + """ + with self._trace("resolve_branches", { + "has_explicit_branches": bool(branches), + "has_branch_hints": bool(hints.branch_hints), + "has_version_hints": bool(hints.version_hints), + "has_llm": bool(llm), + }) as span: + if branches: + logger.debug("Using explicit branches: %s", branches) + span.set_output({"resolution": "explicit", "branches": branches}) + return branches + + if llm and (hints.version_hints or hints.branch_hints): + selected = await self._llm_select_branches( + repo_url=repo_url, + version_hints=hints.version_hints or [], + llm=llm, + package_name=package_name, + cve_description=cve_description, + branch_hints=hints.branch_hints, + ) + if selected: + span.set_output({"resolution": "llm_or_exact_match", "branches": selected}) + return selected + + default_branch = await self.repo_manager.get_default_branch(repo_path) + defaults = [default_branch] + if default_branch not in ("main", "master"): + defaults.extend(["main", "master"]) + + logger.debug("Using default branches: %s", defaults) + span.set_output({"resolution": "default", "branches": defaults}) + return defaults + + async def _llm_select_branches( + self, + repo_url: str, + version_hints: list[str], + llm: "BaseChatModel", + package_name: str | None, + cve_description: str | None, + branch_hints: list[str] | None = None, + ) -> list[str]: + """Use LLM to select best branches from available options. + + Fast path: If branch_hints exactly match available branches, return + them directly without LLM invocation. + """ + with self._trace("select_branches", { + "branch_hints": branch_hints, + "version_hints": version_hints, + }) as span: + try: + available_branches = await self.repo_manager.list_remote_branches(repo_url) + if not available_branches: + logger.debug("No remote branches found for branch selection") + span.set_output({"result": "no_branches_available"}) + return [] + + available_set = set(available_branches) + + if branch_hints: + exact_matches = [b for b in branch_hints if b in available_set] + if exact_matches: + logger.info( + "Branch hints exactly matched available branches: %s", + exact_matches, + ) + span.set_output({ + "method": "exact_match", + "branches": exact_matches[:2], + "available_count": len(available_branches), + }) + return exact_matches[:2] + logger.debug( + "Branch hints %s did not match available branches, falling back to LLM", + branch_hints, + ) + + branches_text = "\n".join(f"- {b}" for b in available_branches[:50]) + + prompt = BRANCH_SELECTION_PROMPT.format( + package_name=package_name or "unknown", + cve_description=(cve_description or "")[:500], + version_hints=", ".join(version_hints), + available_branches=branches_text, + ) + + branch_llm = llm.with_structured_output(BranchSelection) + result = await branch_llm.ainvoke([SystemMessage(content=prompt)]) + if not isinstance(result, BranchSelection): + logger.warning("LLM returned unexpected type: %s", type(result)) + span.set_output({"result": "llm_invalid_response", "type": str(type(result))}) + return [] + + valid_branches = [b for b in result.selected_branches if b in available_branches] + if valid_branches: + logger.info( + "LLM selected branches: %s (reason: %s)", + valid_branches, + result.reasoning[:100], + ) + span.set_output({ + "method": "llm", + "branches": valid_branches[:2], + "reasoning": result.reasoning[:200], + "available_count": len(available_branches), + }) + return valid_branches[:2] + + span.set_output({"result": "llm_no_valid_branches"}) + + except Exception as e: + logger.warning("LLM branch selection failed: %s", e) + span.set_output({"error": str(e)}) + + return [] + + async def _search_by_revision( + self, + repo_path: Path, + revision_hint: str, + branches: list[str], + repo_url: str | None = None, + ) -> CommitSearchResult | None: + """Search for a single SVN/Git revision. + + Strategy: + 1. If looks like git hash → git show + 2. If looks like SVN revision → git log --grep on branches + + Args: + repo_path: Path to the git repository. + revision_hint: Single revision string (e.g., "r1934982" or "abc123"). + branches: Branches to search on for SVN revisions. + repo_url: Repository URL for building commit URL in trace output. + + Returns: + CommitSearchResult if found, None otherwise. + """ + hint_clean = revision_hint.strip() + + with self._trace("search_by_revision", { + "revision_hint": hint_clean, + "branches": branches, + }) as span: + result: CommitSearchResult | None = None + hint_type: str | None = None + + if GIT_HASH_FULL_PATTERN.match(hint_clean) or GIT_HASH_SHORT_PATTERN.match(hint_clean): + hint_type = "git_hash" + result = await self._search_direct_hash(repo_path, hint_clean) + + elif SVN_REVISION_PATTERN.match(hint_clean): + hint_type = "svn_revision" + svn_match = SVN_REVISION_PATTERN.match(hint_clean) + if svn_match: + revision_num = svn_match.group(1) + branch_results = await self._search_revision_on_branches( + repo_path, revision_num, branches + ) + if branch_results: + result = branch_results[0] + + commit_url = None + if result and repo_url: + patch_url = build_patch_url_from_repo(repo_url, result.commit_hash) + if patch_url: + commit_url = patch_url.removesuffix(".patch") + + span.set_output({ + "revision_hint": hint_clean, + "hint_type": hint_type, + "found": result is not None, + "commit": result.commit_hash_short if result else None, + "commit_url": commit_url, + }) + + return result + + async def _search_direct_hash( + self, + repo_path: Path, + commit_hash: str, + ) -> CommitSearchResult | None: + """Direct lookup of a git commit hash.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, f"--format={GIT_LOG_FORMAT}", "-s"], + cwd=repo_path, + ) + + lines = output.strip().split("\n") + if not lines or not lines[0]: + return None + + parts = lines[0].split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + return None + + files = await self._get_commit_files(repo_path, parts[0]) + + return CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_DIRECT, + match_details=f"Direct hash lookup: {commit_hash}", + ) + + except GitCommandError as e: + logger.debug("Direct hash lookup failed for %s: %s", commit_hash, e.stderr) + return None + + async def _search_revision_on_branches( + self, + repo_path: Path, + revision_num: str, + branches: list[str], + ) -> list[CommitSearchResult]: + """Search commit messages for references to an SVN revision number. + + When projects migrate from SVN to Git, fix commits often reference + the original SVN revision in their message (e.g., "Backport fix from + r1234567"). Uses `git log --grep` to find such references. + """ + results: list[CommitSearchResult] = [] + + for branch in branches: + fetched = await self.repo_manager.fetch_branch(repo_path, branch) + if not fetched: + continue + + try: + output = await self.repo_manager.run_git_command( + [ + "log", "FETCH_HEAD", + f"--grep={revision_num}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ], + cwd=repo_path, + ) + + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_BRANCH, + match_details=f"grep matched r{revision_num} on branch {branch}", + )) + + if results: + logger.info("Found %d commits for r%s on branch %s", len(results), revision_num, branch) + return results + + except GitCommandError as e: + logger.debug("Revision search failed on branch %s: %s", branch, e.stderr) + + if not results: + results = await self._search_revision_all_branches(repo_path, revision_num) + + return results + + async def _search_revision_all_branches( + self, + repo_path: Path, + revision_num: str, + ) -> list[CommitSearchResult]: + """Fallback: search all branches for revision.""" + try: + output = await self.repo_manager.run_git_command( + [ + "log", "--all", + f"--grep={revision_num}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ], + cwd=repo_path, + ) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_FALLBACK, + match_details=f"grep matched r{revision_num} on --all branches (fallback)", + )) + + return results + + except GitCommandError as e: + logger.debug("Revision search --all failed: %s", e.stderr) + return [] + + async def _search_by_functions( + self, + repo_path: Path, + function_hints: list[str], + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search by function name in commit messages or code changes.""" + with self._trace("search_by_functions", { + "function_hints": function_hints, + "cve_date": cve_date, + }) as span: + results: list[CommitSearchResult] = [] + hints_processed = [] + + for func_name in function_hints: + hint_info = {"function": func_name, "method": None, "found": False} + + message_results = await self._search_function_in_messages( + repo_path, func_name, cve_date + ) + if message_results: + hint_info["method"] = "message" + hint_info["found"] = True + hint_info["commits"] = [r.commit_hash_short for r in message_results] + results.extend(message_results) + else: + pickaxe_results = await self._search_function_pickaxe( + repo_path, func_name, cve_date + ) + if pickaxe_results: + hint_info["method"] = "pickaxe" + hint_info["found"] = True + hint_info["commits"] = [r.commit_hash_short for r in pickaxe_results] + results.extend(pickaxe_results) + + hints_processed.append(hint_info) + + span.set_output({ + "hints_processed": hints_processed, + "total_results": len(results), + }) + return results + + async def _search_function_in_messages( + self, + repo_path: Path, + function_name: str, + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search for function name in commit messages.""" + try: + args = [ + "log", + f"--grep={function_name}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ] + + if cve_date: + args.extend(["--after", f"{cve_date[:10]} - {DATE_RANGE_MONTHS} months"]) + args.extend(["--before", f"{cve_date[:10]} + 3 months"]) + + output = await self.repo_manager.run_git_command(args, cwd=repo_path) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=CONFIDENCE_FUNCTION_MESSAGE, + match_details=f"Message grep matched: {function_name}", + )) + + return results + + except GitCommandError as e: + logger.debug("Function message search failed for %s: %s", function_name, e.stderr) + return [] + + async def _search_function_pickaxe( + self, + repo_path: Path, + function_name: str, + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search for function name in code changes (pickaxe).""" + try: + args = [ + "log", + f"-S{function_name}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ] + + if cve_date: + args.extend(["--after", f"{cve_date[:10]} - {DATE_RANGE_MONTHS} months"]) + args.extend(["--before", f"{cve_date[:10]} + 3 months"]) + + output = await self.repo_manager.run_git_command(args, cwd=repo_path) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_FUNCTION_PICKAXE, + confidence=CONFIDENCE_FUNCTION_PICKAXE, + match_details=f"Pickaxe matched: {function_name}", + )) + + return results + + except GitCommandError as e: + logger.debug("Function pickaxe search failed for %s: %s", function_name, e.stderr) + return [] + + async def _get_commit_files( + self, + repo_path: Path, + commit_hash: str, + ) -> list[str]: + """Get list of files changed in a commit.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, "--format=", "--name-only"], + cwd=repo_path, + ) + return [f for f in output.strip().split("\n") if f] + + except GitCommandError: + return [] + + def _rank_results( + self, + results: list[CommitSearchResult], + file_hints: list[str] | None, + function_hints: list[str] | None, + ) -> list[CommitSearchResult]: + """Rank results by confidence with boosting for matches.""" + if not results: + return [] + + seen_hashes: set[str] = set() + unique_results: list[CommitSearchResult] = [] + for r in results: + if r.commit_hash not in seen_hashes: + seen_hashes.add(r.commit_hash) + unique_results.append(r) + + for result in unique_results: + result.confidence = self._compute_boosted_confidence( + result, file_hints, function_hints + ) + + unique_results.sort(key=lambda r: r.confidence, reverse=True) + + return unique_results + + def _compute_boosted_confidence( + self, + result: CommitSearchResult, + file_hints: list[str] | None, + function_hints: list[str] | None, + ) -> float: + """Compute confidence with security keyword and hint boosting.""" + score = result.confidence + message_lower = result.commit_message.lower() + + for keyword, boost in SECURITY_KEYWORDS: + if keyword.lower() in message_lower: + score += boost + + if file_hints: + matched_files = set(result.files_changed) & set(file_hints) + score += 0.05 * len(matched_files) + + if function_hints: + for func in function_hints: + if func.lower() in message_lower: + score += 0.10 + + if message_lower.startswith("merge"): + score -= MERGE_COMMIT_PENALTY + + return min(score, MAX_CONFIDENCE) + + async def get_commit_diff( + self, + repo_path: Path, + commit_hash: str, + max_diff_size: int = 100000, + ) -> str: + """Get the full diff for a commit, truncated if too large.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, "--format=", "-p"], + cwd=repo_path, + ) + + if len(output) > max_diff_size: + return output[:max_diff_size] + "\n... (truncated)" + + return output + + except GitCommandError as e: + logger.warning("Failed to get diff for %s: %s", commit_hash, e.stderr) + return "" + + async def commit_to_parsed_patch( + self, + repo_path: Path, + commit_hash: str, + max_diff_size: int = 50000, + repo_url: str | None = None, + ) -> ParsedPatch | None: + """Convert a git commit to ParsedPatch format. + + This makes the discovered commit compatible with existing + L1 agent code that expects ParsedPatch from web_patch_fetcher. + + Strategy: + 1. Try HTTP fetch first (GitHub, etc.) - gets complete patch + 2. Fall back to git show (shallow clone may have incomplete diff) + + Args: + repo_path: Path to local repository + commit_hash: Full commit hash + max_diff_size: Maximum diff size in characters + repo_url: Repository URL (for HTTP fetch) + + Returns: + ParsedPatch or None if parsing fails + """ + patch_filename = f"git_search_{commit_hash[:GIT_HASH_LENGTH_SHORT]}.patch" + diff_text = None + + if repo_url: + diff_text = await self._fetch_patch_via_http(repo_url, commit_hash, max_diff_size) + if diff_text: + logger.info( + "Fetched patch for %s via HTTP (%d chars)", + commit_hash[:GIT_HASH_LENGTH_SHORT], + len(diff_text), + ) + + if not diff_text: + logger.debug("HTTP fetch unavailable, falling back to git show") + diff_text = await self.get_commit_diff(repo_path, commit_hash, max_diff_size) + + if not diff_text: + return None + + parsed = _parse_patch_content(diff_text, patch_filename) + + if parsed: + logger.info( + "Parsed commit %s into %d files", + commit_hash[:GIT_HASH_LENGTH_SHORT], + len(parsed.files), + ) + + return parsed + + async def _fetch_patch_via_http( + self, + repo_url: str, + commit_hash: str, + max_size: int, + ) -> str | None: + """Fetch patch content via HTTP from GitHub/GitLab. + + This bypasses shallow clone limitations by fetching directly from the web. + Uses WebPatchFetcher for robust fetching with retries and rate limiting. + """ + patch_url = build_patch_url_from_repo(repo_url, commit_hash) + if not patch_url: + logger.debug("Cannot build patch URL for %s", repo_url) + return None + + try: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + fetcher = WebPatchFetcher(session) + result = await fetcher.fetch_from_url( + url=patch_url, + cve_id="git_search", + source="git_search", + ) + if result and result.patch_content: + content = result.patch_content + if len(content) > max_size: + content = content[:max_size] + "\n... (truncated)" + return content + + except Exception as e: + logger.debug("HTTP fetch error for %s: %s", patch_url, e) + + return None diff --git a/src/vuln_analysis/utils/git_repo_manager.py b/src/vuln_analysis/utils/git_repo_manager.py new file mode 100644 index 000000000..ac06748fd --- /dev/null +++ b/src/vuln_analysis/utils/git_repo_manager.py @@ -0,0 +1,424 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Git Repository Manager for commit discovery. + +Manages git repository cloning, updating, and caching for CVE fix commit search. +Handles shallow cloning for performance, local caching to avoid re-cloning, +branch fetching for specific branches, and concurrent access with locking. +""" + +from __future__ import annotations + +import asyncio +import shutil +import subprocess +import uuid +from pathlib import Path +from urllib.parse import unquote, urlparse + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +DEFAULT_CACHE_DIR = ".cache/am_cache/checker/upstream_repos" +DEFAULT_CLONE_DEPTH = 1000 +DEFAULT_FETCH_TIMEOUT_SECONDS = 120 +DEFAULT_CLONE_TIMEOUT_SECONDS = 180 + +GIT_HASH_LENGTH_SHORT = 7 +GIT_HASH_LENGTH_FULL = 40 +GIT_SUFFIX = ".git" +UNSAFE_PATH_SEGMENTS = frozenset({".", ".."}) + + +class GitCommandError(Exception): + """Raised when a git command fails.""" + + def __init__(self, command: str, returncode: int, stderr: str): + self.command = command + self.returncode = returncode + self.stderr = stderr + super().__init__(f"Git command failed: {command} (exit {returncode}): {stderr}") + + +class GitRepoManager: + """Manage git repository cloning, updating, and caching. + + Features: + - Shallow cloning for performance + - Local caching to avoid re-cloning + - Branch fetching for specific branches + - Concurrent access handling with asyncio locks + """ + + def __init__( + self, + cache_dir: str | Path = DEFAULT_CACHE_DIR, + default_depth: int = DEFAULT_CLONE_DEPTH, + fetch_timeout_seconds: int = DEFAULT_FETCH_TIMEOUT_SECONDS, + clone_timeout_seconds: int = DEFAULT_CLONE_TIMEOUT_SECONDS, + ): + self.cache_dir = Path(cache_dir) + self.default_depth = default_depth + self.fetch_timeout = fetch_timeout_seconds + self.clone_timeout = clone_timeout_seconds + self.cache_dir.mkdir(parents=True, exist_ok=True) + + self._repo_locks: dict[str, asyncio.Lock] = {} + self._locks_lock = asyncio.Lock() + + async def _get_repo_lock(self, repo_url: str) -> asyncio.Lock: + """Get or create a lock for a specific repo URL.""" + async with self._locks_lock: + if repo_url not in self._repo_locks: + self._repo_locks[repo_url] = asyncio.Lock() + return self._repo_locks[repo_url] + + def get_repo_cache_path(self, repo_url: str) -> Path: + """Get deterministic local path for a repo URL. + + Example: + https://github.com/apache/httpd + → .cache/am_cache/checker/upstream_repos/github.com/apache/httpd + """ + parsed = urlparse(repo_url) + host = (parsed.hostname or parsed.netloc or "unknown").lower() + if not host or "/" in host or "\\" in host: + raise ValueError(f"Invalid repository host in URL: {repo_url!r}") + + path_parts = [ + segment + for segment in unquote(parsed.path).strip("/").split("/") + if segment + ] + if any(segment in UNSAFE_PATH_SEGMENTS for segment in path_parts): + raise ValueError(f"Repository URL contains unsafe path segment: {repo_url!r}") + if not path_parts: + raise ValueError(f"Repository URL has no path: {repo_url!r}") + + if path_parts[-1].endswith(GIT_SUFFIX): + path_parts[-1] = path_parts[-1][: -len(GIT_SUFFIX)] + if not path_parts[-1]: + raise ValueError(f"Repository URL has empty repo name: {repo_url!r}") + + cache_path = self.cache_dir / host + for segment in path_parts: + cache_path /= segment + + cache_root = self.cache_dir.resolve() + if not cache_path.resolve(strict=False).is_relative_to(cache_root): + raise ValueError(f"Repository cache path escapes cache root: {repo_url!r}") + + return cache_path + + def _is_valid_repo(self, repo_path: Path) -> bool: + """Check if path contains a valid git repository.""" + git_dir = repo_path / ".git" + return git_dir.is_dir() and (git_dir / "HEAD").exists() + + async def run_git_command( + self, + args: list[str], + cwd: Path | None = None, + timeout: int | None = None, + ) -> str: + """Run a git command asynchronously. + + Uses subprocess.run with asyncio.to_thread to avoid Python 3.12/uvloop + child watcher incompatibility with asyncio.create_subprocess_exec. + + Args: + args: Git command arguments (without 'git' prefix) + cwd: Working directory for the command + timeout: Command timeout in seconds + + Returns: + Command stdout as string + + Raises: + GitCommandError: If command fails + asyncio.TimeoutError: If command times out + """ + cmd = ["git"] + args + cmd_str = " ".join(cmd) + effective_timeout = timeout or self.fetch_timeout + + logger.debug("Running git command: %s (cwd=%s)", cmd_str, cwd) + + def _run_sync() -> tuple[str, str, int]: + """Execute git command synchronously.""" + result = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + errors="replace", + timeout=effective_timeout, + ) + return (result.stdout, result.stderr, result.returncode) + + try: + stdout, stderr, returncode = await asyncio.to_thread(_run_sync) + + if returncode != 0: + raise GitCommandError( + command=cmd_str, + returncode=returncode, + stderr=stderr, + ) + + return stdout + + except subprocess.TimeoutExpired: + logger.error("Git command timed out after %ds: %s", effective_timeout, cmd_str) + raise asyncio.TimeoutError(f"Git command timed out after {effective_timeout}s: {cmd_str}") + except Exception as e: + logger.error("Git command failed unexpectedly: %s", cmd_str, exc_info=True) + raise GitCommandError( + command=cmd_str, + returncode=-1, + stderr=str(e), + ) + + async def clone_or_update( + self, + repo_url: str, + depth: int | None = None, + ) -> tuple[Path, bool]: + """Clone a repo or update if it exists. + + Args: + repo_url: Git repository URL + depth: Clone depth (None for full clone) + + Returns: + (repo_path, is_fresh_clone) + """ + lock = await self._get_repo_lock(repo_url) + + async with lock: + cache_path = self.get_repo_cache_path(repo_url) + + if cache_path.exists() and self._is_valid_repo(cache_path): + logger.info("Repo exists in cache, fetching updates: %s", repo_url) + await self._git_fetch(cache_path, depth=depth) + return (cache_path, False) + + logger.info("Cloning repository: %s", repo_url) + temp_dir = self.cache_dir / ".tmp" / str(uuid.uuid4()) + + try: + await self._git_clone(repo_url, temp_dir, depth=depth) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + + temp_dir.rename(cache_path) + logger.info("Successfully cloned to: %s", cache_path) + return (cache_path, True) + + except Exception: + shutil.rmtree(temp_dir, ignore_errors=True) + raise + + async def _git_clone( + self, + repo_url: str, + target_dir: Path, + depth: int | None = None, + ) -> None: + """Clone a repository with optional shallow depth.""" + effective_depth = depth or self.default_depth + target_dir.parent.mkdir(parents=True, exist_ok=True) + + args = [ + "clone", + "--depth", str(effective_depth), + "--single-branch", + repo_url, + str(target_dir), + ] + + await self.run_git_command(args, timeout=self.clone_timeout) + + async def _git_fetch( + self, + repo_path: Path, + depth: int | None = None, + ) -> None: + """Fetch updates for an existing repository.""" + effective_depth = depth or self.default_depth + + args = ["fetch", "origin", f"--depth={effective_depth}"] + await self.run_git_command(args, cwd=repo_path) + + async def fetch_branch( + self, + repo_path: Path, + branch: str, + depth: int | None = None, + ) -> bool: + """Fetch a specific branch with depth limit. + + Critical for SVN revision search on stable branches. + + Example: + git fetch origin 2.4.x --depth=1000 + + Args: + repo_path: Path to local repository + branch: Branch name to fetch + depth: Fetch depth + + Returns: + True if fetch succeeded, False otherwise + """ + effective_depth = depth or self.default_depth + + try: + args = ["fetch", "origin", branch, f"--depth={effective_depth}"] + await self.run_git_command(args, cwd=repo_path) + logger.info("Fetched branch %s with depth %d", branch, effective_depth) + return True + + except GitCommandError as e: + logger.warning("Failed to fetch branch %s: %s", branch, e.stderr) + return False + + async def get_default_branch(self, repo_path: Path) -> str: + """Get the default branch name (main, master, trunk, etc.).""" + try: + output = await self.run_git_command( + ["symbolic-ref", "refs/remotes/origin/HEAD", "--short"], + cwd=repo_path, + ) + branch = output.strip() + if branch.startswith("origin/"): + branch = branch[len("origin/"):] + return branch + + except GitCommandError: + logger.debug("Could not get default branch via symbolic-ref, trying fallback") + + for candidate in ["main", "master", "trunk", "develop"]: + try: + await self.run_git_command( + ["rev-parse", "--verify", f"origin/{candidate}"], + cwd=repo_path, + ) + return candidate + except GitCommandError: + continue + + return "main" + + async def list_remote_branches(self, repo_url: str) -> list[str]: + """List available remote branches without cloning. + + Uses git ls-remote --heads to list branches. + + Args: + repo_url: Repository URL + + Returns: + List of branch names (without refs/heads/ prefix) + """ + try: + output = await self.run_git_command( + ["ls-remote", "--heads", repo_url], + ) + + branches = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split("\t") + if len(parts) >= 2: + ref = parts[1] + if ref.startswith("refs/heads/"): + branches.append(ref[len("refs/heads/"):]) + + logger.debug("Found %d branches for %s", len(branches), repo_url) + return branches + + except GitCommandError as e: + logger.warning("Failed to list remote branches: %s", e.stderr) + return [] + + async def list_local_branches(self, repo_path: Path) -> list[str]: + """List branches available in local repository. + + Args: + repo_path: Path to local repository + + Returns: + List of remote tracking branch names + """ + try: + output = await self.run_git_command( + ["branch", "-r", "--format=%(refname:short)"], + cwd=repo_path, + ) + + branches = [] + for line in output.strip().split("\n"): + branch = line.strip() + if branch and not branch.endswith("/HEAD"): + if branch.startswith("origin/"): + branch = branch[len("origin/"):] + branches.append(branch) + + return branches + + except GitCommandError as e: + logger.warning("Failed to list local branches: %s", e.stderr) + return [] + + def cleanup_cache(self, max_age_days: int = 7) -> int: + """Remove repos not accessed in N days. + + Args: + max_age_days: Maximum age in days before removal + + Returns: + Number of repos removed + """ + import time + + max_age_seconds = max_age_days * 24 * 60 * 60 + now = time.time() + removed_count = 0 + + if not self.cache_dir.exists(): + return 0 + + for repo_dir in self.cache_dir.rglob(".git"): + repo_path = repo_dir.parent + try: + mtime = repo_path.stat().st_mtime + if now - mtime > max_age_seconds: + logger.info("Removing stale cached repo: %s", repo_path) + shutil.rmtree(repo_path, ignore_errors=True) + removed_count += 1 + except OSError: + continue + + return removed_count diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index 8e3311f96..215629fb9 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -22,11 +22,13 @@ from exploit_iq_commons.data_models.cve_intel import CveIntel, IntelPluginData from exploit_iq_commons.data_models.cve_intel import CveIntelEpss from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelOsidb from exploit_iq_commons.data_models.cve_intel import CveIntelRhsa from exploit_iq_commons.data_models.cve_intel import CveIntelUbuntu from .clients.first_client import FirstClient from .clients.ghsa_client import GHSAClient from .clients.nvd_client import NVDClient +from .clients.osidb_client import OsidbClient from .clients.rhsa_client import RHSAClient from .clients.ubuntu_client import UbuntuClient from ..data_models.plugin import PluginConfig, IntelPluginSchema @@ -47,13 +49,19 @@ def __init__(self, lang_code: str = 'en', max_retries: int = 10, plugins_config: list[PluginConfig] | None = None, - retry_on_client_errors: bool = True): + retry_on_client_errors: bool = True, + rpm_user_type: str = "external"): """ Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. + + Parameters + ---------- + rpm_user_type : str + User profile type: "internal" (Red Hat VPN) enables OSIDB, "external" disables it. """ - # Create a shared session object for all requests self._session = session + self._rpm_user_type = rpm_user_type self._nvd_client = NVDClient(api_key=os.environ.get('NVD_API_KEY', nist_api_key), session=self._session, @@ -73,6 +81,13 @@ def __init__(self, self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries, retry_on_client_errors=retry_on_client_errors) + + self._osidb_client = None + if self._rpm_user_type == "internal": + self._osidb_client = OsidbClient(session=self._session, + retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) + self._intel_plugins = [] if plugins_config is not None: self._intel_plugins = [IntelPluginSchema.locate( @@ -167,6 +182,23 @@ async def _get_epss_score(self, intel: CveIntel) -> CveIntelEpss | None: return None + async def _get_osidb_intel(self, intel: CveIntel) -> CveIntelOsidb | None: + """Fetch OSIDB intel for internal Red Hat users.""" + if self._osidb_client is None: + return None + + if not intel.has_cve_id(): + logger.debug("Skipping OSIDB retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.osidb = await self._osidb_client.get_intel(cve_id=intel.cve_id) + return intel.osidb + + except Exception as e: + logger.error("Error fetching OSIDB intel for %s: %s", intel.vuln_id, e, exc_info=True) + return None + async def append_plugin_data(self, plugin: IntelPluginSchema, intel: CveIntel) -> IntelPluginData: try: @@ -202,7 +234,6 @@ async def retrieve(self, vuln_id: str) -> CveIntel: vuln_id) return intel - # Run all the coroutines concurrently coros = [ self._get_nvd_intel(intel=intel), self._get_ubuntu_intel(intel=intel), @@ -210,6 +241,9 @@ async def retrieve(self, vuln_id: str) -> CveIntel: self._get_epss_score(intel=intel) ] + if self._rpm_user_type == "internal": + coros.append(self._get_osidb_intel(intel=intel)) + coros.extend([self.append_plugin_data(intel=intel, plugin=plugin) for plugin in self._intel_plugins]) await asyncio.gather(*coros) diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 7be93359f..b45be5649 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ipaddress import re from pathlib import Path +from urllib.parse import urlparse from packaging.version import InvalidVersion from packaging.version import parse as parse_version @@ -594,6 +596,191 @@ def _strip_rejected_package_token(entry: str, rn: str) -> str: # Chromium issue tracker URL pattern - captures the bug ID CHROMIUM_ISSUE_PATTERN = re.compile(r"https?://issues\.chromium\.org/issues/(\d+)") +# --------------------------------------------------------------------------- +# Advisory URL extraction (for Reference Mining) +# --------------------------------------------------------------------------- + +# Advisory source patterns with (source_type, priority) - lower priority = higher quality +# Priority 1-2: Detailed technical mailing lists with commit refs, patches +# Priority 3-4: Vendor advisories with fix details +# Priority 5+: Generic advisories, less likely to have commit hints +ADVISORY_SOURCE_PATTERNS: dict[str, tuple[str, int]] = { + # High priority - detailed technical info, often has commit refs + "openwall.com/lists/oss-security": ("openwall", 1), + "seclists.org": ("mailing_list", 2), + "marc.info": ("mailing_list", 2), + "mail-archive.com": ("mailing_list", 2), + "lore.kernel.org": ("mailing_list", 2), + # Medium priority - vendor advisories with fix info + "apache.org/security": ("vendor_advisory", 3), + "httpd.apache.org/security": ("vendor_advisory", 3), + "kernel.org/doc": ("vendor_advisory", 3), + "debian.org/security": ("vendor_advisory", 3), + "access.redhat.com/errata": ("vendor_advisory", 4), + "access.redhat.com/security": ("vendor_advisory", 4), + "ubuntu.com/security": ("vendor_advisory", 4), + "security.gentoo.org": ("vendor_advisory", 4), + # Lower priority - general advisories (less likely to have commit hints) + "cve.org": ("cve_org", 5), + "cve.mitre.org": ("cve_org", 5), + "nvd.nist.gov": ("nvd", 5), +} + +# URL patterns that indicate commit/patch content (should be excluded from advisory mining) +_COMMIT_URL_PATTERNS = frozenset({ + "/commit/", "/commits/", "/pull/", "/merge_requests/", + ".patch", ".diff", "/raw/", "/blob/", +}) + + +def is_advisory_url(url: str) -> bool: + """Check if URL is an advisory page (not a commit/PR/patch link). + + Returns True if the URL does NOT contain patterns indicating + direct source code or patch content. + + Args: + url: URL string to check + + Returns: + True if URL appears to be an advisory page, False if it's a commit/patch URL + """ + if not url: + return False + url_lower = url.lower() + return not any(pattern in url_lower for pattern in _COMMIT_URL_PATTERNS) + + +# Allowed URL schemes for advisory fetching +_ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) + + +def _is_safe_url(url: str) -> bool: + """Validate URL for safe fetching (SSRF protection). + + Ensures the URL uses http/https scheme and targets a DNS hostname + rather than a raw IP address. This blocks SSRF vectors including + localhost, private IPs, and cloud metadata endpoints. + + Args: + url: URL string to validate + + Returns: + True if URL is safe to fetch, False otherwise + """ + if not url: + return False + + try: + parsed = urlparse(url) + + if parsed.scheme not in _ALLOWED_URL_SCHEMES: + logger.debug("Rejected URL with disallowed scheme: %s", url) + return False + + hostname = parsed.hostname + if not hostname: + logger.debug("Rejected URL with missing hostname: %s", url) + return False + + # Reject raw IP addresses - only allow DNS hostnames + try: + ipaddress.ip_address(hostname) + logger.debug("Rejected IP-based URL (SSRF protection): %s", url) + return False + except ValueError: + # Not an IP address - this is a DNS hostname, which is allowed + pass + + return True + + except Exception: + logger.debug("Rejected malformed URL: %s", url, exc_info=True) + return False + + +def _classify_advisory_url(url: str) -> tuple[str, int]: + """Classify an advisory URL by source type and priority. + + Args: + url: URL string to classify + + Returns: + Tuple of (source_type, priority). Returns ("unknown", 10) if no pattern matches. + """ + url_lower = url.lower() + for pattern, (source_type, priority) in ADVISORY_SOURCE_PATTERNS.items(): + if pattern in url_lower: + return source_type, priority + return "unknown", 10 + + +def extract_advisory_urls(intel: CveIntel) -> list[tuple[str, str, int]]: + """Extract non-commit reference URLs from intel for advisory mining. + + Scans GHSA, NVD, RHSA, and Ubuntu references for URLs that are NOT + direct commit/patch links but may contain advisory content with hints + about fix commits, vulnerable functions, or affected files. + + This is the inverse of extract_commit_url_candidates() - it finds + advisory pages rather than direct commit URLs. + + Args: + intel: CveIntel object containing intel from various providers + + Returns: + List of (url, source_type, priority) tuples, sorted by priority (ascending). + Lower priority number = higher quality source. + Example: [("https://openwall.com/lists/oss-security/...", "openwall", 1), ...] + """ + seen_urls: set[str] = set() + results: list[tuple[str, str, int]] = [] + + def _process_refs(refs: list | None) -> None: + if not refs: + return + for ref in refs: + url = ref if isinstance(ref, str) else (ref.get("url", "") if isinstance(ref, dict) else "") + if not url or url in seen_urls: + continue + if not _is_safe_url(url): + continue + if not is_advisory_url(url): + continue + seen_urls.add(url) + source_type, priority = _classify_advisory_url(url) + results.append((url, source_type, priority)) + + # GHSA references + if intel.ghsa: + ghsa_refs = getattr(intel.ghsa, "references", None) + _process_refs(ghsa_refs) + + # NVD references + if intel.nvd: + _process_refs(intel.nvd.references) + + # RHSA references (may be newline-separated) + if intel.rhsa: + rhsa_refs = getattr(intel.rhsa, "references", None) + if isinstance(rhsa_refs, list): + flat_refs = [] + for ref in rhsa_refs: + if isinstance(ref, str) and "\n" in ref: + flat_refs.extend(ref.split("\n")) + else: + flat_refs.append(ref) + _process_refs(flat_refs) + + # Ubuntu references + if intel.ubuntu: + ubuntu_refs = getattr(intel.ubuntu, "references", None) + _process_refs(ubuntu_refs) + + # Sort by priority (ascending - lower is better) + results.sort(key=lambda x: x[2]) + return results + def extract_commit_url_candidates(intel: CveIntel) -> dict[str, list[str]]: """Extract URLs from intel references that may contain commit/patch information. diff --git a/src/vuln_analysis/utils/package_identifier.py b/src/vuln_analysis/utils/package_identifier.py index e355d8609..614c820d2 100644 --- a/src/vuln_analysis/utils/package_identifier.py +++ b/src/vuln_analysis/utils/package_identifier.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import re from univers import versions @@ -65,6 +66,11 @@ def _extract_rhel_version(distro_tag: str | None) -> str | None: return m.group(1) if m else None +def _is_fedora_profile() -> bool: + """Check if running in Fedora/external profile mode via RPM_USER_TYPE env var.""" + return os.environ.get("RPM_USER_TYPE", "internal").lower() == "external" + + def _match_package_state_for_distro( package_states: list, target_name: str, @@ -73,7 +79,11 @@ def _match_package_state_for_distro( """Find the PackageState entry matching target package name and distro. Returns the matching PackageState or None if no match found. + Returns None for Fedora packages (RHSA data is RHEL-specific). """ + if _is_fedora_profile(): + return None + rhel_version = _extract_rhel_version(target_distro) for ps in package_states: @@ -108,6 +118,48 @@ def _interpret_fix_state(fix_state: str | None) -> EnumIdentifyResult | None: return None # Unknown state, fall through to other checks +def extract_nvd_version_info(intel: CveIntel, target_name: str) -> tuple[str, str]: + """Extract version range and fixed version from NVD configurations. + + Standalone function for reuse across modules without instantiating PackageIdentifier. + + Args: + intel: CveIntel object containing NVD data. + target_name: Target package name for filtering configurations. + + Returns: + Tuple of (affected_version_range, fixed_version). + Returns ("", "") if no NVD data is available. + """ + if intel.nvd is None or not intel.nvd.configurations: + return "", "" + + ranges = [] + fixed_version = "" + + for config in intel.nvd.configurations: + if not package_names_match(target_name, config.package): + continue + + parts = [] + if config.versionStartIncluding: + parts.append(f">={config.versionStartIncluding}") + if config.versionStartExcluding: + parts.append(f">{config.versionStartExcluding}") + if config.versionEndIncluding: + parts.append(f"<={config.versionEndIncluding}") + if config.versionEndExcluding: + parts.append(f"<{config.versionEndExcluding}") + if not fixed_version: + fixed_version = config.versionEndExcluding + + if parts: + ranges.append(" && ".join(parts)) + + affected_range = " OR ".join(ranges) if ranges else "" + return affected_range, fixed_version + + class PackageIdentifier: """ Deterministic PackageIdentify phase: resolves package identity from intel, @@ -181,10 +233,25 @@ def _find_and_locate_rpm(self, intel: CveIntel) -> list[str]: def _is_cve_for_target_package(self, intel: CveIntel) -> bool: """Step 1: target is in scope for this CVE per Red Hat package lists. - Returns True if RHSA has no package lists or the target appears in - ``package_state`` (any fix_state) or ``affected_release`` (patched builds). - Returns False when RHSA lists packages but the target matches neither bucket. + Returns True if: + - OSIDB affects list contains a matching ps_component, OR + - RHSA has no package lists, OR + - Target appears in RHSA package_state or affected_release. + Returns False when intel lists packages but target matches none. """ + target_name = self._target_package.name + + # Priority 1: Check OSIDB affects (most reliable source when available) + if intel.osidb and intel.osidb.affects: + for affect in intel.osidb.affects: + if affect.ps_component and package_names_match(target_name, affect.ps_component): + logger.debug( + "OSIDB match: target=%s matches ps_component=%s", + target_name, affect.ps_component + ) + return True + + # Priority 2: Fall back to RHSA package lists if not intel.rhsa: return True @@ -195,7 +262,6 @@ def _is_cve_for_target_package(self, intel: CveIntel) -> bool: if not has_package_state and not has_affected_release: return True - target_name = self._target_package.name if has_package_state: for ps in intel.rhsa.package_state: if ps.package_name and package_names_match(target_name, ps.package_name): @@ -295,26 +361,8 @@ def _is_target_package_affected( def _format_nvd_version_range(self, intel: CveIntel, target_name: str) -> str: """Format NVD version range for human-readable output.""" - if intel.nvd is None or not intel.nvd.configurations: - return "unknown" - - ranges = [] - for config in intel.nvd.configurations: - if not package_names_match(target_name, config.package): - continue - parts = [] - if config.versionStartIncluding: - parts.append(f">={config.versionStartIncluding}") - if config.versionStartExcluding: - parts.append(f">{config.versionStartExcluding}") - if config.versionEndIncluding: - parts.append(f"<={config.versionEndIncluding}") - if config.versionEndExcluding: - parts.append(f"<{config.versionEndExcluding}") - if parts: - ranges.append(" && ".join(parts)) - - return " OR ".join(ranges) if ranges else "any version (no range specified)" + affected_range, _ = extract_nvd_version_info(intel, target_name) + return affected_range if affected_range else "any version (no range specified)" def _is_target_package_fixed(self, intel: CveIntel, package_identify: PackageIdentifyResult) -> EnumIdentifyResult: """Determine whether the target package is already running the fixed version. diff --git a/src/vuln_analysis/utils/reference_fetcher.py b/src/vuln_analysis/utils/reference_fetcher.py new file mode 100644 index 000000000..d73aa4be7 --- /dev/null +++ b/src/vuln_analysis/utils/reference_fetcher.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reference Fetcher for Advisory URL content retrieval. + +Fetches advisory pages (mailing lists, security bulletins) for LLM hint extraction +as part of the Reference Mining pipeline. +""" + +from __future__ import annotations + +import re + +import aiohttp +from bs4 import BeautifulSoup + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import AdvisoryContent +from vuln_analysis.utils.async_http_utils import request_with_retry + +logger = LoggingFactory.get_agent_logger(__name__) + +DEFAULT_FETCH_TIMEOUT_SECONDS = 30 + + +class ReferenceFetcher: + """Fetches advisory page content for Reference Mining. + + Retrieves content from advisory URLs (mailing lists, vendor security pages) + and converts HTML to plain text for LLM extraction. + """ + + def __init__( + self, + timeout_seconds: int = DEFAULT_FETCH_TIMEOUT_SECONDS, + max_content_length: int = 500_000, + ) -> None: + """Initialize the fetcher. + + Args: + timeout_seconds: HTTP request timeout in seconds. + max_content_length: Maximum content length to fetch (bytes). + """ + self._timeout_seconds = timeout_seconds + self._max_content_length = max_content_length + + async def fetch_advisory( + self, + session: aiohttp.ClientSession, + url: str, + source_type: str, + ) -> AdvisoryContent: + """Fetch advisory content from a URL. + + Args: + session: aiohttp ClientSession for making requests. + url: The advisory URL to fetch. + source_type: Classification of the source (e.g., "openwall", "vendor_advisory"). + + Returns: + AdvisoryContent with fetched text or error information. + """ + logger.info("Fetching advisory: %s (type: %s)", url, source_type) + + try: + request_kwargs = { + "method": "GET", + "url": url, + "timeout": aiohttp.ClientTimeout(total=self._timeout_seconds), + "headers": { + "User-Agent": "Mozilla/5.0 (compatible; CVE-Advisory-Fetcher/1.0)", + "Accept": "text/html,text/plain,application/xhtml+xml,*/*", + }, + } + + async with request_with_retry( + session, + request_kwargs, + max_retries=3, + retry_on_client_errors=False, + ) as response: + content_type = response.headers.get("Content-Type", "") + content_length = response.content_length or 0 + + if content_length > self._max_content_length: + logger.warning( + "Advisory content too large (%d bytes), truncating: %s", + content_length, + url, + ) + + raw_bytes = await response.content.read(self._max_content_length) + raw_text = self._decode_content(raw_bytes, content_type) + + if "html" in content_type.lower(): + text = _extract_text_from_html(raw_text) + else: + text = raw_text + + logger.info( + "Fetched advisory: %s (%d chars)", + url, + len(text), + ) + + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text=text, + fetch_success=True, + error_message=None, + ) + + except aiohttp.ClientError as e: + error_msg = f"HTTP error fetching {url}: {e}" + logger.warning(error_msg) + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text="", + fetch_success=False, + error_message=error_msg, + ) + except Exception as e: + error_msg = f"Unexpected error fetching {url}: {e}" + logger.error(error_msg, exc_info=True) + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text="", + fetch_success=False, + error_message=error_msg, + ) + + def _decode_content(self, raw_bytes: bytes, content_type: str) -> str: + """Decode raw bytes to string, handling encoding detection. + + Args: + raw_bytes: Raw response bytes. + content_type: Content-Type header value. + + Returns: + Decoded string content. + """ + encoding = "utf-8" + if "charset=" in content_type: + charset_match = re.search(r"charset=([^\s;]+)", content_type) + if charset_match: + encoding = charset_match.group(1).strip('"\'') + + try: + return raw_bytes.decode(encoding) + except (UnicodeDecodeError, LookupError): + return raw_bytes.decode("utf-8", errors="replace") + + +def _extract_text_from_html(html: str) -> str: + """Extract readable text from HTML content using BeautifulSoup. + + Args: + html: Raw HTML content. + + Returns: + Extracted plain text with normalized whitespace. + """ + soup = BeautifulSoup(html, "html.parser") + + # Remove script and style elements + for element in soup(["script", "style", "noscript", "header", "footer", "nav"]): + element.decompose() + + # Get text with separator for block elements + text = soup.get_text(separator="\n") + + # Normalize whitespace + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n\s*\n+", "\n\n", text) + text = text.strip() + + return text diff --git a/src/vuln_analysis/utils/reference_parser.py b/src/vuln_analysis/utils/reference_parser.py new file mode 100644 index 000000000..5b017c974 --- /dev/null +++ b/src/vuln_analysis/utils/reference_parser.py @@ -0,0 +1,575 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reflective Reference Parser for CVE hint extraction. + +Uses LLM with reflection pattern to extract hints from advisory content, +with chunking support for limited context window LLMs. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +from langchain_core.messages import HumanMessage, SystemMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + HINT_STRENGTH_NONE, + HINT_STRENGTH_STRONG, + HINT_STRENGTH_MEDIUM, + HINT_STRENGTH_WEAK, + ReferenceHints, +) + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Token estimation and chunking utilities +# --------------------------------------------------------------------------- + +CHARS_PER_TOKEN_ESTIMATE = 4 +DEFAULT_CHUNK_OVERLAP_TOKENS = 200 +DEFAULT_MIN_CHUNK_TOKENS = 500 + + +def estimate_tokens(text: str) -> int: + """Estimate token count using simple character heuristic. + + Args: + text: Input text. + + Returns: + Estimated token count (chars / 4). + """ + return len(text) // CHARS_PER_TOKEN_ESTIMATE + + +def split_into_chunks( + text: str, + max_tokens: int, + overlap_tokens: int = DEFAULT_CHUNK_OVERLAP_TOKENS, + min_chunk_tokens: int = DEFAULT_MIN_CHUNK_TOKENS, +) -> list[str]: + """Split text into chunks respecting token limits and paragraph boundaries. + + Args: + text: Text to split. + max_tokens: Maximum tokens per chunk. + overlap_tokens: Token overlap between chunks for context preservation. + min_chunk_tokens: Minimum chunk size (don't create tiny chunks). + + Returns: + List of text chunks. + """ + if estimate_tokens(text) <= max_tokens: + return [text] + + max_chars = max_tokens * CHARS_PER_TOKEN_ESTIMATE + overlap_chars = overlap_tokens * CHARS_PER_TOKEN_ESTIMATE + min_chars = min_chunk_tokens * CHARS_PER_TOKEN_ESTIMATE + + paragraphs = re.split(r"\n\s*\n", text) + + chunks: list[str] = [] + current_chunk: list[str] = [] + current_length = 0 + + for para in paragraphs: + para_length = len(para) + + if current_length + para_length > max_chars and current_chunk: + chunk_text = "\n\n".join(current_chunk) + chunks.append(chunk_text) + + overlap_text = _get_overlap_text(current_chunk, overlap_chars) + current_chunk = [overlap_text] if overlap_text else [] + current_length = len(overlap_text) if overlap_text else 0 + + current_chunk.append(para) + current_length += para_length + 2 + + if current_chunk: + chunk_text = "\n\n".join(current_chunk) + if chunks and estimate_tokens(chunk_text) < min_chunk_tokens: + chunks[-1] = chunks[-1] + "\n\n" + chunk_text + else: + chunks.append(chunk_text) + + return chunks if chunks else [text] + + +def _get_overlap_text(paragraphs: list[str], max_chars: int) -> str: + """Get trailing text from paragraphs up to max_chars for overlap.""" + if not paragraphs: + return "" + + result: list[str] = [] + total = 0 + + for para in reversed(paragraphs): + if total + len(para) > max_chars: + break + result.insert(0, para) + total += len(para) + 2 + + return "\n\n".join(result) + + +def merge_chunk_hints(chunk_hints: list[ReferenceHints]) -> ReferenceHints: + """Merge hints from multiple chunks, keeping first revision and deduplicating others. + + For revision_hint: keeps the first non-None value found (earlier chunks + are more likely to contain the primary CVE information). + + For other hints: deduplicates across chunks. + + Args: + chunk_hints: List of ReferenceHints from individual chunks. + + Returns: + Merged ReferenceHints with single revision and deduplicated lists. + """ + if not chunk_hints: + return ReferenceHints() + + if len(chunk_hints) == 1: + return chunk_hints[0] + + merged = ReferenceHints() + + for hints in chunk_hints: + if merged.revision_hint is None and hints.revision_hint is not None: + merged.revision_hint = hints.revision_hint + logger.debug("merge_chunk_hints: keeping revision_hint=%s", hints.revision_hint) + + merged.function_hints = list(set(merged.function_hints + hints.function_hints)) + merged.file_hints = list(set(merged.file_hints + hints.file_hints)) + merged.branch_hints = list(set(merged.branch_hints + hints.branch_hints)) + merged.version_hints = list(set(merged.version_hints + hints.version_hints)) + + if hints.confidence > merged.confidence: + merged.confidence = hints.confidence + merged.hint_strength = hints.hint_strength + + return merged + + +# --------------------------------------------------------------------------- +# LLM response models +# --------------------------------------------------------------------------- + + +class HintWithContext(BaseModel): + """A hint value with extraction context.""" + value: str + context: str = "" + + +class ExtractionResult(BaseModel): + """LLM extraction output model.""" + revision_hint: HintWithContext | None = Field(default=None) + function_hints: list[HintWithContext] = Field(default_factory=list) + file_hints: list[HintWithContext] = Field(default_factory=list) + branch_hints: list[HintWithContext] = Field(default_factory=list) + version_hints: list[HintWithContext] = Field(default_factory=list) + extraction_notes: str = "" + + +class QualityIssue(BaseModel): + """A quality issue identified during reflection.""" + field: str + issue: str + severity: str = "medium" + + +class MissedHint(BaseModel): + """A hint that was missed in extraction.""" + type: str + value: str + evidence: str = "" + + +class ReflectionResult(BaseModel): + """LLM reflection output model.""" + quality_issues: list[QualityIssue] = Field(default_factory=list) + missed_hints: list[MissedHint] = Field(default_factory=list) + hint_strength: str = HINT_STRENGTH_NONE + confidence: float = 0.0 + needs_more_sources: bool = True + decision_reasoning: str = "" + + +# --------------------------------------------------------------------------- +# Prompts +# --------------------------------------------------------------------------- + +REFERENCE_EXTRACT_PROMPT = """You are analyzing a CVE advisory to extract hints for finding the fix commit. + +## Context + +CVE: {vuln_id} +CVE Description: {cve_description} +Affected Package: {package_name} + +## Advisory Source + +URL: {advisory_url} +Source Type: {source_type} +{chunk_info} + +## Advisory Content + +``` +{advisory_content} +``` + +## Task + +Extract hints from the advisory text to help find the fix commit: + +1. **revision_hint**: The SINGLE SVN revision (r1935008) or git commit hash for THIS CVE's fix. + - IMPORTANT: Select only ONE revision that is CLEARLY for {vuln_id} + - If multiple revisions appear, pick the one that matches the CVE description + - Return null if no revision clearly relates to this specific CVE +2. **function_hints**: Function names mentioned as vulnerable or patched +3. **file_hints**: Source file paths (.c, .h, .py, .go, etc.) +4. **branch_hints**: Branch names where the fix was applied (e.g., "2.4.x", "main", "stable") +5. **version_hints**: Version numbers that contain the fix (e.g., "2.4.68", "1.2.3") + +## Rules + +- Only extract hints that are CLEARLY related to THIS CVE ({vuln_id}) +- For revision_hint: if the advisory mentions multiple CVEs/fixes, select ONLY the revision for {vuln_id} +- Do NOT hallucinate - if no hints found, return null/empty +- Distinguish between bug IDs and revision numbers + +## Output Format + +Return valid JSON: + +```json +{{ + "revision_hint": {{"value": "r1935008", "context": "fix commit for {vuln_id}"}}, + "function_hints": [ + {{"value": "ap_proxy_cookie_reverse", "context": "vulnerable function"}} + ], + "file_hints": [ + {{"value": "proxy_util.c", "context": "file containing the fix"}} + ], + "branch_hints": [ + {{"value": "2.4.x", "context": "branch where fix was backported"}} + ], + "version_hints": [ + {{"value": "2.4.68", "context": "first fixed version"}} + ], + "extraction_notes": "Any observations about the advisory quality or ambiguity" +}} +```""" + +REFERENCE_REFLECT_PROMPT = """You are reviewing your own extraction of CVE fix hints. +Your task is to CRITIQUE the quality AND DECIDE if more sources are needed. + +## Context + +CVE: {vuln_id} +CVE Description: {cve_description} +Package: {package_name} + +## Current Advisory + +URL: {current_url} + +## Your Extraction + +```json +{extraction_json} +``` + +## Other Advisory URLs Available (not yet processed) + +{remaining_urls} + +--- + +## Part 1: Quality Critique + +Evaluate your extraction: + +1. **Relevance Check**: Is the extracted revision_hint clearly about THIS CVE, not other CVEs? +2. **Type Validation**: + - Is "revision_hint" actually a revision (not a bug ID, ticket number)? + - Are "function_hints" valid function names (not variables, types)? + - Are "file_hints" actual file paths (not directories, URLs)? +3. **Completeness Check**: Did you miss any obvious hints in the text? + +## Part 2: Hint Strength Assessment + +Rate the strength of hints for finding a git commit: + +- **strong**: Have revision/commit hash, OR specific function + file combination +- **medium**: Have function OR file hints with version info +- **weak**: Only have version hints or generic terms +- **none**: No useful hints extracted + +## Part 3: Control Flow Decision + +Based on hint strength and remaining URLs, decide: + +| Condition | Decision | +|-----------|----------| +| Hints are strong | `needs_more_sources: false` (proceed to git search) | +| Hints are weak/none + URLs remain | `needs_more_sources: true` (fetch more) | +| Hints are medium + high-quality URLs remain | `needs_more_sources: true` (try to improve) | +| Hints are medium + low-quality URLs remain | `needs_more_sources: false` (good enough) | +| No URLs remain | `needs_more_sources: false` (can't fetch more) | + +## Output Format + +Return valid JSON: + +```json +{{ + "quality_issues": [ + {{"field": "revision_hint", "issue": "might be a bug ID, not SVN revision", "severity": "medium"}} + ], + "missed_hints": [ + {{"type": "version_hints", "value": "2.4.68", "evidence": "text says 'fixed in 2.4.68'"}} + ], + "hint_strength": "strong", + "confidence": 0.85, + "needs_more_sources": false, + "decision_reasoning": "We have a revision hint (r1935008) which is strong enough to search git log directly." +}} +```""" + + +# --------------------------------------------------------------------------- +# Parser class +# --------------------------------------------------------------------------- + + +@dataclass +class ParserConfig: + """Configuration for ReflectiveReferenceParser.""" + max_chunk_tokens: int = 4400 + chunk_overlap_tokens: int = DEFAULT_CHUNK_OVERLAP_TOKENS + min_chunk_tokens: int = DEFAULT_MIN_CHUNK_TOKENS + + +class ReflectiveReferenceParser: + """Extracts hints from advisory content using LLM with reflection pattern. + + Handles large content via chunking, extracts hints, and uses reflection + to assess quality and decide if more sources are needed. + """ + + def __init__(self, llm: BaseChatModel, config: ParserConfig | None = None) -> None: + """Initialize the parser. + + Args: + llm: LangChain LLM for extraction and reflection. + config: Parser configuration (uses defaults if None). + """ + self._llm = llm + self._config = config or ParserConfig() + + async def extract_hints( + self, + advisory_content: str, + vuln_id: str, + cve_description: str, + package_name: str, + advisory_url: str, + source_type: str, + ) -> ReferenceHints: + """Extract hints from advisory content, handling chunking if needed. + + Args: + advisory_content: Raw text content from advisory page. + vuln_id: CVE identifier (e.g., "CVE-2024-1234"). + cve_description: CVE description text. + package_name: Affected package name. + advisory_url: URL of the advisory. + source_type: Classification of the source. + + Returns: + ReferenceHints with extracted values. + """ + chunks = split_into_chunks( + advisory_content, + max_tokens=self._config.max_chunk_tokens, + overlap_tokens=self._config.chunk_overlap_tokens, + min_chunk_tokens=self._config.min_chunk_tokens, + ) + + logger.info( + "Extracting hints from %s (%d chunks)", + advisory_url, + len(chunks), + ) + + chunk_hints: list[ReferenceHints] = [] + + for i, chunk in enumerate(chunks): + chunk_info = f"Chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else "" + + extraction = await self._extract_from_chunk( + chunk=chunk, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=package_name, + advisory_url=advisory_url, + source_type=source_type, + chunk_info=chunk_info, + ) + + hints = self._extraction_to_hints(extraction) + chunk_hints.append(hints) + + merged = merge_chunk_hints(chunk_hints) + merged.sources_processed.append(advisory_url) + + logger.info( + "Extracted hints: revision=%s, %d functions, %d files", + merged.revision_hint, + len(merged.function_hints), + len(merged.file_hints), + ) + + return merged + + async def reflect_on_extraction( + self, + extraction: ReferenceHints, + vuln_id: str, + cve_description: str, + package_name: str, + current_url: str, + remaining_urls: list[tuple[str, str, int]], + ) -> ReflectionResult: + """Reflect on extraction quality and decide if more sources needed. + + Args: + extraction: Current extracted hints. + vuln_id: CVE identifier. + cve_description: CVE description. + package_name: Package name. + current_url: URL that was just processed. + remaining_urls: List of (url, source_type, priority) not yet processed. + + Returns: + ReflectionResult with quality assessment and control flow decision. + """ + extraction_dict = { + "revision_hint": extraction.revision_hint, + "function_hints": extraction.function_hints, + "file_hints": extraction.file_hints, + "branch_hints": extraction.branch_hints, + "version_hints": extraction.version_hints, + } + + if remaining_urls: + urls_text = "\n".join( + f"- {url} (type: {stype}, priority: {priority})" + for url, stype, priority in remaining_urls + ) + else: + urls_text = "None - all available URLs have been processed." + + prompt = REFERENCE_REFLECT_PROMPT.format( + vuln_id=vuln_id, + cve_description=cve_description[:500] if cve_description else "Not available", + package_name=package_name, + current_url=current_url, + extraction_json=json.dumps(extraction_dict, indent=2), + remaining_urls=urls_text, + ) + + try: + structured_llm = self._llm.with_structured_output(ReflectionResult) + result: ReflectionResult = await structured_llm.ainvoke([ + SystemMessage(content=prompt), + HumanMessage(content="Reflect on the extraction quality."), + ]) # type: ignore[assignment] + + logger.info( + "Reflection: strength=%s, confidence=%.2f, needs_more=%s", + result.hint_strength, + result.confidence, + result.needs_more_sources, + ) + + return result + + except Exception as e: + logger.error("Reflection failed: %s", e, exc_info=True) + return ReflectionResult( + hint_strength=HINT_STRENGTH_WEAK, + confidence=0.3, + needs_more_sources=bool(remaining_urls), + decision_reasoning=f"Reflection failed: {e}", + ) + + async def _extract_from_chunk( + self, + chunk: str, + vuln_id: str, + cve_description: str, + package_name: str, + advisory_url: str, + source_type: str, + chunk_info: str, + ) -> ExtractionResult: + """Extract hints from a single chunk using structured LLM output.""" + prompt = REFERENCE_EXTRACT_PROMPT.format( + vuln_id=vuln_id, + cve_description=cve_description[:500] if cve_description else "Not available", + package_name=package_name, + advisory_url=advisory_url, + source_type=source_type, + chunk_info=f"\n{chunk_info}" if chunk_info else "", + advisory_content=chunk, + ) + + try: + structured_llm = self._llm.with_structured_output(ExtractionResult) + result: ExtractionResult = await structured_llm.ainvoke([ + SystemMessage(content=prompt), + HumanMessage(content="Extract hints from the advisory."), + ]) # type: ignore[assignment] + return result + + except Exception as e: + logger.error("Extraction failed for chunk: %s", e, exc_info=True) + return ExtractionResult() + + def _extraction_to_hints(self, extraction: ExtractionResult) -> ReferenceHints: + """Convert ExtractionResult to ReferenceHints.""" + return ReferenceHints( + revision_hint=extraction.revision_hint.value if extraction.revision_hint else None, + function_hints=[h.value for h in extraction.function_hints], + file_hints=[h.value for h in extraction.file_hints], + branch_hints=[h.value for h in extraction.branch_hints], + version_hints=[h.value for h in extraction.version_hints], + ) diff --git a/src/vuln_analysis/utils/repo_resolver.py b/src/vuln_analysis/utils/repo_resolver.py new file mode 100644 index 000000000..35fb33a17 --- /dev/null +++ b/src/vuln_analysis/utils/repo_resolver.py @@ -0,0 +1,512 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Repository Resolver - map package names to upstream git repository URLs. + +Resolves package names (e.g., "curl", "httpd", "openssl") to their upstream +git repository URLs using multiple strategies in priority order: + +1. OSIDB upstream_purls (confidence 0.95) +2. Curated package → repo mapping (confidence 0.90, 0.85 for aliases) +3. PURL parsing from any intel source (confidence 0.80) + +Only public repositories are supported. If resolution fails, returns +a RepoResolution with method="not_found" and logs the package for +future mapping expansion. +""" + +from __future__ import annotations + +import json +import re +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING + +from packageurl import PackageURL + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + RESOLUTION_METHOD_CURATED_MAPPING, + RESOLUTION_METHOD_NOT_FOUND, + RESOLUTION_METHOD_OSIDB_PURL, + RESOLUTION_METHOD_PURL_PARSE, + RepoResolution, +) + +if TYPE_CHECKING: + from exploit_iq_commons.data_models.cve_intel import CveIntel + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MAPPING_FILE_PATH = Path(__file__).parent.parent.parent / "exploit_iq_commons" / "data" / "package_repo_mapping.json" + +CONFIDENCE_OSIDB_PURL = 0.95 +CONFIDENCE_CURATED_DIRECT = 0.90 +CONFIDENCE_CURATED_ALIAS = 0.85 +CONFIDENCE_PURL_PARSE = 0.80 + +PURL_TYPE_GITHUB = "github" +PURL_TYPE_GITLAB = "gitlab" +PURL_TYPE_BITBUCKET = "bitbucket" + +PLATFORM_GITHUB = "github" +PLATFORM_GITLAB = "gitlab" +PLATFORM_BITBUCKET = "bitbucket" +PLATFORM_KERNEL_ORG = "kernel.org" +PLATFORM_SOURCEWARE = "sourceware" +PLATFORM_SAVANNAH = "savannah" +PLATFORM_UNKNOWN = "unknown" + +# Prefixes to strip during package name normalization +_PACKAGE_PREFIXES = ( + "lib", + "python-", + "python3-", + "py-", + "node-", + "golang-", + "rubygem-", + "perl-", + "php-", +) + +# Pattern to strip version suffixes (e.g., libcurl4 → libcurl) +_VERSION_SUFFIX_PATTERN = re.compile(r"\d+$") + + +# --------------------------------------------------------------------------- +# Mapping Loader (with caching) +# --------------------------------------------------------------------------- + + +@lru_cache(maxsize=1) +def load_package_repo_mapping() -> dict: + """Load the curated package-to-repo mapping from JSON file. + + Uses module-level caching to avoid repeated file reads. + + Returns: + Dictionary with "packages" and "aliases" keys. + + Raises: + FileNotFoundError: If mapping file doesn't exist. + json.JSONDecodeError: If mapping file is invalid JSON. + """ + logger.debug("Loading package repo mapping from %s", MAPPING_FILE_PATH) + + if not MAPPING_FILE_PATH.exists(): + logger.error("Package repo mapping file not found: %s", MAPPING_FILE_PATH) + raise FileNotFoundError(f"Package repo mapping not found: {MAPPING_FILE_PATH}") + + with MAPPING_FILE_PATH.open("r", encoding="utf-8") as f: + mapping = json.load(f) + + package_count = len(mapping.get("packages", {})) + alias_count = len(mapping.get("aliases", {})) + logger.info( + "Loaded package repo mapping: %d packages, %d aliases", + package_count, + alias_count, + ) + + return mapping + + +def clear_mapping_cache() -> None: + """Clear the cached mapping (useful for testing).""" + load_package_repo_mapping.cache_clear() + + +# --------------------------------------------------------------------------- +# Package Name Normalization +# --------------------------------------------------------------------------- + + +def normalize_package_name(name: str) -> list[str]: + """Generate normalized variants of a package name for mapping lookup. + + Strips common prefixes (lib, python-, etc.) and version suffixes + to increase matching probability. + + Args: + name: Original package name (e.g., "python3-urllib3", "libcurl4") + + Returns: + List of name variants to try, most specific first. + + Examples: + >>> normalize_package_name("python3-urllib3") + ['python3-urllib3', 'urllib3'] + >>> normalize_package_name("libcurl4") + ['libcurl4', 'curl4', 'libcurl', 'curl'] + """ + if not name: + return [] + + variants = [name] + name_lower = name.lower() + + # Strip version suffix from original first (libcurl4 → libcurl) + no_version = _VERSION_SUFFIX_PATTERN.sub("", name_lower) + if no_version and no_version != name_lower and no_version not in variants: + variants.append(no_version) + + # Strip common prefixes from original (libcurl4 → curl4) + for prefix in _PACKAGE_PREFIXES: + if name_lower.startswith(prefix): + stripped = name_lower[len(prefix):] + if stripped and stripped not in variants: + variants.append(stripped) + # Also strip version from this (curl4 → curl) + stripped_no_ver = _VERSION_SUFFIX_PATTERN.sub("", stripped) + if stripped_no_ver and stripped_no_ver != stripped and stripped_no_ver not in variants: + variants.append(stripped_no_ver) + break + + return variants + + +# --------------------------------------------------------------------------- +# PURL Utilities +# --------------------------------------------------------------------------- + + +_PURL_TYPE_TO_PLATFORM = { + PURL_TYPE_GITHUB: (PLATFORM_GITHUB, "https://github.com"), + PURL_TYPE_GITLAB: (PLATFORM_GITLAB, "https://gitlab.com"), + PURL_TYPE_BITBUCKET: (PLATFORM_BITBUCKET, "https://bitbucket.org"), +} + + +def purl_to_repo_url(purl: str | None) -> tuple[str | None, str | None]: + """Convert Package URL to repository URL. + + Supported PURL types: + - pkg:github/owner/repo → https://github.com/owner/repo + - pkg:gitlab/group/project → https://gitlab.com/group/project + - pkg:bitbucket/owner/repo → https://bitbucket.org/owner/repo + + Args: + purl: Package URL string (e.g., "pkg:github/curl/curl") + + Returns: + Tuple of (repo_url, platform) or (None, None) if unsupported. + """ + if not purl: + return None, None + + try: + parsed = PackageURL.from_string(purl) + + platform_info = _PURL_TYPE_TO_PLATFORM.get(parsed.type) + if not platform_info: + return None, None + + platform, base_url = platform_info + + if not parsed.namespace or not parsed.name: + logger.debug("PURL missing namespace or name: %s", purl) + return None, None + + repo_url = f"{base_url}/{parsed.namespace}/{parsed.name}" + return repo_url, platform + + except Exception as e: + logger.debug("Failed to parse PURL %s: %s", purl, e) + return None, None + + +def detect_platform(repo_url: str) -> str: + """Detect the hosting platform from a repository URL. + + Args: + repo_url: Repository URL + + Returns: + Platform identifier string. + """ + if not repo_url: + return PLATFORM_UNKNOWN + + url_lower = repo_url.lower() + + if "github.com" in url_lower: + return PLATFORM_GITHUB + if "gitlab.com" in url_lower or "gitlab." in url_lower: + return PLATFORM_GITLAB + if "bitbucket.org" in url_lower: + return PLATFORM_BITBUCKET + if "kernel.org" in url_lower: + return PLATFORM_KERNEL_ORG + if "sourceware.org" in url_lower: + return PLATFORM_SOURCEWARE + if "savannah.gnu.org" in url_lower: + return PLATFORM_SAVANNAH + + return PLATFORM_UNKNOWN + + +# --------------------------------------------------------------------------- +# PURL Collection from Intel +# --------------------------------------------------------------------------- + + +def collect_all_purls(intel: CveIntel | None) -> list[str]: + """Collect PURLs from all intel sources (OSIDB, NVD, GHSA). + + Args: + intel: CveIntel object containing data from various sources + + Returns: + Deduplicated list of PURL strings found in intel. + """ + if not intel: + return [] + + purls: set[str] = set() + + # OSIDB upstream_purls + if intel.osidb and intel.osidb.upstream_purls: + for purl_info in intel.osidb.upstream_purls: + if hasattr(purl_info, "purl") and purl_info.purl: + purls.add(purl_info.purl) + + # OSIDB affects (may contain PURLs) + if intel.osidb and intel.osidb.affects: + for affect in intel.osidb.affects: + if hasattr(affect, "purl") and affect.purl: + purls.add(affect.purl) + + return list(purls) + + +# --------------------------------------------------------------------------- +# Repository Resolver +# --------------------------------------------------------------------------- + + +class RepoResolver: + """Resolve package names to upstream git repository URLs. + + Uses multiple strategies in priority order: + 1. OSIDB upstream_purls (highest confidence) + 2. Curated package → repo mapping (PRIMARY for C/C++) + 3. PURL parsing from any intel source + + Example: + resolver = RepoResolver() + result = resolver.resolve("curl", intel=vulnerability_intel) + if result.repo_url: + print(f"Found repo: {result.repo_url} via {result.resolution_method}") + """ + + def __init__(self) -> None: + """Initialize the resolver.""" + self._mapping: dict | None = None + + @property + def mapping(self) -> dict: + """Lazily load and cache the package mapping.""" + if self._mapping is None: + try: + self._mapping = load_package_repo_mapping() + except FileNotFoundError: + logger.warning("Package mapping file not found, using empty mapping") + self._mapping = {"packages": {}, "aliases": {}} + return self._mapping + + def resolve( + self, + package_name: str, + intel: CveIntel | None = None, + ) -> RepoResolution: + """Resolve package name to upstream git repository URL. + + Tries strategies in order until one succeeds: + 1. OSIDB upstream_purls + 2. Curated mapping + 3. PURL parsing + + Args: + package_name: Package name to resolve (e.g., "curl", "httpd") + intel: Optional vulnerability intel containing PURLs + + Returns: + RepoResolution with repo_url if found, or method="not_found". + """ + strategies_tried: list[str] = [] + error_messages: list[str] = [] + alternative_repos: list[str] = [] + + # Strategy 1: OSIDB upstream_purls + strategies_tried.append(RESOLUTION_METHOD_OSIDB_PURL) + result = self._try_osidb_purls(intel) + if result: + repo_url, platform = result + logger.info( + "repo_resolution: resolved %s via OSIDB PURL → %s", + package_name, + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_OSIDB_PURL, + confidence=CONFIDENCE_OSIDB_PURL, + is_alias=False, + strategies_tried=strategies_tried, + ) + logger.debug( + "repo_resolution: OSIDB PURL strategy failed for %s, trying curated mapping", + package_name, + ) + + # Strategy 2: Curated mapping (PRIMARY) + strategies_tried.append(RESOLUTION_METHOD_CURATED_MAPPING) + result = self._try_curated_mapping(package_name) + if result: + repo_url, is_alias = result + platform = detect_platform(repo_url) + confidence = CONFIDENCE_CURATED_ALIAS if is_alias else CONFIDENCE_CURATED_DIRECT + logger.info( + "repo_resolution: resolved %s via curated mapping%s → %s", + package_name, + " (alias)" if is_alias else "", + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_CURATED_MAPPING, + confidence=confidence, + is_alias=is_alias, + strategies_tried=strategies_tried, + ) + logger.debug( + "repo_resolution: curated mapping strategy failed for %s, trying PURL parse", + package_name, + ) + + # Strategy 3: PURL parsing from any intel source + strategies_tried.append(RESOLUTION_METHOD_PURL_PARSE) + result = self._try_purl_parse(intel) + if result: + repo_url, platform = result + logger.info( + "repo_resolution: resolved %s via PURL parse → %s", + package_name, + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_PURL_PARSE, + confidence=CONFIDENCE_PURL_PARSE, + is_alias=False, + strategies_tried=strategies_tried, + ) + + # All strategies failed - log for mapping expansion + logger.info( + "repo_resolution: no repo found for '%s' - consider adding to mapping", + package_name, + ) + return RepoResolution( + repo_url=None, + repo_platform=None, + resolution_method=RESOLUTION_METHOD_NOT_FOUND, + confidence=0.0, + is_alias=False, + strategies_tried=strategies_tried, + error_messages=error_messages, + alternative_repos=alternative_repos, + ) + + def _try_osidb_purls( + self, + intel: CveIntel | None, + ) -> tuple[str, str] | None: + """Strategy 1: Extract repo URL from OSIDB upstream_purls. + + Returns: + Tuple of (repo_url, platform) if found, None otherwise. + """ + if not intel or not intel.osidb: + return None + + if not intel.osidb.upstream_purls: + return None + + for purl_info in intel.osidb.upstream_purls: + purl = getattr(purl_info, "purl", None) + if not purl: + continue + + repo_url, platform = purl_to_repo_url(purl) + if repo_url and platform: + return repo_url, platform + + return None + + def _try_curated_mapping( + self, + package_name: str, + ) -> tuple[str, bool] | None: + """Strategy 2: Look up package in curated mapping. + + Returns: + Tuple of (repo_url, is_alias) if found, None otherwise. + """ + packages = self.mapping.get("packages", {}) + aliases = self.mapping.get("aliases", {}) + + # Generate name variants + variants = normalize_package_name(package_name) + + for name in variants: + # Check direct mapping + if name in packages: + return packages[name], False + + # Check aliases + if name in aliases: + canonical = aliases[name] + if canonical in packages: + return packages[canonical], True + + return None + + def _try_purl_parse( + self, + intel: CveIntel | None, + ) -> tuple[str, str] | None: + """Strategy 3: Parse PURLs from any intel source. + + Returns: + Tuple of (repo_url, platform) if found, None otherwise. + """ + purls = collect_all_purls(intel) + + for purl in purls: + repo_url, platform = purl_to_repo_url(purl) + if repo_url and platform: + return repo_url, platform + + return None diff --git a/src/vuln_analysis/utils/rpm_checker_prompts.py b/src/vuln_analysis/utils/rpm_checker_prompts.py index bdbcff0e2..3a7b82bf3 100644 --- a/src/vuln_analysis/utils/rpm_checker_prompts.py +++ b/src/vuln_analysis/utils/rpm_checker_prompts.py @@ -70,10 +70,14 @@ 1. affected_files: Extract file paths from patch headers (strip a/ b/ prefixes) -2. vulnerable_functions: Extract function names from: - - Removed lines (- lines) in patch - - Function names mentioned in CVE description - - Functions or buffers mentioned in VENDOR_MITIGATIONS +2. vulnerable_functions: Extract function names ONLY from: + - Removed lines (- lines) in patch (HIGHEST PRIORITY - these are the actual vulnerable functions) + - Explicit function names mentioned VERBATIM in CVE description (e.g., "vulnerability in parse_rockridge()") + - Functions or buffers EXPLICITLY named in VENDOR_MITIGATIONS + ANTI-HALLUCINATION: Do NOT infer or construct function names from descriptions. + - BAD: CVE says "HTML directory list generation" -> guessing "mod_proxy_ftp_html_dir_list" (WRONG - made up) + - GOOD: CVE says "vulnerability in ap_proxy_ftp_handler()" -> extract "ap_proxy_ftp_handler" (CORRECT - verbatim) + - If no explicit function names exist in patch or CVE, leave this field EMPTY rather than guessing 3. vulnerable_variables: Extract variable names from: - Removed lines that are key to the vulnerability - Variables/buffers explicitly mentioned in VENDOR_MITIGATIONS (e.g., "sum2 buffer") @@ -82,6 +86,11 @@ - Include enough context to be unique 5. fix_patterns: Extract distinctive code snippets from added lines (+ lines) - These indicate the fix is present + - MUST be actual CODE patterns that can be grepped in source files + NO-PATCH RULE: If no patch is provided, leave fix_patterns EMPTY. + - Do NOT substitute version strings (e.g., "Apache HTTP Server 2.4.68") - these do not appear in source code + - Do NOT guess fix patterns from CVE description prose + - Empty fix_patterns is valid and expected when no patch is available 6. root_cause: Explain WHY the code is vulnerable in 1-2 sentences - Incorporate insights from VENDOR_MITIGATIONS if provided 7. vulnerability_type: Classify as one of: buffer_overflow, integer_overflow, use_after_free, @@ -89,19 +98,26 @@ 8. search_keywords: List 3-5 grep patterns ordered by specificity: - Start with most specific (unique variable/function names from patch or mitigations) - End with broader patterns (file names, component names) -9. affected_bitness: Determine which bitness is affected: +9. component_names: Extract module or component names ONLY if explicitly mentioned in CVE description or advisory. + - ONLY include names that appear verbatim (e.g., "mod_http2", "mod_proxy_http2", "libxml2_sax") + - Do NOT infer component names from context (e.g., do NOT guess "mod_ssl" just because CVE mentions TLS) + - Do NOT include the package name itself (e.g., do NOT add "httpd" for an httpd package CVE) + - If CVE mentions a broad component (like "kernel" or "glibc"), leave empty - too generic to help file discovery + - Examples of GOOD extractions: ["mod_http2", "mod_proxy_http2"], ["nf_tables"], ["bpf_verifier"] + - Examples of BAD extractions: ["httpd"] (package name), ["network"] (too vague), ["memory"] (not a component) +10. affected_bitness: Determine which bitness is affected: - "32-bit": Look for "32-bit systems", "i386", "i686", "on 32-bit" - "64-bit": Look for "64-bit only", "x86_64 only" (rare) - "both": DEFAULT when not explicitly stated NOTE: Do NOT assume bitness based on the vulnerability type. Default to "both" unless explicitly stated. -10. affected_architectures: Determine which CPU families are affected (or null for all): +11. affected_architectures: Determine which CPU families are affected (or null for all): - Look for: "x86", "Intel", "AMD" -> ["x86"] - Look for: "ARM", "aarch64", "arm64" -> ["arm"] - Look for: "PowerPC", "POWER", "ppc64" -> ["ppc"] - Look for: "s390", "z/Architecture", "IBM Z" -> ["s390"] - If none mentioned or "all architectures" -> null (affects all) NOTE: Most CVEs affect all architectures. Only extract specific families if explicitly mentioned. -11. known_mitigations: Copy vendor-provided mitigations verbatim if present (e.g., compiler flags like "-ftrivial-auto-var-init=zero", configuration changes). Empty if none provided. +12. known_mitigations: Copy vendor-provided mitigations verbatim if present (e.g., compiler flags like "-ftrivial-auto-var-init=zero", configuration changes). Empty if none provided. @@ -366,6 +382,15 @@ - ALL AFFECTED_FILES have been searched - ALL VULNERABLE_FUNCTIONS have been located - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If your code searches found NO conclusive evidence (no vulnerable pattern, no fix pattern): + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + - The absence of a verified fix, combined with version range membership, + is strong evidence of vulnerability. @@ -416,7 +441,10 @@ {{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} -""" + + +{{"thought": "Code searches found no vulnerable or fix patterns - affected module not located in source. However, TARGET_IN_VULNERABLE_RANGE is YES. No fix was verified in code. Version evidence indicates vulnerability.", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range per VULNERABILITY_INTEL. Code search could not locate the affected module or verify a fix. Based on version evidence and absence of verified fix, the package is vulnerable."}} +""" L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS = """ You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). @@ -511,24 +539,48 @@ PHASE 1 - INTELLIGENCE (PRE-COMPLETED): Review VULNERABILITY_INTEL above. It contains: - - AFFECTED_FILES: Files to verify (may be inferred from CVE description) + - COMPONENT_NAMES: Module/component names to use for file discovery (PRIORITY - use these FIRST) + - AFFECTED_FILES: Files to verify (may be empty in no-patch mode) - VULNERABLE_FUNCTIONS: Functions to search for - VULNERABLE_PATTERNS: Code patterns indicating vulnerability - FIX_PATTERNS: Code patterns indicating the fix - - SEARCH_KEYWORDS: Terms to grep for + - SEARCH_KEYWORDS: Terms to grep for (includes COMPONENT_NAMES at start) - ROOT_CAUSE: Description of the vulnerability mechanism PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): - For EACH item in VULNERABLE_FUNCTIONS and SEARCH_KEYWORDS: - 1. Search for vulnerable code patterns - 2. Search for defensive/fix patterns (bounds checks, validation, etc.) - IMPORTANT: Do NOT stop after finding the first file. Check ALL potential locations. + **STEP 1 - FILE DISCOVERY (DO THIS FIRST):** + If COMPONENT_NAMES is available (e.g., "mod_http2", "nf_tables"): + - Search for files containing the component name: "mod_http2" or "mod_http2.c" + - This locates the relevant source files for the CVE + - Record discovered file paths for STEP 2 + + If COMPONENT_NAMES is empty or search returns no results: + - Use SEARCH_KEYWORDS to find related files + - Try function names from VULNERABLE_FUNCTIONS + - Try combinations like ".c" or header file names + + **STEP 2 - CODE ANALYSIS (AFTER finding files):** + Once you have found relevant files from STEP 1: + - Search WITHIN those files for VULNERABLE_PATTERNS + - Search WITHIN those files for FIX_PATTERNS or defensive code + - Use file-scoped searches: "pattern,discovered_file.c" + + IMPORTANT: Do NOT skip STEP 1. File discovery FIRST, then code analysis. PHASE 3 - VERDICT: Only conclude when: - - Key files have been searched - - Vulnerable functions have been located + - Relevant files have been discovered (STEP 1 complete) + - Vulnerable/fix patterns searched within those files (STEP 2 complete) - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If your code searches found NO conclusive evidence (no vulnerable pattern, no fix pattern): + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + - The absence of a verified fix, combined with version range membership, + is strong evidence of vulnerability. RESPONSE FORMAT (JSON): @@ -545,23 +597,60 @@ -If a search returned results: -- Narrow down by searching within that specific file (e.g., "pattern,filename.c") -- Search for related symbols or defensive patterns in the found code -If a pattern wasn't found: -- Try simpler substrings or partial patterns -- Try a different tool (Source Grep <-> Code Keyword Search) -- Search for SEARCH_KEYWORDS from VULNERABILITY_INTEL +**STEP 1 - FILE DISCOVERY:** +If COMPONENT_NAMES available (e.g., "mod_http2"): +- First search: "mod_http2" to find files containing this component +- This returns file paths like modules/http2/mod_http2.c +If COMPONENT_NAMES empty or no results: +- Try SEARCH_KEYWORDS from VULNERABILITY_INTEL +- Try function names with .c suffix: "function_name.c" + +**STEP 2 - CODE ANALYSIS (after files found):** +Once you have file paths from STEP 1: +- Search WITHIN those files: "vulnerable_pattern,mod_http2.c" +- Search for fix patterns: "fix_pattern,mod_http2.c" + +**MANDATORY FALLBACK - If Source Grep returns NO MATCHES:** +When Source Grep returns empty for a function name or pattern search: +1. MUST try Code Keyword Search with semantic terms from ROOT_CAUSE or CVE description + - Example: CVE describes "XSS in HTML directory listing" → try "HTML directory" or "directory listing" +2. MUST try simpler substrings within the discovered file + - Example: instead of "mod_proxy_ftp_html_dir_list", try just "directory" or "html" in mod_proxy_ftp.c +3. Do NOT give up after 2-3 empty Source Grep searches - switch tools first! - -{{"thought": "No prior searches in KNOWLEDGE. Search for key function from CVE description", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate code related to CVE vulnerability"}}, "final_answer": null}} - + +{{"thought": "COMPONENT_NAMES shows mod_http2. Start with file discovery - search for files containing this module", "mode": "act", "actions": {{"tool": "Source Grep", "query": "mod_http2", "reason": "FILE DISCOVERY: Locate source files for the mod_http2 component"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows mod_http2.c found at modules/http2/. Now search within that file for vulnerable pattern", "mode": "act", "actions": {{"tool": "Source Grep", "query": "h2_stream_reset,mod_http2.c", "reason": "CODE ANALYSIS: Search for vulnerable function within discovered file"}}, "final_answer": null}} + + +{{"thought": "COMPONENT_NAMES is empty. Use SEARCH_KEYWORDS for file discovery", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "FILE DISCOVERY: Locate relevant files using CVE keywords"}}, "final_answer": null}} + + +{{"thought": "Source Grep for function name returned empty. MANDATORY FALLBACK: Try Code Keyword Search with semantic terms from CVE description (XSS in HTML directory listing)", "mode": "act", "actions": {{"tool": "Code Keyword Search", "query": "HTML directory listing", "reason": "FALLBACK: Source Grep failed, using semantic search for vulnerability-related code"}}, "final_answer": null}} + + +{{"thought": "Source Grep for specific function returned empty. Try simpler pattern within discovered file", "mode": "act", "actions": {{"tool": "Source Grep", "query": "directory,mod_proxy_ftp.c", "reason": "FALLBACK: Broader search within known-relevant file"}}, "final_answer": null}} + + +When concluding, follow this STRICT priority order: +1. If fix code found in target → LIKELY PATCHED +2. If vulnerable code found and no fix → VULNERABLE +3. If code search inconclusive AND TARGET_IN_VULNERABLE_RANGE=YES → VULNERABLE (version-based) +4. ONLY use INCONCLUSIVE when TARGET_IN_VULNERABLE_RANGE=NO or UNKNOWN +IMPORTANT: You MUST check TARGET_IN_VULNERABLE_RANGE before concluding INCONCLUSIVE. + + -{{"thought": "KNOWLEDGE shows defensive code found, no vulnerability indicators", "mode": "finish", "actions": null, "final_answer": "The package is LIKELY PATCHED. Found defensive code at [file:line]: [quote code]. The fix behavior described in the CVE appears to be present."}} +{{"thought": "KNOWLEDGE shows defensive code found in discovered file, no vulnerability indicators", "mode": "finish", "actions": null, "final_answer": "The package is LIKELY PATCHED. Found defensive code at [file:line]: [quote code]. The fix behavior described in the CVE appears to be present."}} + +{{"thought": "Code searches found no vulnerable or fix patterns - affected module not located in source. However, TARGET_IN_VULNERABLE_RANGE is YES. No fix was verified in code. Version evidence indicates vulnerability.", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range per VULNERABILITY_INTEL. Code search could not locate the affected module or verify a fix. Based on version evidence and absence of verified fix, the package is vulnerable."}} + -{{"thought": "KNOWLEDGE shows insufficient evidence to determine patch status", "mode": "finish", "actions": null, "final_answer": "INCONCLUSIVE. Could not find definitive evidence of fix or vulnerability. Manual review recommended."}} +{{"thought": "KNOWLEDGE shows insufficient evidence AND TARGET_IN_VULNERABLE_RANGE is NO/UNKNOWN - cannot determine patch status from version alone", "mode": "finish", "actions": null, "final_answer": "INCONCLUSIVE. Could not find definitive evidence of fix or vulnerability, and version range is unknown. Manual review recommended."}} """ # --------------------------------------------------------------------------- @@ -589,10 +678,42 @@ If RAW_PATCH_DIFF contains content: - Lines starting with "-" are REMOVED by the fix (vulnerable code) - Lines starting with "+" are ADDED by the fix (patched code) -- CRITICAL: A pattern appearing in BOTH "-" and "+" lines is NOT a unique indicator -- Only patterns EXCLUSIVELY in "-" lines indicate vulnerable code -- Only patterns EXCLUSIVELY in "+" lines indicate the fix is present -- Compare grep results against these exact lines, not the summarized patterns above + +PATTERN CLASSIFICATION (do this for EACH grep match): +1. Check if the matched pattern appears ONLY in "-" lines → VULNERABLE_ONLY pattern +2. Check if the matched pattern appears ONLY in "+" lines → FIX_ONLY pattern +3. Check if the matched pattern appears in BOTH "-" and "+" lines → AMBIGUOUS pattern (not a unique indicator) + +DECISION RULES: +- Finding a VULNERABLE_ONLY pattern in source = code is VULNERABLE (fix removed this, but it still exists) +- Finding a FIX_ONLY pattern in source = fix IS APPLIED (fix added this, and it exists) +- Finding an AMBIGUOUS pattern = INCONCLUSIVE (exists in both versions, proves nothing) +- CRITICAL: If you find a VULNERABLE_ONLY pattern, the code is VULNERABLE even if you also see AMBIGUOUS patterns nearby + +WORKED EXAMPLE - Follow this exact reasoning: +Given RAW_PATCH_DIFF: + - vulnerable_call(); + + safe_call(); + - shared_code(); + + shared_code(); + +Step 1: Grep finds 'vulnerable_call()' in source + → Check RAW_PATCH_DIFF: In "-" lines? YES. In "+" lines? NO. + → Classification: VULNERABLE_ONLY (the fix REMOVES this line) + → Correct finding: "VULNERABLE_ONLY_FOUND: vulnerable_call()" + → Meaning: Code is VULNERABLE because this line should have been removed + +Step 2: Grep finds 'shared_code()' in source + → Check RAW_PATCH_DIFF: In "-" lines? YES. In "+" lines? YES. + → Classification: AMBIGUOUS (exists in both versions) + → Correct finding: "AMBIGUOUS_PATTERN: shared_code() - not evidence" + → Meaning: This proves nothing about vulnerability status + +Step 3: Grep does NOT find 'safe_call()' in source + → This would be FIX_ONLY (only in "+" lines) + → Its absence confirms fix is NOT applied + +REMEMBER: Lines with "-" are REMOVED by fix. If you find them, the vulnerable code still exists! If RAW_PATCH_DIFF is empty: - Fall back to using VULNERABLE_PATTERNS and FIX_PATTERNS from VULNERABILITY_INTEL @@ -611,18 +732,24 @@ 4. The tool_outcome MUST accurately reflect what NEW OUTPUT shows, not what you expect. CODE ANALYSIS RULES (only if NEW OUTPUT has content): -1. READ the actual code snippets in NEW OUTPUT. Compare against VULNERABLE_PATTERNS and FIX_PATTERNS. -2. For each match found: - - Quote the actual line from NEW OUTPUT - - State the file:line where it was found - - Determine if it matches VULNERABLE or FIX pattern -3. If RAW_PATCH_DIFF is available: - - Check if the grep match appears in "-" lines (removed/vulnerable) - - Check if the grep match appears in "+" lines (added/fix) - - Report accordingly: matches "-" lines = VULNERABLE_CODE_FOUND - - A match that appears in BOTH "-" and "+" lines is NOT a unique indicator - note this -4. RECORD file paths and line numbers for all relevant matches. -5. FIX LOCATION: Tag FIX_DEFINITION_FOUND if found outside AFFECTED_FILES or is a function definition. Tag FIX_APPLIED_AT_CALL_SITE only when fix is CALLED and result USED in an AFFECTED_FILE. +1. MANDATORY CLASSIFICATION - For EACH pattern found in NEW OUTPUT, classify against RAW_PATCH_DIFF: + a. Pattern appears ONLY in "-" lines → VULNERABLE_ONLY (fix removed this) + b. Pattern appears ONLY in "+" lines → FIX_ONLY (fix added this) + c. Pattern appears in BOTH "-" and "+" lines → AMBIGUOUS (exists in both versions) + +2. LABELING RULES (strict): + - VULNERABLE_ONLY → report as "VULNERABLE_ONLY_FOUND: [pattern] at [file:line]" + - FIX_ONLY → report as "FIX_ONLY_FOUND: [pattern] at [file:line]" + - AMBIGUOUS → report as "AMBIGUOUS_PATTERN: [pattern] - exists in both versions, not evidence" + - NEVER label an AMBIGUOUS pattern as "fix pattern found" or "vulnerable pattern found" + +3. DECISION PRIORITY: + - If ANY VULNERABLE_ONLY pattern is found → code is VULNERABLE (definitive) + - AMBIGUOUS patterns do NOT contradict a VULNERABLE_ONLY finding + - Only FIX_ONLY patterns (without any VULNERABLE_ONLY) indicate fix is applied + +4. Quote actual lines from NEW OUTPUT and record file:line for all matches. +5. FIX LOCATION: Tag FIX_DEFINITION_FOUND if found outside AFFECTED_FILES. Tag FIX_APPLIED_AT_CALL_SITE only when fix is CALLED and result USED in an AFFECTED_FILE. Based on VULNERABILITY_INTEL above, assess investigation completeness: @@ -1345,7 +1472,9 @@ 1. CHECK if tool output shows: - Compilation commands for AFFECTED_FILES (e.g., gcc -c file.c -o file.o) - Feature-disable flags that match the CVE-affected component - - Object files or compilation artifacts for VULNERABLE_FUNCTIONS + - Patching or processing of files containing VULNERABLE_FUNCTIONS + NOTE: Function names do NOT appear in build logs. Only FILE names appear. + If the file containing a function is compiled/patched, the function is compiled. 2. COMPILATION EVIDENCE: - COMPILED: Found gcc/compile commands for affected files diff --git a/tests/test_git_commit_searcher.py b/tests/test_git_commit_searcher.py new file mode 100644 index 000000000..9ad0d7848 --- /dev/null +++ b/tests/test_git_commit_searcher.py @@ -0,0 +1,462 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for git_commit_searcher and git_repo_manager: commit discovery in git repositories.""" + +import pytest +from pathlib import Path +from unittest.mock import MagicMock + +from vuln_analysis.utils.git_repo_manager import ( + GitRepoManager, + DEFAULT_CACHE_DIR, + DEFAULT_CLONE_DEPTH, + GIT_HASH_LENGTH_SHORT, + GIT_HASH_LENGTH_FULL, +) +from vuln_analysis.utils.git_commit_searcher import ( + GitCommitSearcher, + GIT_HASH_FULL_PATTERN, + GIT_HASH_SHORT_PATTERN, + SVN_REVISION_PATTERN, + SECURITY_KEYWORDS, + MAX_RESULTS_PER_STRATEGY, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + CommitSearchResult, + GitSearchReport, + ReferenceHints, + SEARCH_METHOD_REVISION, + SEARCH_METHOD_FUNCTION_MESSAGE, + CONFIDENCE_REVISION_DIRECT, + CONFIDENCE_REVISION_BRANCH, + CONFIDENCE_FUNCTION_MESSAGE, + CONFIDENCE_FUNCTION_PICKAXE, + CONFIDENCE_THRESHOLD_MIN, +) + + +# --------------------------------------------------------------------------- +# GitRepoManager Tests +# --------------------------------------------------------------------------- + + +class TestGitRepoManagerCachePath: + """Tests for cache path generation.""" + + def test_github_url_to_cache_path(self): + """GitHub URL converts to proper cache path.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://github.com/apache/httpd") + + assert path == Path("/tmp/test_cache/github.com/apache/httpd") + + def test_gitlab_url_to_cache_path(self): + """GitLab URL converts to proper cache path.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://gitlab.gnome.org/GNOME/libxml2") + + assert path == Path("/tmp/test_cache/gitlab.gnome.org/GNOME/libxml2") + + def test_git_suffix_stripped(self): + """URL with .git suffix has it stripped.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://github.com/curl/curl.git") + + assert path == Path("/tmp/test_cache/github.com/curl/curl") + + def test_kernel_org_url(self): + """kernel.org URL converts properly.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + ) + + assert "git.kernel.org" in str(path) + assert "linux" in str(path) + + +class TestGitRepoManagerValidation: + """Tests for repository validation.""" + + def test_is_valid_repo_with_git_dir(self, tmp_path): + """Valid repo has .git directory with HEAD.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "test_repo" + git_dir = repo_path / ".git" + git_dir.mkdir(parents=True) + (git_dir / "HEAD").write_text("ref: refs/heads/main") + + assert manager._is_valid_repo(repo_path) is True + + def test_is_valid_repo_without_git_dir(self, tmp_path): + """Directory without .git is not valid.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "not_a_repo" + repo_path.mkdir(parents=True) + + assert manager._is_valid_repo(repo_path) is False + + def test_is_valid_repo_missing_head(self, tmp_path): + """Directory with .git but no HEAD is not valid.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "broken_repo" + git_dir = repo_path / ".git" + git_dir.mkdir(parents=True) + + assert manager._is_valid_repo(repo_path) is False + + +# --------------------------------------------------------------------------- +# Revision Pattern Tests +# --------------------------------------------------------------------------- + + +class TestRevisionPatterns: + """Tests for revision type classification patterns.""" + + def test_full_git_hash_pattern(self): + """40-character hex string matches full hash pattern.""" + full_hash = "a70753d2a61b7a43e11aaa217f8d2844c7d8cfab" + assert GIT_HASH_FULL_PATTERN.match(full_hash) is not None + + def test_short_git_hash_pattern(self): + """7-12 character hex string matches short hash pattern.""" + short_hashes = ["a70753d", "40326939", "a70753d2a61b"] + for h in short_hashes: + assert GIT_HASH_SHORT_PATTERN.match(h) is not None + + def test_svn_revision_with_r_prefix(self): + """SVN revision with r prefix matches.""" + match = SVN_REVISION_PATTERN.match("r1935008") + assert match is not None + assert match.group(1) == "1935008" + + def test_svn_revision_without_prefix(self): + """SVN revision without r prefix matches.""" + match = SVN_REVISION_PATTERN.match("1935008") + assert match is not None + assert match.group(1) == "1935008" + + def test_short_number_does_not_match_svn(self): + """Short numbers (< 5 digits) don't match SVN pattern.""" + assert SVN_REVISION_PATTERN.match("1234") is None + + def test_non_hex_does_not_match_git_hash(self): + """Non-hex strings don't match git hash patterns.""" + assert GIT_HASH_FULL_PATTERN.match("not-a-hash-string") is None + assert GIT_HASH_SHORT_PATTERN.match("xyz1234") is None + + +# --------------------------------------------------------------------------- +# Security Keyword Tests +# --------------------------------------------------------------------------- + + +class TestSecurityKeywords: + """Tests for security keyword boosting.""" + + def test_security_keywords_defined(self): + """Security keywords list is populated.""" + assert len(SECURITY_KEYWORDS) > 0 + + def test_security_keywords_have_boosts(self): + """Each keyword has a positive boost value.""" + for keyword, boost in SECURITY_KEYWORDS: + assert isinstance(keyword, str) + assert isinstance(boost, float) + assert boost > 0 + + def test_common_security_terms_present(self): + """Common security terms are in the list.""" + keyword_names = [k for k, _ in SECURITY_KEYWORDS] + assert "fix" in keyword_names + assert "security" in keyword_names + assert "CVE" in keyword_names + + +# --------------------------------------------------------------------------- +# CommitSearchResult Model Tests +# --------------------------------------------------------------------------- + + +class TestCommitSearchResultModel: + """Tests for CommitSearchResult Pydantic model.""" + + def test_create_valid_result(self): + """Valid CommitSearchResult can be created.""" + result = CommitSearchResult( + commit_hash="a70753d2a61b7a43e11aaa217f8d2844c7d8cfab", + commit_hash_short="a70753d", + commit_message="Fix buffer overflow in parser", + author="Test Author ", + date="2026-01-15T10:30:00+00:00", + files_changed=["src/parser.c", "include/parser.h"], + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + match_details="grep matched r1935008", + ) + + assert result.commit_hash_short == "a70753d" + assert result.confidence == 0.95 + assert len(result.files_changed) == 2 + + def test_confidence_bounds(self): + """Confidence must be between 0 and 1.""" + with pytest.raises(ValueError): + CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="test", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=1.5, # Invalid + ) + + def test_search_method_literal(self): + """Search method must be a valid literal.""" + result = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="test", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + assert result.search_method == "function_message" + + +# --------------------------------------------------------------------------- +# GitSearchReport Model Tests +# --------------------------------------------------------------------------- + + +class TestGitSearchReportModel: + """Tests for GitSearchReport Pydantic model.""" + + def test_create_empty_report(self): + """Empty GitSearchReport can be created.""" + report = GitSearchReport(repo_url="https://github.com/test/repo") + + assert report.repo_url == "https://github.com/test/repo" + assert report.search_success is False + assert report.best_result is None + assert len(report.all_results) == 0 + + def test_report_with_results(self): + """GitSearchReport with results.""" + result = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Fix security issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + report = GitSearchReport( + repo_url="https://github.com/test/repo", + best_result=result, + all_results=[result], + search_success=True, + strategies_tried=[SEARCH_METHOD_REVISION], + strategies_succeeded=[SEARCH_METHOD_REVISION], + ) + + assert report.search_success is True + assert report.best_result is not None + assert len(report.strategies_succeeded) == 1 + + +# --------------------------------------------------------------------------- +# GitCommitSearcher Tests (with mocks) +# --------------------------------------------------------------------------- + + +class TestGitCommitSearcherRanking: + """Tests for result ranking logic.""" + + def test_rank_boosts_security_keywords(self): + """Results with security keywords get boosted.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + result_with_keyword = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Fix security vulnerability in parser", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + result_without_keyword = CommitSearchResult( + commit_hash="b" * 40, + commit_hash_short="b" * 7, + commit_message="Update documentation", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + ranked = searcher._rank_results( + [result_without_keyword, result_with_keyword], + file_hints=None, + function_hints=None, + ) + + # Result with security keyword should be ranked first + assert ranked[0].commit_hash.startswith("a") + assert ranked[0].confidence > 0.75 + + def test_rank_penalizes_merge_commits(self): + """Merge commits get penalized in ranking.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + merge_commit = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Merge branch 'feature' into main", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + regular_commit = CommitSearchResult( + commit_hash="b" * 40, + commit_hash_short="b" * 7, + commit_message="Fix buffer overflow", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + ranked = searcher._rank_results( + [merge_commit, regular_commit], + file_hints=None, + function_hints=None, + ) + + # Regular commit should be ranked first after penalty + assert ranked[0].commit_hash.startswith("b") + + def test_rank_deduplicates_by_hash(self): + """Duplicate commits are removed.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + same_hash = "a" * 40 + result1 = CommitSearchResult( + commit_hash=same_hash, + commit_hash_short="a" * 7, + commit_message="Fix issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + result2 = CommitSearchResult( + commit_hash=same_hash, + commit_hash_short="a" * 7, + commit_message="Fix issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + ranked = searcher._rank_results( + [result1, result2], + file_hints=None, + function_hints=None, + ) + + # Should only have one result + assert len(ranked) == 1 + + +# --------------------------------------------------------------------------- +# ReferenceHints Tests +# --------------------------------------------------------------------------- + + +class TestReferenceHints: + """Tests for ReferenceHints model used in searches.""" + + def test_create_hints_with_revision(self): + """ReferenceHints with single revision hint.""" + hints = ReferenceHints( + revision_hint="r1935008", + function_hints=["ap_proxy_cookie_revers"], + file_hints=["proxy_util.c"], + branch_hints=["2.4.x"], + ) + + assert hints.revision_hint == "r1935008" + assert len(hints.function_hints) == 1 + + def test_empty_hints(self): + """Empty ReferenceHints is valid.""" + hints = ReferenceHints() + + assert hints.revision_hint is None + assert len(hints.function_hints) == 0 + assert hints.hint_strength == "none" + + +# --------------------------------------------------------------------------- +# Confidence Constants Tests +# --------------------------------------------------------------------------- + + +class TestConfidenceConstants: + """Tests for confidence threshold constants.""" + + def test_revision_direct_highest(self): + """Direct revision lookup has highest confidence.""" + assert CONFIDENCE_REVISION_DIRECT > CONFIDENCE_REVISION_BRANCH + assert CONFIDENCE_REVISION_DIRECT > CONFIDENCE_FUNCTION_MESSAGE + + def test_function_message_higher_than_pickaxe(self): + """Function message search has higher confidence than pickaxe.""" + assert CONFIDENCE_FUNCTION_MESSAGE > CONFIDENCE_FUNCTION_PICKAXE + + def test_threshold_is_reasonable(self): + """Minimum threshold is a sensible value.""" + assert CONFIDENCE_THRESHOLD_MIN >= 0.0 + assert CONFIDENCE_THRESHOLD_MIN <= 1.0 + assert CONFIDENCE_THRESHOLD_MIN == 0.50 + + +# --------------------------------------------------------------------------- +# Integration Test Helpers +# --------------------------------------------------------------------------- + + +class TestDefaultConstants: + """Tests for default constant values.""" + + def test_default_cache_dir(self): + """Default cache directory is set.""" + assert DEFAULT_CACHE_DIR == ".cache/am_cache/checker/upstream_repos" + + def test_default_clone_depth(self): + """Default clone depth is reasonable.""" + assert DEFAULT_CLONE_DEPTH == 1000 + + def test_hash_lengths(self): + """Git hash lengths are correct.""" + assert GIT_HASH_LENGTH_SHORT == 7 + assert GIT_HASH_LENGTH_FULL == 40 + + def test_max_results(self): + """Max results per strategy is set.""" + assert MAX_RESULTS_PER_STRATEGY == 5 diff --git a/tests/test_package_identifier.py b/tests/test_package_identifier.py index 67accd3cc..844192f88 100644 --- a/tests/test_package_identifier.py +++ b/tests/test_package_identifier.py @@ -163,6 +163,14 @@ def test_matches_by_product_name(self): assert result is not None assert result.fix_state == "Will not fix" + def test_fedora_profile_skips_rhsa_matching(self, rhel6_package_state, rhel7_package_state, monkeypatch): + """When RPM_USER_TYPE=external (Fedora), RHSA matching is skipped.""" + monkeypatch.setenv("RPM_USER_TYPE", "external") + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", None) + assert result is None + class TestPackageIdentifierWithFixState: """Integration tests for PackageIdentifier with RHSA fix_state.""" diff --git a/tests/test_repo_resolver.py b/tests/test_repo_resolver.py new file mode 100644 index 000000000..bb9311380 --- /dev/null +++ b/tests/test_repo_resolver.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for repo_resolver: package name to git repository URL resolution.""" + +import pytest +from unittest.mock import MagicMock + +from vuln_analysis.utils.repo_resolver import ( + normalize_package_name, + purl_to_repo_url, + detect_platform, + collect_all_purls, + load_package_repo_mapping, + clear_mapping_cache, + RepoResolver, + CONFIDENCE_OSIDB_PURL, + CONFIDENCE_CURATED_DIRECT, + CONFIDENCE_CURATED_ALIAS, + CONFIDENCE_PURL_PARSE, + RESOLUTION_METHOD_OSIDB_PURL, + RESOLUTION_METHOD_CURATED_MAPPING, + RESOLUTION_METHOD_PURL_PARSE, + RESOLUTION_METHOD_NOT_FOUND, + PLATFORM_GITHUB, + PLATFORM_GITLAB, + PLATFORM_BITBUCKET, + PLATFORM_KERNEL_ORG, +) + + +class TestNormalizePackageName: + """Tests for package name normalization.""" + + def test_simple_name_unchanged(self): + """Simple package names return as-is.""" + variants = normalize_package_name("curl") + assert "curl" in variants + + def test_python3_prefix_stripped(self): + """python3- prefix is stripped.""" + variants = normalize_package_name("python3-urllib3") + assert "python3-urllib3" in variants + assert "urllib3" in variants + + def test_lib_prefix_stripped(self): + """lib prefix is stripped.""" + variants = normalize_package_name("libcurl") + assert "libcurl" in variants + assert "curl" in variants + + def test_version_suffix_stripped(self): + """Version suffixes are stripped.""" + variants = normalize_package_name("libcurl4") + assert "libcurl4" in variants + assert "libcurl" in variants + assert "curl" in variants + + def test_golang_prefix_stripped(self): + """golang- prefix is stripped.""" + variants = normalize_package_name("golang-github-foo") + assert "golang-github-foo" in variants + assert "github-foo" in variants + + def test_empty_string_returns_empty_list(self): + """Empty string returns empty list.""" + variants = normalize_package_name("") + assert variants == [] + + def test_no_duplicates(self): + """No duplicate variants returned.""" + variants = normalize_package_name("curl") + assert len(variants) == len(set(variants)) + + +class TestPurlToRepoUrl: + """Tests for PURL to repository URL conversion.""" + + def test_github_purl(self): + """GitHub PURL converts to GitHub URL.""" + url, platform = purl_to_repo_url("pkg:github/curl/curl") + assert url == "https://github.com/curl/curl" + assert platform == PLATFORM_GITHUB + + def test_gitlab_purl(self): + """GitLab PURL converts to GitLab URL.""" + url, platform = purl_to_repo_url("pkg:gitlab/gnome/libxml2") + assert url == "https://gitlab.com/gnome/libxml2" + assert platform == PLATFORM_GITLAB + + def test_bitbucket_purl(self): + """Bitbucket PURL converts to Bitbucket URL.""" + url, platform = purl_to_repo_url("pkg:bitbucket/owner/repo") + assert url == "https://bitbucket.org/owner/repo" + assert platform == PLATFORM_BITBUCKET + + def test_purl_with_version(self): + """PURL with version is handled correctly.""" + url, platform = purl_to_repo_url("pkg:github/curl/curl@7.88.0") + assert url == "https://github.com/curl/curl" + assert platform == PLATFORM_GITHUB + + def test_purl_with_qualifiers(self): + """PURL with qualifiers is handled correctly.""" + url, _ = purl_to_repo_url("pkg:github/curl/curl?foo=bar") + assert url == "https://github.com/curl/curl" + + def test_unsupported_purl_type(self): + """Unsupported PURL type returns None.""" + url, platform = purl_to_repo_url("pkg:pypi/requests") + assert url is None + assert platform is None + + def test_invalid_purl(self): + """Invalid PURL returns None.""" + url, platform = purl_to_repo_url("not-a-purl") + assert url is None + assert platform is None + + def test_empty_purl(self): + """Empty PURL returns None.""" + url, platform = purl_to_repo_url("") + assert url is None + assert platform is None + + def test_none_purl(self): + """None PURL returns None.""" + url, platform = purl_to_repo_url(None) + assert url is None + assert platform is None + + +class TestDetectPlatform: + """Tests for platform detection from URLs.""" + + def test_github_url(self): + """GitHub URLs detected.""" + assert detect_platform("https://github.com/curl/curl") == PLATFORM_GITHUB + + def test_gitlab_url(self): + """GitLab URLs detected.""" + assert detect_platform("https://gitlab.com/gnome/libxml2") == PLATFORM_GITLAB + + def test_gitlab_instance_url(self): + """Self-hosted GitLab URLs detected.""" + assert detect_platform("https://gitlab.gnome.org/GNOME/libxml2") == PLATFORM_GITLAB + + def test_bitbucket_url(self): + """Bitbucket URLs detected.""" + assert detect_platform("https://bitbucket.org/owner/repo") == PLATFORM_BITBUCKET + + def test_kernel_org_url(self): + """kernel.org URLs detected.""" + kernel_url = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + assert detect_platform(kernel_url) == PLATFORM_KERNEL_ORG + + def test_empty_url(self): + """Empty URL returns unknown.""" + assert detect_platform("") == "unknown" + + +class TestCollectAllPurls: + """Tests for collecting PURLs from CveIntel.""" + + def _make_osidb_with_purls(self, purls): + """Create mock OSIDB with upstream_purls.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [ + MagicMock(purl=purl) for purl in purls + ] + mock_intel.osidb.affects = [] + return mock_intel + + def test_collects_osidb_upstream_purls(self): + """PURLs from OSIDB upstream_purls are collected.""" + intel = self._make_osidb_with_purls(["pkg:github/curl/curl"]) + purls = collect_all_purls(intel) + assert "pkg:github/curl/curl" in purls + + def test_deduplicates_purls(self): + """Duplicate PURLs are removed.""" + intel = self._make_osidb_with_purls([ + "pkg:github/curl/curl", + "pkg:github/curl/curl", + ]) + purls = collect_all_purls(intel) + assert len(purls) == 1 + + def test_none_intel_returns_empty(self): + """None intel returns empty list.""" + purls = collect_all_purls(None) + assert purls == [] + + def test_no_osidb_returns_empty(self): + """Intel without OSIDB returns empty list.""" + mock_intel = MagicMock() + mock_intel.osidb = None + purls = collect_all_purls(mock_intel) + assert purls == [] + + +class TestLoadPackageRepoMapping: + """Tests for mapping file loading.""" + + def test_mapping_loads_successfully(self): + """Mapping file loads and contains expected structure.""" + clear_mapping_cache() + mapping = load_package_repo_mapping() + assert "packages" in mapping + assert "aliases" in mapping + assert len(mapping["packages"]) > 50 + + def test_mapping_contains_curl(self): + """Mapping contains common package 'curl'.""" + mapping = load_package_repo_mapping() + assert "curl" in mapping["packages"] + assert "github.com/curl/curl" in mapping["packages"]["curl"] + + def test_mapping_contains_aliases(self): + """Mapping contains aliases like apache -> httpd.""" + mapping = load_package_repo_mapping() + assert "apache" in mapping["aliases"] + assert mapping["aliases"]["apache"] == "httpd" + + def test_cache_works(self): + """Second call returns cached mapping.""" + clear_mapping_cache() + mapping1 = load_package_repo_mapping() + mapping2 = load_package_repo_mapping() + assert mapping1 is mapping2 + + +class TestRepoResolver: + """Tests for the RepoResolver class.""" + + @pytest.fixture + def resolver(self): + """Create a resolver instance.""" + clear_mapping_cache() + return RepoResolver() + + def test_resolve_curated_direct(self, resolver): + """Direct curated mapping returns correct result.""" + result = resolver.resolve("curl") + assert result.repo_url == "https://github.com/curl/curl" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + assert result.confidence == CONFIDENCE_CURATED_DIRECT + assert result.is_alias is False + + def test_resolve_curated_alias(self, resolver): + """Alias curated mapping returns correct result.""" + result = resolver.resolve("apache") + assert result.repo_url == "https://github.com/apache/httpd" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + assert result.confidence == CONFIDENCE_CURATED_ALIAS + assert result.is_alias is True + + def test_resolve_with_prefix_stripping(self, resolver): + """Package with prefix resolves via normalization.""" + result = resolver.resolve("libcurl") + assert result.repo_url == "https://github.com/curl/curl" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + + def test_resolve_not_found(self, resolver): + """Unknown package returns not_found.""" + result = resolver.resolve("nonexistent-package-xyz-123") + assert result.repo_url is None + assert result.resolution_method == RESOLUTION_METHOD_NOT_FOUND + assert result.confidence == 0.0 + + def test_resolve_osidb_purl_priority(self, resolver): + """OSIDB PURL takes priority over curated mapping.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [ + MagicMock(purl="pkg:github/custom/custom-curl") + ] + + result = resolver.resolve("curl", intel=mock_intel) + assert result.repo_url == "https://github.com/custom/custom-curl" + assert result.resolution_method == RESOLUTION_METHOD_OSIDB_PURL + assert result.confidence == CONFIDENCE_OSIDB_PURL + + def test_resolve_strategies_tried_tracked(self, resolver): + """Strategies tried are tracked in result.""" + result = resolver.resolve("nonexistent-xyz") + assert RESOLUTION_METHOD_OSIDB_PURL in result.strategies_tried + assert RESOLUTION_METHOD_CURATED_MAPPING in result.strategies_tried + assert RESOLUTION_METHOD_PURL_PARSE in result.strategies_tried + + def test_resolve_httpd(self, resolver): + """httpd resolves correctly.""" + result = resolver.resolve("httpd") + assert result.repo_url == "https://github.com/apache/httpd" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + + def test_resolve_openssl(self, resolver): + """openssl resolves correctly.""" + result = resolver.resolve("openssl") + assert result.repo_url == "https://github.com/openssl/openssl" + + def test_resolve_linux_kernel(self, resolver): + """linux kernel resolves to kernel.org.""" + result = resolver.resolve("linux") + assert "kernel.org" in result.repo_url + assert result.repo_platform == PLATFORM_KERNEL_ORG + + +class TestRepoResolverPurlParseFallback: + """Tests for PURL parse fallback strategy.""" + + @pytest.fixture + def resolver(self): + """Create a resolver instance.""" + clear_mapping_cache() + return RepoResolver() + + def test_purl_parse_fallback_for_unknown_package(self, resolver): + """PURL parse works as fallback when package not in mapping.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [] + mock_intel.osidb.affects = [ + MagicMock(purl="pkg:github/unknown/unknown-package") + ] + + result = resolver.resolve("unknown-package-not-in-mapping", intel=mock_intel) + assert result.repo_url == "https://github.com/unknown/unknown-package" + assert result.resolution_method == RESOLUTION_METHOD_PURL_PARSE + assert result.confidence == CONFIDENCE_PURL_PARSE From 15342f24d868026eac957ce4f9f6183ea7d3d5af Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:11:14 +0300 Subject: [PATCH 267/286] Rewrite JS parser and Segmenter: (#255) * Rewrite JS parser and Segmenter: - Replaced esprima-based JavaScript segmenter with tree-sitter for reliable parsing of modern JS syntax (optional chaining, nullish coalescing, top-level await) - Fixed JS function name extraction: keyword filtering, position-aware matching, redundant pattern removal, generator/TypeScript/anonymous-export support - Added build-artifact filtering (should_skip) that excludes app-level dist/, build/static/, .min.js while preserving node_modules/*/dist/ as legitimate third-party source - Added empty-name guards in CCA BFS to prevent documents with unextractable function names from entering call-chain analysis - Fixed _get_function_calls regex to detect calls through optional chaining (obj?.method()) * Remove parser_threshold parameter from ExtendedLanguageParser creation * Fix JS get_function_name catastrophic regex backtracking and harden JS parser - Replace 3 catastrophic regex patterns (P4/P9/P11) in get_function_name with safe arrow_header check + simple name extraction, eliminating 12-38s hangs - Add arrow_header extension for destructured params like ({a,b}) => {body} where first '{' is in params, with guard to skip non-arrow functions - Use negative lookahead =(?!>) in P9 to distinguish assignment from arrow - Fix should_skip to only match top-level exclude dirs (dist/, build/) not nested ones (node_modules/*/dist/) - Add thread-safe double-checked locking for tree-sitter language init - Refactor class/method extraction to handle anonymous classes, generators, getters/setters, and export patterns - Fix is_function to use class\b word boundary instead of class\s+[\w$]+ - Strip optional chaining '?' from _get_function_calls results - Guard empty function names in CCA and create_map_of_local_vars with ValueError handling - Add direct export detection (export function/const) in is_function_exported - Add 387 JS parser tests and 139 JS segmenter tests covering backtracking regression, arrow functions, anonymous classes, and should_skip filtering * Harden JS parser and CCA: $-sign support, alias recursion, empty-name guards - Fix _build_class_hierarchy regex to include $ in identifier char class ([\w$]+) - Pass code_documents to recursive _get_function_calls alias resolution calls - Add empty-name guard in CCA BFS to skip documents with unextractable function names - Add try/except + empty guard in function_name_extractor for robust name extraction - Use [-1] instead of [1] for get_package_names to handle variable-length returns - Simplify should_skip with direct index checks instead of loop - Add regex comments describing direct-call and callback-reference patterns - Add 8 tests: $-sign class hierarchy (6) and chained alias resolution (2) - Fix test assertions: generator star spacing, empty-string guard, jQuery dollar detection * - Log source path of documents skipped due to empty get_function_name result in __find_initial_function * Clean up test comments and add _JS_KEYWORDS intent documentation - Add comment explaining _JS_KEYWORDS filters control-flow false positives, not invalid identifiers - Remove all "Fix X", "Fix A-I", and "Pattern N" references from test comments and docstrings - Replace internal references with behavioral descriptions of what each test verifies - Add warning log when skipping documents with empty function names in CCA __find_initial_function Signed-off-by: Theodor Mihalache --- .../utils/chain_of_calls_retriever.py | 30 +- .../utils/document_embedding.py | 4 + .../javascript_functions_parser.py | 394 +-- .../lang_functions_parsers.py | 2 + .../utils/javascript_extended_segmenter.py | 562 ++--- .../utils/function_name_extractor.py | 7 +- tests/test_java_script_extended_segmenter.py | 186 +- tests/test_javascript_functions_parser.py | 2151 ++++++++++++++++- 8 files changed, 2687 insertions(+), 649 deletions(-) diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py index 0233564a4..7f61f5a49 100644 --- a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -227,6 +227,8 @@ def __find_caller_function_dfs(self, document_function: Document, function_packa if parents: direct_parents.extend(parents) function_name_to_search = self.language_parser.get_function_name(document_function) + if not function_name_to_search: + return None if function_name_to_search == self.language_parser.get_constructor_method_name(): function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) function_file_name = document_function.metadata.get('source') @@ -319,6 +321,8 @@ def get_possible_docs(self, function_name_to_search: str, package: str, exclusio (self.language_parser.is_function(doc) or self.language_parser.is_script_language()) and not self._is_doc_excluded(doc, exclusions)] + if not function_name_to_search: + return [] return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] def __find_caller_functions_bfs(self, document_function: Document, function_package: str, @@ -344,6 +348,8 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack # direct_parents.extend([function_package]) # gets list of documents to search in only from parents of function' package. function_name_to_search = self.language_parser.get_function_name(document_function) + if not function_name_to_search: + return [] function_file_name = document_function.metadata.get('source') relevant_docs_to_search_in = list() # Search for caller functions only at parents according to dependency tree. @@ -365,7 +371,12 @@ def __find_caller_functions_bfs(self, document_function: Document, function_pack file_name = doc.metadata.get('source') if doc.metadata.get('state') == "invalid": continue - func_name = self.language_parser.get_function_name(doc) + try: + func_name = self.language_parser.get_function_name(doc) + except ValueError: + continue + if not func_name: + continue # check for same doc if (function_name_to_search == func_name) and (file_name == function_file_name): continue @@ -438,6 +449,8 @@ def _breadth_first_search(self, matching_documents: List[Document], target_funct logger.debug("get_relevant_documents: invalid function %s", target_doc.metadata['source']) continue function_name = self.language_parser.get_function_name(target_doc) + if not function_name: + continue function_file = target_doc.metadata.get('source') hashed_value = calculate_hashable_string_for_function(function_file, function_name) @@ -577,9 +590,9 @@ def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value] prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. - parents = list({self.language_parser.get_package_names(doc)[1] for doc in importing_docs if + parents = list({self.language_parser.get_package_names(doc)[-1] for doc in importing_docs if doc.metadata['source'].startswith( - prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[1] + prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[-1] in self.tree_dict.keys()}) for doc in importing_docs: if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): @@ -651,7 +664,11 @@ def __find_initial_function(self, function_name: str, package_name: str, documen self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=True), self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=False), ): - if function_name.lower() == self.language_parser.get_function_name(document).lower(): + doc_func_name = self.language_parser.get_function_name(document) + if not doc_func_name: + logger.warning("Skipping document with empty function name: %s", document.metadata.get('source', '')) + continue + if function_name.lower() == doc_func_name.lower(): package_exclusions.append(document) return document @@ -684,7 +701,10 @@ def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str] package_name = package_function.metadata['source'] try: function_name = self.language_parser.get_function_name(package_function) - current_level = f"(package={package_name},function={function_name},depth={i})" + if not function_name: + current_level = f"(document={package_name},depth={i})" + else: + current_level = f"(package={package_name},function={function_name},depth={i})" except ValueError: current_level = f"(document={package_name},depth={i})" results.append(current_level) diff --git a/src/exploit_iq_commons/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py index ff0681710..ad805cd1f 100644 --- a/src/exploit_iq_commons/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -147,6 +147,10 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) return + segmenter_cls = self.LANGUAGE_SEGMENTERS.get(language) + if segmenter_cls is not None and hasattr(segmenter_cls, "should_skip") and isinstance(blob.source, str) and segmenter_cls.should_skip(blob.source): + return + if self.parser_threshold >= len(code.splitlines()): yield Document( page_content=code, diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py index 6ed0895d9..e2b376128 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -23,73 +23,119 @@ logger = LoggingFactory.get_agent_logger(__name__) +# Keywords whose control-flow syntax resembles a function call — if(cond), +# for(...), while(...), new Foo(), typeof x — causing false positives in the +# fallback pattern of get_function_name. Keywords like case, default, export, +# import, in, super are deliberately excluded: they never appear with call-like +# name(...) syntax, and they ARE valid method names (obj.default(), query.in()). +_JS_KEYWORDS = frozenset({ + 'if', 'for', 'while', 'switch', 'catch', 'with', 'do', 'else', + 'return', 'throw', 'new', 'delete', 'typeof', 'void', + 'try', 'finally', 'function', 'class', 'extends', + 'break', 'continue', 'var', 'let', 'const', 'yield', 'await', + 'debugger', 'instanceof', +}) + class JavaScriptFunctionsParser(LanguageFunctionsParser): + def __init__(self): + self._class_hierarchy_cache: dict[str, tuple[str | None, str | None]] = {} + self._class_hierarchy_cache_key: int = -1 + def get_function_name(self, function: Document) -> str: - """ - Extract function name from JavaScript code. - - Handles various JavaScript function patterns: - - function myFunc() {...} - - const myFunc = () => {...} - - async function myFunc() {...} - - class methods - - arrow functions without parentheses: const name = param => ... - - computed property methods: [Symbol.iterator]() {...} - """ + """Extract function name from JavaScript/TypeScript code.""" content = function.page_content if not self.is_function(function): raise ValueError('Only function document is supported') - # Try to match function declarations: function name(...) or async function name(...) - match = re.search(r'(?:async\s+)?function\s+(\w+)\s*\(', content) + if re.match(r'export\s+default\s+(?:async\s+)?function\s*\*?\s*\(', content): + return '' + + if re.match(r'export\s+default\s+(?:async\s+)?\(', content): + return '' + + if re.match(r'export\s+default\s+(?:async\s+)?[\w$]+\s*=>', content): + return '' + + body_start = content.find('{') + header = content[:body_start] if body_start != -1 else content + # For arrow functions with destructured params like ({a, b}) => {body}, + # the first '{' is inside the params, not the body start. + # Extend header to include '=>' only when followed by '{' or end (top-level arrow). + arrow_header = header + if body_start != -1 and not header.rstrip().endswith(')') and not re.match(r'(?:(?:async|static|get|set)\s+)*[\w$]+\s*\(', header): + search_from = body_start + while search_from < len(content): + arrow_pos = content.find('=>', search_from) + if arrow_pos == -1 or arrow_pos > body_start + 500: + break + after_arrow = content[arrow_pos + 2:].lstrip() + if after_arrow.startswith('{') or not after_arrow: + arrow_header = content[:arrow_pos + 2] + break + search_from = arrow_pos + 2 + + match = re.search(r'(?:async\s+)?function\b\s*\*?\s*([\w$]+)\s*\(', header) + if match: + return match.group(1) + + if '=>' in arrow_header: + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=', arrow_header) + if match: + return match.group(1) + + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?[\w$]+\s*=>', header) if match: return match.group(1) - # Try to match arrow functions with parentheses: const name = (...) => or const name = async (...) => - match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>', content) + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?function\s*\(', header) if match: return match.group(1) - # Try to match arrow functions without parentheses (single parameter): const name = param => ... - # This pattern matches: const name = identifier => - match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\w+\s*=>', content) + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*[\w$.]+\s*\(', header) if match: return match.group(1) - # Try to match function expressions: const name = function(...) { - match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function\s*\(', content) + match = re.search(r'([\w$]+)\s*=\s*(?:async\s+)?function\s*\*?\s*[\w$]*\s*\(', header) if match: return match.group(1) - # Try to match wrapper calls: var name = wrapper(function(...) { or var name = wrapper(() => - match = re.search(r'(?:const|let|var)\s+(\w+)\s*=\s*\w+\s*\(', content) + if '=>' in arrow_header: + match = re.search(r'([\w$]+)\s*=(?!>)', arrow_header) + if match: + return match.group(1) + + match = re.search(r'([\w$]+)\s*=\s*(?:async\s+)?[\w$]+\s*=>', header) if match: return match.group(1) - # Try to match property assignment functions: obj.name = function(...) { - match = re.search(r'(?:[\w.]+)\.(\w+)\s*=\s*(?:async\s+)?function\s*\(', content) + if '=>' in arrow_header: + match = re.search(r'^([\w$]+)\s*:', arrow_header, re.MULTILINE) + if match: + return match.group(1) + + match = re.search(r'^([\w$]+)\s*:\s*(?:async\s+)?[\w$]+\s*=>', content, re.MULTILINE) if match: return match.group(1) - # Try to match property assignment arrow functions: obj.name = (...) => - match = re.search(r'(?:[\w.]+)\.(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>', content) + match = re.search(r'^([\w$]+)\s*:\s*(?:async\s+)?function\s*\(', content, re.MULTILINE) if match: return match.group(1) - # Try to match computed property methods: [Symbol.iterator]() { or [expr]() { - # Return the computed expression inside brackets - match = re.search(r'^\s*\[([^\]]+)\]\s*\([^)]*\)\s*\{', content, re.MULTILINE) + match = re.search(r"""^["']([\w$.!-]+)["']\s*[:(\s]""", content, re.MULTILINE) if match: return match.group(1) - # Try to match class methods: methodName(...) { or async methodName(...) { - match = re.search(r'(?:async\s+)?(\w+)\s*\([^)]*\)\s*\{', content) + match = re.search(r'^\s*\*?\s*(?:static\s+)?(?:async\s+)?(?:get\s+|set\s+)?\[([^\]]+)\]\s*\(', content, re.MULTILINE) if match: return match.group(1) + for match in re.finditer(r'(?:static\s+)?(?:async\s+)?(?:get\s+|set\s+)?([\w$]+)\s*[<(]', content): + if match.group(1) not in _JS_KEYWORDS: + return match.group(1) + logger.warning(f"Could not extract function name from: {content[:100]}") return '' @@ -119,7 +165,10 @@ def is_function(self, function: Document) -> bool: content = function.page_content.strip() - if re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\s+\w+', content): + if re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\b', content): + return False + + if re.match(r'^\s*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+', content): return False return True @@ -130,31 +179,36 @@ def supported_files_extensions(self) -> list[str]: def _get_function_calls(self, caller_function: Document, callee_function_name: str, code_documents: dict[str, Document] = None) -> list[str]: """ Extract all function calls matching the given callee_function_name from caller_function. - + Detects both: - Direct function calls: functionName() or obj.functionName() - Callback references: setTimeout(functionName, ...) or arr.forEach(functionName) - + Also handles aliased imports in JavaScript: - ES6: import { originalName as alias } from 'module' - CommonJS: const { originalName: alias } = require('module') - + Args: caller_function: Document containing the caller function code callee_function_name: Name of the function to find calls to code_documents: Optional dict mapping source paths to full file Documents (for alias resolution) - + Returns: List of function call patterns found (e.g., ['func', 'obj.func']) """ + if not callee_function_name: + return [] + content = caller_function.page_content content = '\n'.join([line if not self.is_comment_line(line) else ''for line in content.splitlines()]) - direct_call_pattern = rf'((?:[\w.()]+\.)?(? bool: - calls = self._get_function_calls(caller_function, callee_function_name) + calls = self._get_function_calls(caller_function, callee_function_name, code_documents) if not calls: return False @@ -242,10 +296,59 @@ def search_for_called_function(self, caller_function: Document, callee_function_ return False + @staticmethod + def _build_class_hierarchy(code_documents: dict[str, Document]) -> dict[str, tuple[str | None, str | None]]: + """ + Build a class hierarchy index from code_documents in a single pass. + + Returns: + dict mapping child_class -> (extends_clause, immediate_parent) + extends_clause is the raw text after 'extends' (for mixin matching), + immediate_parent is the resolved parent class name. + """ + hierarchy: dict[str, tuple[str | None, str | None]] = {} + + es6_pattern = re.compile(r'class\s+([\w$]+)\s+extends\s+([^{]+)') + proto_patterns = [ + re.compile(r'(?:util\.)?inherits\s*\(\s*([\w$]+)\s*,\s*([\w$]+)\s*\)'), + re.compile(r'([\w$]+)\.prototype\s*=\s*Object\.create\s*\(\s*([\w$]+)\.prototype\s*\)'), + re.compile(r'Object\.setPrototypeOf\s*\(\s*([\w$]+)\.prototype\s*,\s*([\w$]+)\.prototype\s*\)'), + ] + + for doc in code_documents.values(): + content = doc.page_content + + for match in es6_pattern.finditer(content): + child = match.group(1) + extends_clause = match.group(2).strip() + if innermost := re.search(r'([\w$]+)\s*\)+\s*$', extends_clause): + parent = innermost.group(1) + elif simple := re.match(r'^([\w$]+)\s*$', extends_clause): + parent = simple.group(1) + else: + parent = None + hierarchy[child] = (extends_clause, parent) + + for pattern in proto_patterns: + for match in pattern.finditer(content): + child = match.group(1) + parent = match.group(2) + if child not in hierarchy: + hierarchy[child] = (None, parent) + + return hierarchy + + def _get_class_hierarchy(self, code_documents: dict[str, Document]) -> dict[str, tuple[str | None, str | None]]: + cache_key = id(code_documents) + if self._class_hierarchy_cache_key != cache_key: + self._class_hierarchy_cache = self._build_class_hierarchy(code_documents) + self._class_hierarchy_cache_key = cache_key + return self._class_hierarchy_cache + def _is_subclass_of(self, child_class: str, parent_class: str, code_documents: dict[str, Document], visited: set = None) -> bool: """ - Check if child_class extends parent_class by searching simplified_code documents. + Check if child_class extends parent_class using pre-built hierarchy index. Supports: - Simple: class X extends Y - Mixin: class X extends Mixin(Y) @@ -260,95 +363,71 @@ def _is_subclass_of(self, child_class: str, parent_class: str, code_documents: d visited = set() if child_class in visited: - return False # Prevent infinite recursion on circular references + return False visited.add(child_class) - es6_class_pattern = rf'class\s+{re.escape(child_class)}\s+extends\s+([^{{]+)' + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if not entry: + return False - for doc in code_documents.values(): - if match := re.search(es6_class_pattern, doc.page_content): - extends_clause = match.group(1).strip() - # Direct match in extends clause (handles mixins too) - if re.search(rf'\b{re.escape(parent_class)}\b', extends_clause): - return True + extends_clause, immediate_parent = entry + + if extends_clause and re.search(rf'\b{re.escape(parent_class)}\b', extends_clause): + return True - immediate_parent = self._get_parent(child_class, code_documents) if immediate_parent: - # Direct match: immediate parent is the target if immediate_parent == parent_class: return True - # Transitive check: follow the chain return self._is_subclass_of(immediate_parent, parent_class, code_documents, visited) return False def _get_direct_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: - """ - Extract the immediate parent class name from ES6 class extends syntax. - - Handles: - - Simple: class X extends Y -> returns Y - - Mixin: class X extends Mixin(Y) -> returns Y (the innermost/base class) - - Chained: class X extends Mixin1(Mixin2(Y)) -> returns Y - """ - class_pattern = rf'class\s+{re.escape(child_class)}\s+extends\s+([^{{]+)' - - for doc in code_documents.values(): - if match := re.search(class_pattern, doc.page_content): - extends_clause = match.group(1).strip() - # Find the last identifier before closing parentheses - if innermost_match := re.search(r'(\w+)\s*\)+\s*$', extends_clause): - return innermost_match.group(1) - # Simple case: class X extends Y (no parentheses) - if parent_match := re.match(r'^(\w+)\s*$', extends_clause): - return parent_match.group(1) + """Extract the immediate parent from ES6 class extends syntax using the hierarchy index.""" + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry and entry[0] is not None: + return entry[1] return None def _get_prototype_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: - """ - Extract parent class from prototype-based inheritance patterns. - - Handles: - - util.inherits(Child, Parent) - - inherits(Child, Parent) - - Child.prototype = Object.create(Parent.prototype) - - Object.setPrototypeOf(Child.prototype, Parent.prototype) - """ - patterns = [ - # util.inherits(Child, Parent) or inherits(Child, Parent) - rf'(?:util\.)?inherits\s*\(\s*{re.escape(child_class)}\s*,\s*(\w+)\s*\)', - # Child.prototype = Object.create(Parent.prototype) - rf'{re.escape(child_class)}\.prototype\s*=\s*Object\.create\s*\(\s*(\w+)\.prototype\s*\)', - # Object.setPrototypeOf(Child.prototype, Parent.prototype) - rf'Object\.setPrototypeOf\s*\(\s*{re.escape(child_class)}\.prototype\s*,\s*(\w+)\.prototype\s*\)', - ] - - for doc in code_documents.values(): - for pattern in patterns: - if match := re.search(pattern, doc.page_content): - return match.group(1) + """Extract parent from prototype-based inheritance using the hierarchy index.""" + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry and entry[0] is None and entry[1] is not None: + return entry[1] return None def _get_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: - return self._get_direct_parent(child_class, code_documents) or \ - self._get_prototype_parent(child_class, code_documents) + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry: + return entry[1] + return None def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: mappings = {} - + for func_method in functions_methods_documents: - func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + try: + func_name = self.get_function_name(func_method) + except ValueError: + continue + if not func_name: + continue + func_key = f"{func_name}@{func_method.metadata.get('source', '?')}" content = func_method.page_content - + all_vars = {} - + # Extract parameters from function signature param_match = re.search(r'\(([^)]*)\)', content) if param_match: all_vars.update(self._parse_declarations(param_match.group(1), is_param=True)) - + all_vars['return_types'] = [] - + # Extract local variables from function body if param_match: first_brace = content.find('{', param_match.end()) @@ -362,13 +441,13 @@ def create_map_of_local_vars(self, functions_methods_documents: list[Document]) if not self.is_comment_line(statement): if match := re.match(r'(const|let|var)\s+(.+)', statement): all_vars.update(self._parse_declarations(match.group(2), is_param=False)) - + # Add 'this' reference for class/object methods if class_name := self.get_class_name_from_class_function(func_method): all_vars['this'] = {"value": f'{class_name}()', "type": class_name} - + mappings[func_key] = all_vars - + return mappings @staticmethod @@ -724,6 +803,8 @@ def _trace_variable_to_value(self, variable_name: str, lines: list[str], depth: return '' def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + if not identifier: + return False if callee_package and callee_package not in code_content: return False @@ -745,7 +826,7 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package else: # import { template } from 'lodash' # import * as lodash from 'lodash' - before_from = line.split('from')[0] + before_from = line.split(' from ')[0] if re.search(rf'\b{re.escape(identifier)}\b', before_from): return True else: @@ -768,7 +849,7 @@ def is_package_imported(self, code_content: str, identifier: str, callee_package if 'export' in line and 'from' in line: if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): if identifier: - before_from = line.split('from')[0] + before_from = line.split(' from ')[0] if re.search(rf'\b{re.escape(identifier)}\b', before_from): return True else: @@ -911,11 +992,42 @@ def _check_package_reexport(self, function_name: str, source_file: str, document return False + @staticmethod + def _is_exported_by_name(name: str, full_file_content: str) -> bool: + """Check if a name is exported via ES6 named export, export default, or CommonJS.""" + escaped = re.escape(name) + + # ES6: export default name + if re.search(rf'export\s+default\s+{escaped}\b', full_file_content): + return True + + # ES6: export { name } or export { x as name } + if re.search(rf'export\s+\{{[^}}]*\b{escaped}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports = name + if re.search(rf'module\.exports\s*=\s*{escaped}\b', full_file_content): + return True + + # CommonJS: module.exports = { name } or module.exports = { name: name } + if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{escaped}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports.name = ... + if re.search(rf'module\.exports\.{escaped}\s*=', full_file_content): + return True + + # CommonJS: exports.name = ... + if re.search(rf'exports\.{escaped}\s*=', full_file_content): + return True + + return False + @staticmethod def _is_exportable_class(class_name: str, full_file_content: str) -> bool: """ Check if a class is exported using any export syntax. - + Handles: - ES6: export class ClassName - ES6: export default ClassName or export default class ClassName @@ -927,39 +1039,14 @@ def _is_exportable_class(class_name: str, full_file_content: str) -> bool: # ES6: export class ClassName or export default class ClassName if re.search(rf'export\s+(?:default\s+)?class\s+{re.escape(class_name)}\b', full_file_content): return True - - # ES6: export default ClassName (after class definition) - if re.search(rf'export\s+default\s+{re.escape(class_name)}\b', full_file_content): - return True - - # ES6: export { ClassName } - export_pattern = rf'export\s+\{{[^}}]*\b{re.escape(class_name)}\b[^}}]*\}}' - if re.search(export_pattern, full_file_content): - return True - - # CommonJS: module.exports = ClassName - if re.search(rf'module\.exports\s*=\s*{re.escape(class_name)}\b', full_file_content): - return True - - # CommonJS: module.exports = { ClassName } or module.exports = { ClassName: ClassName } - if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{re.escape(class_name)}\b[^}}]*\}}', full_file_content): - return True - - # CommonJS: module.exports.ClassName = ClassName - if re.search(rf'module\.exports\.{re.escape(class_name)}\s*=', full_file_content): - return True - - # CommonJS: exports.ClassName = ClassName - if re.search(rf'exports\.{re.escape(class_name)}\s*=', full_file_content): - return True - - return False + + return JavaScriptFunctionsParser._is_exported_by_name(class_name, full_file_content) @staticmethod def _is_exportable_function(function_name: str, full_file_content: str) -> bool: """ Check if a standalone function is exported using any export syntax. - + Handles: - ES6: export function functionName - ES6: export default functionName @@ -969,32 +1056,15 @@ def _is_exportable_function(function_name: str, full_file_content: str) -> bool: - CommonJS: module.exports.functionName = functionName - CommonJS: exports.functionName = functionName """ - # ES6: export { functionName } or export { func as functionName } - export_pattern = rf'export\s+\{{[^}}]*\b{re.escape(function_name)}\b[^}}]*\}}' - if re.search(export_pattern, full_file_content): - return True - - # ES6: export default functionName - if re.search(rf'export\s+default\s+{re.escape(function_name)}\b', full_file_content): - return True - - # CommonJS: module.exports = functionName - if re.search(rf'module\.exports\s*=\s*{re.escape(function_name)}\b', full_file_content): - return True - - # CommonJS: module.exports = { functionName } or module.exports = { functionName: functionName } - if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{re.escape(function_name)}\b[^}}]*\}}', full_file_content): + # ES6: export function functionName / export async function functionName + if re.search(rf'export\s+(?:async\s+)?function\b\s*\*?\s*{re.escape(function_name)}\b', full_file_content): return True - - # CommonJS: module.exports.functionName = ... - if re.search(rf'module\.exports\.{re.escape(function_name)}\s*=', full_file_content): - return True - - # CommonJS: exports.functionName = ... - if re.search(rf'exports\.{re.escape(function_name)}\s*=', full_file_content): + + # ES6: export const/let/var functionName = + if re.search(rf'export\s+(?:const|let|var)\s+{re.escape(function_name)}\b', full_file_content): return True - - return False + + return JavaScriptFunctionsParser._is_exported_by_name(function_name, full_file_content) def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py index 7c7f71467..65cc2c6d1 100644 --- a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -173,6 +173,8 @@ def is_doc_type(self, doc: Document) -> bool: return doc.page_content.startswith(self.get_type_reserved_word()) def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: + if not function_name: + return [] relevant_docs = [ doc for doc in documents if doc.metadata.get('source').__contains__(package_name) and diff --git a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py index d5168a365..28bb9211f 100644 --- a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py +++ b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py @@ -13,386 +13,264 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any +import threading from typing import List -from typing import Tuple -import esprima -from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter +from tree_sitter_languages import get_language +from langchain_community.document_loaders.parsers.language.tree_sitter_segmenter import TreeSitterSegmenter from exploit_iq_commons.logging.loggers_factory import LoggingFactory logger = LoggingFactory.get_agent_logger(__name__) +_JS_EXCLUDE_DIRS: frozenset[str] = frozenset({ + "dist", "build/static", "coverage", ".nyc_output", +}) -class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): - """Extended JavaScript segmenter that handles shebang and ES optional chaining.""" +_JS_EXCLUDE_SUFFIXES: tuple[str, ...] = ( + ".min.js", ".min.css", ".bundle.js", +) + +# Tree-sitter query matching JavaScript constructs that the agent tools +# need for function extraction and call-chain analysis. +# +# Named exports (export function foo, export class Bar, export const x = () => {}) +# are captured by the inner declaration patterns — the export keyword is NOT +# included in those chunks. Only anonymous default exports need explicit +# export_statement patterns since they have no matching inner declaration. +JS_CHUNK_QUERY = """ +[ + (function_declaration) @function + (generator_function_declaration) @generator + + (class_declaration) @class + + (lexical_declaration + (variable_declarator + value: [(arrow_function) (function)])) @func_expr + + (variable_declaration + (variable_declarator + value: [(arrow_function) (function)])) @var_func_expr + + (variable_declaration + (variable_declarator + value: (call_expression + arguments: (arguments [(function) (arrow_function)])))) @wrapped_func + + (lexical_declaration + (variable_declarator + value: (call_expression + arguments: (arguments [(function) (arrow_function)])))) @wrapped_func_const + + (lexical_declaration + (variable_declarator + value: (object + [(method_definition) (pair value: [(function) (arrow_function)])]))) @obj_with_methods + + (variable_declaration + (variable_declarator + value: (object + [(method_definition) (pair value: [(function) (arrow_function)])]))) @var_obj_with_methods + + (export_statement (function)) @export_default_func + (export_statement (arrow_function)) @export_default_arrow + (export_statement (class)) @export_default_class + + (expression_statement + (assignment_expression + right: [(function) (arrow_function)])) @assign_func +] +""" + + +class ExtendedJavaScriptSegmenter(TreeSitterSegmenter): + """JavaScript segmenter using tree-sitter for fast, modern-syntax-aware parsing. + + Handles optional chaining, nullish coalescing, top-level await, and all + other modern JS syntax that esprima-python cannot parse. Parses each + file once and caches the tree for reuse across extract/simplify calls. + """ + + # Class-level caches — initialized on first instantiation, shared across + # all instances. Double-checked locking for thread safety. + _js_language = None + _js_chunk_query = None + _js_error_query = None + _init_lock = threading.Lock() + + @staticmethod + def should_skip(path: str) -> bool: + """Return True if the file at *path* is a JS build artifact that should not be indexed.""" + parts = path.replace("\\", "/").split("/") + if "node_modules" in parts[:-1]: + return False + if path.endswith(_JS_EXCLUDE_SUFFIXES): + return True + # Only check root-level directories — deeper dist/build are legitimate (e.g. node_modules/pkg/dist/) + if len(parts) > 1: + if len(parts) > 2 and f"{parts[0]}/{parts[1]}" in _JS_EXCLUDE_DIRS: + return True + if parts[0] in _JS_EXCLUDE_DIRS: + return True + return False def __init__(self, code: str): - """Initialize the segmenter with preprocessed code.""" super().__init__(code) - # Skip files with shebang (#!) line since they typically contain non-standard JavaScript syntax + cls = type(self) + if cls._js_language is None: + with cls._init_lock: + if cls._js_language is None: + lang = get_language("javascript") + cls._js_chunk_query = lang.query(JS_CHUNK_QUERY) + cls._js_error_query = lang.query("(ERROR) @error") + cls._js_language = lang + self._cached_tree = None + if code.startswith("#!"): logger.warning("File contains a shebang line. Skipping parsing.") self.skip_file = True - # esprima-python parser limitation: cannot handle JavaScript optional chaining syntax, - # fallback to regular property access else: self.skip_file = False - self.code = self.code.replace("?.", ".") - - def _parse_with_fallback(self) -> Any: - """Try to parse code as script first, then as module if that fails.""" - try: - logger.debug("Attempting to parse as a script...") - return esprima.parseScript(self.code, loc=True) - except esprima.Error: - logger.debug("Script parsing failed. Trying module parsing...") - try: - return esprima.parseModule(self.code, loc=True) - except esprima.Error as e: - logger.error("Module parsing failed: %s", str(e)) - print(f'TW Module parsing error: {self.code=} {e=}') - return None - def extract_functions_classes(self) -> List[str]: - """ - Extract functions, classes, and individual class methods from JavaScript code. - - Handles various JavaScript function patterns: - - Regular function declarations - - Arrow functions (const func = () => {}) - - Async functions - - Generator functions - - Class declarations and their methods - - Exported functions and classes - - Returns: - List of code strings representing all extracted functions and classes - """ - if self.skip_file: - return [] + # -- TreeSitterSegmenter contract ---------------------------------------- - tree = self._parse_with_fallback() - if tree is None: - return [] + def get_language(self): + return self._js_language - functions_classes = [] - for node in tree.body: - # Handle direct function/class declarations (including async and generator) - if isinstance(node, (esprima.nodes.FunctionDeclaration, - esprima.nodes.AsyncFunctionDeclaration, - esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - - # Handle variable declarations that might contain arrow functions or function expressions - elif isinstance(node, esprima.nodes.VariableDeclaration): - arrow_funcs = self._extract_arrow_functions(node) - functions_classes.extend(arrow_funcs) - - # Handle exported declarations - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, - esprima.nodes.AsyncFunctionDeclaration, - esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - elif isinstance(node.declaration, esprima.nodes.VariableDeclaration): - arrow_funcs = self._extract_arrow_functions(node.declaration) - functions_classes.extend(arrow_funcs) - - # Handle default exports - elif isinstance(node, esprima.nodes.ExportDefaultDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, - esprima.nodes.AsyncFunctionDeclaration, - esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - - # Handle property assignments with function expressions, - # e.g. hb.compile = function(input, options) { ... } - elif isinstance(node, esprima.nodes.ExpressionStatement): - if isinstance(node.expression, esprima.nodes.AssignmentExpression): - if isinstance(node.expression.right, ( - esprima.nodes.FunctionExpression, - esprima.nodes.ArrowFunctionExpression - )): - functions_classes.append(self._extract_code(node)) - - # Extract individual class methods - class_methods = self._extract_all_class_methods() - functions_classes.extend(class_methods) - - # Extract individual object methods - object_methods = self._extract_all_object_methods() - functions_classes.extend(object_methods) - - return functions_classes + def get_chunk_query(self) -> str: + return JS_CHUNK_QUERY - def simplify_code(self) -> str: - """Simplify the code by replacing function/class bodies with comments.""" - if self.skip_file: - return self.code + def make_line_comment(self, text: str) -> str: + return f"// {text}" - tree = self._parse_with_fallback() - if tree is None: - return self.code + # -- Cached tree --------------------------------------------------------- - simplified_lines = self.source_lines[:] - indices_to_del: List[Tuple[int, int]] = [] - - for node in tree.body: - if isinstance(node, (esprima.nodes.FunctionDeclaration, - esprima.nodes.AsyncFunctionDeclaration, - esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, - esprima.nodes.AsyncFunctionDeclaration, - esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - - for start, end in reversed(indices_to_del): - del simplified_lines[start:end] - - return "\n".join(line for line in simplified_lines) - - def _extract_arrow_functions(self, var_node: esprima.nodes.VariableDeclaration) -> List[str]: - """ - Extract arrow functions, function expressions, and object literals with methods - from variable declarations. - - Handles patterns like: - - const func = () => {} - - const func = async () => {} - - const func = function() {} - - const func = async function() {} - - const obj = { method() {}, prop: function() {} } - - Args: - var_node: VariableDeclaration node that might contain function expressions or objects - - Returns: - List of function/object code strings - """ - functions = [] - - for declarator in var_node.declarations: - if declarator.init is None: - continue - - # Check if this is an arrow function or function expression - is_function_expr = isinstance(declarator.init, ( - esprima.nodes.ArrowFunctionExpression, - esprima.nodes.FunctionExpression - )) - - # Check if init is a CallExpression wrapping a function, - # e.g. var defaultsDeep = baseRest(function(args) { ... }) - if not is_function_expr and isinstance(declarator.init, esprima.nodes.CallExpression): - is_function_expr = any( - isinstance(arg, (esprima.nodes.FunctionExpression, - esprima.nodes.ArrowFunctionExpression)) - for arg in declarator.init.arguments - ) - - if is_function_expr: - # Extract the entire variable declaration including the assignment - code = self._extract_code(var_node) - functions.append(code) - break # Only extract once per VariableDeclaration - - # Check if this is an object literal with methods - if isinstance(declarator.init, esprima.nodes.ObjectExpression): - if self._object_has_methods(declarator.init): - code = self._extract_code(var_node) - functions.append(code) - break # Only extract once per VariableDeclaration - - return functions - - def _object_has_methods(self, obj_node: esprima.nodes.ObjectExpression) -> bool: - """ - Check if an object literal contains any method properties. - - Args: - obj_node: ObjectExpression AST node - - Returns: - True if the object contains at least one method property - """ - for prop in obj_node.properties: - if isinstance(prop, esprima.nodes.Property): - # Check for shorthand method syntax: { method() {} } - if prop.method: - return True - # Check for function expression: { prop: function() {} } - # or arrow function: { prop: () => {} } - if isinstance(prop.value, ( - esprima.nodes.FunctionExpression, - esprima.nodes.ArrowFunctionExpression - )): - return True - return False + def _get_tree(self): + if self._cached_tree is None: + from tree_sitter import Parser + parser = Parser() + parser.set_language(self._js_language) + self._cached_tree = parser.parse(bytes(self.code, "utf-8")) + return self._cached_tree - def _extract_all_class_methods(self) -> List[str]: - """ - Extract all methods from all classes in the code. + # -- Public API ---------------------------------------------------------- + # Overrides of TreeSitterSegmenter methods to use cached tree/query and + # to extend extract_functions_classes with class/object method extraction. - Each method is extracted as standalone code with a //class: ClassName annotation - appended at the end, enabling later identification of which class the method belongs to. + def is_valid(self) -> bool: + if self.skip_file: + return False + return len(self._js_error_query.captures(self._get_tree().root_node)) == 0 - Returns: - List of method code strings with class annotations - """ - tree = self._parse_with_fallback() - if tree is None: + def extract_functions_classes(self) -> List[str]: + if self.skip_file: return [] - methods = [] + tree = self._get_tree() + captures = self._js_chunk_query.captures(tree.root_node) - for node in tree.body: - if isinstance(node, esprima.nodes.ClassDeclaration): - class_methods = self._extract_class_methods(node) - methods.extend(class_methods) - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, esprima.nodes.ClassDeclaration): - class_methods = self._extract_class_methods(node.declaration) - methods.extend(class_methods) + processed_lines = set() + chunks: List[str] = [] - return methods - - def _extract_class_methods(self, class_node: esprima.nodes.ClassDeclaration) -> List[str]: - """ - Extract all methods from a single class node. - - Args: - class_node: AST node representing a class declaration + for node, _name in captures: + start_line = node.start_point[0] + end_line = node.end_point[0] + lines = range(start_line, end_line + 1) - Returns: - List of method code strings, each annotated with //class: ClassName - """ - class_name = class_node.id.name if class_node.id else "AnonymousClass" - methods = [] - - for method_node in class_node.body.body: - if isinstance(method_node, esprima.nodes.MethodDefinition): - method_code = self._extract_method_code(method_node) - annotated_method = f"{method_code}\n//(class: {class_name})" - methods.append(annotated_method) - - return methods - - def _extract_method_code(self, method_node: esprima.nodes.MethodDefinition) -> str: - """ - Extract the source code for a single method. - - Args: - method_node: AST node representing a method definition - - Returns: - Source code string for the method - """ - if not hasattr(method_node, 'loc') or method_node.loc is None: - logger.warning("Method node has no location information") - return "" - - start_line = method_node.loc.start.line - 1 # Convert to 0-indexed - end_line = method_node.loc.end.line - - method_lines = self.source_lines[start_line:end_line] - return "\n".join(method_lines) - - def _extract_all_object_methods(self) -> List[str]: - """ - Extract all methods from all object literals in the code. - - Each method is extracted as standalone code with a //(object: objectName) annotation - appended at the end, enabling later identification of which object the method belongs to. - - Returns: - List of method code strings with object annotations - """ - tree = self._parse_with_fallback() - if tree is None: - return [] + if any(line in processed_lines for line in lines): + continue - methods = [] + processed_lines.update(lines) + chunks.append(node.text.decode("utf-8")) - for node in tree.body: - if isinstance(node, esprima.nodes.VariableDeclaration): - for declarator in node.declarations: - if declarator.init is None: - continue - if isinstance(declarator.init, esprima.nodes.ObjectExpression): - if self._object_has_methods(declarator.init): - object_name = declarator.id.name if hasattr(declarator.id, 'name') else "AnonymousObject" - object_methods = self._extract_object_methods(declarator.init, object_name) - methods.extend(object_methods) - - # Handle exported object literals - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, esprima.nodes.VariableDeclaration): - for declarator in node.declaration.declarations: - if declarator.init is None: - continue - if isinstance(declarator.init, esprima.nodes.ObjectExpression): - if self._object_has_methods(declarator.init): - object_name = declarator.id.name if hasattr(declarator.id, 'name') else "AnonymousObject" - object_methods = self._extract_object_methods(declarator.init, object_name) - methods.extend(object_methods) + chunks.extend(self._extract_class_methods(tree)) + chunks.extend(self._extract_object_methods(tree)) - return methods + return chunks - def _extract_object_methods(self, obj_node: esprima.nodes.ObjectExpression, object_name: str) -> List[str]: - """ - Extract all methods from a single object literal. + def simplify_code(self) -> str: + if self.skip_file: + return self.code - Args: - obj_node: AST node representing an object expression - object_name: Name of the variable holding the object + tree = self._get_tree() + processed_lines: set = set() - Returns: - List of method code strings, each annotated with //(object: objectName) - """ - methods = [] + simplified_lines = self.source_lines[:] + for node, _name in self._js_chunk_query.captures(tree.root_node): + start_line = node.start_point[0] + end_line = node.end_point[0] - for prop in obj_node.properties: - if not isinstance(prop, esprima.nodes.Property): + lines = range(start_line, end_line + 1) + if any(line in processed_lines for line in lines): continue - # Check if this property is a method - is_method = prop.method or isinstance(prop.value, ( - esprima.nodes.FunctionExpression, - esprima.nodes.ArrowFunctionExpression - )) - - if is_method: - method_code = self._extract_property_method_code(prop) - if method_code: - # Use same annotation pattern as classes for consistency - annotated_method = f"{method_code}\n//(class: {object_name})" - methods.append(annotated_method) - + simplified_lines[start_line] = self.make_line_comment( + f"Code for: {self.source_lines[start_line]}" + ) + for line_num in range(start_line + 1, end_line + 1): + simplified_lines[line_num] = None # type: ignore[call-overload] + + processed_lines.update(lines) + + return "\n".join(line for line in simplified_lines if line is not None) + + # -- Private helpers ----------------------------------------------------- + + def _extract_class_methods(self, tree) -> List[str]: + """Extract individual methods from class declarations, annotated with class name.""" + methods: List[str] = [] + for node_type in ("class_declaration", "class"): + for node in self._walk_type(tree.root_node, node_type): + class_name = self._child_text(node, "identifier") or "AnonymousClass" + body = self._child_by_type(node, "class_body") + if body is None: + continue + for method_node in body.children: + if method_node.type == "method_definition": + method_code = method_node.text.decode("utf-8") + methods.append(f"{method_code}\n//(class: {class_name})") return methods - def _extract_property_method_code(self, prop_node: esprima.nodes.Property) -> str: - """ - Extract the source code for a single object property method. - - Args: - prop_node: AST node representing a property with a method value - - Returns: - Source code string for the method - """ - if not hasattr(prop_node, 'loc') or prop_node.loc is None: - logger.warning("Property node has no location information") - return "" - - start_line = prop_node.loc.start.line - 1 # Convert to 0-indexed - end_line = prop_node.loc.end.line + def _extract_object_methods(self, tree) -> List[str]: + """Extract individual methods from object literals in variable declarations.""" + methods: List[str] = [] + for decl_type in ("lexical_declaration", "variable_declaration"): + for decl_node in self._walk_type(tree.root_node, decl_type): + for declarator in decl_node.children: + if declarator.type != "variable_declarator": + continue + obj_name = self._child_text(declarator, "identifier") or "AnonymousObject" + obj_node = self._child_by_type(declarator, "object") + if obj_node is None: + continue + for prop in obj_node.children: + if prop.type == "method_definition": + methods.append(f"{prop.text.decode('utf-8')}\n//(class: {obj_name})") + elif prop.type == "pair": + value = self._child_by_type(prop, "function") or self._child_by_type(prop, "arrow_function") + if value is not None: + methods.append(f"{prop.text.decode('utf-8')}\n//(class: {obj_name})") + return methods - method_lines = self.source_lines[start_line:end_line] - return "\n".join(method_lines) + @staticmethod + def _walk_type(root, node_type: str): + """Yield top-level children matching node_type, including inside export_statement.""" + for child in root.children: + if child.type == node_type: + yield child + elif child.type == "export_statement": + for grandchild in child.children: + if grandchild.type == node_type: + yield grandchild + + @staticmethod + def _child_by_type(node, child_type: str): + for child in node.children: + if child.type == child_type: + return child + return None + + @staticmethod + def _child_text(node, child_type: str) -> str | None: + child = ExtendedJavaScriptSegmenter._child_by_type(node, child_type) + return child.text.decode("utf-8") if child is not None else None \ No newline at end of file diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py index 6d789561a..b3d19e467 100644 --- a/src/vuln_analysis/utils/function_name_extractor.py +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -144,7 +144,12 @@ def fetch_list(self, query: str) -> list[str]: if func.page_content[current_offset: match.start()].strip().startswith(comment_line_character): pass else: - new_function_name = self.lang_parser.get_function_name(func) + try: + new_function_name = self.lang_parser.get_function_name(func) + except ValueError: + continue + if not new_function_name: + continue containing_functions.append(f"{package},{new_function_name}") return containing_functions diff --git a/tests/test_java_script_extended_segmenter.py b/tests/test_java_script_extended_segmenter.py index 637f0d1f6..41dfef7a5 100644 --- a/tests/test_java_script_extended_segmenter.py +++ b/tests/test_java_script_extended_segmenter.py @@ -103,13 +103,12 @@ def test_code_simplification(test_case): assert "// Code for:" in simplified -def test_optional_chaining_replacement(): - """Test that optional chaining is correctly replaced.""" +def test_optional_chaining_preservation(): + """Test that optional chaining is preserved (tree-sitter handles it natively).""" code = "const name = user?.profile?.name;" segmenter = ExtendedJavaScriptSegmenter(code) assert not segmenter.skip_file - assert "?." not in segmenter.code - assert "." in segmenter.code + assert "?." in segmenter.code def test_invalid_js(): @@ -195,8 +194,9 @@ def test_extract_async_function(): segmenter = ExtendedJavaScriptSegmenter(code) functions = segmenter.extract_functions_classes() - assert len(functions) == 1 + assert len(functions) == 2 assert 'fetchData' in functions[0] and 'async' in functions[0] + assert 'asyncArrow' in functions[1] @@ -506,7 +506,7 @@ def test_shebang_file(): }; """, "expected_count": 3, - "object_var": "export const config", + "object_var": "const config", "annotation_name": "config", "expected_methods": 2, }, @@ -591,3 +591,177 @@ class UserService { object_methods = [f for f in functions if '//(class: apiUtils)' in f] assert len(object_methods) == 2 + +def test_anonymous_default_export_class_methods(): + """Test that methods from 'export default class { ... }' get //(class:) annotations.""" + code = """ +export default class { + connect() { + return this.db.connect(); + } + + disconnect() { + this.db.close(); + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: the class chunk + 2 methods with AnonymousClass annotation + method_functions = [f for f in functions if '//(class: AnonymousClass)' in f] + assert len(method_functions) == 2 + assert any('connect' in f for f in method_functions) + assert any('disconnect' in f for f in method_functions) + + +def test_named_default_export_class_methods(): + """Test that methods from 'export default class Foo { ... }' get //(class: Foo) annotations.""" + code = """ +export default class Foo { + bar() { + return 1; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + method_functions = [f for f in functions if '//(class: Foo)' in f] + assert len(method_functions) == 1 + assert 'bar' in method_functions[0] + + +# ============================================================================= +# should_skip too broad — app-level dist/ directories excluded +# ============================================================================= + +class TestShouldSkipEdgeCases: + """should_skip currently excludes any 'dist/' outside node_modules, even + when it's nested inside application source like 'src/dist/'.""" + + def test_node_modules_dist_preserved(self): + """Third-party dist/ inside node_modules should NOT be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/lodash/dist/lodash.js" + ) + + def test_node_modules_nested_dist_preserved(self): + """Nested dist/ inside node_modules should NOT be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/@babel/core/dist/index.js" + ) + + def test_root_dist_skipped(self): + """Root-level dist/ IS a build artifact — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("dist/bundle.js") + + def test_root_build_static_skipped(self): + """Root-level build/static/ IS a build artifact — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("build/static/js/main.js") + + def test_minified_file_skipped(self): + """.min.js files should always be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("lib/utils.min.js") + + def test_src_dist_should_not_be_skipped(self): + """dist/ inside src/ is likely application source, not a build artifact. + Currently should_skip returns True here — this is the bug.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "src/dist/utils.js" + ), "src/dist/utils.js is likely app source, not a build artifact" + + def test_lib_dist_should_not_be_skipped(self): + """dist/ inside lib/ is likely a sub-module, not a build artifact.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "lib/dist/helpers.js" + ), "lib/dist/helpers.js is likely app source, not a build artifact" + + def test_regular_app_file_not_skipped(self): + """Regular application files should not be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip("src/app/main.js") + + def test_bundle_file_skipped(self): + """.bundle.js files should always be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("assets/app.bundle.js") + + +# ============================================================================= +# Thread-safe class-level init +# ============================================================================= + +class TestThreadSafeInit: + """Class-level caches (_js_language, _js_chunk_query, _js_error_query) + are initialized on first instantiation with a None-check but no lock. + Two threads hitting __init__ simultaneously could see _js_language set + but _js_chunk_query still None.""" + + def test_concurrent_init_produces_valid_segmenters(self): + """Create segmenters from multiple threads and verify they all parse correctly.""" + from concurrent.futures import ThreadPoolExecutor, as_completed + + code = "function hello() { return 42; }" + results = [] + errors = [] + + def create_and_extract(idx): + seg = ExtendedJavaScriptSegmenter(code) + funcs = seg.extract_functions_classes() + return idx, funcs + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(create_and_extract, i) for i in range(20)] + for f in as_completed(futures): + try: + idx, funcs = f.result() + results.append((idx, funcs)) + except Exception as exc: + errors.append(exc) + + assert not errors, f"Concurrent init raised: {errors}" + assert len(results) == 20 + for idx, funcs in results: + assert len(funcs) == 1, f"Thread {idx}: expected 1 function, got {len(funcs)}" + + def test_class_caches_are_set_after_init(self): + """After any instantiation, all three class caches must be non-None.""" + ExtendedJavaScriptSegmenter("var x = 1;") + assert ExtendedJavaScriptSegmenter._js_language is not None + assert ExtendedJavaScriptSegmenter._js_chunk_query is not None + assert ExtendedJavaScriptSegmenter._js_error_query is not None + + +# ============================================================================= +# Bug: should_skip .min.js check runs before node_modules path analysis +# Lines 102-104: .min.js suffix check returns True before we check node_modules +# ============================================================================= + +class TestShouldSkipMinJsInNodeModules: + """Third-party .min.js files inside node_modules should NOT be skipped.""" + + def test_minified_in_node_modules_should_not_be_skipped(self): + """node_modules/jquery/dist/jquery.min.js is legitimate third-party source.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/jquery/dist/jquery.min.js" + ), ".min.js inside node_modules should not be skipped" + + def test_minified_bundle_in_node_modules_should_not_be_skipped(self): + """node_modules/lodash/lodash.min.js is legitimate.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/lodash/lodash.min.js" + ) + + def test_bundle_in_node_modules_should_not_be_skipped(self): + """node_modules/chart.js/dist/chart.bundle.js is legitimate.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/chart.js/dist/chart.bundle.js" + ) + + def test_app_level_minified_still_skipped(self): + """App-level .min.js files are build artifacts — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("lib/utils.min.js") + + def test_app_level_bundle_still_skipped(self): + """App-level .bundle.js files are build artifacts — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("assets/app.bundle.js") + diff --git a/tests/test_javascript_functions_parser.py b/tests/test_javascript_functions_parser.py index 68973d857..fda6ea5bb 100644 --- a/tests/test_javascript_functions_parser.py +++ b/tests/test_javascript_functions_parser.py @@ -2,6 +2,7 @@ from langchain_core.documents import Document from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter @pytest.fixture @@ -254,6 +255,137 @@ def test_get_function_name_valid_patterns(parser, code, expected_name): assert result == expected_name +@pytest.mark.parametrize("code,expected_name", [ + # Object literal property with arrow function + ( + "handler: (req, res) => {\n res.send('ok');\n}", + "handler" + ), + ( + "onClick: async (event) => {\n await handleClick(event);\n}", + "onClick" + ), + # Object literal property with single-param arrow (no parens) + ( + "transform: item => item.toUpperCase()", + "transform" + ), + ( + "callback: async result => {\n await process(result);\n}", + "callback" + ), + # Object literal property with function expression + ( + "initialize: function() {\n this.setup();\n}", + "initialize" + ), + ( + "process: async function(data) {\n await save(data);\n}", + "process" + ), + # Property assignment with single-param arrow + ( + "module.exports.handler = event => {\n return event.body;\n}", + "handler" + ), + ( + "self.callback = async result => {\n await log(result);\n}", + "callback" + ), + # Dollar sign in identifiers + ( + "function $apply(scope) {\n scope.$digest();\n}", + "$apply" + ), + ( + "const $timeout = (fn, delay) => {\n setTimeout(fn, delay);\n}", + "$timeout" + ), + ( + "function not$(value) {\n return !value;\n}", + "not$" + ), + ( + "const in$ = (key, obj) => key in obj;", + "in$" + ), + # String-keyed object property + ( + '"content-type"(value) {\n return value.toLowerCase();\n}', + "content-type" + ), + ( + "'set-cookie': (value) => {\n return parse(value);\n}", + "set-cookie" + ), + # Static getter/setter with computed property + ( + "static [Symbol.species]() {\n return Array;\n}", + "Symbol.species" + ), + ( + "static async [Symbol.asyncIterator]() {\n yield 1;\n}", + "Symbol.asyncIterator" + ), + ( + "get [Symbol.toStringTag]() {\n return 'Custom';\n}", + "Symbol.toStringTag" + ), + # TypeScript generics in function signatures + ( + "function assert(value: T): asserts value {\n if (!value) throw new Error();\n}", + "assert" + ), + ( + "function identity(arg: T): T {\n return arg;\n}", + "identity" + ), + # Constructor with destructured default params + ( + "constructor({name = 'default', age = 0} = {}) {\n this.name = name;\n}//(class: Config)", + "constructor" + ), + # Wrapper call pattern (const name = wrapper(...)) + ( + "const memoized = memoize(computeValue)", + "memoized" + ), + ( + "const debounced = debounce(handleInput, 300)", + "debounced" + ), +]) +def test_get_function_name_new_patterns(parser, code, expected_name): + """Test get_function_name with generator and TypeScript patterns.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == expected_name + + +@pytest.mark.parametrize("code", [ + # TypeScript interface declarations + "interface UserConfig {\n name: string;\n age: number;\n}", + "export interface ApiResponse {\n data: any;\n status: number;\n}", + "declare interface Window {\n customProp: string;\n}", + # TypeScript enum declarations + "enum Color {\n Red,\n Green,\n Blue\n}", + "export enum Direction {\n Up,\n Down,\n Left,\n Right\n}", + "const enum HttpStatus {\n OK = 200,\n NotFound = 404\n}", + "export const enum LogLevel {\n Debug,\n Info,\n Error\n}", + "declare enum Platform {\n Web,\n Mobile\n}", +]) +def test_is_function_rejects_typescript_constructs(parser, code): + """Test that is_function returns False for TypeScript interface/enum declarations.""" + doc = Document( + page_content=code, + metadata={'source': 'test.ts', 'content_type': 'functions_classes'} + ) + assert parser.is_function(doc) is False + + @pytest.mark.parametrize("code", [ # Class declarations should raise ValueError "class MyClass {\n constructor() {}\n}", @@ -1440,7 +1572,7 @@ class TypeTest { ), ]) def test_is_exported_function(parser, description, function_content, file_content, expected): - source = "node_modules/packageurl-js/index.js", + source = "node_modules/packageurl-js/index.js" func_doc = Document( page_content=function_content, metadata={'source': source, 'content_type': 'functions_classes'} @@ -2194,18 +2326,18 @@ def test_is_exportable_function(parser, function_name, file_content, expected, d True, ), ( - "Dynamic import without await - .then pattern", + "Dynamic import without await - .then pattern (empty identifier → False)", """import('lodash').then(mod => mod.template());""", "", "lodash", - True, + False, ), ( - "Dynamic import side-effect only", + "Dynamic import side-effect only (empty identifier → False)", """await import('side-effect-pkg');""", "", "side-effect-pkg", - True, + False, ), ( "Dynamic import with backticks", @@ -2399,11 +2531,11 @@ def test_is_exportable_function(parser, function_name, file_content, expected, d True, ), ( - "Dynamic import without assignment", + "Dynamic import without assignment (empty identifier → False)", """(await import('lodash')).template();""", "", "lodash", - True, + False, ), ( "Multiple dynamic imports - check first", @@ -2449,169 +2581,1922 @@ def test_is_package_imported_dynamic_imports(parser, description, code_content, # Tests for is_package_imported - Multi-function imports and word boundary edge cases # ============================================================================ -@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ - # ======================================================================== - # Multi-function imports - should match exact identifiers - # ======================================================================== - ( - "Multi-function import - exact match for function1", - """import { function1, function2 } from "./utils.js";""", - "function1", - "./utils.js", - True, - ), - ( - "Multi-function import - exact match for function2", - """import { function1, function2 } from "./utils.js";""", - "function2", - "./utils.js", - True, - ), - # ======================================================================== - # Substring edge cases - should NOT match partial identifiers - # ======================================================================== +# ============================================================================ +# Tests for empty-name CCA guards +# ============================================================================ + +def test_get_function_calls_empty_name_returns_empty(parser): + """_get_function_calls must return [] when callee_function_name is empty.""" + caller = Document( + page_content="function test() {\n anything();\n something.method();\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser._get_function_calls(caller, '') == [] + assert parser._get_function_calls(caller, None) == [] + + +def test_is_package_imported_empty_identifier_returns_false(parser): + """is_package_imported must return False when identifier is empty.""" + code = "import { template } from 'lodash';\nconst x = require('express');" + assert parser.is_package_imported(code, '', 'lodash') is False + assert parser.is_package_imported(code, '', '') is False + + +def test_filter_docs_by_func_pkg_name_empty_name_returns_empty(parser): + """filter_docs_by_func_pkg_name must return [] when function_name is empty.""" + docs = [ + Document(page_content="function template() {}", metadata={'source': 'node_modules/lodash/template.js'}), + Document(page_content="function parse() {}", metadata={'source': 'node_modules/lodash/parse.js'}), + ] + assert parser.filter_docs_by_func_pkg_name('', 'lodash', docs) == [] + + +def test_search_for_called_function_empty_callee_name(parser): + """search_for_called_function returns False when callee_function_name is empty. + + Without the guard on _get_function_calls, an empty callee name would + cause the regex to match spuriously and return True for any caller that imports + the callee package. + """ + caller = Document( + page_content="function handler() {\n lodash.template('hello');\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + callee = Document( + page_content="function template(str) { return str; }", + metadata={'source': 'node_modules/lodash/template.js', 'content_type': 'functions_classes'} + ) + code_docs = { + 'app/handler.js': Document( + page_content="import { template } from 'lodash';\nfunction handler() { lodash.template('hello'); }", + metadata={'source': 'app/handler.js', 'content_type': 'simplified_code'} + ) + } + + result = parser.search_for_called_function( + caller_function=caller, + callee_function_name='', + callee_function=callee, + callee_function_package='lodash', + code_documents=code_docs, + type_documents=[], + callee_function_file_name='node_modules/lodash/template.js', + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + type_inheritance={} + ) + assert result is False + + +# ============================================================================ +# Tests for dotted call chain RHS in variable declarations +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ ( - "Multi-function import - 'func' should NOT match 'function1'", - """import { function1, function2 } from "./utils.js";""", - "func", - "./utils.js", - False, + "var changes = ts.textChanges.ChangeTracker.with(context, function(tracker) {\n tracker.apply();\n})", + "changes" ), ( - "Multi-function import - 'add' should NOT match 'addHandler'", - """import { addHandler, removeHandler } from "./utils.js";""", - "add", - "./utils.js", - False, + "const result = obj.method.call(this, arg)", + "result" ), ( - "Single function import - 'parse' should NOT match 'parseAsync'", - """import { parseAsync } from "some-lib";""", - "parse", - "some-lib", - False, + "let parser = JSON.parse(data)", + "parser" ), - # ======================================================================== - # CommonJS require - substring edge cases - # ======================================================================== +]) +def test_get_function_name_dotted_rhs(parser, code, expected_name): + """Variable declarations with dotted call chains on the RHS should extract the variable name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for bare assignment function +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ ( - "CommonJS destructured require - exact match", - """const { function1, function2 } = require('./utils.js');""", - "function1", - "./utils.js", - True, + "name = function(x, y) {\n return x + y;\n}", + "name" ), ( - "CommonJS destructured require - 'func' should NOT match 'function1'", - """const { function1, function2 } = require('./utils.js');""", - "func", - "./utils.js", - False, + "callback = function() {\n return 1;\n}", + "callback" ), - # ======================================================================== - # Re-exports - substring edge cases - # ======================================================================== ( - "Re-export - exact match", - """export { function1, function2 } from "./utils.js";""", - "function1", - "./utils.js", - True, + "handler = async function(req, res) {\n await handle(req);\n}", + "handler" ), ( - "Re-export - 'func' should NOT match 'function1'", - """export { function1, function2 } from "./utils.js";""", - "func", - "./utils.js", - False, + "gen = function*() {\n yield 1;\n}", + "gen" ), ]) -def test_is_package_imported_word_boundaries(parser, description, code_content, identifier, callee_package, expected): - """Test is_package_imported with word boundary edge cases to prevent substring false positives.""" - result = parser.is_package_imported(code_content, identifier, callee_package) - assert result == expected, f"{description}: expected {expected}, got {result}" +def test_get_function_name_bare_assignment_function(parser, code, expected_name): + """Bare assignment to function expression (no const/let/var) should extract the LHS name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name -@pytest.mark.parametrize("function_content,expected_name", [ - # Object methods now use //(class: name) pattern +# ============================================================================ +# Tests for bare assignment arrow +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ ( - "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", - "apiUtils" + "resolve = (id, options) => {\n return lookup(id);\n}", + "resolve" ), ( - "format: function(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", - "helpers" + "process = async (data) => {\n await save(data);\n}", + "process" ), - # Standalone function (no annotation) ( - "function regularFunc() {\n return true;\n}", - None + "transform = x => x * 2", + "transform" ), ]) -def test_get_class_name_for_object_methods(parser, function_content, expected_name): - """Test that object methods use the same //(class: name) pattern as classes.""" +def test_get_function_name_bare_assignment_arrow(parser, code, expected_name): + """Bare assignment to arrow function (no const/let/var) should extract the LHS name.""" doc = Document( - page_content=function_content, + page_content=code, metadata={'source': 'test.js', 'content_type': 'functions_classes'} ) - result = parser.get_class_name_from_class_function(doc) - assert result == expected_name + assert parser.get_function_name(doc) == expected_name -@pytest.mark.parametrize("description,function_content,file_content,expected", [ - ( - "Object method should be exported when object is in module.exports", - "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", - """const apiUtils = { - fetchData(url) { - return fetch(url); - } -}; +# ============================================================================ +# Tests for anonymous export default +# ============================================================================ -module.exports = { apiUtils }; -""", - True, +@pytest.mark.parametrize("code", [ + "export default function() {\n return 42;\n}", + "export default async function() {\n await fetch('/api');\n}", + "export default function*() {\n yield 1;\n}", +]) +def test_get_function_name_anonymous_export_default(parser, code): + """Anonymous export default should return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for generator computed property +# ============================================================================ + +def test_get_function_name_generator_computed_property(parser): + """Generator computed property with * prefix should extract the symbol expression.""" + doc = Document( + page_content="*[Symbol.iterator]() {\n for (const item of this.items) {\n yield item;\n }\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == 'Symbol.iterator' + + +# ============================================================================ +# Tests for keyword rejection in fallback pattern +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ + ( + "doStuff() {\n if (x) {\n return;\n }\n}", + "doStuff" ), ( - "Object method should be exported with ES6 named export", - "format(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", - """const helpers = { - format(data) { - return JSON.stringify(data); - } -}; - -export { helpers }; -""", - True, + "render() {\n for (let i = 0; i < 10; i++) {\n items.push(i);\n }\n}", + "render" ), ( - "Object method should NOT be exported when object is not exported", - "internalMethod() {\n return 'internal';\n}\n//(class: privateUtils)", - """const privateUtils = { - internalMethod() { - return 'internal'; - } -}; - -// No exports -""", - False, + "handle() {\n while (queue.length) {\n process(queue.shift());\n }\n}", + "handle" + ), + ( + "execute() {\n switch(type) {\n case('a'):\n break;\n }\n}", + "execute" ), ]) -def test_is_exported_function_for_objects(parser, description, function_content, file_content, expected): - """Test is_exported_function correctly handles object methods with //(class: name) annotation.""" - source = "node_modules/test-pkg/index.js" - func_doc = Document( - page_content=function_content, - metadata={'source': source, 'content_type': 'functions_classes'} +def test_get_function_name_keyword_rejection(parser, code, expected_name): + """Keyword filtering should skip JS keywords and find the real method name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} ) - full_file_doc = Document( - page_content=file_content, - metadata={'source': source, 'content_type': 'simplified_code'} + assert parser.get_function_name(doc) == expected_name + + +@pytest.mark.parametrize("code", [ + "var reporter = {\n onTestStart: function(test) { if (test.pending) return; }\n}", + "if (x) {\n for (y in z) {\n while (true) { break; }\n }\n}", +]) +def test_get_function_name_all_keywords_returns_empty(parser, code): + """When only keywords match the fallback pattern, return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} ) - documents_of_full_sources = {source: full_file_doc} - - result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for keyword set trimming: removed keywords that are valid method names +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # 'default' removed from _JS_KEYWORDS — valid method in commander.js etc. + ("default(value, description) {\n this._defaultValue = value;\n}", "default"), + # 'case' removed — can be a method name (e.g., case(value) in switch builders) + ("case(value) {\n this.cases.push(value);\n}", "case"), + # 'export' removed — can be a method name (e.g., export(data) in serializers) + ("export(data) {\n return JSON.stringify(data);\n}", "export"), + # 'import' removed — can be a method name + ("import(module) {\n return require(module);\n}", "import"), + # 'in' removed — can be a method name (e.g., in(collection) in query builders) + ("in(collection) {\n return this.where(collection);\n}", "in"), + # 'of' removed — can be a method name + ("of(items) {\n return new Collection(items);\n}", "of"), + # 'super' removed — can be used as method name in some patterns + ("super(args) {\n return parent.call(this, args);\n}", "super"), + # 'this' removed — can be used as method name + ("this(args) {\n return self.init(args);\n}", "this"), +]) +def test_get_function_name_removed_keywords_are_valid_methods(parser, code, expected_name): + """Keywords removed from _JS_KEYWORDS should be extractable as method names.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +@pytest.mark.parametrize("code", [ + # Keywords still in _JS_KEYWORDS should NOT be extracted (no non-keyword calls in body) + "if (x) { return (y) }", + "for (var y in z) { break; }", + "while (true) { continue; }", + "switch (x) { break; }", + "return (x)", + "throw (x)", + "new (x)", + "typeof (x)", + "delete (x)", +]) +def test_get_function_name_retained_keywords_still_rejected(parser, code): + """Keywords still in _JS_KEYWORDS should return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for variable declaration matching restricted to header +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # Should NOT match const/let/var assignments inside function body + ("async method() {\n const x = obj.call(y);\n return x;\n}", "method"), + ("render() {\n const sanitized = this.sanitize(input);\n return sanitized;\n}", "render"), + # Should match in header (no body yet) + ("const result = helper.create(args)", "result"), + # Multi-line header before body + ("const handler =\n middleware.wrap(\n fn\n )", "handler"), +]) +def test_get_function_name_pattern5_body_restriction(parser, code, expected_name): + """Variable declaration matching should only search before the first '{' to avoid body matches.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for body-matching prevention: patterns should not match inside bodies +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # Function declaration inside body should not hijack name + ("method() { function inner() {} }", "method"), + ("render() { async function fetchData() {} }", "render"), + # Const single-param arrow inside body + ("method() { const helper = x => x }", "method"), + # Property function assignment inside body + ("method() { this.handler = function() {} }", "method"), + # Property single-param arrow inside body + ("method() { this.cb = val => val }", "method"), + # Bare assignment function inside body + ("method() { callback = function() {} }", "method"), + # Bare single-param arrow inside body + ("method() { transform = x => x * 2 }", "method"), +]) +def test_get_function_name_body_matching_prevention(parser, code, expected_name): + """Patterns restricted to header should not match declarations inside function bodies.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for generator name without space: function*name() +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + ("function*myGen() { yield 1; }", "myGen"), + ("function *myGen() { yield 1; }", "myGen"), + ("function * myGen() { yield 1; }", "myGen"), + ("async function*streamGen() { yield 1; }", "streamGen"), +]) +def test_get_function_name_generator_no_space(parser, code, expected_name): + """Generator declarations should match function*name() without space after *.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +def test_get_function_name_no_functionName_false_match(parser): + """Word boundary should not match 'functionName(' as 'Name'.""" + doc = Document( + page_content="functionName(x) { return x; }", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == 'functionName' + + +# ============================================================================ +# Tests for function_called_from_caller_body regex fixes +# ============================================================================ + +def test_function_called_from_caller_body_finds_call(): + """Should return truthy when function is called (not declared) in body.""" + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever + from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + + doc = Document( + page_content="function handler(req) {\n const result = process(req.body);\n return result;\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + assert retriever.function_called_from_caller_body(doc, "process") + + +# ============================================================================ +# Tests for function_called_from_caller_body empty-name pass-through +# ============================================================================ + +def test_function_called_from_caller_body_empty_name_passthrough(): + """function_called_from_caller_body returns True for empty function_to_search (pass-through). + + Empty function_to_search is used by __find_initial_function as a "no filter" — + all documents pass through. Empty-name defense is handled by BFS guards. + """ + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever + from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + + doc = Document( + page_content="function handler() {\n someLib.process(data);\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + assert retriever.function_called_from_caller_body(doc, "") is True + assert retriever.function_called_from_caller_body(doc, " ") is True + + +# ============================================================================ +# Tests for _breadth_first_search empty-name guard +# ============================================================================ + +def test_bfs_skips_empty_name_docs(): + """BFS should skip documents whose get_function_name returns empty string.""" + from unittest.mock import MagicMock, patch + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + retriever.documents = [] + retriever.sort_docs = {} + + empty_name_doc = Document( + page_content="var reporter = {\n if (x) {}\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes', 'state': None} + ) + + ctx = _SearchCtx() + result, found = retriever._breadth_first_search([], empty_name_doc, 'test-pkg', ctx) + assert found is False + assert result == [] + + +# ============================================================================ +# Tests for __find_caller_functions_bfs candidate empty-name guard +# ============================================================================ + +def test_bfs_caller_skips_empty_name_candidates(): + """__find_caller_functions_bfs should skip candidate docs with empty function names.""" + from unittest.mock import MagicMock, patch + from collections import defaultdict + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + retriever.ecosystem = 'javascript' + + target_doc = Document( + page_content="function vulnerable() {\n return 'exploit';\n}", + metadata={'source': 'node_modules/lib/index.js', 'content_type': 'functions_classes'} + ) + + empty_name_candidate = Document( + page_content="var reporter = {\n if (x) {}\n}", + metadata={'source': 'node_modules/parent/index.js', 'content_type': 'functions_classes', 'state': None} + ) + + retriever.sort_docs = defaultdict(list, {'parent-pkg': [empty_name_candidate]}) + retriever.documents = [empty_name_candidate] + retriever.documents_of_full_sources = {} + retriever.documents_of_types = [] + retriever.types_classes_fields_mapping = {} + retriever.functions_local_variables_index = {} + retriever.documents_of_functions = [] + + ctx = _SearchCtx() + + with patch.object(retriever, '_get_parents', return_value=['parent-pkg']): + result = retriever._ChainOfCallsRetriever__find_caller_functions_bfs( + document_function=target_doc, + function_package='lib', + ctx=ctx + ) + + assert result == [] + + +@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ + # ======================================================================== + # Multi-function imports - should match exact identifiers + # ======================================================================== + ( + "Multi-function import - exact match for function1", + """import { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Multi-function import - exact match for function2", + """import { function1, function2 } from "./utils.js";""", + "function2", + "./utils.js", + True, + ), + # ======================================================================== + # Substring edge cases - should NOT match partial identifiers + # ======================================================================== + ( + "Multi-function import - 'func' should NOT match 'function1'", + """import { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), + ( + "Multi-function import - 'add' should NOT match 'addHandler'", + """import { addHandler, removeHandler } from "./utils.js";""", + "add", + "./utils.js", + False, + ), + ( + "Single function import - 'parse' should NOT match 'parseAsync'", + """import { parseAsync } from "some-lib";""", + "parse", + "some-lib", + False, + ), + # ======================================================================== + # CommonJS require - substring edge cases + # ======================================================================== + ( + "CommonJS destructured require - exact match", + """const { function1, function2 } = require('./utils.js');""", + "function1", + "./utils.js", + True, + ), + ( + "CommonJS destructured require - 'func' should NOT match 'function1'", + """const { function1, function2 } = require('./utils.js');""", + "func", + "./utils.js", + False, + ), + # ======================================================================== + # Re-exports - substring edge cases + # ======================================================================== + ( + "Re-export - exact match", + """export { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Re-export - 'func' should NOT match 'function1'", + """export { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), +]) +def test_is_package_imported_word_boundaries(parser, description, code_content, identifier, callee_package, expected): + """Test is_package_imported with word boundary edge cases to prevent substring false positives.""" + result = parser.is_package_imported(code_content, identifier, callee_package) assert result == expected, f"{description}: expected {expected}, got {result}" + +@pytest.mark.parametrize("function_content,expected_name", [ + # Object methods now use //(class: name) pattern + ( + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + "apiUtils" + ), + ( + "format: function(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + "helpers" + ), + # Standalone function (no annotation) + ( + "function regularFunc() {\n return true;\n}", + None + ), +]) +def test_get_class_name_for_object_methods(parser, function_content, expected_name): + """Test that object methods use the same //(class: name) pattern as classes.""" + doc = Document( + page_content=function_content, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_class_name_from_class_function(doc) + assert result == expected_name + + +@pytest.mark.parametrize("description,function_content,file_content,expected", [ + ( + "Object method should be exported when object is in module.exports", + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + """const apiUtils = { + fetchData(url) { + return fetch(url); + } +}; + +module.exports = { apiUtils }; +""", + True, + ), + ( + "Object method should be exported with ES6 named export", + "format(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + """const helpers = { + format(data) { + return JSON.stringify(data); + } +}; + +export { helpers }; +""", + True, + ), + ( + "Object method should NOT be exported when object is not exported", + "internalMethod() {\n return 'internal';\n}\n//(class: privateUtils)", + """const privateUtils = { + internalMethod() { + return 'internal'; + } +}; + +// No exports +""", + False, + ), +]) +def test_is_exported_function_for_objects(parser, description, function_content, file_content, expected): + """Test is_exported_function correctly handles object methods with //(class: name) annotation.""" + source = "node_modules/test-pkg/index.js" + func_doc = Document( + page_content=function_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + full_file_doc = Document( + page_content=file_content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + documents_of_full_sources = {source: full_file_doc} + + result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +# ============================================================================= +# should_skip — node_modules/dist should NOT be skipped +# ============================================================================= + +@pytest.mark.parametrize("path,expected", [ + ("node_modules/express/dist/router.js", False), + ("node_modules/@babel/core/dist/index.js", False), + ("node_modules/pkg/dist/build/static/app.js", False), + ("dist/bundle.js", True), + ("src/dist/helper.js", False), + ("build/static/app.js", True), + ("app.min.js", True), + ("src/index.js", False), + ("coverage/report.js", True), +]) +def test_should_skip_node_modules_dist(path, expected): + """dist/ inside node_modules/ is legitimate source, not a build artifact.""" + assert ExtendedJavaScriptSegmenter.should_skip(path) == expected + + +# ============================================================================= +# Optional chaining in _get_function_calls +# ============================================================================= + +def test_get_function_calls_optional_chaining(parser): + """_get_function_calls should detect calls through optional chaining (?.).""" + caller = Document( + page_content="function test() {\n obj?.method(arg);\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "method") + assert any("method" in c for c in calls), f"Expected 'method' call, got {calls}" + + +def test_get_function_calls_chained_optional(parser): + """_get_function_calls should handle chained optional access a?.b?.method().""" + caller = Document( + page_content="function test() {\n a?.b?.method(x);\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "method") + assert any("method" in c for c in calls), f"Expected 'method' call, got {calls}" + + +# ============================================================================= +# Position check — destructuring defaults inside bodies +# ============================================================================= + +@pytest.mark.parametrize("code,expected_name", [ + ( + "method({callback = (x) => x}) {\n return callback;\n}", + "method", + ), + ( + "const handler = ({a, b}) => { return a + b; }", + "handler", + ), + ( + "resolve = ({id}) => { doStuff(); }", + "resolve", + ), + ( + "function outer() {\n const inner = (x) => x;\n}", + "outer", + ), +]) +def test_position_check_destructuring_in_body(parser, code, expected_name): + """Patterns with [^)]* should not match destructuring defaults inside bodies.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================= +# Function expression with destructuring uses header +# ============================================================================= + +def test_pattern5_function_expr_with_destructuring(parser): + """Function expressions with destructuring params should extract the variable name.""" + doc = Document( + page_content="const fn = function({a, b}) { return a + b; }", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "fn" + + +# ============================================================================= +# Bare assignment and property patterns +# ============================================================================= + +@pytest.mark.parametrize("code,expected_name", [ + ( + "obj.method = function() { return 1; }", + "method", + ), + ( + "obj.handler = (x) => x * 2", + "handler", + ), + ( + "exports.default = async function(req, res) { res.send(); }", + "default", + ), + ( + "module.exports.init = function() {}", + "init", + ), +]) +def test_property_assignment_after_pattern_removal(parser, code, expected_name): + """Property assignments (obj.name = ...) should still work after removing patterns 6-8.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================= +# Optional chaining '?' leaks into identifiers in _get_function_calls +# ============================================================================= + +class TestOptionalChainingInFunctionCalls: + """_get_function_calls regex captures '?' from optional chaining, which + then leaks into search_for_called_function's identifier split, preventing + import resolution and local-var lookup.""" + + def test_optional_chaining_direct_call_detected(self, parser): + """_get_function_calls should detect calls through optional chaining.""" + caller = Document( + page_content="function process(res) {\n return res?.json();\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "json") + assert len(calls) >= 1, "Should detect json() through optional chaining" + + def test_optional_chaining_identifier_has_no_question_mark(self, parser): + """After splitting a qualified optional-chaining call by '.', identifiers + should NOT contain '?'. The '?' is syntax, not part of the name.""" + caller = Document( + page_content="function process(res) {\n return res?.json();\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "json") + for call in calls: + parts = call.split('.') + for part in parts: + clean = part.rstrip('(') + assert '?' not in clean, ( + f"Identifier '{clean}' still contains '?' from optional chaining" + ) + + def test_optional_chaining_search_for_called_function(self, parser): + """search_for_called_function should find calls through optional chaining + when the identifier matches an imported package.""" + caller = Document( + page_content=( + "import axios from 'axios';\n" + "function fetchData() {\n" + " return axios?.get('/api/data');\n" + "}" + ), + metadata={'source': 'app/api.js', 'content_type': 'functions_classes'} + ) + callee = Document( + page_content="get(url) {\n return this.request('GET', url);\n}\n//(class: axios)", + metadata={'source': 'node_modules/axios/index.js', 'content_type': 'functions_classes'} + ) + code_docs = { + 'app/api.js': Document( + page_content="import axios from 'axios';\nfunction fetchData() { return axios?.get('/api/data'); }", + metadata={'source': 'app/api.js', 'content_type': 'simplified_code'} + ) + } + result = parser.search_for_called_function( + caller_function=caller, + callee_function_name="get", + callee_function=callee, + callee_function_package="axios", + code_documents=code_docs, + type_documents=[], + callee_function_file_name='node_modules/axios/index.js', + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + ) + assert result is True, ( + "search_for_called_function should match 'axios?.get()' — " + "the '?' is syntax, not part of the identifier" + ) + + +# ============================================================================= +# Named export chunks lose 'export' prefix — is_exported_function +# fallback doesn't check for 'export function name' pattern +# ============================================================================= + +class TestNamedExportWithoutPrefix: + """Tree-sitter extracts 'function compile(...)' without the 'export' prefix. + is_exported_function should still detect it as exported via the fallback.""" + + def test_named_export_function_without_prefix(self, parser): + """A chunk 'function compile(input) {...}' from an 'export function compile' + should be detected as exported by checking the full source.""" + func_doc = Document( + page_content="function compile(input) {\n return transform(input);\n}", + metadata={'source': 'node_modules/handlebars/index.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content=( + "import { transform } from './transform';\n" + "export function compile(input) {\n" + " return transform(input);\n" + "}\n" + "export function precompile(input) {\n" + " return parse(input);\n" + "}\n" + ), + metadata={'source': 'node_modules/handlebars/index.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/handlebars/index.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is True, ( + "Function 'compile' is exported via 'export function compile' in source, " + "even though the chunk itself lacks the 'export' prefix" + ) + + def test_named_export_async_function_without_prefix(self, parser): + """Async export: 'async function fetchData(...)' without 'export' in chunk.""" + func_doc = Document( + page_content="async function fetchData(url) {\n return await fetch(url);\n}", + metadata={'source': 'node_modules/my-lib/api.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content="export async function fetchData(url) {\n return await fetch(url);\n}\n", + metadata={'source': 'node_modules/my-lib/api.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/my-lib/api.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is True, ( + "Async function 'fetchData' is exported via 'export async function fetchData' in source" + ) + + def test_non_exported_function_still_false(self, parser): + """A function that is genuinely not exported should still return False.""" + func_doc = Document( + page_content="function internal() {\n return 'private';\n}", + metadata={'source': 'node_modules/pkg/utils.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content=( + "function internal() {\n return 'private';\n}\n" + "function external() {\n return internal();\n}\n" + "module.exports = external;\n" + ), + metadata={'source': 'node_modules/pkg/utils.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/pkg/utils.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is False + + +# ============================================================================= +# BFS ValueError on class docs — get_function_name raises for classes +# ============================================================================= + +class TestGetFunctionNameOnClassDocs: + """get_function_name raises ValueError on class documents. In BFS, this + aborts the entire search loop.""" + + def test_named_class_raises_valueerror(self, parser): + """get_function_name should raise ValueError for named class docs.""" + doc = Document( + page_content="class Router {\n constructor() {}\n route(path) {}\n}", + metadata={'source': 'app/router.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + with pytest.raises(ValueError, match="Only function document"): + parser.get_function_name(doc) + + def test_anonymous_class_is_function_returns_false(self, parser): + """is_function should return False for anonymous class expressions, + preventing ValueError in get_function_name downstream.""" + doc = Document( + page_content="export default class {\n connect() {}\n disconnect() {}\n}", + metadata={'source': 'node_modules/pkg/db.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc), ( + "Anonymous class 'export default class { ... }' should NOT pass is_function. " + "The regex requires class\\s+[\\w$]+ (a named identifier after 'class')." + ) + + +# ============================================================================= +# Nested parens in arrow function default parameters +# ============================================================================= + +class TestNestedParensInArrowDefaults: + """Arrow functions with nested parens in default params (e.g. getDefaults()) should extract the variable name.""" + + def test_arrow_with_function_call_default(self, parser): + """Arrow function with a function call as default parameter value.""" + doc = Document( + page_content="const handler = (options = getDefaults()) => {\n process(options);\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "handler", ( + f"Expected 'handler' but got '{name}'. [^)]* in arrow regex stops at " + "the inner ')' of getDefaults(), causing fallthrough to catch-all." + ) + + def test_arrow_with_nested_method_call_default(self, parser): + """Arrow with obj.method() call in default parameter.""" + doc = Document( + page_content="const processor = (cfg = Config.load()) => {\n return cfg.run();\n}", + metadata={'source': 'app/process.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "processor", f"Expected 'processor' but got '{name}'" + + def test_arrow_with_simple_default_still_works(self, parser): + """Ensure simple defaults (no nested parens) still work after the fix.""" + doc = Document( + page_content="const greet = (name = 'world') => {\n console.log(name);\n}", + metadata={'source': 'app/greet.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "greet" + + def test_arrow_with_numeric_default_still_works(self, parser): + """Numeric default: no nested parens.""" + doc = Document( + page_content="const delay = (ms = 1000) => {\n return new Promise(r => setTimeout(r, ms));\n}", + metadata={'source': 'app/delay.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "delay" + + +# ============================================================================= +# Anonymous export default arrow returns inner function name +# ============================================================================= + +class TestAnonymousExportDefaultArrow: + """export default (x) => {...} with inner named functions should return ''.""" + + def test_anonymous_default_arrow_with_inner_function(self, parser): + """An anonymous export default arrow should return '' even when the body + contains named functions.""" + doc = Document( + page_content=( + "export default async (req) => {\n" + " function validate(r) { return r.ok; }\n" + " return validate(req);\n" + "}" + ), + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', ( + f"Expected '' for anonymous export default arrow, but got '{name}'. " + "The catch-all matched an inner function name from the body." + ) + + def test_anonymous_default_arrow_simple(self, parser): + """Simple anonymous export default arrow with no inner functions.""" + doc = Document( + page_content="export default (x, y) => {\n return x + y;\n}", + metadata={'source': 'node_modules/pkg/add.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for anonymous arrow, got '{name}'" + + def test_named_default_function_still_works(self, parser): + """Named export default function should still return the name.""" + doc = Document( + page_content="export default function handler(req) {\n return req.body;\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "handler" + + +# ============================================================================= +# is_function returns True for anonymous class expressions +# ============================================================================= + +class TestGetFunctionNamePerformance: + """get_function_name must not catastrophically backtrack on large function bodies + containing many parenthesized expressions (e.g., compiled TypeScript output).""" + + def test_large_prototype_method_completes_fast(self, parser): + """Reproduces XMLSerializerImpl.js — prototype method assignment with a + 17K+ char body full of var declarations and parenthesized calls. + Before fix, get_function_name took 34.8s on this content because + arrow-function regex patterns searched the full content.""" + import time + body_lines = [] + for i in range(300): + body_lines.append(f" var markup{i} = this._serialize(node, (opts{i} || {{}}));") + body_lines.append(f" if (requireWellFormed && (node.localName.indexOf(':') !== -1 ||") + body_lines.append(f" !algorithm_1.xml_isName(node.localName))) {{") + body_lines.append(f" throw new Error('not well-formed: ' + node.localName);") + body_lines.append(f" }}") + body = "\n".join(body_lines) + content = ( + f"XMLSerializerImpl.prototype._serializeElementNS = " + f"function (node, namespace, prefixMap, prefixIndex, requireWellFormed) {{\n" + f" var e_1, _a;\n{body}\n }}" + ) + assert len(content) > 15000 + doc = Document( + page_content=content, + metadata={'source': 'node_modules/@oozcitak/dom/lib/serializer/XMLSerializerImpl.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert name == "_serializeElementNS" + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + + def test_arrow_with_destructured_params_still_works(self, parser): + """Arrow function with destructured params — first '{' is in params, + not in the body. Ensures arrow_header covers the full signature.""" + import time + content = ( + "const handleRequest = ({method, url, headers}) => {\n" + " const response = fetch(url);\n" + " let status = 200;\n" + " return response;\n" + "}" + ) + doc = Document( + page_content=content, + metadata={'source': 'app/request.js', 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert name == "handleRequest" + assert elapsed < 1.0 + + def test_small_doc_with_parens_in_body(self, parser): + """Reproduces lodash LazyWrapper — anonymous function assigned to + computed property, body has var = (expr) patterns that triggered + catastrophic regex backtracking even on 556 chars.""" + import time + content = ( + "LazyWrapper.prototype[methodName] = function(n) {\n" + " n = n === undefined ? 1 : nativeMax(toInteger(n), 0);\n" + " var result = (this.__filtered__ && !index)\n" + " ? new LazyWrapper(this)\n" + " : this.clone();\n" + " if (result.__filtered__) {\n" + " result.__takeCount__ = nativeMin(n, result.__takeCount__);\n" + " } else {\n" + " result.__views__.push({\n" + " 'size': nativeMin(n, MAX_ARRAY_LENGTH),\n" + " 'type': methodName + (result.__dir__ < 0 ? 'Right' : '')\n" + " });\n" + " }\n" + " return result;\n" + " };" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/lodash/lodash.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + + def test_babel_iife_wrapped_arrow_completes_fast(self, parser): + """Reproduces babel package.js — IIFE-wrapped arrow with nested parens + in the header like (0, _utils.makeStaticFileCache)((filepath, content) => + caused catastrophic regex backtracking (38.8s on 662 chars).""" + import time + content = ( + "const readConfigPackage = " + "(0, _utils.makeStaticFileCache)((filepath, content) => {\n" + " let options;\n" + " try {\n" + " options = (0, _json.parse)(content);\n" + " } catch (err) {\n" + " throw new _configError.default(\n" + " `Error while parsing JSON - ${err.message}`, filepath);\n" + " }\n" + " if (!options) throw new _configError.default(\n" + " `No config detected in ${filepath}`, filepath);\n" + " if (typeof options !== 'object') throw new _configError.default(\n" + " `Config returned typeof ${typeof options}`, filepath);\n" + " if (Array.isArray(options)) throw new _configError.default(\n" + " `Config returned an array`, filepath);\n" + " delete options['$schema'];\n" + " return {\n" + " filepath,\n" + " dirname: _path().dirname(filepath),\n" + " options\n" + " };\n" + "});" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/@babel/core/lib/config/files/package.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + assert name == "readConfigPackage", f"Expected 'readConfigPackage', got '{name}'" + + def test_constructor_with_inner_arrow_not_confused(self, parser): + """Reproduces PackageURL constructor — body contains arrow functions + like `key => {` which caused arrow_header to extend into the body, + then P9 matched `required =` instead of extracting `constructor`.""" + content = ( + "constructor(type, namespace, name, version, qualifiers, subpath) {\n" + " let required = { 'type': type, 'name': name };\n" + " Object.keys(required).forEach(key => {\n" + " if (!required[key]) {\n" + " throw new Error('Invalid purl: \"' + key + '\" is a required field.');\n" + " }\n" + " });\n" + " }\n" + "//(class: PackageURL)" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/packageurl-js/lib/package-url.js', + 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "constructor", f"Expected 'constructor', got '{name}'" + + +class TestIsFunctionAnonymousClass: + """is_function regex requires class\\s+[\\w$]+ (a named class). Anonymous + classes like 'export default class { ... }' bypass the check.""" + + def test_anonymous_class_is_not_function(self, parser): + """Anonymous default export class should not be considered a function.""" + doc = Document( + page_content="export default class {\n connect() {}\n}", + metadata={'source': 'node_modules/pkg/db.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc), ( + "Anonymous class should not pass is_function" + ) + + def test_anonymous_class_with_methods_is_not_function(self, parser): + """Anonymous class with multiple methods.""" + doc = Document( + page_content=( + "export default class {\n" + " constructor() { this.x = 1; }\n" + " render() { return this.x; }\n" + "}" + ), + metadata={'source': 'node_modules/pkg/component.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + + def test_named_class_still_not_function(self, parser): + """Named classes should still return False (existing behavior).""" + doc = Document( + page_content="class MyClass {\n constructor() {}\n}", + metadata={'source': 'app/my.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + + def test_regular_function_still_is_function(self, parser): + """Regular functions should still return True.""" + doc = Document( + page_content="function doStuff() {\n return 1;\n}", + metadata={'source': 'app/util.js', 'content_type': 'functions_classes'} + ) + assert parser.is_function(doc) + + +# ============================================================================= +# Generator export regex misses function* (no space) +# ============================================================================= + +class TestGeneratorExportDetection: + """_is_exportable_function must detect generator functions exported as + 'export function* genName()' where * immediately follows 'function'.""" + + def test_export_function_star_no_space(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function* genValues() { yield 1; }" + ) + assert result is True, "function* (no space) should be detected as exported" + + def test_export_function_space_star(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function *genValues() { yield 1; }" + ) + assert result is True, "function * (space before star) should be detected" + + def test_export_function_star_space(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function * genValues() { yield 1; }" + ) + assert result is True, "function * genValues (space around star) should be detected" + + def test_export_async_function_star(self, parser): + result = parser._is_exportable_function( + "genValues", + "export async function* genValues() { yield 1; }" + ) + assert result is True, "async function* should be detected" + + def test_non_generator_still_works(self, parser): + result = parser._is_exportable_function( + "myFunc", + "export function myFunc() { return 1; }" + ) + assert result is True, "regular export function should still work" + + +# ============================================================================= +# Anonymous export-default arrow without parens +# ============================================================================= + +class TestAnonymousExportDefaultArrowNoParens: + """export default x => { innerFunc(); } should return '' not 'innerFunc'.""" + + def test_single_param_no_parens(self, parser): + doc = Document( + page_content="export default x => {\n innerFunc();\n}", + metadata={'source': 'node_modules/pkg/index.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for anonymous single-param arrow, got '{name}'" + + def test_single_param_async_no_parens(self, parser): + doc = Document( + page_content="export default async req => {\n validate(req);\n return respond();\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for async single-param arrow, got '{name}'" + + def test_single_param_expression_body(self, parser): + doc = Document( + page_content="export default x => x + 1", + metadata={'source': 'node_modules/pkg/inc.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for expression-body arrow, got '{name}'" + + +# ============================================================================= +# arrow_header extends into inner arrows in destructured params +# ============================================================================= + +class TestArrowHeaderInnerArrowExtension: + """Method definitions with destructured params containing inner arrows + should not have arrow_header extend into the params.""" + + def test_method_with_inner_arrow_default(self, parser): + doc = Document( + page_content="method({cb = x => {return x}}) {\n doStuff();\n}", + metadata={'source': 'node_modules/pkg/util.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == 'method', f"Expected 'method' but got '{name}'" + + def test_method_with_complex_arrow_default(self, parser): + doc = Document( + page_content="handle({onError = err => {log(err)}, timeout = 5000}) {\n process();\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == 'handle', f"Expected 'handle' but got '{name}'" + + +# ============================================================================= +# Anonymous function name collisions in local-var map +# ============================================================================= + +class TestLocalVarMapCollisions: + """Multiple anonymous functions from same file should not overwrite each other.""" + + def test_multiple_anonymous_same_file_no_collision(self, parser): + """Anonymous functions (empty name) should be skipped rather than collide.""" + docs = [ + Document( + page_content="(x) => {\n const a = x + 1;\n return a;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="(y) => {\n const b = y * 2;\n return b;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + ] + mappings = parser.create_map_of_local_vars(docs) + assert len(mappings) == 0, ( + f"Expected 0 entries (anonymous functions skipped), got {len(mappings)}. " + "Empty function names cannot be looked up and should not be indexed." + ) + + def test_named_functions_still_indexed(self, parser): + """Named functions should still be indexed normally.""" + docs = [ + Document( + page_content="function add(x) {\n const result = x + 1;\n return result;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="function multiply(y) {\n const product = y * 2;\n return product;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + ] + mappings = parser.create_map_of_local_vars(docs) + assert len(mappings) == 2 + assert 'add@app/utils.js' in mappings + assert 'multiply@app/utils.js' in mappings + + +# ============================================================================= +# _is_exportable duplication +# ============================================================================= + +class TestIsExportableSharedLogic: + """_is_exportable_class and _is_exportable_function should share CommonJS checks.""" + + def test_class_commonjs_module_exports_direct(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports = MyClass;") + + def test_class_commonjs_module_exports_object(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports = { MyClass };") + + def test_class_commonjs_module_exports_property(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports.MyClass = MyClass;") + + def test_class_commonjs_exports_property(self, parser): + assert parser._is_exportable_class("MyClass", "exports.MyClass = MyClass;") + + def test_function_commonjs_module_exports_direct(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports = myFunc;") + + def test_function_commonjs_module_exports_object(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports = { myFunc };") + + def test_function_commonjs_module_exports_property(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports.myFunc = myFunc;") + + def test_function_commonjs_exports_property(self, parser): + assert parser._is_exportable_function("myFunc", "exports.myFunc = myFunc;") + + +# ============================================================================= +# Bug: is_package_imported split('from') corrupts identifiers containing "from" +# Lines 799, 822: line.split('from')[0] splits on substrings like "fromString" +# ============================================================================= + +class TestIsPackageImportedFromSplit: + """split('from') breaks identifiers that start with or contain 'from'.""" + + def test_es6_import_identifier_starting_with_from(self, parser): + """import { fromString } from 'packageurl-js' should detect 'fromString'.""" + code = "import { fromString } from 'packageurl-js';" + assert parser.is_package_imported(code, "fromString", "packageurl-js") + + def test_es6_import_identifier_from_inside_name(self, parser): + """import { transformData } from 'utils' should detect 'transformData'.""" + code = "import { transformData } from 'utils';" + assert parser.is_package_imported(code, "transformData", "utils") + + def test_reexport_identifier_starting_with_from(self, parser): + """export { fromBuffer } from 'uuid' should detect 'fromBuffer'.""" + code = "export { fromBuffer } from 'uuid';" + assert parser.is_package_imported(code, "fromBuffer", "uuid") + + def test_normal_import_still_works(self, parser): + """import { template } from 'lodash' should still work after fix.""" + code = "import { template } from 'lodash';" + assert parser.is_package_imported(code, "template", "lodash") + + +# ============================================================================= +# Bug: _get_function_calls called without code_documents, alias resolution dead +# Line 236: search_for_called_function doesn't pass code_documents +# ============================================================================= + +class TestAliasResolutionPassthrough: + """search_for_called_function must pass code_documents so alias resolution works.""" + + def test_aliased_import_detected_via_search_for_called_function(self, parser): + """When an aliased import exists, search_for_called_function should detect it.""" + caller_code = "function handler() {\n myAlias();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "handlers/main.js"} + ) + + callee_code = "function original() { return 1; }" + callee_doc = Document( + page_content=callee_code, + metadata={"source": "node_modules/somelib/index.js"} + ) + + full_file_code = "import { original as myAlias } from 'somelib';\n\n" + caller_code + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "handlers/main.js"} + ) + + code_documents = {"handlers/main.js": full_file_doc} + + result = parser.search_for_called_function( + caller_function=caller_doc, + callee_function_name="original", + callee_function=callee_doc, + callee_function_package="somelib", + code_documents=code_documents, + type_documents=[], + callee_function_file_name="index.js", + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + ) + assert result is True, "Alias 'myAlias' for 'original' should be detected" + + +# ============================================================================= +# Bug: $ missing from regex lookbehind in _get_function_calls +# Lines 196, 199: (? 0, "template() should match" + + def test_jquery_dollar_qualified_call_detected(self, parser): + """$.ajax() is a call to ajax through qualifier $ — should be detected.""" + code = "function fetch() {\n $.ajax('/api');\n}" + doc = Document(page_content=code, metadata={"source": "app.js"}) + calls = parser._get_function_calls(doc, "ajax") + # $.ajax() is a legitimate call to ajax. The $ qualifier isn't in the + # qualifier char class [\w.?()], so the match is 'ajax' not '$.ajax'. + # This is correct — the call IS to ajax. + assert len(calls) > 0, "$.ajax() should be detected as a call to ajax" + + +# ============================================================================= +# Bug: get_package_names returns single-element list for JS +# chain_of_calls_retriever.py:593 accesses [1] → IndexError +# ============================================================================= + +class TestGetPackageNamesSingleElement: + """JS get_package_names returns single-element list, so [1] index crashes.""" + + def test_third_party_package_returns_single_element(self, parser): + """Third-party packages return ['package_name'] — only one element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/lodash/index.js"}) + result = parser.get_package_names(doc) + assert result == ["lodash"] + assert len(result) == 1 + + def test_third_party_scoped_returns_single_element(self, parser): + """Scoped packages return ['@scope/pkg'] — still one element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/@babel/core/index.js"}) + result = parser.get_package_names(doc) + assert result == ["@babel/core"] + assert len(result) == 1 + + def test_root_project_returns_single_element(self, parser): + """Root project files return ['root_project'] — single element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "src/app.js"}) + result = parser.get_package_names(doc) + assert result == ["root_project"] + assert len(result) == 1 + + def test_index_0_always_valid(self, parser): + """Accessing [0] should always work for any document.""" + for source in ["node_modules/express/index.js", "src/app.js", + "node_modules/@types/node/index.d.ts"]: + doc = Document(page_content="var x;", metadata={"source": source}) + names = parser.get_package_names(doc) + assert len(names) >= 1 + _ = names[0] # Should never raise + + def test_index_1_raises_for_js(self, parser): + """Accessing [1] should raise IndexError — this is the bug CCA hits.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/lodash/index.js"}) + result = parser.get_package_names(doc) + with pytest.raises(IndexError): + _ = result[1] + + +# ============================================================================= +# Bug: print_call_hierarchy doesn't handle empty function_name string +# chain_of_calls_retriever.py:702 — ValueError is caught but empty string isn't +# ============================================================================= + +class TestPrintCallHierarchyEmptyName: + """print_call_hierarchy catches ValueError but doesn't check for empty string.""" + + def test_get_function_name_empty_string_handled(self, parser): + """Documents where get_function_name returns '' should not crash hierarchy.""" + doc = Document( + page_content="// just a comment block\n/* nothing here */", + metadata={"source": "utils.js"} + ) + try: + name = parser.get_function_name(doc) + except ValueError: + name = None + + # The bug: if get_function_name returns '' instead of raising, + # print_call_hierarchy formats it as (package=...,function=,depth=0) + # which is a meaningless entry in the call hierarchy. + # The fix in chain_of_calls_retriever.py guards against empty strings. + assert name is None or isinstance(name, str) + + +# ============================================================================= +# _build_class_hierarchy index and _is_subclass_of optimization +# ============================================================================= + +class TestBuildClassHierarchy: + """Verify the class hierarchy index is built correctly from code_documents.""" + + def test_simple_es6_extends(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { bark() {} }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Dog" in hierarchy + extends_clause, parent = hierarchy["Dog"] + assert parent == "Animal" + assert "Animal" in extends_clause + + def test_mixin_extends(self, parser): + docs = { + "a.js": Document(page_content="class MyComponent extends EventEmitter(Base) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "MyComponent" in hierarchy + extends_clause, parent = hierarchy["MyComponent"] + assert parent == "Base" + assert "EventEmitter" in extends_clause + + def test_chained_mixin(self, parser): + docs = { + "a.js": Document(page_content="class Widget extends Draggable(Resizable(Component)) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + extends_clause, parent = hierarchy["Widget"] + assert parent == "Component" + assert "Draggable" in extends_clause + assert "Resizable" in extends_clause + + def test_prototype_inherits(self, parser): + docs = { + "a.js": Document(page_content="util.inherits(ReadStream, EventEmitter);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "ReadStream" in hierarchy + extends_clause, parent = hierarchy["ReadStream"] + assert extends_clause is None + assert parent == "EventEmitter" + + def test_object_create_prototype(self, parser): + docs = { + "a.js": Document(page_content="Child.prototype = Object.create(Parent.prototype);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Child" in hierarchy + assert hierarchy["Child"] == (None, "Parent") + + def test_set_prototype_of(self, parser): + docs = { + "a.js": Document(page_content="Object.setPrototypeOf(Sub.prototype, Super.prototype);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Sub" in hierarchy + assert hierarchy["Sub"] == (None, "Super") + + def test_multiple_classes_across_files(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends C { }", metadata={"source": "b.js"}), + "c.js": Document(page_content="class C { }", metadata={"source": "c.js"}), + } + hierarchy = parser._build_class_hierarchy(docs) + assert hierarchy["A"] == ("B", "B") + assert hierarchy["B"] == ("C", "C") + assert "C" not in hierarchy + + def test_es6_takes_precedence_over_prototype(self, parser): + """If both ES6 and prototype patterns exist, ES6 wins (indexed first).""" + docs = { + "a.js": Document( + page_content="class X extends Y { }\nutil.inherits(X, Z);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert hierarchy["X"][1] == "Y" + + def test_empty_documents(self, parser): + hierarchy = parser._build_class_hierarchy({}) + assert hierarchy == {} + + +class TestIsSubclassOfOptimized: + """Verify _is_subclass_of correctness with the hierarchy index.""" + + def test_direct_subclass(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("Dog", "Animal", docs) + + def test_not_subclass(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { }", + metadata={"source": "a.js"}) + } + assert not parser._is_subclass_of("Dog", "Vehicle", docs) + + def test_transitive_chain(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends C { }", metadata={"source": "b.js"}), + } + assert parser._is_subclass_of("A", "C", docs) + + def test_mixin_match(self, parser): + docs = { + "a.js": Document(page_content="class X extends Mixin(Base) { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("X", "Mixin", docs) + assert parser._is_subclass_of("X", "Base", docs) + + def test_circular_reference_no_infinite_loop(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends A { }", metadata={"source": "b.js"}), + } + assert not parser._is_subclass_of("A", "Z", docs) + + def test_empty_child_or_parent(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert not parser._is_subclass_of("", "Y", docs) + assert not parser._is_subclass_of("X", "", docs) + assert not parser._is_subclass_of(None, "Y", docs) + + def test_prototype_inheritance(self, parser): + docs = { + "a.js": Document(page_content="util.inherits(ReadStream, EventEmitter);", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("ReadStream", "EventEmitter", docs) + + def test_hierarchy_cache_reused(self, parser): + """Same code_documents dict should reuse cached hierarchy.""" + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}) + } + parser._is_subclass_of("A", "B", docs) + cache_key_1 = parser._class_hierarchy_cache_key + + parser._is_subclass_of("A", "C", docs) + cache_key_2 = parser._class_hierarchy_cache_key + + assert cache_key_1 == cache_key_2 + + def test_different_docs_rebuilds_cache(self, parser): + docs1 = {"a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"})} + docs2 = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + + parser._is_subclass_of("A", "B", docs1) + key1 = parser._class_hierarchy_cache_key + + parser._is_subclass_of("X", "Y", docs2) + key2 = parser._class_hierarchy_cache_key + + assert key1 != key2 + + +class TestGetParentOptimized: + """Verify _get_direct_parent, _get_prototype_parent, _get_parent with index.""" + + def test_get_direct_parent(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) == "Y" + + def test_get_direct_parent_mixin(self, parser): + docs = {"a.js": Document(page_content="class X extends Mixin(Base) { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) == "Base" + + def test_get_direct_parent_not_found(self, parser): + docs = {"a.js": Document(page_content="class X { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) is None + + def test_get_prototype_parent(self, parser): + docs = {"a.js": Document(page_content="util.inherits(Child, Parent);", metadata={"source": "a.js"})} + assert parser._get_prototype_parent("Child", docs) == "Parent" + + def test_get_prototype_parent_not_found(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert parser._get_prototype_parent("X", docs) is None + + def test_get_parent_prefers_es6(self, parser): + docs = {"a.js": Document( + page_content="class X extends Y { }\nutil.inherits(X, Z);", + metadata={"source": "a.js"})} + assert parser._get_parent("X", docs) == "Y" + + +# ============================================================================= +# Bug: _build_class_hierarchy uses \w+ which misses $ in JS identifiers +# ============================================================================= + +class TestDollarSignInClassHierarchy: + """$ is a valid JS identifier char — hierarchy builder must handle it.""" + + def test_dollar_prefixed_es6_class(self, parser): + """class $Component extends React.Component should be in the hierarchy.""" + docs = { + "a.js": Document(page_content="class $Component extends Base { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Component" in hierarchy, f"$Component not found in hierarchy: {hierarchy}" + _, parent = hierarchy["$Component"] + assert parent == "Base" + + def test_dollar_prefixed_parent(self, parser): + """class Child extends $Base should resolve $Base as parent.""" + docs = { + "a.js": Document(page_content="class Child extends $Base { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Child" in hierarchy + _, parent = hierarchy["Child"] + assert parent == "$Base" + + def test_dollar_prefixed_mixin_parent(self, parser): + """class Foo extends Mixin($Bar) should resolve $Bar as innermost parent.""" + docs = { + "a.js": Document(page_content="class Foo extends Mixin($Bar) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Foo" in hierarchy + _, parent = hierarchy["Foo"] + assert parent == "$Bar" + + def test_dollar_prefixed_prototype_inherits(self, parser): + """util.inherits($Stream, EventEmitter) should capture $Stream.""" + docs = { + "a.js": Document(page_content="util.inherits($Stream, EventEmitter);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Stream" in hierarchy + assert hierarchy["$Stream"] == (None, "EventEmitter") + + def test_is_subclass_of_with_dollar(self, parser): + """_is_subclass_of should work with $-prefixed class names.""" + docs = { + "a.js": Document(page_content="class $Widget extends Component { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("$Widget", "Component", docs) + + def test_dollar_in_both_child_and_parent(self, parser): + """Both child and parent have $ prefix.""" + docs = { + "a.js": Document(page_content="class $Child extends $Parent { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Child" in hierarchy + _, parent = hierarchy["$Child"] + assert parent == "$Parent" + + +# ============================================================================= +# Bug: recursive _get_function_calls omits code_documents — chained alias broken +# ============================================================================= + +class TestChainedAliasResolution: + """Recursive _get_function_calls must pass code_documents for chained aliases.""" + + def test_alias_of_alias_es6(self, parser): + """import { orig as mid } then import { mid as final } — final() should match orig.""" + caller_code = "function handler() {\n final();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "handler.js"} + ) + + full_file_code = ( + "import { orig as mid } from 'pkg';\n" + "import { mid as final } from './re-export';\n" + "\n" + caller_code + ) + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "handler.js"} + ) + + code_documents = {"handler.js": full_file_doc} + + calls = parser._get_function_calls(caller_doc, "orig", code_documents) + assert "final" in calls, ( + f"Chained alias orig→mid→final should resolve. Got: {calls}" + ) + + def test_alias_commonjs_chain(self, parser): + """const { orig: mid } = require('a') then const { mid: local } = require('b').""" + caller_code = "function run() {\n local();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "run.js"} + ) + + full_file_code = ( + "const { orig: mid } = require('a');\n" + "const { mid: local } = require('b');\n" + "\n" + caller_code + ) + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "run.js"} + ) + + code_documents = {"run.js": full_file_doc} + + calls = parser._get_function_calls(caller_doc, "orig", code_documents) + assert "local" in calls, ( + f"Chained CommonJS alias orig→mid→local should resolve. Got: {calls}" + ) + From 759d6d881be784f72428edcf50e8f81ddc5860ab Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 23 Jun 2026 16:24:38 +0300 Subject: [PATCH 268/286] chore(kustomize): remove exploit-iq-pull-secret from all manifests and docs --- kustomize/README.md | 51 +++++++++------------ kustomize/base/argilla/deployment.yaml | 2 - kustomize/base/ips-patch-client.json | 6 --- kustomize/base/ips-patch.json | 6 --- kustomize/base/kustomization.yaml | 17 ++----- kustomize/overlays/tests/kustomization.yaml | 5 -- 6 files changed, 27 insertions(+), 60 deletions(-) delete mode 100644 kustomize/base/ips-patch-client.json delete mode 100644 kustomize/base/ips-patch.json diff --git a/kustomize/README.md b/kustomize/README.md index 7b0b7db5a..275253eda 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -21,16 +21,17 @@ limitations under the License. Install the following tools and verify that all binaries are available on your system path: -- [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) — OpenShift CLI, logged in to your target cluster -- [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) — version 5 or later -- `openssl` — for generating secrets and verifying certificates +- [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) +- [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) +- `openssl` Obtain credentials for the following services before you begin. For instructions on creating each credential, refer to the main [README](../README.md#obtain-api-keys): -- SerpAPI key - GitHub Advisory Database (GHSA) API key +- National Vulnerability Database (NVD) API key - NVIDIA API key (required for Remote NIM deployments; a placeholder value is sufficient for self-hosted variants) - Red Hat registry credentials (`registry.redhat.io`) +- SerpAPI key > [!NOTE] > All commands in this guide assume that your current working directory is `kustomize/`. @@ -41,7 +42,14 @@ Obtain credentials for the following services before you begin. For instructions Complete the following steps before running any deployment command. -### Step 1. Create API Credentials +### Step 1. Create a Project Namespace + +```shell +export YOUR_NAMESPACE_NAME= +oc new-project $YOUR_NAMESPACE_NAME +``` + +### Step 2. Create API Credentials Create the `base/secrets.env` file with your API keys. The following table shows which keys are required for each deployment variant: @@ -50,6 +58,7 @@ Create the `base/secrets.env` file with your API keys. The following table shows | `serpapi_api_key` | Required | Required | Web search for patch intelligence | | `ghsa_api_key` | Required | Required | GitHub token for advisory lookups and repository scanning | | `nvidia_api_key` | Placeholder | Required | A placeholder value is sufficient for self-hosted variants | +| `nvd_api_key` | Required | Required | National Vulnerability Database API key | | `registry_redhat_username` | Required | Required | Red Hat registry credentials for container image scanning | | `registry_redhat_password` | Required | Required | Red Hat registry credentials for container image scanning | @@ -58,29 +67,13 @@ cat > base/secrets.env << EOF serpapi_api_key= ghsa_api_key= nvidia_api_key= +nvd_api_key= registry_redhat_username= registry_redhat_password= EOF ``` -### Step 2. Create a Project Namespace - -```shell -export YOUR_NAMESPACE_NAME= -oc new-project $YOUR_NAMESPACE_NAME -``` - -### Step 3. Create an Image Pull Secret - -Create the secret that grants permission to pull Exploit Intelligence and Argilla container images: - -```shell -oc create secret generic exploit-iq-pull-secret \ - --from-file=.dockerconfigjson= \ - --type=kubernetes.io/dockerconfigjson -``` - -### Step 4. Configure Image Registry Credentials +### Step 3. Configure Image Registry Credentials Create the `base/image-registry-credentials.env` file with credentials for the registries that product scanning uses to pull component images. The `auth` value is the base64 encoding of `:`. @@ -99,7 +92,7 @@ EOF > [!IMPORTANT] > Product scanning requires valid registry credentials to pull component images. If you skip this step, the deployment succeeds but authenticated image pulls fail. -### Step 5. Configure Argilla Feedback Credentials +### Step 4. Configure Argilla Feedback Credentials Create the `base/argilla/feedback_secret.env` file with credentials for the Argilla feedback service: @@ -111,7 +104,7 @@ argilla_api_key= EOF ``` -### Step 6. Configure OAuth Authentication +### Step 5. Configure OAuth Authentication Exploit Intelligence uses OpenShift OAuth for user authentication. Select the option that matches your situation. @@ -161,7 +154,7 @@ openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') EOF ``` -### Step 7. Create Database Credentials +### Step 6. Create Database Credentials ```shell cat > base/mongodb-credentials.env << EOF @@ -172,14 +165,14 @@ exploit-iq-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) EOF ``` -### Step 8. Set the Application Callback URL +### Step 7. Set the Application Callback URL ```shell export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` -### Step 9. Configure a Custom Git Server CA +### Step 8. Configure a Custom Git Server CA Complete this step if your Git server uses a certificate signed by a custom Certificate Authority (CA). @@ -223,7 +216,7 @@ openssl crl2pkcs7 -nocrl -certfile base/ca-certs/ca-bundle.crt | \ openssl pkcs7 -print_certs -noout ``` -### Step 10. Configure the OAuth Endpoint CA +### Step 9. Configure the OAuth Endpoint CA Complete this step if your OCP ingress router uses a CA that is not trusted by default. diff --git a/kustomize/base/argilla/deployment.yaml b/kustomize/base/argilla/deployment.yaml index 0e1c6601d..4854af9ad 100644 --- a/kustomize/base/argilla/deployment.yaml +++ b/kustomize/base/argilla/deployment.yaml @@ -17,8 +17,6 @@ spec: app: morpheus-feedback-api spec: restartPolicy: Always - imagePullSecrets: - - name: exploit-iq-pull-secret serviceAccountName: argilla securityContext: fsGroup: 1000 diff --git a/kustomize/base/ips-patch-client.json b/kustomize/base/ips-patch-client.json deleted file mode 100644 index e2b53b801..000000000 --- a/kustomize/base/ips-patch-client.json +++ /dev/null @@ -1,6 +0,0 @@ -[{ - "op": "add", - "path": "/spec/template/spec/imagePullSecrets/0", - "value": {"name": "exploit-iq-pull-secret"} -} -] diff --git a/kustomize/base/ips-patch.json b/kustomize/base/ips-patch.json deleted file mode 100644 index e2b53b801..000000000 --- a/kustomize/base/ips-patch.json +++ /dev/null @@ -1,6 +0,0 @@ -[{ - "op": "add", - "path": "/spec/template/spec/imagePullSecrets/0", - "value": {"name": "exploit-iq-pull-secret"} -} -] diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 16abd4d8e..6b2da8d3c 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -80,18 +80,11 @@ configMapGenerator: options: disableNameSuffixHash: true -patches: - - path: ips-patch.json - - target: - name: exploit-iq - kind: Deployment - - - path: ips-patch-client.json - - target: - name: exploit-iq-client - kind: Deployment + - name: oidc-ca-bundle + files: + - oidc-ca/ca-bundle.crt + options: + disableNameSuffixHash: true images: - name: quay.io/ecosystem-appeng/agent-morpheus-rh diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml index 39e29f0d7..74ea831cc 100644 --- a/kustomize/overlays/tests/kustomization.yaml +++ b/kustomize/overlays/tests/kustomization.yaml @@ -14,11 +14,6 @@ resources: - secrets/exploit-iq-automation-token.yaml secretGenerator: - - name: exploit-iq-pull-secret - files: - - .dockerconfigjson=secrets/exploit-iq-ips.json - type: kubernetes.io/dockerconfigjson - - name: ecosystem-appeng-morpheus-quay files: - .dockerconfigjson=secrets/exploit-iq-ips.json From 620c9673a388993417d2bd035ca7fe8f5f72b55f Mon Sep 17 00:00:00 2001 From: Gal Netanel Date: Wed, 24 Jun 2026 11:42:24 +0300 Subject: [PATCH 269/286] Appeng 5327 create vex on none vulnerable and updating missing fields (#252) * APPENG-5327: generate VEX report even when no vulnerabilities are found * Update flags labels with thh correct value * Add uuid to tracking-id to add uniquness to the id * Add purl to vex report * Bugfix in purl for oci-image * Remove namespace from oci purl * Update full image name in repository_url --------- Co-authored-by: Gal Netanel --- pyproject.toml | 1 + .../functions/cve_generate_vex.py | 9 ++- .../vex/implementations/csaf_generator.py | 62 ++++++++++++++++--- .../tests/test_csaf_generator_integration.py | 57 +++++++++++++++++ src/vuln_analysis/utils/vex/vex_utils.py | 53 +++++++++++++++- 5 files changed, 171 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4a11258c4..7a72008a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "nemollm", "openinference-instrumentation-langchain~=0.1.31", "ordered_set", + "packageurl-python", "pydpkg==1.9.4", "rank_bm25==0.2.2", "tantivy==0.22.2", diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py index 62a9d0598..a7d2ee4b9 100644 --- a/src/vuln_analysis/functions/cve_generate_vex.py +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +import uuid from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo @@ -43,12 +43,15 @@ async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: return state if not any(justification.get("justification_label") == "vulnerable" for justification in state.justifications.values()): - logger.info("No vulnerable CVE(s) found. Skipping VEX generation.") - return state + logger.info("No vulnerable CVE(s) found. Generating VEX with known_not_affected status.") try: generator = load_vex_generator(config.vex_format) vex_doc = generator.generate(state) + if vex_doc: + tracking = vex_doc.get("document", {}).get("tracking") + if tracking and tracking.get("id"): + tracking["id"] = f"{tracking['id']}-{uuid.uuid4()}" state.vex = vex_doc except ValueError as e: logger.error("VEX generator initialization failed: %s", e) diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py index 605c37192..c5fe5f300 100644 --- a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -26,7 +26,7 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from ..vex_generator_base import VexGenerator -from ..vex_utils import get_vex_validator, build_patch_recommendation +from ..vex_utils import build_oci_image_purl, get_vex_validator, build_patch_recommendation from csaf.generator import CSAFGenerator from exploit_iq_commons.logging.loggers_factory import LoggingFactory @@ -71,6 +71,22 @@ # Justification labels JUSTIFICATION_LABEL_VULNERABLE = "vulnerable" +# ExploitIQ justification labels mapped to CSAF 2.0 VEX flag labels +EXPLOITIQ_TO_CSAF_JUSTIFICATION_MAP: dict[str, str] = { + "false_positive": "component_not_present", + "code_not_present": "vulnerable_code_not_present", + "code_not_reachable": "vulnerable_code_not_in_execute_path", + "requires_configuration": "vulnerable_code_cannot_be_controlled_by_adversary", + "requires_dependency": "component_not_present", + "requires_environment": "vulnerable_code_cannot_be_controlled_by_adversary", + "protected_by_compiler": "inline_mitigations_already_exist", + "protected_at_runtime": "inline_mitigations_already_exist", + "protected_at_perimeter": "vulnerable_code_cannot_be_controlled_by_adversary", + "protected_by_mitigating_control": "inline_mitigations_already_exist", + "uncertain": "component_not_present", +} +DEFAULT_CSAF_JUSTIFICATION = "component_not_present" + # Vulnerability statuses STATUS_KNOWN_AFFECTED = "known_affected" STATUS_KNOWN_NOT_AFFECTED = "known_not_affected" @@ -90,6 +106,12 @@ CSAF_SCHEMA_PATH = Path(__file__).resolve().parents[3] / "configs" / "vex" / "csaf" / "v2.0" / "csaf_json_schema.json" +def _map_justification_to_csaf_label(exploitiq_label: str | None) -> str: + if not exploitiq_label: + return DEFAULT_CSAF_JUSTIFICATION + return EXPLOITIQ_TO_CSAF_JUSTIFICATION_MAP.get(exploitiq_label, DEFAULT_CSAF_JUSTIFICATION) + + def _enrich_vulnerabilities_with_notes( csaf_json: Dict[str, Any], intel_map: Dict[str, CveIntel], @@ -162,6 +184,25 @@ def _enrich_vulnerabilities_with_notes( v["notes"] = notes +def _enrich_product_tree_with_purl(csaf_json: Dict[str, Any], purl: str | None) -> None: + """Add product_identification_helper.purl to each product in the product tree.""" + if not purl: + return + + def visit(obj: Any) -> None: + if isinstance(obj, dict): + if "product_id" in obj and "name" in obj: + helper = obj.setdefault("product_identification_helper", {}) + helper["purl"] = purl + for value in obj.values(): + visit(value) + elif isinstance(obj, list): + for item in obj: + visit(item) + + visit(csaf_json.get("product_tree", {})) + + class CsafVexGenerator(VexGenerator): """ CSAF VEX generator. Builds a CSAF JSON document and validates it with the csaf-tool. @@ -202,8 +243,10 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: ci = intel_map.get(vuln_id) impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else DEFAULT_IMPACT - is_vulnerable = justification.get("justification_label") == JUSTIFICATION_LABEL_VULNERABLE - + justification_label = justification.get("justification_label") + is_vulnerable = justification_label == JUSTIFICATION_LABEL_VULNERABLE + csaf_justification = _map_justification_to_csaf_label(justification_label) + if is_vulnerable: patch_recommendation = build_patch_recommendation(ci, sbom_names) comment = ( @@ -222,7 +265,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: action=comment ) - else: + else: csaf_gen.add_vulnerability( product_name=product_name, release=product_tag, @@ -230,6 +273,7 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: status=STATUS_KNOWN_NOT_AFFECTED, description="", comment=impact, + justification=csaf_justification, ) csaf_gen.generate_csaf() @@ -243,9 +287,13 @@ def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: csaf_json = json.load(f) # Enrich the CSAF in memory - _enrich_vulnerabilities_with_notes( - csaf_json, intel_map, state.final_summaries, state.justifications - ) + image = message.input.image + if image.analysis_type == "image": + product_purl = build_oci_image_purl(image.name, image.tag, image.digest) + _enrich_product_tree_with_purl(csaf_json, product_purl) + _enrich_vulnerabilities_with_notes( + csaf_json, intel_map, state.final_summaries, state.justifications + ) # Validate the CSAF document against the JSON schema errors = list(get_vex_validator(CSAF_SCHEMA_PATH).iter_errors(csaf_json)) diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py index 5bca102d3..4418a7663 100644 --- a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -36,6 +36,7 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator +from vuln_analysis.utils.vex.vex_utils import build_oci_image_purl _DEFAULT_SOURCE_INFO = [ @@ -161,6 +162,42 @@ def test_product_tree_contains_product(self, mock_state): product_tree = result["product_tree"] assert _DEFAULT_PRODUCT_NAME in product_tree.get("branches")[0].get("branches")[0].get("name") assert _DEFAULT_PRODUCT_TAG in product_tree.get("branches")[0].get("branches")[0].get("branches")[0].get("name") + + def test_product_tree_includes_oci_purl(self, mock_state): + """Test that product tree includes product_identification_helper with OCI purl.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + product = ( + result["product_tree"] + .get("branches")[0] + .get("branches")[0] + .get("branches")[0] + .get("product") + ) + helper = product.get("product_identification_helper", {}) + assert helper.get("purl") == build_oci_image_purl(_DEFAULT_PRODUCT_NAME, _DEFAULT_PRODUCT_TAG) + + def test_product_tree_purl_prefers_digest_over_tag(self): + """Test that explicit digest is used in purl instead of tag.""" + oci_digest = "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + state = create_mock_state(product_tag="v1.0.0") + state.original_input.input.image.digest = oci_digest + + generator = CsafVexGenerator() + result = generator.generate(state) + + product = ( + result["product_tree"] + .get("branches")[0] + .get("branches")[0] + .get("branches")[0] + .get("product") + ) + helper = product.get("product_identification_helper", {}) + assert helper.get("purl") == build_oci_image_purl( + _DEFAULT_PRODUCT_NAME, "v1.0.0", oci_digest + ) def test_vulnerable_cve_has_known_affected_status(self, mock_state): """Test that vulnerable CVEs get 'known_affected' status.""" @@ -184,6 +221,26 @@ def test_not_vulnerable_cve_has_known_not_affected_status(self): product_status = vuln.get("product_status", {}) assert "known_not_affected" in product_status + def test_code_not_reachable_maps_to_csaf_execute_path_flag(self): + """Test that code_not_reachable maps to the CSAF execute-path flag.""" + state = create_mock_state( + justification={ + "justification": "Vulnerable function exists but is not called.", + "justification_label": "code_not_reachable", + }, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + assert vuln["flags"][0]["label"] == "vulnerable_code_not_in_execute_path" + label_notes = [ + n for n in vuln.get("notes", []) + if n.get("title") == "ExploitIQ Analysis Justification Label" + ] + assert label_notes[0]["text"] == "code_not_reachable" + def test_vulnerable_cve_includes_remediation(self): """Test that vulnerable CVEs include remediation information when patch is available.""" ghsa = CveIntelGhsa( diff --git a/src/vuln_analysis/utils/vex/vex_utils.py b/src/vuln_analysis/utils/vex/vex_utils.py index e27bf227d..f41c92048 100644 --- a/src/vuln_analysis/utils/vex/vex_utils.py +++ b/src/vuln_analysis/utils/vex/vex_utils.py @@ -20,9 +20,11 @@ from pathlib import Path from jsonschema import Draft202012Validator +from packageurl import PackageURL from exploit_iq_commons.data_models.cve_intel import CveIntel from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from urllib.parse import urlparse logger = LoggingFactory.get_agent_logger(__name__) @@ -58,6 +60,56 @@ def get_patched_package(vuln: dict) -> tuple[str | None, str | None]: return pkg.get("name"), vuln.get("first_patched_version") +def build_oci_image_purl( + image_name: str, + tag: str | None = None, + digest: str | None = None, +) -> str | None: + """ + Build an OCI package URL (purl) for a container image. + + Prefers an explicit digest, then falls back to the image tag. + """ + image_path = image_name + parsed = urlparse(f"//{image_path}") + registry = parsed.netloc + # qualifiers include registry and full name which all already exist in image_path + qualifiers = {"repository_url": image_path} if image_path else {} + path_parts = [part for part in parsed.path.strip("/").split("/") if part] + if path_parts: + if len(path_parts) > 1: + name = path_parts[-1] + namespace = "/".join(path_parts[:-1]) + else: + name = path_parts[0] + namespace = None + elif parsed.netloc: + name = parsed.netloc + namespace = None + else: + name = image_path + namespace = None + + version = digest or tag + # oci purl specification required to emit namespace, therefor it is set to None + purl = PackageURL( + type="oci", + namespace=None, + name=name, + version=version, + qualifiers=qualifiers if qualifiers else None, + ) + logger.debug( + "Building OCI image purl components: registry=%s, qualifiers=%s, name=%s, version=%s", + registry, + qualifiers, + name, + version, + ) + logger.debug("Resulting OCI image purl: %s", purl.to_string()) + return purl.to_string() + + def build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: """ Build a patch recommendation string from GHSA data. @@ -93,4 +145,3 @@ def build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None if not name_to_version: return "" return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) - From 03bc5e91b6d00cd89eec1cf934d4b8ffdbed7765 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 24 Jun 2026 11:50:52 +0300 Subject: [PATCH 270/286] fix: prevent documents loading to run twice Signed-off-by: Zvi Grinberg --- .../tools/tests/test_concurrency.py | 45 ++++++++--------- .../tests/test_transitive_code_search.py | 1 - .../tools/transitive_code_search.py | 48 ++++++++++++++----- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py index 4d0df58c5..880a627c5 100644 --- a/src/vuln_analysis/tools/tests/test_concurrency.py +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -139,7 +139,7 @@ def _make_slow_builder(build_log, sleep_secs=0.2, java=True): """ lock = threading.Lock() - def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): tag = query.split(",")[0] start = time.monotonic() time.sleep(sleep_secs) @@ -234,8 +234,8 @@ async def test_java_same_repo_different_packages_are_serialized(): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=_make_slow_builder(build_log, java=True)): - task1 = asyncio.create_task(_build_or_get_cached(si, "pkg-a:art-a:1.0,ClassA.foo", _DEFAULT_THRESHOLD)) - task2 = asyncio.create_task(_build_or_get_cached(si, "pkg-b:art-b:2.0,ClassB.bar", _DEFAULT_THRESHOLD)) + task1 = asyncio.create_task(_build_or_get_cached(si=si, query="pkg-a:art-a:1.0,ClassA.foo", uber_jar_file_threshold=_DEFAULT_THRESHOLD )) + task2 = asyncio.create_task(_build_or_get_cached(si=si, query="pkg-b:art-b:2.0,ClassB.bar", uber_jar_file_threshold=_DEFAULT_THRESHOLD)) await asyncio.gather(task1, task2) assert len(build_log) == 2, f"Expected 2 Java builds (different packages), got {len(build_log)}" @@ -256,7 +256,7 @@ async def test_different_repos_can_build_concurrently(): si_b = _make_si("https://github.com/example/repo-b") build_log = [] - def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): tag = si[0].git_repo.split("/")[-1] start = time.monotonic() time.sleep(0.2) @@ -266,8 +266,8 @@ def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=slow_build): - task1 = asyncio.create_task(_build_or_get_cached(si_a, "pkg-a:art-a:1.0,Foo.bar", _DEFAULT_THRESHOLD)) - task2 = asyncio.create_task(_build_or_get_cached(si_b, "pkg-b:art-b:2.0,Baz.qux", _DEFAULT_THRESHOLD)) + task1 = asyncio.create_task(_build_or_get_cached(si_a, query="pkg-a:art-a:1.0,Foo.bar", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si_b, query="pkg-b:art-b:2.0,Baz.qux", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) await asyncio.gather(task1, task2) assert len(build_log) == 2 @@ -288,7 +288,7 @@ async def test_same_key_deduplicates_build(): build_count = 0 count_lock = threading.Lock() - def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): nonlocal build_count with count_lock: build_count += 1 @@ -297,8 +297,8 @@ def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - task1 = asyncio.create_task(_build_or_get_cached(si, query, _DEFAULT_THRESHOLD)) - task2 = asyncio.create_task(_build_or_get_cached(si, query, _DEFAULT_THRESHOLD)) + task1 = asyncio.create_task(_build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) results = await asyncio.gather(task1, task2) assert build_count == 1, f"Expected 1 build (deduplicated), got {build_count}" @@ -319,14 +319,14 @@ async def test_cache_hit_skips_build(): build_count = 0 - def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): nonlocal build_count build_count += 1 return _make_nonjava_searcher() with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - result = await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) + result = await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) assert build_count == 0, "Build should not run when cache hit exists" assert result is pre_cached, "Should return the pre-cached searcher" @@ -345,14 +345,14 @@ async def test_java_cache_hit_skips_build(): build_count = 0 - def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): nonlocal build_count build_count += 1 return _make_java_searcher() with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - result = await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) + result = await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) assert build_count == 0, "Build should not run when Java cache hit exists" assert result is pre_cached, "Should return the pre-cached Java searcher" @@ -367,13 +367,13 @@ async def test_build_failure_cleans_up_building_marker(): full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") repo_key = ("https://github.com/example/repo", "main") - def failing_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def failing_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): raise RuntimeError("Maven failed") with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=failing_build): with pytest.raises(RuntimeError, match="Maven failed"): - await _build_or_get_cached(si, query, _DEFAULT_THRESHOLD) + await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) assert full_key not in _searcher_building, "Building marker not cleaned up after failure" assert full_key not in _searcher_cache, "Failed build should not be cached" @@ -394,7 +394,7 @@ async def test_java_repo_lock_recheck_avoids_redundant_build(): build_count = 0 count_lock = threading.Lock() - def build_that_precaches_b(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def build_that_precaches_b(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): nonlocal build_count with count_lock: build_count += 1 @@ -406,9 +406,9 @@ def build_that_precaches_b(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHO with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=build_that_precaches_b): - task1 = asyncio.create_task(_build_or_get_cached(si, query_a, _DEFAULT_THRESHOLD)) + task1 = asyncio.create_task(_build_or_get_cached(si, query=query_a, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) await asyncio.sleep(0.01) - task2 = asyncio.create_task(_build_or_get_cached(si, query_b, _DEFAULT_THRESHOLD)) + task2 = asyncio.create_task(_build_or_get_cached(si, query=query_b, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) await asyncio.gather(task1, task2) assert build_count == 1, ( @@ -429,7 +429,7 @@ async def test_nonjava_same_repo_different_packages_share_cache(): build_count = 0 count_lock = threading.Lock() - def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): nonlocal build_count with count_lock: build_count += 1 @@ -438,8 +438,9 @@ def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", side_effect=counting_build): - task1 = asyncio.create_task(_build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey", _DEFAULT_THRESHOLD)) - task2 = asyncio.create_task(_build_or_get_cached(si, "net/http,ListenAndServe", _DEFAULT_THRESHOLD)) + task1 = asyncio.create_task( + _build_or_get_cached(si, query="crypto/x509,ParsePKCS1PrivateKey", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si, query="net/http,ListenAndServe", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) results = await asyncio.gather(task1, task2) # Non-Java: both should resolve to the same (repo, ref) cache entry. @@ -461,7 +462,7 @@ async def test_nonjava_caches_under_repo_key(): with patch("vuln_analysis.tools.transitive_code_search._build_searcher", return_value=_make_nonjava_searcher()): - await _build_or_get_cached(si, "crypto/x509,ParsePKCS1PrivateKey", _DEFAULT_THRESHOLD) + await _build_or_get_cached(si, query="crypto/x509,ParsePKCS1PrivateKey", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) assert repo_key in _searcher_cache, "Non-Java should cache under repo_key" assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py index b6b5180c1..80d3dd8f9 100644 --- a/src/vuln_analysis/tools/tests/test_transitive_code_search.py +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -24,7 +24,6 @@ from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path, get_repo_path_with_ref from pathlib import Path - @pytest.fixture(autouse=True) def patch_repo_path_with_fallback(): """ diff --git a/src/vuln_analysis/tools/transitive_code_search.py b/src/vuln_analysis/tools/transitive_code_search.py index ec5e7c281..e030dbcfd 100644 --- a/src/vuln_analysis/tools/transitive_code_search.py +++ b/src/vuln_analysis/tools/transitive_code_search.py @@ -38,6 +38,7 @@ from exploit_iq_commons.logging.loggers_factory import LoggingFactory from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever, _release_repo_data +from ..functions.cve_segmentation import CVESegmentationConfig PACKAGE_AND_FUNCTION_LOCATOR_TOOL_NAME = "package_and_function_locator" @@ -77,6 +78,14 @@ "Please retry with the correct format." ) +async def get_git_and_pickle_base_dirs(builder: Builder) ->tuple: + if builder is None: + return () + segmentation_config = builder.get_function_config("cve_segmentation") + if isinstance(segmentation_config, CVESegmentationConfig): + return segmentation_config.base_git_dir, segmentation_config.base_pickle_dir + else: + return () def _summarize_call_chain(call_hierarchy_list: list[Document]) -> list[str]: """Summarize a call chain into concise strings for the agent scratchpad. @@ -209,7 +218,7 @@ def _get_cache_keys(si, query: str) -> tuple[tuple | None, tuple | None]: return None, None -def _build_searcher(si, query: str, uber_jar_file_threshold: int) -> TransitiveCodeSearcher: +def _build_searcher(si, query: str, uber_jar_file_threshold: int, base_dirs : tuple = ()) -> TransitiveCodeSearcher: """Synchronous helper that builds a TransitiveCodeSearcher. Separated so it can be offloaded to a thread via asyncio.to_thread(), @@ -218,12 +227,21 @@ def _build_searcher(si, query: str, uber_jar_file_threshold: int) -> TransitiveC package-specific retrievers via _JavaRepoData (in-memory cache keyed by (git_repo, ref), backed by pickle sub-caches on disk). """ - documents_embedder = DocumentEmbedding(embedding=None) + if len(base_dirs) == 2: + git_base_dir_config, pickle_base_dir_config = base_dirs + documents_embedder = DocumentEmbedding( + embedding=None, + pickle_cache_directory=pickle_base_dir_config, + git_directory=git_base_dir_config, + ) + else: + documents_embedder = DocumentEmbedding(embedding=None) + coc_retriever = get_call_of_chains_retriever(documents_embedder, si, query, uber_jar_file_threshold) return TransitiveCodeSearcher(chain_of_calls_retriever=coc_retriever) -async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int) -> TransitiveCodeSearcher: +async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int, base_dirs: tuple = ()) -> TransitiveCodeSearcher: """Build a TransitiveCodeSearcher, or return a cached one. Cache keys: @@ -239,6 +257,7 @@ async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int) -> - Per-repo serialization via asyncio.Lock (protects shared filesystem: git checkout, install_dependencies, document creation). - The expensive build runs in a thread via asyncio.to_thread(). + :param base_dirs: a tuple contains base dirs of cache - (git_base_dir, pickle_base_dir) """ repo_key, full_key = _get_cache_keys(si, query) @@ -296,7 +315,7 @@ async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int) -> # other tasks can read/write the cache while this build runs, # but no concurrent build on the same repo's filesystem. logger.info("Building TransitiveCodeSearcher for %s", full_key) - searcher = await asyncio.to_thread(_build_searcher, si, query, uber_jar_file_threshold) + searcher = await asyncio.to_thread(_build_searcher, si, query, uber_jar_file_threshold, base_dirs) async with _searcher_cache_lock: # Cache under the appropriate key based on ecosystem @@ -328,13 +347,13 @@ async def _build_or_get_cached(si, query: str, uber_jar_file_threshold: int) -> raise -async def get_transitive_code_searcher(query: str): +async def get_transitive_code_searcher(query: str, base_dirs: tuple): state: AgentMorpheusEngineState = ctx_state.get() si = state.original_input.input.image.source_info threshold = state.uber_jar_file_threshold if state.transitive_code_searcher is None: - state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold) + state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold, base_dirs) elif isinstance(state.transitive_code_searcher.chain_of_calls_retriever, JavaChainOfCallsRetriever): # Java: different queries produce different dep trees (build_tree uses # -DtargetIncludes for GAV queries), so rebuild when the package changes. @@ -346,7 +365,7 @@ async def get_transitive_code_searcher(query: str): if cached is not None and cached is state.transitive_code_searcher: pass # Same searcher, no change needed else: - state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold) + state.transitive_code_searcher = await _build_or_get_cached(si, query, threshold, base_dirs) # Both Java and non-Java retrievers use per-search context objects (_JavaSearchCtx / _SearchCtx) # for mutable state, so the retriever instance is immutable after __init__ and needs no deep copy. @@ -376,7 +395,9 @@ async def _arun(query: str) -> tuple: if not is_valid: return False, [validation_result] transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = await get_transitive_code_searcher(validation_result) + base_dirs = await get_git_and_pickle_base_dirs(builder) + + transitive_code_searcher = await get_transitive_code_searcher(validation_result, base_dirs) found_path, call_hierarchy_list = transitive_code_searcher.search(validation_result) # Return concise call chain summary instead of full Document objects # to avoid blowing up the agent's context window with source code. @@ -427,7 +448,8 @@ async def functions_usage_search(config: CallingFunctionNameExtractorToolConfig, async def _arun(query: str) -> list: coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = await get_transitive_code_searcher(query) + base_dirs = await get_git_and_pickle_base_dirs(builder) + transitive_code_searcher = await get_transitive_code_searcher(query, base_dirs) coc_retriever = transitive_code_searcher.chain_of_calls_retriever function_name_extractor = FunctionNameExtractor(coc_retriever) result = function_name_extractor.fetch_list(query) @@ -458,7 +480,8 @@ async def _arun(query: str) -> dict: return {"error": validation_result} coc_retriever: ChainOfCallsRetrieverBase transitive_code_searcher: TransitiveCodeSearcher - transitive_code_searcher = await get_transitive_code_searcher(validation_result) + base_dirs = await get_git_and_pickle_base_dirs(builder) + transitive_code_searcher = await get_transitive_code_searcher(validation_result, base_dirs) coc_retriever = transitive_code_searcher.chain_of_calls_retriever locator = FunctionNameLocator(coc_retriever) result = await locator.locate_functions(validation_result) @@ -491,7 +514,8 @@ async def library_version_finder(config: FunctionLibraryVersionFinderToolConfig, @catch_tool_errors(FUNCTION_LIBRARY_VERSION_FINDER_TOOL_NAME) async def _arun(query: str) -> dict: - transitive_code_searcher = await get_transitive_code_searcher(query) + base_dirs = await get_git_and_pickle_base_dirs(builder) + transitive_code_searcher = await get_transitive_code_searcher(query, base_dirs) coc_retriever = transitive_code_searcher.chain_of_calls_retriever # Clean the query: strip whitespace, trailing junk after newlines, then quotes (including unicode smart quotes) @@ -535,4 +559,4 @@ async def _arun(query: str) -> dict: Returns: {'ecosystem': str, 'found': bool, 'message': str, 'matching_packages': list}. For Java, matching_packages contains Maven GAV coordinates like 'groupId:artifactId:version'. -""")) \ No newline at end of file +""")) From 7d18be4c2537dd6bad82e03b17d5095d37e5b120 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 24 Jun 2026 12:13:36 +0300 Subject: [PATCH 271/286] test: Adding tests for - 'prevent documents loading to run twice' Signed-off-by: Zvi Grinberg --- .../tools/tests/test_concurrency.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py index 880a627c5..4e62fc9f7 100644 --- a/src/vuln_analysis/tools/tests/test_concurrency.py +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -19,6 +19,8 @@ from vuln_analysis.data_models.state import AgentMorpheusEngineState from vuln_analysis.tools.transitive_code_search import ( _build_or_get_cached, + _build_searcher, + get_git_and_pickle_base_dirs, _searcher_cache, _searcher_building, _repo_build_locks, @@ -468,6 +470,85 @@ async def test_nonjava_caches_under_repo_key(): assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" +# --------------------------------------------------------------------------- +# Base Dirs Cache Path Alignment Tests +# --------------------------------------------------------------------------- + +class TestGetGitAndPickleBaseDirs: + """Tests for get_git_and_pickle_base_dirs — reads config from Builder.""" + + @pytest.mark.asyncio + async def test_returns_empty_tuple_when_builder_is_none(self): + result = await get_git_and_pickle_base_dirs(None) + assert result == () + + @pytest.mark.asyncio + async def test_returns_empty_tuple_when_config_is_not_segmentation(self): + builder = MagicMock() + builder.get_function_config.return_value = "some_other_config" + result = await get_git_and_pickle_base_dirs(builder) + assert result == () + builder.get_function_config.assert_called_once_with("cve_segmentation") + + @pytest.mark.asyncio + async def test_returns_dirs_from_segmentation_config(self): + from vuln_analysis.functions.cve_segmentation import CVESegmentationConfig + config = CVESegmentationConfig( + agent_name="cve_agent_executor", + embedder_name="nim_embedder", + base_git_dir="/data/git", + base_pickle_dir="/data/pickle", + ) + builder = MagicMock() + builder.get_function_config.return_value = config + result = await get_git_and_pickle_base_dirs(builder) + assert result == ("/data/git", "/data/pickle") + + +class TestBuildSearcherBaseDirs: + """Tests for _build_searcher — ensures configured base_dirs reach DocumentEmbedding.""" + + def test_empty_base_dirs_uses_defaults(self): + """Empty tuple should create DocumentEmbedding with default dirs.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=()) + mock_de.assert_called_once_with(embedding=None) + + def test_valid_base_dirs_passed_to_document_embedding(self): + """Two-element tuple should pass git_directory and pickle_cache_directory.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=("/custom/git", "/custom/pickle")) + mock_de.assert_called_once_with( + embedding=None, + pickle_cache_directory="/custom/pickle", + git_directory="/custom/git", + ) + + def test_single_element_tuple_falls_back_to_defaults(self): + """Tuple with wrong length should fall back to defaults, not crash.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=("/only/one",)) + mock_de.assert_called_once_with(embedding=None) + + def test_default_parameter_uses_defaults(self): + """Omitting base_dirs entirely should use defaults.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD) + mock_de.assert_called_once_with(embedding=None) + + # --------------------------------------------------------------------------- # Split Clone/Segmentation Pipeline Tests # --------------------------------------------------------------------------- From 34cafb9147ee3e4e49260d003d39bf3458f9d07d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 24 Jun 2026 18:43:52 +0300 Subject: [PATCH 272/286] chore: add missing workflow stages, LLM Tools and config settings Signed-off-by: Zvi Grinberg --- kustomize/base/exploit-iq-config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml index 819084623..e5a9363bd 100644 --- a/kustomize/base/exploit-iq-config.yml +++ b/kustomize/base/exploit-iq-config.yml @@ -58,6 +58,11 @@ functions: cve_process_sbom: _type: cve_process_sbom + cve_verify_vuln_package: + _type: cve_verify_vuln_package + skip: false + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + llm_name: checklist_llm cve_checklist: _type: cve_checklist llm_name: checklist_llm @@ -96,6 +101,13 @@ functions: max_retries: 5 Container Analysis Data: _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm @@ -108,6 +120,8 @@ functions: - Function Caller Finder - Function Locator - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false @@ -116,6 +130,7 @@ functions: return_intermediate_steps: false # transitive_search_tool_enabled: false cve_web_search_enabled: true + uber_jar_file_threshold: 600 verbose: false cve_generate_cvss: _type: cve_generate_cvss @@ -195,6 +210,8 @@ functions: tool_names: - Source Grep - Code Keyword Search + cve_fetch_patches: + _type: cve_fetch_patches health_check: _type: health_check @@ -280,6 +297,7 @@ workflow: cve_fetch_intel_name: cve_fetch_intel cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom + cve_verify_vuln_package_name: cve_verify_vuln_package cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor cve_generate_cvss_name: cve_generate_cvss @@ -292,6 +310,7 @@ workflow: cve_package_code_agent_name: cve_package_code_agent cve_checker_report_name: cve_checker_report cve_build_agent_name: cve_build_agent + cve_fetch_patches_name: cve_fetch_patches eval: general: From 3758c2daf9cd79a875725a31efc36dc6521d9668 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 18:46:07 +0300 Subject: [PATCH 273/286] chore(kustomize): move OIDC CA trust to optional overlay --- kustomize/base/kustomization.yaml | 7 ------ kustomize/overlays/oidc-ca/kustomization.yaml | 11 ++++++++++ kustomize/overlays/oidc-ca/oidc-ca-patch.yaml | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 kustomize/overlays/oidc-ca/kustomization.yaml create mode 100644 kustomize/overlays/oidc-ca/oidc-ca-patch.yaml diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 6b2da8d3c..26f9fc41a 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -79,13 +79,6 @@ configMapGenerator: - ca-certs/ca-bundle.crt options: disableNameSuffixHash: true - - - name: oidc-ca-bundle - files: - - oidc-ca/ca-bundle.crt - options: - disableNameSuffixHash: true - images: - name: quay.io/ecosystem-appeng/agent-morpheus-rh newTag: latest diff --git a/kustomize/overlays/oidc-ca/kustomization.yaml b/kustomize/overlays/oidc-ca/kustomization.yaml new file mode 100644 index 000000000..3f3d7bc60 --- /dev/null +++ b/kustomize/overlays/oidc-ca/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: oidc-ca-patch.yaml + target: + name: exploit-iq-client + kind: Deployment diff --git a/kustomize/overlays/oidc-ca/oidc-ca-patch.yaml b/kustomize/overlays/oidc-ca/oidc-ca-patch.yaml new file mode 100644 index 000000000..e61335c2f --- /dev/null +++ b/kustomize/overlays/oidc-ca/oidc-ca-patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client +spec: + template: + spec: + containers: + - name: exploit-iq-client + env: + - name: QUARKUS_TLS__OIDC__TRUST_STORE_PEM_CERTS + value: /etc/oidc-ca/ca-bundle.crt + - name: QUARKUS_OIDC_TLS_TLS_CONFIGURATION_NAME + value: oidc + volumeMounts: + - name: oidc-ca + mountPath: /etc/oidc-ca + readOnly: true + volumes: + - name: oidc-ca + configMap: + name: oidc-ca-bundle From 25e94e1177fd5885c06eb90316db587b7d51b4b0 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 18:46:26 +0300 Subject: [PATCH 274/286] fix(kustomize): add NIM embed upstream to self-hosted nginx patch --- .../overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml index b3d598543..8e3fb5e70 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml @@ -10,5 +10,7 @@ spec: env: - name: NGINX_UPSTREAM_NIM_LLM value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 - name: NGINX_UPSTREAM_OPENAI value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 From da3f99ffa5d1b8add7168bf0ccacf2a3ce442b5e Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 20:41:53 +0300 Subject: [PATCH 275/286] feat(kustomize): add deployer-rbac.yaml for non-cluster-admin deployments --- kustomize/deployer-rbac.yaml | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 kustomize/deployer-rbac.yaml diff --git a/kustomize/deployer-rbac.yaml b/kustomize/deployer-rbac.yaml new file mode 100644 index 000000000..8420fb9d1 --- /dev/null +++ b/kustomize/deployer-rbac.yaml @@ -0,0 +1,94 @@ +# deployer-rbac.yaml +# +# Grants a non-cluster-admin user the minimum permissions required to +# deploy Exploit Intelligence on OpenShift Container Platform. +# +# Please replace the following placeholders: +# — the OpenShift username of the deployer (e.g. jdoe) +# — the target namespace (e.g. exploit-iq) +# +# NOTE: The OAuthClient permissions (get, patch) can be further restricted +# to specific resource names using the resourceNames field. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: exploit-intelligence-oauthclient-deploy +rules: + - apiGroups: + - oauth.openshift.io + resources: + - oauthclients + verbs: + - get + - list + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: exploit-intelligence-oauthclient-deploy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: exploit-intelligence-oauthclient-deploy +subjects: + - kind: User + name: +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exploit-intelligence-rbac-deploy + namespace: +rules: + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - get + - create + - update + - patch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create + - update + - patch + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - anyuid + verbs: + - use + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exploit-intelligence-rbac-deploy + namespace: +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exploit-intelligence-rbac-deploy +subjects: + - kind: User + name: From c2f7fc78f44522b55d898d4d025a8735153c2911 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 20:42:16 +0300 Subject: [PATCH 276/286] docs(kustomize): revamp deployment guide and update gitignore --- .gitignore | 5 + kustomize/README.md | 382 ++++++++++++++++++++++++++------------------ 2 files changed, 229 insertions(+), 158 deletions(-) diff --git a/.gitignore b/.gitignore index 3ef4a4930..84537bbeb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ ###### Place new entries directly below this line! ###### +kustomize/overlays/oidc-ca/ca-bundle.crt +kustomize/overlays/oidc-ca/oidc-oauth-ca.crt +kustomize/overlays/oidc-ca/oidc-api-ca.crt +kustomize/overlays/oidc-ca/*.crt + # Ignore anything in the ./.tmp directory .tmp/ diff --git a/kustomize/README.md b/kustomize/README.md index 275253eda..9b9002f9e 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -15,27 +15,70 @@ See the License for the specific language governing permissions and limitations under the License. --> +- [Deploying Exploit Intelligence on OpenShift Container Platform](#deploying-exploit-intelligence-on-openshift-container-platform) + - [Prerequisites](#prerequisites) + - [Required Tools](#required-tools) + - [Cluster Access](#cluster-access) + - [API Credentials](#api-credentials) + - [Preparing Your Deployment](#preparing-your-deployment) + - [Step 1. Create a Project Namespace](#step-1-create-a-project-namespace) + - [Step 2. Create API Credentials](#step-2-create-api-credentials) + - [Step 3. Configure Image Registry Credentials](#step-3-configure-image-registry-credentials) + - [Step 4. Configure Argilla Feedback Credentials](#step-4-configure-argilla-feedback-credentials) + - [Step 5. Configure OAuth Authentication](#step-5-configure-oauth-authentication) + - [Option A: Create a New OAuthClient](#option-a-create-a-new-oauthclient) + - [Option B: Reuse an Existing OAuthClient](#option-b-reuse-an-existing-oauthclient) + - [Step 6. Create Database Credentials](#step-6-create-database-credentials) + - [Step 7. Set the Application Callback URL](#step-7-set-the-application-callback-url) + - [Selecting a Deployment Variant](#selecting-a-deployment-variant) + - [Deploying Exploit Intelligence](#deploying-exploit-intelligence) + - [Deploy with a Self-Hosted LLM](#deploy-with-a-self-hosted-llm) + - [Deploy with a Self-Hosted LLM and MLOps Observability](#deploy-with-a-self-hosted-llm-and-mlops-observability) + - [Deploy with Remote NIM](#deploy-with-remote-nim) + - [Post-Deployment Configuration](#post-deployment-configuration) + - [Enable OAuth for the MCP Server](#enable-oauth-for-the-mcp-server) + - [Configure the Swagger UI Endpoint](#configure-the-swagger-ui-endpoint) + - [Configure a Custom Git Server CA](#configure-a-custom-git-server-ca) + - [Configure the OAuth Endpoint CA](#configure-the-oauth-endpoint-ca) + - [Uninstalling Exploit Intelligence](#uninstalling-exploit-intelligence) + - [Running Exploit Intelligence Locally](#running-exploit-intelligence-locally) + - [Running the Application Locally](#running-the-application-locally) + - [Enabling Container Source Download](#enabling-container-source-download) + - [Deploying the Test Variant](#deploying-the-test-variant) + - [Deploying the Test Overlay](#deploying-the-test-overlay) + - [Tearing Down the Test Environment](#tearing-down-the-test-environment) + - [Installing Pipelines as Code](#installing-pipelines-as-code) + + # Deploying Exploit Intelligence on OpenShift Container Platform ## Prerequisites +> [!NOTE] +> All commands in this guide assume that your current working directory is `kustomize/`. + +### Required Tools + Install the following tools and verify that all binaries are available on your system path: - [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) - [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) - `openssl` -Obtain credentials for the following services before you begin. For instructions on creating each credential, refer to the main [README](../README.md#obtain-api-keys): +### Cluster Access + +Cluster-admin privileges are required. `OAuthClient` creation, anyuid SCC grants, and `ServiceMonitor` access are all cluster-scoped operations. If cluster-admin access is not available, a cluster administrator can optionally apply `deployer-rbac.yaml` to delegate the minimum required permissions. + +### API Credentials + +Obtain the following credentials before you begin. For instructions on creating each credential, refer to the main [README](../README.md#obtain-api-keys): - GitHub Advisory Database (GHSA) API key - National Vulnerability Database (NVD) API key -- NVIDIA API key (required for Remote NIM deployments; a placeholder value is sufficient for self-hosted variants) +- NVIDIA API key (required for Remote NIM; a placeholder value is sufficient for self-hosted variants) - Red Hat registry credentials (`registry.redhat.io`) - SerpAPI key -> [!NOTE] -> All commands in this guide assume that your current working directory is `kustomize/`. - --- ## Preparing Your Deployment @@ -75,8 +118,13 @@ EOF ### Step 3. Configure Image Registry Credentials +> [!IMPORTANT] +> Product scanning requires valid registry credentials to pull product component images. If you skip this step, the deployment succeeds but authenticated image pulls fail. + Create the `base/image-registry-credentials.env` file with credentials for the registries that product scanning uses to pull component images. The `auth` value is the base64 encoding of `:`. +For `registry.redhat.io`, Red Hat recommends creating a registry service account at [https://access.redhat.com/terms-based-registry/](https://access.redhat.com/terms-based-registry/) + ```shell cat > base/image-registry-credentials.env << 'EOF' { @@ -89,12 +137,11 @@ cat > base/image-registry-credentials.env << 'EOF' EOF ``` -> [!IMPORTANT] -> Product scanning requires valid registry credentials to pull component images. If you skip this step, the deployment succeeds but authenticated image pulls fail. - ### Step 4. Configure Argilla Feedback Credentials -Create the `base/argilla/feedback_secret.env` file with credentials for the Argilla feedback service: +Argilla runs as an internal cluster service and does not connect to any external account. Use any values for the username, password, and API key. + +Create the `base/argilla/feedback_secret.env` file: ```shell cat > base/argilla/feedback_secret.env << EOF @@ -106,29 +153,44 @@ EOF ### Step 5. Configure OAuth Authentication -Exploit Intelligence uses OpenShift OAuth for user authentication. Select the option that matches your situation. +By default Exploit Intelligence uses OpenShift OAuth for user authentication. The OAuth client secret must be at least 32 bytes (256 bits) because Exploit Intelligence uses it to sign internal session tokens with HS256, which requires a minimum key length of 256 bits. + +Select the option that matches your situation. #### Option A: Create a New OAuthClient -Use this option when deploying Exploit Intelligence for the first time. Generate a new client secret and write the credentials file: +Use this option when deploying Exploit Intelligence for the first time. Generate a new client secret, write the credentials file, and create the `OAuthClient` resource: ```shell -export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32) export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') cat > base/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET session-secret=$OAUTH_SESSION_SECRET -openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +openshift-domain=$OCP_DOMAIN EOF ``` -> [!IMPORTANT] -> Save the value of `$OAUTH_CLIENT_SECRET`. You need it when you create the `OAuthClient` resource after deployment. +```shell +oc create -f - < base/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET session-secret=$OAUTH_SESSION_SECRET -openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +openshift-domain=$OCP_DOMAIN EOF ``` +```shell +oc patch oauthclient exploit-iq-client -p "{\"redirectURIs\":[ + \"http://exploit-iq-client:8080\", + \"https://exploit-iq-client.${YOUR_NAMESPACE_NAME}.apps.${OCP_DOMAIN}\", + \"http://exploit-iq-client.${YOUR_NAMESPACE_NAME}.apps.${OCP_DOMAIN}\" +]}" +``` + ### Step 6. Create Database Credentials ```shell cat > base/mongodb-credentials.env << EOF admin-user=mongoadmin -admin-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) +admin-password=$(openssl rand -base64 24) exploit-iq-user=exploit-iq-user -exploit-iq-password=$(openssl rand -base64 24 | tr -d '/+=' | head -c 32) +exploit-iq-password=$(openssl rand -base64 24) EOF ``` @@ -172,86 +243,6 @@ export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + ``` -### Step 8. Configure a Custom Git Server CA - -Complete this step if your Git server uses a certificate signed by a custom Certificate Authority (CA). - -> [!IMPORTANT] -> Complete this step if you access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`. - -**1.** Create the certificate directory: - -```shell -mkdir -p base/ca-certs -``` - -**2.** Obtain your CA certificates. - -For Red Hat internal Git repositories: - -```shell -curl -o base/ca-certs/internal-root-ca.pem \ - https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem - -curl -o base/ca-certs/rhcs-intermediate-ca.crt \ - https://certs.corp.redhat.com/chains/rhcs-ca-chain-2022-self-signed.crt -``` - -For other custom CAs: - -```shell -cp /path/to/your-custom-ca.pem base/ca-certs/ -``` - -**3.** Create the CA bundle: - -```shell -cat base/ca-certs/*.{pem,crt} > base/ca-certs/ca-bundle.crt -``` - -**4.** Verify the bundle: - -```shell -openssl crl2pkcs7 -nocrl -certfile base/ca-certs/ca-bundle.crt | \ - openssl pkcs7 -print_certs -noout -``` - -### Step 9. Configure the OAuth Endpoint CA - -Complete this step if your OCP ingress router uses a CA that is not trusted by default. - -> [!IMPORTANT] -> If you see the error `PKIX path building failed` in the `exploit-iq-client` pod logs after deployment, complete this step and redeploy. - -**1.** Create the certificate directory: - -```shell -mkdir -p base/oidc-ca -``` - -**2.** Fetch the CA chain from the OAuth endpoint: - -```shell -openssl s_client \ - -connect oauth-openshift.apps.$(oc get dns cluster -o jsonpath='{.spec.baseDomain}'):443 \ - -showcerts 2>/dev/null | awk '/BEGIN CERT/,/END CERT/' \ - > base/oidc-ca/ca-bundle.crt -``` - -Or fetch the CA bundle from the cluster: - -```shell -kubectl get cm default-ingress-cert -n openshift-config-managed \ - -o jsonpath='{.data.ca-bundle\.crt}' > base/oidc-ca/ca-bundle.crt -``` - -**3.** Verify the bundle: - -```shell -openssl crl2pkcs7 -nocrl -certfile base/oidc-ca/ca-bundle.crt | \ - openssl pkcs7 -print_certs -noout -``` - --- ## Selecting a Deployment Variant @@ -260,8 +251,8 @@ Exploit Intelligence supports the following deployment variants. Run only one de | Variant | Overlay | LLM | Use When | | --- | --- | --- | --- | -| Self-Hosted LLM | `self-hosted-llama3.1-70b-4bit` | On-cluster GPU node | You have GPU capacity in the cluster | -| Self-Hosted LLM + MLOps | `mlops` | On-cluster GPU node | You need Grafana and Tempo observability | +| Self-Hosted LLM | `self-hosted-llama3.1-70b-4bit` | Any network-accessible LLM endpoint | You run your own LLM (on-cluster or external) | +| Self-Hosted LLM + MLOps | `mlops` | Any network-accessible LLM endpoint | You also need Grafana and Tempo observability | | Remote NIM | `remote-nim-all` | NVIDIA-hosted NIM | You use NVIDIA-hosted inference | --- @@ -270,6 +261,8 @@ Exploit Intelligence supports the following deployment variants. Run only one de ### Deploy with a Self-Hosted LLM +This overlay assumes the LLM is deployed using the [exploit-iq-models](https://github.com/RHEcosystemAppEng/exploit-iq-models) Helm chart, which creates the `llama3-1-70b-instruct-4bit` service in the `exploit-iq-models` namespace. If your LLM is deployed differently, update the upstream URLs in `overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml` to point to your model endpoint before deploying. + ```shell oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` @@ -319,93 +312,166 @@ oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME --- -## Configuring OpenShift OAuth - -> [!WARNING] -> Complete this step before attempting to log in to the Exploit Intelligence UI. Authentication fails if the `OAuthClient` resource is not configured correctly. +## Post-Deployment Configuration -After the deployment completes and the `exploit-iq-client` route is available, configure the OpenShift OAuth client. +### Enable OAuth for the MCP Server -### Create the OAuthClient Resource +Complete this step if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. -Create the `OAuthClient` resource using the secret from [Step 6, Option A](#option-a-create-a-new-oauthclient): +If the `OAuthClient` resource does not exist, create it: ```shell oc create -f - < | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +To restrict MCP access to specific OpenShift groups, set the `EXPLOITIQ_ALLOWED_GROUPS` environment variable in `base/exploitiq_mcp_server.yaml`. The default value is `exploitiq-users`. To allow any authenticated user, remove the environment variable entirely. -oc patch oauthclient exploit-iq-client \ - -p '{"redirectURIs":["'$HTTP_ROUTE'","'$HTTPS_ROUTE'"]}' +### Configure the Swagger UI Endpoint + +Complete this step to enable the Swagger UI endpoint for API testing. This configuration applies to non-production environments only. + +```shell +oc set env deployment -l component=exploit-iq-client \ + QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') ``` ---- +### Configure a Custom Git Server CA -## Post-Deployment Configuration +Complete this step if your Git server uses a certificate signed by a custom Certificate Authority (CA). -### Enable OAuth for the MCP Server +> [!IMPORTANT] +> Complete this step if you access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`. -Complete this step if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. +**1.** Create the certificate directory: -If the `OAuthClient` resource does not exist, create it: +```shell +mkdir -p base/ca-certs +``` + +**2.** Obtain your CA certificates. + +For Red Hat internal Git repositories: ```shell -oc create -f - < base/ca-certs/ca-bundle.crt ``` -Uncomment the OAuth environment variables in `base/exploitiq_mcp_server.yaml` and redeploy: +**4.** Verify the bundle: + +```shell +openssl crl2pkcs7 -nocrl -certfile base/ca-certs/ca-bundle.crt | \ + openssl pkcs7 -print_certs -noout +``` + +Then redeploy: ```shell oc kustomize overlays/ | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -To restrict MCP access to specific OpenShift groups, set the `EXPLOITIQ_ALLOWED_GROUPS` environment variable in `base/exploitiq_mcp_server.yaml`. The default value is `exploitiq-users`. To allow any authenticated user, remove the environment variable entirely. +### Configure the OAuth Endpoint CA -### Configure the Swagger UI Endpoint +Complete this step if you see the error `PKIX path building failed` in the `exploit-iq-client` pod logs after deployment. -Complete this step to enable the Swagger UI endpoint for API testing. This configuration applies to non-production environments only. +**1.** Fetch and merge CA chains from both OpenShift OIDC endpoints: + +- `oauth-openshift.apps.` (authorization/token endpoint chain) +- `api.:6443` (JWKS and user-info endpoint chain) + +The command extracts only CA certificates (skips endpoint leaf certificates, which are not trust anchors): ```shell -oc set env deployment -l component=exploit-iq-client \ - QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') +if [ -d overlays/oidc-ca ]; then + export OIDC_CA_DIR=overlays/oidc-ca +else + export OIDC_CA_DIR=kustomize/overlays/oidc-ca +fi +mkdir -p "${OIDC_CA_DIR}" + +openssl s_client -connect oauth-openshift.apps.${OCP_DOMAIN}:443 \ + -servername oauth-openshift.apps.${OCP_DOMAIN} \ + -showcerts /dev/null \ + | awk '/BEGIN CERT/,/END CERT/' \ + | awk 'BEGIN{n=0;buf=""} /BEGIN CERT/{n++;buf=""} {buf=buf $0"\n"} /END CERT/{if(n>1) printf buf}' \ + > "${OIDC_CA_DIR}/oidc-oauth-ca.crt" + +openssl s_client -connect api.${OCP_DOMAIN}:6443 \ + -servername api.${OCP_DOMAIN} \ + -showcerts /dev/null \ + | awk '/BEGIN CERT/,/END CERT/' \ + | awk 'BEGIN{n=0;buf=""} /BEGIN CERT/{n++;buf=""} {buf=buf $0"\n"} /END CERT/{if(n>1) printf buf}' \ + > "${OIDC_CA_DIR}/oidc-api-ca.crt" +``` + +**2.** Verify the bundle: + +```shell +cat "${OIDC_CA_DIR}/oidc-oauth-ca.crt" "${OIDC_CA_DIR}/oidc-api-ca.crt" > "${OIDC_CA_DIR}/ca-bundle.crt" + +openssl crl2pkcs7 -nocrl -certfile "${OIDC_CA_DIR}/ca-bundle.crt" | \ + openssl pkcs7 -print_certs -noout +``` + +**3.** Create or update the `oidc-ca-bundle` ConfigMap in your namespace (no certificate files are stored in this repository): + +```shell +oc -n $YOUR_NAMESPACE_NAME create configmap oidc-ca-bundle \ + --from-file=ca-bundle.crt="${OIDC_CA_DIR}/ca-bundle.crt" \ + --dry-run=client -o yaml | oc apply -f - ``` +**4.** Apply the `oidc-ca` overlay. It patches the `exploit-iq-client` deployment to use `oidc-ca-bundle` as the named OIDC TLS trust store: + +```shell +oc kustomize overlays/oidc-ca | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +> [!NOTE] +> **Dev/temporary workaround:** If you need to unblock quickly without the certificate setup, set the following environment variable on the deployment. This disables TLS verification entirely and must not be used in production environments. +> +> ```shell +> oc set env deployment/exploit-iq-client QUARKUS_TLS_TRUST_ALL=true -n $YOUR_NAMESPACE_NAME +> ``` + --- ## Uninstalling Exploit Intelligence @@ -569,15 +635,17 @@ export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{ Otherwise, generate a new secret (minimum 32 characters): ```shell -export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32 | tr -d '/+=' | head -c 40) +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32) ``` Write the credentials file: ```shell +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') + cat > secrets/oauth-secrets.env << EOF client-secret=$OAUTH_CLIENT_SECRET -openshift-domain=$(oc get dns cluster -o jsonpath='{.spec.baseDomain}') +openshift-domain=$OCP_DOMAIN EOF ``` @@ -610,19 +678,19 @@ grantMethod: prompt secret: $OAUTH_CLIENT_SECRET redirectURIs: - "http://exploit-iq-client:8080" - - "https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')" - - "http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}')" + - "https://exploit-iq-client.${PROJECT_NAME}.apps.${OCP_DOMAIN}" + - "http://exploit-iq-client.${PROJECT_NAME}.apps.${OCP_DOMAIN}" EOF ``` If the resource already exists, add the new route to the redirect URIs: ```shell -export HTTPS_ROUTE=https://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') -export HTTP_ROUTE=http://$(oc get route exploit-iq-client -o jsonpath='{.spec.host}') - -oc patch oauthclient exploit-iq-client \ - -p '{"redirectURIs":["'$HTTP_ROUTE'","'$HTTPS_ROUTE'"]}' +oc patch oauthclient exploit-iq-client -p "{\"redirectURIs\":[ + \"http://exploit-iq-client:8080\", + \"https://exploit-iq-client.${PROJECT_NAME}.apps.${OCP_DOMAIN}\", + \"http://exploit-iq-client.${PROJECT_NAME}.apps.${OCP_DOMAIN}\" +]}" ``` **9.** Complete this step if you want MCP clients to authenticate with OpenShift OAuth. @@ -638,17 +706,15 @@ metadata: grantMethod: prompt secret: $OAUTH_CLIENT_SECRET redirectURIs: - - "https://$(oc get route exploitiq-mcp-server -o jsonpath='{.spec.host}')/oauth/callback" + - "https://exploitiq-mcp-server.${PROJECT_NAME}.apps.${OCP_DOMAIN}/oauth/callback" EOF ``` If it already exists, add the callback URI: ```shell -export MCP_CALLBACK=https://$(oc get route exploitiq-mcp-server -o jsonpath='{.spec.host}')/oauth/callback - oc patch oauthclient exploitiq-mcp-server \ - -p '{"redirectURIs":["'$MCP_CALLBACK'"]}' + -p "{\"redirectURIs\":[\"https://exploitiq-mcp-server.${PROJECT_NAME}.apps.${OCP_DOMAIN}/oauth/callback\"]}" ``` Then redeploy: From 3df693eda4792dc4ce1e54962ffc0b2efdebdb7f Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 20:53:18 +0300 Subject: [PATCH 277/286] docs(kustomize): add UI access section and remove TOC --- kustomize/README.md | 43 +++++++++---------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 9b9002f9e..577a13d32 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -15,40 +15,6 @@ See the License for the specific language governing permissions and limitations under the License. --> -- [Deploying Exploit Intelligence on OpenShift Container Platform](#deploying-exploit-intelligence-on-openshift-container-platform) - - [Prerequisites](#prerequisites) - - [Required Tools](#required-tools) - - [Cluster Access](#cluster-access) - - [API Credentials](#api-credentials) - - [Preparing Your Deployment](#preparing-your-deployment) - - [Step 1. Create a Project Namespace](#step-1-create-a-project-namespace) - - [Step 2. Create API Credentials](#step-2-create-api-credentials) - - [Step 3. Configure Image Registry Credentials](#step-3-configure-image-registry-credentials) - - [Step 4. Configure Argilla Feedback Credentials](#step-4-configure-argilla-feedback-credentials) - - [Step 5. Configure OAuth Authentication](#step-5-configure-oauth-authentication) - - [Option A: Create a New OAuthClient](#option-a-create-a-new-oauthclient) - - [Option B: Reuse an Existing OAuthClient](#option-b-reuse-an-existing-oauthclient) - - [Step 6. Create Database Credentials](#step-6-create-database-credentials) - - [Step 7. Set the Application Callback URL](#step-7-set-the-application-callback-url) - - [Selecting a Deployment Variant](#selecting-a-deployment-variant) - - [Deploying Exploit Intelligence](#deploying-exploit-intelligence) - - [Deploy with a Self-Hosted LLM](#deploy-with-a-self-hosted-llm) - - [Deploy with a Self-Hosted LLM and MLOps Observability](#deploy-with-a-self-hosted-llm-and-mlops-observability) - - [Deploy with Remote NIM](#deploy-with-remote-nim) - - [Post-Deployment Configuration](#post-deployment-configuration) - - [Enable OAuth for the MCP Server](#enable-oauth-for-the-mcp-server) - - [Configure the Swagger UI Endpoint](#configure-the-swagger-ui-endpoint) - - [Configure a Custom Git Server CA](#configure-a-custom-git-server-ca) - - [Configure the OAuth Endpoint CA](#configure-the-oauth-endpoint-ca) - - [Uninstalling Exploit Intelligence](#uninstalling-exploit-intelligence) - - [Running Exploit Intelligence Locally](#running-exploit-intelligence-locally) - - [Running the Application Locally](#running-the-application-locally) - - [Enabling Container Source Download](#enabling-container-source-download) - - [Deploying the Test Variant](#deploying-the-test-variant) - - [Deploying the Test Overlay](#deploying-the-test-overlay) - - [Tearing Down the Test Environment](#tearing-down-the-test-environment) - - [Installing Pipelines as Code](#installing-pipelines-as-code) - # Deploying Exploit Intelligence on OpenShift Container Platform @@ -314,6 +280,15 @@ oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ## Post-Deployment Configuration +### Grant Users Access to the Exploit Intelligence UI + +Access to the Exploit Intelligence UI is controlled by OpenShift group membership. Add users to the `exploit-iq-view` group to grant UI access. Create the group if it does not exist: + +```shell +oc adm groups new exploit-iq-view +oc adm groups add-users exploit-iq-view +``` + ### Enable OAuth for the MCP Server Complete this step if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. From a794fb1d5885f62d7ccea5f2b3d793145d831118 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Wed, 24 Jun 2026 21:01:34 +0300 Subject: [PATCH 278/286] chore: simplify oidc-ca gitignore to wildcard pattern --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 84537bbeb..eec16236b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ ###### Place new entries directly below this line! ###### -kustomize/overlays/oidc-ca/ca-bundle.crt -kustomize/overlays/oidc-ca/oidc-oauth-ca.crt -kustomize/overlays/oidc-ca/oidc-api-ca.crt kustomize/overlays/oidc-ca/*.crt # Ignore anything in the ./.tmp directory From 030d3eb6467446398d34dec1436e28227b20fc4d Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 25 Jun 2026 12:49:49 +0300 Subject: [PATCH 279/286] fix(kustomize): restrict OAuthClient RBAC to specific resource names --- kustomize/deployer-rbac.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/kustomize/deployer-rbac.yaml b/kustomize/deployer-rbac.yaml index 8420fb9d1..8be695176 100644 --- a/kustomize/deployer-rbac.yaml +++ b/kustomize/deployer-rbac.yaml @@ -7,8 +7,6 @@ # — the OpenShift username of the deployer (e.g. jdoe) # — the target namespace (e.g. exploit-iq) # -# NOTE: The OAuthClient permissions (get, patch) can be further restricted -# to specific resource names using the resourceNames field. --- apiVersion: rbac.authorization.k8s.io/v1 @@ -16,15 +14,24 @@ kind: ClusterRole metadata: name: exploit-intelligence-oauthclient-deploy rules: + # get and patch scoped to the two project OAuthClients only. - apiGroups: - oauth.openshift.io resources: - oauthclients + resourceNames: + - exploit-iq-client + - exploitiq-mcp-server verbs: - get - - list - - create - patch + # create cannot be restricted by resourceNames (resource does not exist yet). + - apiGroups: + - oauth.openshift.io + resources: + - oauthclients + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From f26ad4d3e5cf2fabb05ca00fa6c7a983a10ef20b Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 25 Jun 2026 12:50:05 +0300 Subject: [PATCH 280/286] refactor(kustomize): convert oidc-ca overlay to reusable component --- .gitignore | 2 +- kustomize/README.md | 11 ++++------- kustomize/components/oidc-ca/kustomization.yaml | 15 +++++++++++++++ .../oidc-ca/oidc-ca-patch.yaml | 0 kustomize/overlays/oidc-ca/kustomization.yaml | 11 ----------- 5 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 kustomize/components/oidc-ca/kustomization.yaml rename kustomize/{overlays => components}/oidc-ca/oidc-ca-patch.yaml (100%) delete mode 100644 kustomize/overlays/oidc-ca/kustomization.yaml diff --git a/.gitignore b/.gitignore index eec16236b..97c68c84f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ ###### Place new entries directly below this line! ###### -kustomize/overlays/oidc-ca/*.crt +kustomize/components/oidc-ca/*.crt # Ignore anything in the ./.tmp directory .tmp/ diff --git a/kustomize/README.md b/kustomize/README.md index 577a13d32..ab36c90e3 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -395,11 +395,7 @@ The command extracts only CA certificates (skips endpoint leaf certificates, whi ```shell export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') -if [ -d overlays/oidc-ca ]; then - export OIDC_CA_DIR=overlays/oidc-ca -else - export OIDC_CA_DIR=kustomize/overlays/oidc-ca -fi +export OIDC_CA_DIR=components/oidc-ca mkdir -p "${OIDC_CA_DIR}" openssl s_client -connect oauth-openshift.apps.${OCP_DOMAIN}:443 \ @@ -434,10 +430,11 @@ oc -n $YOUR_NAMESPACE_NAME create configmap oidc-ca-bundle \ --dry-run=client -o yaml | oc apply -f - ``` -**4.** Apply the `oidc-ca` overlay. It patches the `exploit-iq-client` deployment to use `oidc-ca-bundle` as the named OIDC TLS trust store: +**4.** Add the `oidc-ca` component to your deployment variant and redeploy. The component patches the `exploit-iq-client` deployment to mount the CA bundle and configure the named OIDC TLS trust store: ```shell -oc kustomize overlays/oidc-ca | oc apply -f - -n $YOUR_NAMESPACE_NAME +(cd overlays/ && kustomize edit add component ../../components/oidc-ca) +oc kustomize overlays/ | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` > [!NOTE] diff --git a/kustomize/components/oidc-ca/kustomization.yaml b/kustomize/components/oidc-ca/kustomization.yaml new file mode 100644 index 000000000..e0887cfe9 --- /dev/null +++ b/kustomize/components/oidc-ca/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +configMapGenerator: + - name: oidc-ca-bundle + files: + - ca-bundle.crt + options: + disableNameSuffixHash: true + +patches: + - path: oidc-ca-patch.yaml + target: + name: exploit-iq-client + kind: Deployment diff --git a/kustomize/overlays/oidc-ca/oidc-ca-patch.yaml b/kustomize/components/oidc-ca/oidc-ca-patch.yaml similarity index 100% rename from kustomize/overlays/oidc-ca/oidc-ca-patch.yaml rename to kustomize/components/oidc-ca/oidc-ca-patch.yaml diff --git a/kustomize/overlays/oidc-ca/kustomization.yaml b/kustomize/overlays/oidc-ca/kustomization.yaml deleted file mode 100644 index 3f3d7bc60..000000000 --- a/kustomize/overlays/oidc-ca/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../base - -patches: - - path: oidc-ca-patch.yaml - target: - name: exploit-iq-client - kind: Deployment From 5812de1426e85dcaa6505d88c20fa5e29b3e0c67 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 25 Jun 2026 12:50:14 +0300 Subject: [PATCH 281/286] refactor(kustomize): extract inline patches to separate files across overlays --- .../exploit-iq-client-batch-patch.yaml | 31 ++++ .../exploit-iq-resources-patch.yaml | 32 +++++ .../batch-processing/kustomization.yaml | 90 ++---------- .../remote-nim-all/exploit-iq-nim-patch.yaml | 56 ++++++++ .../remote-nim-all/kustomization.yaml | 81 +---------- .../exploit-iq-llm-patch.yaml | 88 ++++++++++++ .../kustomization.yaml | 136 +----------------- 7 files changed, 228 insertions(+), 286 deletions(-) create mode 100644 kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml create mode 100644 kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml create mode 100644 kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml create mode 100644 kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml diff --git a/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml b/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml new file mode 100644 index 000000000..a753910a2 --- /dev/null +++ b/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client + spec: + imagePullSecrets: [] + serviceAccountName: exploit-iq-client-sa + containers: + - name: exploit-iq-client + imagePullPolicy: Always + env: + - name: MORPHEUS_QUEUE_TIMEOUT + value: 60m + - name: MORPHEUS_QUEUE_MAX_ACTIVE + value: "5" diff --git a/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml b/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml new file mode 100644 index 000000000..195e8792d --- /dev/null +++ b/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq-service + imagePullPolicy: Always + workingDir: /workspace/ + # Deploy with QoS(Guaranteed) + resources: + limits: + memory: "12Gi" + cpu: "1000m" + nvidia.com/gpu: "1" + requests: + memory: "12Gi" + cpu: "1000m" + nvidia.com/gpu: "1" diff --git a/kustomize/overlays/batch-processing/kustomization.yaml b/kustomize/overlays/batch-processing/kustomization.yaml index d2d4de172..5d7cc1acd 100644 --- a/kustomize/overlays/batch-processing/kustomization.yaml +++ b/kustomize/overlays/batch-processing/kustomization.yaml @@ -1,86 +1,22 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- ../local-llama3.1-70b-4bit/ +resources: +- ../self-hosted-llama3.1-70b-4bit commonAnnotations: deployment-variant: batch-processing -patchesStrategicMerge: - - |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: exploit-iq - labels: - app: exploit-iq - component: exploit-iq - spec: - selector: - matchLabels: - app: exploit-iq - component: exploit-iq - template: - metadata: - labels: - app: exploit-iq - component: exploit-iq - spec: - containers: - - name: exploit-iq-service - imagePullPolicy: Always - workingDir: /workspace/ - # Deploy with QoS(Guaranteed) - resources: - limits: - memory: "12Gi" - cpu: "1000m" - nvidia.com/gpu: "1" - requests: - memory: "12Gi" - cpu: "1000m" - nvidia.com/gpu: "1" - - |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: exploit-iq-client - labels: - app: exploit-iq - component: exploit-iq-client - spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: exploit-iq - component: exploit-iq-client - template: - metadata: - labels: - app: exploit-iq - component: exploit-iq-client - spec: - imagePullSecrets: [] - serviceAccountName: exploit-iq-client-sa - containers: - - name: exploit-iq-client - imagePullPolicy: Always - env: - - name: MORPHEUS_QUEUE_TIMEOUT - value: 60m - - name: MORPHEUS_QUEUE_MAX_ACTIVE - value: "5" +patches: +- path: exploit-iq-resources-patch.yaml +- path: exploit-iq-client-batch-patch.yaml configMapGenerator: - - behavior: replace - - name: nginx-cache-routes - files: - - nginx/templates/routes/intel.conf.template - - nginx/templates/routes/nemo.conf.template - - nginx/templates/routes/nim.conf.template - - nginx/templates/routes/nvidia.conf.template - - nginx/templates/routes/openai.conf.template +- behavior: replace + name: nginx-cache-routes + files: + - nginx/templates/routes/intel.conf.template + - nginx/templates/routes/nemo.conf.template + - nginx/templates/routes/nim.conf.template + - nginx/templates/routes/nvidia.conf.template + - nginx/templates/routes/openai.conf.template diff --git a/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml b/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml new file mode 100644 index 000000000..15b18a3a6 --- /dev/null +++ b/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq + annotations: + api-key: &api-key "EXPLOIT_IQ" + nim-llm-baseurl: &nim-llm-baseurl "http://nginx-cache:8080/nim_llm/v1" +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq + imagePullPolicy: Always + workingDir: /workspace/ + env: + - name: LLM_API_KEY_CHECKLIST + value: *api-key + - name: LLM_API_KEY_CODE_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_DOC_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_AGENT_EXECUTOR + value: *api-key + - name: LLM_API_KEY_SUMMARIZE + value: *api-key + - name: LLM_API_KEY_JUSTIFY + value: *api-key + - name: LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key + - name: CHECKLIST_LLM_API_BASE + value: *nim-llm-baseurl + - name: CODE_VDB_RETRIEVER_API_BASE + value: *nim-llm-baseurl + - name: DOC_VDB_RETRIEVER_API_BASE + value: *nim-llm-baseurl + - name: AGENT_EXECUTOR_LLM_API_BASE + value: *nim-llm-baseurl + - name: GENERATE_CVSS_LLM_API_BASE + value: *nim-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE + value: *nim-llm-baseurl + - name: JUSTIFY_LLM_API_BASE + value: *nim-llm-baseurl + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *nim-llm-baseurl diff --git a/kustomize/overlays/remote-nim-all/kustomization.yaml b/kustomize/overlays/remote-nim-all/kustomization.yaml index 994d919e1..56689c402 100644 --- a/kustomize/overlays/remote-nim-all/kustomization.yaml +++ b/kustomize/overlays/remote-nim-all/kustomization.yaml @@ -2,84 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../base +- ../../base commonAnnotations: deployment-variant: remote-nim -patchesStrategicMerge: - - nginx-patch.yaml - - |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: exploit-iq - labels: - app: exploit-iq - component: exploit-iq - annotations: - api-key: &api-key "EXPLOIT_IQ" - nim-llm-baseurl: &nim-llm-baseurl "http://nginx-cache:8080/nim_llm/v1" - spec: - selector: - matchLabels: - app: exploit-iq - component: exploit-iq - template: - metadata: - labels: - app: exploit-iq - component: exploit-iq - spec: - containers: - - name: exploit-iq - imagePullPolicy: Always - workingDir: /workspace/ - env: - - name: LLM_API_KEY_CHECKLIST - value: *api-key - - - name: LLM_API_KEY_CODE_VDB_RETRIEVER - value: *api-key - - - name: LLM_API_KEY_DOC_VDB_RETRIEVER - value: *api-key - - - name: LLM_API_KEY_AGENT_EXECUTOR - value: *api-key - - - name: LLM_API_KEY_SUMMARIZE - value: *api-key - - - name: LLM_API_KEY_JUSTIFY - value: *api-key - - - name: LLM_API_KEY_INTEL_SOURCE_SCORE - value: *api-key - - - - name: CHECKLIST_LLM_API_BASE - value: *nim-llm-baseurl - - - name: CODE_VDB_RETRIEVER_API_BASE - value: *nim-llm-baseurl - - - - name: DOC_VDB_RETRIEVER_API_BASE - value: *nim-llm-baseurl - - - name: AGENT_EXECUTOR_LLM_API_BASE - value: *nim-llm-baseurl - - - name: GENERATE_CVSS_LLM_API_BASE - value: *nim-llm-baseurl - - - name: SUMMARIZE_LLM_API_BASE - value: *nim-llm-baseurl - - - name: JUSTIFY_LLM_API_BASE - value: *nim-llm-baseurl - - - name: INTEL_SOURCE_SCORE_LLM_API_BASE - value: *nim-llm-baseurl - +patches: +- path: nginx-patch.yaml +- path: exploit-iq-nim-patch.yaml diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml new file mode 100644 index 000000000..1aad91928 --- /dev/null +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq + annotations: + llm-type: &llm-type openai + api-key: &api-key "EMPTY" + openai-llm-baseurl: &openai-llm-baseurl http://nginx-cache:8080/openai/v1 + model-name: &model-name hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq + imagePullPolicy: Always + workingDir: /workspace/ + env: + - name: LLM_TYPE_CHECKLIST + value: *llm-type + - name: LLM_TYPE_VDB_CODE_RETRIEVER + value: *llm-type + - name: LLM_TYPE_VDB_DOC_RETRIEVER + value: *llm-type + - name: LLM_TYPE_AGENT_EXECUTOR + value: *llm-type + - name: LLM_TYPE_SUMMARIZE + value: *llm-type + - name: LLM_TYPE_JUSTIFY + value: *llm-type + - name: LLM_TYPE_INTEL_SOURCE_SCORE + value: *llm-type + - name: LLM_API_KEY_CHECKLIST + value: *api-key + - name: LLM_API_KEY_CODE_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_DOC_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_AGENT_EXECUTOR + value: *api-key + - name: LLM_API_KEY_SUMMARIZE + value: *api-key + - name: LLM_API_KEY_JUSTIFY + value: *api-key + - name: LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key + - name: CHECKLIST_LLM_API_BASE + value: *openai-llm-baseurl + - name: CODE_VDB_RETRIEVER_API_BASE + value: *openai-llm-baseurl + - name: DOC_VDB_RETRIEVER_API_BASE + value: *openai-llm-baseurl + - name: AGENT_EXECUTOR_LLM_API_BASE + value: *openai-llm-baseurl + - name: GENERATE_CVSS_LLM_API_BASE + value: *openai-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE + value: *openai-llm-baseurl + - name: JUSTIFY_LLM_API_BASE + value: *openai-llm-baseurl + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *openai-llm-baseurl + - name: CHECKLIST_MODEL_NAME + value: *model-name + - name: CODE_VDB_RETRIEVER_MODEL_NAME + value: *model-name + - name: DOC_VDB_RETRIEVER_MODEL_NAME + value: *model-name + - name: AGENT_EXECUTOR_MODEL_NAME + value: *model-name + - name: SUMMARIZE_MODEL_NAME + value: *model-name + - name: JUSTIFY_MODEL_NAME + value: *model-name + - name: INTEL_SOURCE_SCORE_MODEL_NAME + value: *model-name + - name: GENERATE_CVSS_MODEL_NAME + value: *model-name diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml index f44ae4703..c0948da4c 100644 --- a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -2,139 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../base +- ../../base commonAnnotations: deployment-variant: local-llama3.1-70b-4bit -patchesStrategicMerge: - - nginx-patch.yaml - - |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: exploit-iq - labels: - app: exploit-iq - component: exploit-iq - annotations: - llm-type: &llm-type openai - api-key: &api-key "EMPTY" - openai-llm-baseurl: &openai-llm-baseurl http://nginx-cache:8080/openai/v1 - model-name: &model-name hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 - spec: - selector: - matchLabels: - app: exploit-iq - component: exploit-iq - template: - metadata: - labels: - app: exploit-iq - component: exploit-iq - spec: - containers: - - name: exploit-iq - imagePullPolicy: Always - workingDir: /workspace/ - env: - - name: LLM_TYPE_CHECKLIST - value: *llm-type - - - - name: LLM_TYPE_VDB_CODE_RETRIEVER - value: *llm-type - - - - name: LLM_TYPE_VDB_DOC_RETRIEVER - value: *llm-type - - - - name: LLM_TYPE_AGENT_EXECUTOR - value: *llm-type - - - name: LLM_TYPE_SUMMARIZE - value: *llm-type - - - - name: LLM_TYPE_JUSTIFY - value: *llm-type - - - name: LLM_TYPE_INTEL_SOURCE_SCORE - value: *llm-type - - - name: LLM_API_KEY_CHECKLIST - value: *api-key - - - name: LLM_API_KEY_CODE_VDB_RETRIEVER - value: *api-key - - - name: LLM_API_KEY_DOC_VDB_RETRIEVER - value: *api-key - - - name: LLM_API_KEY_AGENT_EXECUTOR - value: *api-key - - - name: LLM_API_KEY_SUMMARIZE - value: *api-key - - - name: LLM_API_KEY_JUSTIFY - value: *api-key - - - name: LLM_API_KEY_INTEL_SOURCE_SCORE - value: *api-key - - - - name: CHECKLIST_LLM_API_BASE - value: *openai-llm-baseurl - - - name: CODE_VDB_RETRIEVER_API_BASE - value: *openai-llm-baseurl - - - - name: DOC_VDB_RETRIEVER_API_BASE - value: *openai-llm-baseurl - - - name: AGENT_EXECUTOR_LLM_API_BASE - value: *openai-llm-baseurl - - - name: GENERATE_CVSS_LLM_API_BASE - value: *openai-llm-baseurl - - - name: SUMMARIZE_LLM_API_BASE - value: *openai-llm-baseurl - - - name: JUSTIFY_LLM_API_BASE - value: *openai-llm-baseurl - - - name: INTEL_SOURCE_SCORE_LLM_API_BASE - value: *openai-llm-baseurl - - - name: CHECKLIST_MODEL_NAME - value: *model-name - - - name: CODE_VDB_RETRIEVER_MODEL_NAME - value: *model-name - - - name: DOC_VDB_RETRIEVER_MODEL_NAME - value: *model-name - - - name: AGENT_EXECUTOR_MODEL_NAME - value: *model-name - - - name: SUMMARIZE_MODEL_NAME - value: *model-name - - - name: JUSTIFY_MODEL_NAME - value: *model-name - - - name: INTEL_SOURCE_SCORE_MODEL_NAME - value: *model-name - - - name: GENERATE_CVSS_MODEL_NAME - value: *model-name - - - - - +patches: +- path: nginx-patch.yaml +- path: exploit-iq-llm-patch.yaml From 1c1f4e10b7382fa891f8f53fadd1f39f5c99a675 Mon Sep 17 00:00:00 2001 From: Shimon Tanny <162278968+RedTanny@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:32:59 +0300 Subject: [PATCH 282/286] Bug fix rpm analysis prep demo * BugFix 1.support package flavors kernel for example rt or debug need to parse and find correct config file * BugFix 2. in case grep tool return empty results need to clasify the results correctly. * BugFix 3. grep tool failed in multi query make robust --- .../functions/build_agent_graph_defs.py | 149 ++++++++++++-- .../functions/cve_build_agent.py | 53 +++-- .../functions/cve_package_code_agent.py | 161 +++++++++------ .../functions/react_internals.py | 68 +++++-- src/vuln_analysis/tools/source_grep.py | 38 +++- .../utils/rpm_checker_prompts.py | 183 +++++++++++++++++- .../test_vulnerability_intel_sanitizer.py | 80 +++++++- src/vuln_analysis/utils/token_utils.py | 45 +++++ .../utils/vulnerability_intel_sanitizer.py | 39 ++++ 9 files changed, 692 insertions(+), 124 deletions(-) diff --git a/src/vuln_analysis/functions/build_agent_graph_defs.py b/src/vuln_analysis/functions/build_agent_graph_defs.py index 698ff1b84..5f98c5dbb 100644 --- a/src/vuln_analysis/functions/build_agent_graph_defs.py +++ b/src/vuln_analysis/functions/build_agent_graph_defs.py @@ -48,6 +48,12 @@ ) logger = logging.getLogger(__name__) +BASE_KERNEL_PACKAGE = "kernel" +KERNEL_RHEL_CONFIG_SUFFIX = "rhel.config" +KERNEL_DEBUG_FLAVOR = "debug" +KERNEL_RT_FLAVOR = "rt" +KERNEL_RT_DEBUG_FLAVOR = "rt-debug" + # --------------------------------------------------------------------------- # Data Models @@ -407,16 +413,82 @@ def _is_kernel_package(source_path: Path) -> bool: return False -def _find_kernel_config_file(source_path: Path, arch: str) -> Path | None: - """Find the kernel config file for a specific architecture. +def _kernel_package_flavor(package_name: str | None) -> str | None: + """Return the kernel sub-flavor from an RPM name (e.g. kernel-rt -> rt).""" + if not package_name or package_name == BASE_KERNEL_PACKAGE: + return None + prefix = f"{BASE_KERNEL_PACKAGE}-" + if not package_name.startswith(prefix): + return None + return package_name.removeprefix(prefix) + + +def _kernel_config_candidate_names(arch: str, package_name: str | None) -> list[str]: + """Build ordered RHEL kernel config basenames for a package and architecture.""" + flavor = _kernel_package_flavor(package_name) + candidates: list[str] = [] + + if flavor == KERNEL_RT_DEBUG_FLAVOR: + # kernel-rt-debug -> support both layouts seen in RHEL sources: + # kernel-rt-x86_64-debug-rhel.config + # kernel-x86_64-rt-debug-rhel.config + candidates.append( + f"kernel-{KERNEL_RT_FLAVOR}-{arch}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + candidates.append( + f"kernel-{arch}-{KERNEL_RT_FLAVOR}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + elif flavor and KERNEL_DEBUG_FLAVOR in flavor: + # kernel-debug -> kernel-x86_64-debug-rhel.config + candidates.append( + f"kernel-{arch}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + elif flavor: + # kernel-rt -> kernel-rt-x86_64-rhel.config; also try flavor-after-arch variant + candidates.append(f"kernel-{flavor}-{arch}-{KERNEL_RHEL_CONFIG_SUFFIX}") + candidates.append(f"kernel-{arch}-{flavor}-{KERNEL_RHEL_CONFIG_SUFFIX}") + else: + candidates.append(f"kernel-{arch}-{KERNEL_RHEL_CONFIG_SUFFIX}") + + return list(dict.fromkeys(candidates)) + + +def _score_kernel_config_basename(name: str, arch: str, flavor: str | None) -> int: + """Rank glob matches: exact segment match wins over substring.""" + if arch not in name: + return -1 + + score = 0 + if flavor: + if f"-{flavor}-" in name or name.startswith(f"kernel-{flavor}-"): + score += 50 + elif flavor in name: + score += 25 + + wants_debug = flavor is not None and KERNEL_DEBUG_FLAVOR in flavor + is_debug = f"-{KERNEL_DEBUG_FLAVOR}-" in name + if is_debug: + score += 20 if wants_debug else -40 - RHEL kernel packages store config files in the source root with naming - pattern: kernel-{arch}-rhel.config (base flavor) or - kernel-{arch}-{flavor}-rhel.config (debug, rt, etc.) + return score + + +def _find_kernel_config_file( + source_path: Path, + arch: str, + package_name: str | None = None, +) -> Path | None: + """Find the kernel config file for a specific architecture and package flavor. + + RHEL SRPMs use several naming layouts in the source root, for example: + - kernel-x86_64-rhel.config (base kernel) + - kernel-rt-x86_64-rhel.config (kernel-rt: flavor before arch) + - kernel-x86_64-debug-rhel.config (kernel-debug: flavor after arch) Args: source_path: Path to the source directory arch: Target architecture (e.g., 'x86_64', 'aarch64') + package_name: RPM package name (e.g., 'kernel', 'kernel-rt') Returns: Path to the config file, or None if not found @@ -424,20 +496,58 @@ def _find_kernel_config_file(source_path: Path, arch: str) -> Path | None: if not source_path or not source_path.exists() or not arch: return None - # Try base flavor first: kernel-{arch}-rhel.config - config_path = source_path / f"kernel-{arch}-rhel.config" - if config_path.exists(): - logger.info("_find_kernel_config_file: found config at %s", config_path) - return config_path + flavor = _kernel_package_flavor(package_name) + for candidate_name in _kernel_config_candidate_names(arch, package_name): + config_path = source_path / candidate_name + if config_path.is_file(): + logger.info( + "_find_kernel_config_file: matched candidate %s for package=%s arch=%s", + candidate_name, + package_name, + arch, + ) + return config_path + + glob_pattern = f"kernel*{arch}*{KERNEL_RHEL_CONFIG_SUFFIX}" + glob_matches = [path for path in source_path.glob(glob_pattern) if path.is_file()] + if not glob_matches: + logger.warning( + "_find_kernel_config_file: no config found for package=%s arch=%s in %s", + package_name, + arch, + source_path, + ) + return None - # Fallback: any kernel-{arch}*.config - for config in source_path.glob(f"kernel-{arch}*.config"): - if config.is_file(): - logger.info("_find_kernel_config_file: found fallback config at %s", config) - return config + best_match = max( + glob_matches, + key=lambda path: _score_kernel_config_basename(path.name, arch, flavor), + ) + best_score = _score_kernel_config_basename(best_match.name, arch, flavor) + if best_score < 0: + logger.warning( + "_find_kernel_config_file: glob results did not contain arch %s in %s", + arch, + source_path, + ) + return None + if flavor and best_score <= 0: + logger.warning( + "_find_kernel_config_file: fallback %s does not match flavor=%s for package=%s arch=%s", + best_match, + flavor, + package_name, + arch, + ) + return None - logger.warning("_find_kernel_config_file: no config found for arch %s in %s", arch, source_path) - return None + logger.info( + "_find_kernel_config_file: selected fallback %s for package=%s arch=%s", + best_match, + package_name, + arch, + ) + return best_match def _find_kernel_source_root(source_path: Path) -> Path | None: @@ -528,9 +638,10 @@ async def harvest_build_data( logger.info("harvest_build_data: detected kernel package") # Find kernel config file for the target architecture if arch: - config_file = _find_kernel_config_file(source_path, arch) + config_file = _find_kernel_config_file(source_path, arch, package_name) if config_file: - kernel_config_path = str(config_file) + # Basename only: L2 Source Grep uses it as a file_glob under source root. + kernel_config_path = config_file.name # Find kernel source root (contains Kconfig, Makefiles) source_root = _find_kernel_source_root(source_path) if source_root: diff --git a/src/vuln_analysis/functions/cve_build_agent.py b/src/vuln_analysis/functions/cve_build_agent.py index dfca3dce4..ad52e3709 100644 --- a/src/vuln_analysis/functions/cve_build_agent.py +++ b/src/vuln_analysis/functions/cve_build_agent.py @@ -77,7 +77,7 @@ L2_KERNEL_THOUGHT_INSTRUCTIONS, ) from vuln_analysis.runtime_context import ctx_state -from vuln_analysis.utils.token_utils import truncate_tool_output +from vuln_analysis.utils.token_utils import truncate_tool_output, truncate_tool_output_list import uuid import tiktoken logger = LoggingFactory.get_agent_logger(__name__) @@ -455,7 +455,7 @@ async def observation_node(state: BuildAgentState) -> dict: tool_output_for_llm = tool_message.content # Check for empty/error outputs - bypass LLM if so to prevent hallucination - empty_findings = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) + empty_findings, _ = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) if empty_findings: # Build-specific: empty grep for file in logs = NOT_COMPILED evidence if tool_used == "Source Grep" and "logs:" in tool_input_detail: @@ -469,22 +469,39 @@ async def observation_node(state: BuildAgentState) -> dict: ) code_findings = empty_findings else: - # Step 1: Comprehension - extract findings from tool output - comp_prompt = L2_COMPREHENSION_PROMPT.format( - vuln_id=vuln_id, - target_package=target_package_name, - vulnerability_intel=vulnerability_intel_str, - disabled_features=", ".join(harvest_report.disabled_features) if harvest_report.disabled_features else "None", - spec_disabled_features=", ".join(harvest_report.spec_disabled_features) if harvest_report.spec_disabled_features else "None", - enabled_features=", ".join(harvest_report.enabled_features) if harvest_report.enabled_features else "None", - spec_enabled_features=", ".join(harvest_report.spec_enabled_features) if harvest_report.spec_enabled_features else "None", - tool_used=tool_used, - tool_input=tool_input_detail, - last_thought=last_thought_text, - tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), - ) - code_findings: CodeFindings = await invoke_comprehension( - comprehension_llm, comp_prompt, tool_used, tool_input_detail, tool_output_for_llm, agent_label="L2", + # Step 1: Comprehension - split into chunks and process each + chunks = truncate_tool_output_list(tool_output_for_llm, tool_used, max_tokens=1000) + all_findings = [] + best_tool_outcome = "" + + for chunk in chunks: + comp_prompt = L2_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=vulnerability_intel_str, + disabled_features=", ".join(harvest_report.disabled_features) if harvest_report.disabled_features else "None", + spec_disabled_features=", ".join(harvest_report.spec_disabled_features) if harvest_report.spec_disabled_features else "None", + enabled_features=", ".join(harvest_report.enabled_features) if harvest_report.enabled_features else "None", + spec_enabled_features=", ".join(harvest_report.spec_enabled_features) if harvest_report.spec_enabled_features else "None", + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=chunk, + ) + chunk_findings: CodeFindings = await invoke_comprehension( + comprehension_llm, comp_prompt, tool_used, tool_input_detail, chunk, agent_label="L2", + ) + all_findings.extend(chunk_findings.findings) + # Keep tool_outcome from chunk with actual findings (not FAILED) + if not best_tool_outcome or ( + chunk_findings.findings and + not any("FAILED" in f for f in chunk_findings.findings) + ): + best_tool_outcome = chunk_findings.tool_outcome + + code_findings = CodeFindings( + findings=all_findings, + tool_outcome=best_tool_outcome or "No matches found" ) findings_text = "\n".join(f"- {f}" for f in code_findings.findings) diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py index 251a0d17a..035396c69 100644 --- a/src/vuln_analysis/functions/cve_package_code_agent.py +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -51,17 +51,17 @@ ) from vuln_analysis.utils.rpm_checker_prompts import ( L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, - L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, L1_AGENT_SYS_PROMPT_REBASE_FIX, L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, L1_AGENT_PROMPT_TEMPLATE, L1_AGENT_PROMPT_TEMPLATE_NO_PATCH, L1_AGENT_THOUGHT_INSTRUCTIONS, - L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + select_upstream_prompt_and_instructions, L1_COMPREHENSION_PROMPT, L1_MEMORY_UPDATE_PROMPT, + L1_EMPTY_RESULT_CLASSIFICATION_PROMPT, VULNERABILITY_INTEL_EXTRACTION_PROMPT, ) from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile @@ -72,7 +72,7 @@ from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer from vuln_analysis.utils.reference_fetcher import ReferenceFetcher from vuln_analysis.utils.reference_parser import ReflectiveReferenceParser, ParserConfig -from vuln_analysis.utils.token_utils import truncate_tool_output +from vuln_analysis.utils.token_utils import truncate_tool_output, truncate_tool_output_list from vuln_analysis.runtime_context import ctx_state logger = LoggingFactory.get_agent_logger(__name__) @@ -425,7 +425,7 @@ async def create_graph_code_agent(config: CVEPackageCodeAgentConfig, builder: Bu tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) thought_llm = llm.with_structured_output(CheckerThought) - comprehension_llm = llm.with_structured_output(CodeFindings) + structured_comprehension_llm = llm.with_structured_output(CodeFindings) observation_llm = llm.with_structured_output(Observation) vulnerability_intel_llm = llm.with_structured_output(VulnerabilityIntel) # Get tool names after filtering for dynamic guidance @@ -693,30 +693,37 @@ async def L1_agent(state: CodeAgentState) -> dict: }) # use case 3: in target patch was not found but patch is found in the rpm that was mention in cve that is fixed elif upstream_report and upstream_report.fixed_parsed_patch: + sys_prompt, tool_instructions = select_upstream_prompt_and_instructions( + vulnerability_intel.vulnerable_patterns + ) runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( - sys_prompt=L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + sys_prompt=sys_prompt, vuln_id=vuln_id, target_package=target_package.name, vulnerability_intel=vulnerability_intel.format_for_prompt(), tools=tools_str, tool_selection_strategy=tool_strategy, - tool_instructions=L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + tool_instructions=tool_instructions, ) span.set_output({ "mode": "upstream_patch_verification", "patch_filename": upstream_report.fixed_srpm_file_name, + "prompt_variant": "case_b" if not vulnerability_intel.vulnerable_patterns else "case_a", }) # use case 4: Fix commit discovered via git search elif git_search_report and git_search_report.parsed_patch: + sys_prompt, tool_instructions = select_upstream_prompt_and_instructions( + vulnerability_intel.vulnerable_patterns + ) runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( - sys_prompt=L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + sys_prompt=sys_prompt, vuln_id=vuln_id, target_package=target_package.name, vulnerability_intel=vulnerability_intel.format_for_prompt(), tools=tools_str, tool_selection_strategy=tool_strategy, - tool_instructions=L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + tool_instructions=tool_instructions, ) span.set_output({ @@ -724,6 +731,7 @@ async def L1_agent(state: CodeAgentState) -> dict: "commit_hash": git_search_report.best_result.commit_hash_short if git_search_report.best_result else "", "confidence": git_search_report.best_result.confidence if git_search_report.best_result else 0, "search_method": git_search_report.best_result.search_method if git_search_report.best_result else "", + "prompt_variant": "case_b" if not vulnerability_intel.vulnerable_patterns else "case_a", }) else: # Use case 4: Default prompt - no patch context, use VulnerabilityIntel from CVE description @@ -1320,60 +1328,99 @@ async def observation_node(state: CodeAgentState) -> dict: with tracer.push_active_function("observation node", input_data=f"tool used:{tool_used} + {tool_input_detail}") as span: tool_output_for_llm = tool_message.content - # Check for empty/error outputs - bypass LLM if so to prevent hallucination - empty_findings = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) + # Check for empty/error outputs + # For empty Source Grep on source code, let LLM classify against RAW_PATCH_DIFF + # to distinguish VULNERABLE_CODE_ABSENT vs FIX_CODE_ABSENT + empty_findings, needs_llm_classification = check_empty_output( + tool_output_for_llm, tool_used, tool_input_detail, + allow_llm_classification=True + ) + + # Get parsed_patch from state for raw diff context (needed for classification and comprehension) + downstream_report = state.get("downstream_report") + upstream_report = state.get("upstream_report") + parsed_patch = None + + if downstream_report: + if isinstance(downstream_report, dict): + parsed_patch = downstream_report.get('parsed_patch') + else: + parsed_patch = getattr(downstream_report, 'parsed_patch', None) + + if not parsed_patch and upstream_report: + if isinstance(upstream_report, dict): + parsed_patch = upstream_report.get('fixed_parsed_patch') + else: + parsed_patch = getattr(upstream_report, 'fixed_parsed_patch', None) + + # If parsed_patch is a dict, convert it to ParsedPatch model + if parsed_patch and isinstance(parsed_patch, dict): + try: + parsed_patch = ParsedPatch(**parsed_patch) + except Exception as e: + logger.warning("Failed to parse parsed_patch dict: %s", e) + parsed_patch = None + + logger.debug("observation_node: parsed_patch=%s, downstream=%s, upstream=%s", + parsed_patch is not None, + downstream_report is not None, + upstream_report is not None) + + # Extract relevant hunks based on grep target file + raw_patch_diff = "" + if tool_used == "Source Grep" and parsed_patch: + raw_patch_diff = get_relevant_hunks(parsed_patch, tool_input_detail) + if empty_findings: code_findings = empty_findings - else: - # Get parsed_patch from state for raw diff context - # Reports may be Pydantic models or dicts depending on state serialization - downstream_report = state.get("downstream_report") - upstream_report = state.get("upstream_report") - parsed_patch = None - - if downstream_report: - if isinstance(downstream_report, dict): - parsed_patch = downstream_report.get('parsed_patch') - else: - parsed_patch = getattr(downstream_report, 'parsed_patch', None) - - if not parsed_patch and upstream_report: - if isinstance(upstream_report, dict): - parsed_patch = upstream_report.get('fixed_parsed_patch') - else: - parsed_patch = getattr(upstream_report, 'fixed_parsed_patch', None) - - # If parsed_patch is a dict, convert it to ParsedPatch model - if parsed_patch and isinstance(parsed_patch, dict): - try: - parsed_patch = ParsedPatch(**parsed_patch) - except Exception as e: - logger.warning("Failed to parse parsed_patch dict: %s", e) - parsed_patch = None - - logger.debug("observation_node: parsed_patch=%s, downstream=%s, upstream=%s", - parsed_patch is not None, - downstream_report is not None, - upstream_report is not None) - - # Extract relevant hunks based on grep target file - raw_patch_diff = "" - if tool_used == "Source Grep" and parsed_patch: - raw_patch_diff = get_relevant_hunks(parsed_patch, tool_input_detail) - - # Step 1: Comprehension - extract key findings from raw tool output - comp_prompt = L1_COMPREHENSION_PROMPT.format( - vuln_id=vuln_id, - target_package=target_package_name, - vulnerability_intel=intel_formatted, - raw_patch_diff=raw_patch_diff, + elif needs_llm_classification: + # Empty source grep - use classification prompt to determine meaning + classification_prompt = L1_EMPTY_RESULT_CLASSIFICATION_PROMPT.format( tool_used=tool_used, - tool_input=tool_input_detail, last_thought=last_thought_text, - tool_output=truncate_tool_output(tool_output_for_llm, tool_used, max_tokens=1000), + tool_input=tool_input_detail, + raw_patch_diff=raw_patch_diff if raw_patch_diff else "No patch diff available", ) - code_findings: CodeFindings = await invoke_comprehension( - comprehension_llm, comp_prompt, tool_used, tool_input_detail, tool_output_for_llm, agent_label="L1", + code_findings = await structured_comprehension_llm.ainvoke( + [SystemMessage(content=classification_prompt)] + ) + logger.debug("Empty source grep classified: %s", code_findings.findings) + else: + # Has actual content - split into chunks and process each + chunks = truncate_tool_output_list(tool_output_for_llm, tool_used, max_tokens=1000) + all_findings = [] + best_tool_outcome = "" + + for chunk in chunks: + comp_prompt = L1_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=intel_formatted, + raw_patch_diff=raw_patch_diff, + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=chunk, + ) + chunk_findings = await invoke_comprehension( + structured_comprehension_llm, + comp_prompt, + tool_used, + tool_input_detail, + chunk, + agent_label="L1", + ) + all_findings.extend(chunk_findings.findings) + # Keep tool_outcome from chunk with actual findings (not FAILED) + if not best_tool_outcome or ( + chunk_findings.findings and + not any("FAILED" in f for f in chunk_findings.findings) + ): + best_tool_outcome = chunk_findings.tool_outcome + + code_findings = CodeFindings( + findings=all_findings, + tool_outcome=best_tool_outcome or "No matches found" ) findings_text = "\n".join(f"- {f}" for f in code_findings.findings) diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py index 22300de74..4a7c74913 100644 --- a/src/vuln_analysis/functions/react_internals.py +++ b/src/vuln_analysis/functions/react_internals.py @@ -107,13 +107,25 @@ def check_empty_output( tool_output: str | list, tool_used: str, tool_input: str, -) -> CodeFindings | None: + allow_llm_classification: bool = False, +) -> tuple[CodeFindings | None, bool]: """Check if tool output is empty or an error, returning factual CodeFindings if so. - This bypasses LLM comprehension for empty/error outputs to prevent hallucination. + For empty Source Grep on source code, we may want the LLM to classify the result + against the RAW_PATCH_DIFF (to distinguish VULNERABLE_CODE_ABSENT vs FIX_CODE_ABSENT). + + Args: + tool_output: The raw output from the tool. + tool_used: Name of the tool (e.g., "Source Grep"). + tool_input: The input/query passed to the tool. + allow_llm_classification: If True, signal that empty Source Grep on source + code needs LLM classification against the diff. Returns: - CodeFindings with factual empty/error message, or None if output has content. + Tuple of (CodeFindings | None, needs_llm_classification: bool). + - If CodeFindings is returned, use it directly. + - If CodeFindings is None and needs_llm_classification is True, call classification LLM. + - If CodeFindings is None and needs_llm_classification is False, output has content. """ is_empty = ( not tool_output @@ -126,29 +138,49 @@ def check_empty_output( and any(m in tool_output for m in ["Error:", "error:", "Failed:", "Exception:", "Traceback"]) ) - if is_empty: - return CodeFindings( - findings=[f"{tool_used} for '{tool_input}' returned empty - no matches found"], - tool_outcome=f"CALLED: {tool_used} with {tool_input} -> EMPTY (no results)" + if is_error: + return ( + CodeFindings( + findings=[ + f"FAILED: {tool_used} [{tool_input}] - tool error", + f"Details: {str(tool_output)[:150]}" + ], + tool_outcome=f"FAILED: {tool_used} with {tool_input} -> ERROR" + ), + False, ) - if is_error: - return CodeFindings( - findings=[ - f"FAILED: {tool_used} [{tool_input}] - tool error", - f"Details: {str(tool_output)[:150]}" - ], - tool_outcome=f"FAILED: {tool_used} with {tool_input} -> ERROR" + if is_empty: + is_source_grep_on_source = ( + tool_used == "Source Grep" + and not tool_input.startswith("logs:") + and not tool_input.startswith("patch:") + ) + if allow_llm_classification and is_source_grep_on_source: + return (None, True) + + return ( + CodeFindings( + findings=[f"{tool_used} for '{tool_input}' returned empty - no matches found"], + tool_outcome=f"CALLED: {tool_used} with {tool_input} -> EMPTY (no results)" + ), + False, ) - return None + return (None, False) -async def invoke_comprehension(llm, prompt: str, tool_used: str, tool_input: str, tool_output: str, - agent_label: str = "") -> CodeFindings: +async def invoke_comprehension( + structured_comprehension_llm, + prompt: str, + tool_used: str, + tool_input: str, + tool_output: str, + agent_label: str = "", +) -> CodeFindings: """Invoke comprehension LLM with fallback on token limit overflow.""" try: - return await llm.ainvoke([SystemMessage(content=prompt)]) + return await structured_comprehension_llm.ainvoke([SystemMessage(content=prompt)]) except LengthFinishReasonError: logger.warning("%s comprehension LLM hit token limit (tool=%s), using fallback", agent_label, tool_used) summary = tool_output[:500] if isinstance(tool_output, str) else str(tool_output)[:500] diff --git a/src/vuln_analysis/tools/source_grep.py b/src/vuln_analysis/tools/source_grep.py index b549df58f..9220937e0 100644 --- a/src/vuln_analysis/tools/source_grep.py +++ b/src/vuln_analysis/tools/source_grep.py @@ -55,6 +55,9 @@ class SourceGrepToolConfig(FunctionBaseConfig, name=SOURCE_GREP): VALID_TARGETS = ("source", "logs", "patch") +QUERY_FILE_SEPARATOR = "," +QUERY_MULTI_PATTERN_SEPARATOR = ";" +FILE_GLOB_WILDCARDS = ("*", "?", "/") TARGET_EXTENSIONS: dict[str, list[str]] = { "source": ["*.c", "*.h", "*.cpp", "*.hpp", "*.py", "*.go", "*.java", "*.spec", "*.cmake", "Makefile", "*.mk", "*.config"], @@ -63,6 +66,14 @@ class SourceGrepToolConfig(FunctionBaseConfig, name=SOURCE_GREP): } +def _looks_like_file_glob(token: str) -> bool: + """Return True when token appears to be a filename or glob.""" + if not token: + return False + filename = Path(token).name + return any(ch in token for ch in FILE_GLOB_WILDCARDS) or "." in filename + + def _parse_query(query: str) -> tuple[str | list[str], str | None, str, bool]: """Parse query string into (pattern(s), file_glob, target, word_boundary). @@ -74,6 +85,7 @@ def _parse_query(query: str) -> tuple[str | list[str], str | None, str, bool]: - "pattern -w" -> search with word boundary (whole words only) - "target:pattern,file_glob -w" -> full format with word boundary - "pattern1;pattern2,file.c" -> multiple patterns (only with file_glob) + - "pattern1,pattern2,file.c" -> compatibility syntax (normalized to multi-pattern + file) Valid targets: source, logs, patch @@ -94,16 +106,32 @@ def _parse_query(query: str) -> tuple[str | list[str], str | None, str, bool]: target = prefix query = rest - if "," in query: - parts = query.split(",", 1) + if QUERY_FILE_SEPARATOR in query: + parts = query.split(QUERY_FILE_SEPARATOR, 1) pattern_part = parts[0].strip() file_glob = parts[1].strip() if len(parts) > 1 else None # Multi-pattern support: only when file_glob is provided - if file_glob and ";" in pattern_part: - patterns = [p.strip() for p in pattern_part.split(";") if p.strip()] + if file_glob and QUERY_MULTI_PATTERN_SEPARATOR in pattern_part: + patterns = [p.strip() for p in pattern_part.split(QUERY_MULTI_PATTERN_SEPARATOR) if p.strip()] return patterns, file_glob, target, word_boundary + # Compatibility mode: + # LLMs sometimes emit "pattern1,pattern2,file.c" instead of + # the documented "pattern1;pattern2,file.c". Normalize it. + if file_glob and QUERY_FILE_SEPARATOR in file_glob: + trailing_parts = [p.strip() for p in file_glob.split(QUERY_FILE_SEPARATOR) if p.strip()] + candidate_file_glob = trailing_parts[-1] if trailing_parts else "" + extra_patterns = trailing_parts[:-1] + + if _looks_like_file_glob(candidate_file_glob): + patterns = [pattern_part, *extra_patterns] + patterns = [p for p in patterns if p] + if len(patterns) > 1: + return patterns, candidate_file_glob, target, word_boundary + if len(patterns) == 1: + return patterns[0], candidate_file_glob, target, word_boundary + return pattern_part, file_glob, target, word_boundary return query, None, target, word_boundary @@ -145,12 +173,14 @@ async def _arun(query: str) -> str: Options: - -w: Match whole words only (word boundary) - Multiple patterns: use ';' separator ONLY with a specific file + - Compatibility: "pattern1,pattern2,file.c" is accepted and normalized Examples: - 'archive_read_open' - search source files - 'archive_read_open,*.c' - search only .c source files - 'archive_read_open -w' - search for whole word only - 'unsigned int cursor;unsigned int nodes,archive_read.c' - multiple patterns in one file + - 'sum2,s2length,match.c' - compatibility syntax for multiple patterns in one file - 'logs:undefined reference' - search build logs for link errors - 'logs:error:' - search build logs for error messages - 'patch:CVE-2026-5121' - find patch for specific CVE diff --git a/src/vuln_analysis/utils/rpm_checker_prompts.py b/src/vuln_analysis/utils/rpm_checker_prompts.py index 3a7b82bf3..6007e8ad8 100644 --- a/src/vuln_analysis/utils/rpm_checker_prompts.py +++ b/src/vuln_analysis/utils/rpm_checker_prompts.py @@ -180,6 +180,30 @@ "- State confidence level based on evidence quality." ) +L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B = ( + "You are a security analyst verifying whether a package remains VULNERABLE to a CVE.\n" + "The TARGET package does NOT contain a CVE-specific patch file.\n" + "Patterns were extracted from a FIXED RPM patch, but VULNERABLE_PATTERNS is EMPTY.\n\n" + "YOUR TASK: Determine whether the FIX is present in the target source.\n\n" + "VERIFICATION STRATEGY (CASE B - EMPTY VULNERABLE_PATTERNS):\n" + "1. FIRST search FIX_PATTERNS inside each file from AFFECTED_FILES.\n" + "2. If a fix marker is found, verify fix application at the call site.\n" + " - Definition-only matches are insufficient.\n" + " - The fix must be called and its result must be used.\n" + "3. CONCLUSION:\n" + " - If fix is applied at call site → Package is PATCHED via rebase.\n" + " - If fix is absent or unverified → investigate remaining affected files and use version fallback rules.\n\n" + "CRITICAL RULES:\n" + "- Do NOT start with vulnerable-pattern search in this mode.\n" + "- Do NOT assume vulnerability or safety without tool evidence.\n" + "- Base conclusions ONLY on tool results and version-range evidence when required.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Clearly state whether fix is present at call site, absent, or unverified.\n" + "- State confidence level based on evidence quality." +) + L1_AGENT_SYS_PROMPT_REBASE_FIX = ( "You are a security analyst verifying that a CVE fix is PRESENT in a rebased package.\n" "The TARGET package was REBASED to a newer upstream version that claims to fix this CVE.\n\n" @@ -372,9 +396,14 @@ - SEARCH_KEYWORDS: Terms to grep for PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): - For EACH item in VULNERABLE_FUNCTIONS and AFFECTED_FILES: + CASE A - VULNERABLE_PATTERNS is NOT empty: 1. Search for vulnerable pattern - it SHOULD exist in unpatched target 2. Search for fix pattern - it should NOT exist in unpatched target + + CASE B - VULNERABLE_PATTERNS is empty: + 1. Search for FIX_PATTERNS in AFFECTED_FILES first + 2. Verify fix is applied at call site (not definition-only) + 3. Do NOT force a vulnerable-pattern search in this case IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. PHASE 3 - VERDICT: @@ -384,7 +413,9 @@ - Evidence is sufficient for confident verdict **VERSION-BASED FALLBACK (when code search is inconclusive):** - If your code searches found NO conclusive evidence (no vulnerable pattern, no fix pattern): + If your code searches found NO conclusive evidence: + - CASE A: no vulnerable pattern and no fix pattern found + - CASE B: no fix pattern or no fix call-site verification found - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: → Conclude VULNERABLE based on version evidence @@ -400,11 +431,12 @@ 4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). 5. Code Keyword Search: use query field for broader searches. 6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. -7. FIRST search for VULNERABLE code - it SHOULD exist in target. -8. THEN search for FIX code - it should NOT exist in target. -9. If a pattern contains special regex characters, escape them or use literal substrings. -10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. -11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. +7. IF VULNERABLE_PATTERNS is non-empty: search VULNERABLE code first. +8. IF VULNERABLE_PATTERNS is empty: search FIX_PATTERNS first in AFFECTED_FILES. +9. Do NOT classify as vulnerable solely because VULNERABLE_PATTERNS is empty. +10. If a pattern contains special regex characters, escape them or use literal substrings. +11. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +12. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. @@ -424,6 +456,9 @@ {{"thought": "No prior searches in KNOWLEDGE. Search for the vulnerable code pattern from the patch", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate vulnerable code that should exist in unpatched target"}}, "final_answer": null}} + +{{"thought": "VULNERABLE_PATTERNS is empty. Start with FIX_PATTERNS in affected files.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "CASE B: verify additive fix markers are present at call site"}}, "final_answer": null}} + {{"thought": "KNOWLEDGE shows function found at iso9660.c:2074. Now verify the fix is NOT present", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Check if fix code is absent (confirms vulnerability)"}}, "final_answer": null}} @@ -442,10 +477,96 @@ {{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} + +{{"thought": "VULNERABLE_PATTERNS is empty and fix application is unverified", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - vulnerable patterns were unavailable and fix call-site evidence is insufficient. Manual review required."}} + {{"thought": "Code searches found no vulnerable or fix patterns - affected module not located in source. However, TARGET_IN_VULNERABLE_RANGE is YES. No fix was verified in code. Version evidence indicates vulnerability.", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range per VULNERABILITY_INTEL. Code search could not locate the affected module or verify a fix. Based on version evidence and absence of verified fix, the package is vulnerable."}} """ +L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (CASE B ONLY): + VULNERABLE_PATTERNS is empty: + 1. Search FIX_PATTERNS in AFFECTED_FILES first + 2. Verify fix is applied at call site (not definition-only) + 3. Do NOT force a vulnerable-pattern search in this case + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If code searches found no fix pattern or no fix call-site verification: + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. FIRST action in this mode MUST search FIX_PATTERNS in AFFECTED_FILES. +8. Do NOT perform vulnerable-pattern-first search in this mode. +9. If a pattern contains special regex characters, escape them or use literal substrings. +10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. CASE B requires fix-first verification in affected file.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "CASE B: verify whether fix marker exists in affected file"}}, "final_answer": null}} + + +{{"thought": "Fix marker search completed for one file. Continue to remaining affected files and verify call-site usage where found.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "Ensure all affected files are checked before verdict"}}, "final_answer": null}} + + +{{"thought": "Fix was verified at call site in affected code", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix pattern is present and applied at call site in affected files."}} + + +{{"thought": "Fix not verified in code and target remains in affected version range", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range and no fix was verified at call site in affected files."}} +""" + +def select_upstream_prompt_and_instructions(vulnerable_patterns: list[str]) -> tuple[str, str]: + """Select upstream patch prompts based on vulnerable-pattern availability.""" + if vulnerable_patterns: + return L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS + return L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B, L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS + L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS = """ You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). BEFORE ACTING, you MUST: @@ -653,6 +774,54 @@ {{"thought": "KNOWLEDGE shows insufficient evidence AND TARGET_IN_VULNERABLE_RANGE is NO/UNKNOWN - cannot determine patch status from version alone", "mode": "finish", "actions": null, "final_answer": "INCONCLUSIVE. Could not find definitive evidence of fix or vulnerability, and version range is unknown. Manual review recommended."}} """ +# --------------------------------------------------------------------------- +# L1 Empty Result Classification Prompt +# --------------------------------------------------------------------------- + +L1_EMPTY_RESULT_CLASSIFICATION_PROMPT = """Classify an empty grep result for CVE patch verification. + +The grep search returned NO MATCHES. Determine what this means for patch verification. + +TOOL USED: {tool_used} +AGENT THOUGHT (why the search was performed): +{last_thought} + +SEARCHED PATTERN: {tool_input} + +RAW_PATCH_DIFF (for reference): +{raw_patch_diff} + +CLASSIFICATION RULES: +1. Read AGENT THOUGHT to understand what the agent was searching for +2. Determine if the agent was searching for VULNERABLE code or FIX code: + - Keywords like "verify vulnerable", "check if vulnerable", "removed", "absent" → searching for VULNERABLE code + - Keywords like "search for fix", "find fix", "added", "patched" → searching for FIX code +3. Classify the empty result: + - Searching for VULNERABLE code + NOT FOUND → VULNERABLE_CODE_ABSENT (good - fix removed it) + - Searching for FIX code + NOT FOUND → FIX_CODE_ABSENT (bad - fix not applied) + - Intent unclear → INCONCLUSIVE + +OUTPUT FORMAT (JSON - CodeFindings): +{{ + "findings": [": not found - "], + "tool_outcome": "{tool_used} [{tool_input}] -> NO MATCHES ()" +}} + +CLASSIFICATION VALUES for findings[0]: +- VULNERABLE_CODE_ABSENT: Agent searched for vulnerable code, it's not found = fix likely applied +- FIX_CODE_ABSENT: Agent searched for fix code, it's not found = fix NOT applied +- INCONCLUSIVE: Cannot determine search intent from agent thought + +EXAMPLE OUTPUTS: +If searching for vulnerable code: +{{"findings": ["VULNERABLE_CODE_ABSENT: switch (data->verdict.code & NF_VERDICT_MASK) not found - vulnerable pattern removed by fix"], "tool_outcome": "{tool_used} [pattern] -> NO MATCHES (vulnerable code absent = fix applied)"}} + +If searching for fix code: +{{"findings": ["FIX_CODE_ABSENT: safe_function() not found - fix pattern missing"], "tool_outcome": "{tool_used} [pattern] -> NO MATCHES (fix code absent = fix NOT applied)"}} + +RESPONSE: +{{""" + # --------------------------------------------------------------------------- # L1 Observation Prompts (Comprehension + Memory Update) # --------------------------------------------------------------------------- diff --git a/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py index d98ccbc35..5185ccacf 100644 --- a/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py +++ b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py @@ -5,7 +5,7 @@ from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel -from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile +from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile, PatchHunk from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer @@ -22,6 +22,58 @@ def _patch_with_util_c() -> ParsedPatch: ) +def _additive_only_patch() -> ParsedPatch: + return ParsedPatch( + patch_filename="additive.patch", + files=[ + PatchFile( + source_path="a/net/sched/act_ct.c", + target_path="b/net/sched/act_ct.c", + hunks=[ + PatchHunk( + source_start=100, + source_length=0, + target_start=100, + target_length=4, + section_header="tcf_ct_init", + context_lines=[], + removed_lines=[], + added_lines=[ + "if (bind && !(flags & TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT)) {", + "return -EOPNOTSUPP;", + "}", + ], + ) + ], + ) + ], + ) + + +def _patch_with_removed_lines() -> ParsedPatch: + return ParsedPatch( + patch_filename="mixed.patch", + files=[ + PatchFile( + source_path="a/foo.c", + target_path="b/foo.c", + hunks=[ + PatchHunk( + source_start=10, + source_length=1, + target_start=10, + target_length=1, + section_header="foo", + context_lines=[], + removed_lines=["unsafe_call();"], + added_lines=["safe_call();"], + ) + ], + ) + ], + ) + + class TestSanitizeAffectedFiles: def test_clears_affected_files_when_no_patch(self): raw = VulnerabilityIntel(affected_files=["generator.c", "tar/util.c"]) @@ -83,6 +135,32 @@ def test_keeps_keyword_with_and(self): assert result.search_keywords == ["foo AND bar"] +class TestSanitizeAdditiveOnlyPatchIntel: + def test_clears_vulnerable_fields_for_additive_only_patch(self): + raw = VulnerabilityIntel( + vulnerable_functions=["classify"], + vulnerable_variables=["skb"], + vulnerable_patterns=["TC_ACT_CONSUMED"], + fix_patterns=["TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT"], + ) + result = VulnerabilityIntelSanitizer(_additive_only_patch()).apply(raw) + assert result.vulnerable_functions == [] + assert result.vulnerable_variables == [] + assert result.vulnerable_patterns == [] + assert result.fix_patterns == ["TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT"] + + def test_keeps_vulnerable_fields_when_patch_has_removed_lines(self): + raw = VulnerabilityIntel( + vulnerable_functions=["classify"], + vulnerable_variables=["skb"], + vulnerable_patterns=["TC_ACT_CONSUMED"], + ) + result = VulnerabilityIntelSanitizer(_patch_with_removed_lines()).apply(raw) + assert result.vulnerable_functions == ["classify"] + assert result.vulnerable_variables == ["skb"] + assert result.vulnerable_patterns == ["TC_ACT_CONSUMED"] + + class TestRsyncStyleNoPatch: def test_strips_hallucinated_paths_and_prose(self): raw = VulnerabilityIntel( diff --git a/src/vuln_analysis/utils/token_utils.py b/src/vuln_analysis/utils/token_utils.py index 3624e3e80..fb29bbf6b 100644 --- a/src/vuln_analysis/utils/token_utils.py +++ b/src/vuln_analysis/utils/token_utils.py @@ -112,3 +112,48 @@ def truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400 tail_tokens += lt truncated = token_count - head_tokens - tail_tokens return '\n'.join(head_lines) + f"\n[... truncated {truncated} tokens ...]\n" + '\n'.join(tail_lines) + + +def truncate_tool_output_list( + tool_output: str, + tool_name: str, + max_tokens: int = 1000, + max_chunks: int = 2 +) -> list[str]: + """Split tool output into chunks, each up to max_tokens. Returns up to max_chunks. + + Use this when you want to process large outputs in multiple passes rather than + truncating to a single smaller output. Each chunk can be processed separately + and findings merged. + + Calls truncate_tool_output internally to apply tool-specific truncation logic. + """ + total_tokens = count_tokens(tool_output) + if total_tokens <= max_tokens: + return [truncate_tool_output(tool_output, tool_name, max_tokens)] + + # Split by lines and create raw chunks + lines = tool_output.split('\n') + chunks = [] + current_lines = [] + current_tokens = 0 + + for line in lines: + line_tokens = count_tokens(line) + if current_tokens + line_tokens > max_tokens and current_lines: + # Apply tool-specific truncation to each chunk + raw_chunk = '\n'.join(current_lines) + chunks.append(truncate_tool_output(raw_chunk, tool_name, max_tokens)) + if len(chunks) >= max_chunks: + break + current_lines = [] + current_tokens = 0 + current_lines.append(line) + current_tokens += line_tokens + + # Handle remaining lines as final chunk + if current_lines and len(chunks) < max_chunks: + raw_chunk = '\n'.join(current_lines) + chunks.append(truncate_tool_output(raw_chunk, tool_name, max_tokens)) + + return chunks if chunks else [truncate_tool_output(tool_output, tool_name, max_tokens)] diff --git a/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py index a703d0e96..99632b919 100644 --- a/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py +++ b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py @@ -27,6 +27,23 @@ def _has_boolean_operator(keyword: str) -> bool: return bool(_BOOLEAN_OP_RE.search(keyword)) +def _patch_line_presence(parsed_patch: ParsedPatch) -> tuple[bool, bool]: + """Return (has_removed_lines, has_added_lines) across all hunks.""" + has_removed_lines = False + has_added_lines = False + + for patch_file in parsed_patch.files: + for hunk in patch_file.hunks: + if hunk.removed_lines: + has_removed_lines = True + if hunk.added_lines: + has_added_lines = True + if has_removed_lines and has_added_lines: + return has_removed_lines, has_added_lines + + return has_removed_lines, has_added_lines + + class VulnerabilityIntelSanitizer: """Apply shape rules to L1 VulnerabilityIntel; extensible one method per rule.""" @@ -37,8 +54,17 @@ def __init__(self, parsed_patch: ParsedPatch | None = None) -> None: def _has_trusted_patch(self) -> bool: return self._parsed_patch is not None and bool(self._parsed_patch.files) + @property + def _is_additive_only_patch(self) -> bool: + if not self._has_trusted_patch: + return False + + has_removed_lines, has_added_lines = _patch_line_presence(self._parsed_patch) + return has_added_lines and not has_removed_lines + def apply(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: intel = self.sanitize_affected_files(intel) + intel = self.sanitize_additive_only_patch_intel(intel) intel = self.filter_vulnerable_functions(intel) return self.filter_search_keywords(intel) @@ -59,6 +85,19 @@ def filter_vulnerable_functions(self, intel: VulnerabilityIntel) -> Vulnerabilit kept = [name for name in intel.vulnerable_functions if " " not in name] return intel.model_copy(update={"vulnerable_functions": kept}) + def sanitize_additive_only_patch_intel(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + """Drop vulnerable-side intel when patch has only added lines.""" + if not self._is_additive_only_patch: + return intel + + return intel.model_copy( + update={ + "vulnerable_functions": [], + "vulnerable_variables": [], + "vulnerable_patterns": [], + } + ) + def filter_search_keywords(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: kept = [ kw From 244998c929c091f62d0e5f5e1fd6458e4e43386d Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Thu, 25 Jun 2026 18:44:07 +0300 Subject: [PATCH 283/286] =?UTF-8?q?docs(kustomize):=20fix=20OAuth=20flow?= =?UTF-8?q?=20=E2=80=94=20move=20OAuthClient=20to=20post-deploy,=20restore?= =?UTF-8?q?=20oc=20get=20route?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kustomize/README.md | 120 ++++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index ab36c90e3..373017ef7 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -117,15 +117,24 @@ argilla_api_key= EOF ``` -### Step 5. Configure OAuth Authentication +### Step 5. Configure OAuth Credentials -By default Exploit Intelligence uses OpenShift OAuth for user authentication. The OAuth client secret must be at least 32 bytes (256 bits) because Exploit Intelligence uses it to sign internal session tokens with HS256, which requires a minimum key length of 256 bits. +Exploit Intelligence uses OpenShift OAuth for user authentication. The OAuth client secret must be at least 32 bytes (256 bits) because Exploit Intelligence uses it to sign internal session tokens with HS256, which requires a minimum key length of 256 bits. -Select the option that matches your situation. +> [!IMPORTANT] +> Save the value of `$OAUTH_CLIENT_SECRET` after running the commands below. You need it after deployment to create or update the `OAuthClient` resource. + +#### First-Time Deployment -#### Option A: Create a New OAuthClient +Use this procedure only if no `OAuthClient` named `exploit-iq-client` exists on the cluster. If another Exploit Intelligence installation already uses that `OAuthClient`, you must use the [Reusing an Existing OAuthClient](#reusing-an-existing-oauthclient) procedure instead — generating a new secret overwrites the existing one and breaks authentication for all users of that installation. -Use this option when deploying Exploit Intelligence for the first time. Generate a new client secret, write the credentials file, and create the `OAuthClient` resource: +Verify that the `OAuthClient` does not exist before proceeding: + +```shell +oc get oauthclient exploit-iq-client 2>&1 | grep -q "not found" && echo "Safe to proceed" || echo "OAuthClient exists — use Reusing an Existing OAuthClient" +``` + +Generate a new client secret and write the credentials file: ```shell export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32) @@ -139,40 +148,12 @@ openshift-domain=$OCP_DOMAIN EOF ``` -```shell -oc create -f - < [!WARNING] +> Complete this step before attempting to log in to the Exploit Intelligence UI. Authentication fails if the `OAuthClient` resource is not configured correctly. + +After the deployment completes and the `exploit-iq-client` route is available, configure the OpenShift OAuth client. Select the procedure that matches your situation. + +#### Create the OAuthClient Resource + +Complete this procedure if you followed Step 5, "First-Time Deployment": + +```shell +oc create -f - < Date: Sun, 28 Jun 2026 12:35:52 +0300 Subject: [PATCH 284/286] Fixed Bugs in RPM and Transitive for demo * bugFix rpm check: fine tune prompt for grep kernel config flags * BugFix remove .git/index files from lexical search or docs * BugFix separate observation seeding and add token-bounded critical-context merge in BaseGraphAgent --- .../utils/source_code_git_loader.py | 13 +++- .../functions/base_graph_agent.py | 77 +++++++++++++++---- .../functions/code_understanding_agent.py | 2 +- .../functions/reachability_agent.py | 5 +- .../tools/import_usage_analyzer.py | 3 + src/vuln_analysis/utils/full_text_search.py | 19 ++++- .../utils/rpm_checker_prompts.py | 10 +-- 7 files changed, 105 insertions(+), 24 deletions(-) diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py index edc1bd6a5..0d2e6f1d7 100644 --- a/src/exploit_iq_commons/utils/source_code_git_loader.py +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -55,6 +55,15 @@ "doc", }) +# Patterns always excluded from code indexing regardless of include/exclude config. +# These are internal directories or files that contain binary/metadata content +# and should never be searched or indexed. +_ALWAYS_EXCLUDED_PATTERNS: tuple[str, ...] = ( + ".git/**/*", # Files inside .git/ + ".git/**", # Directories inside .git/ + ".git", # The .git directory itself +) + PathLike = typing.Union[str, os.PathLike] @@ -454,7 +463,9 @@ def yield_blobs(self) -> typing.Iterator[Blob]: for inc in self.include or ["**/*"]: include_files = include_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(inc))) - for exc in self.exclude or {}: + # Combine user-provided excludes with always-excluded internal patterns + all_excludes = list(self.exclude or []) + list(_ALWAYS_EXCLUDED_PATTERNS) + for exc in all_excludes: exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) # Always include installed_packages.txt when present so that Code diff --git a/src/vuln_analysis/functions/base_graph_agent.py b/src/vuln_analysis/functions/base_graph_agent.py index 9d142a319..05ab11387 100644 --- a/src/vuln_analysis/functions/base_graph_agent.py +++ b/src/vuln_analysis/functions/base_graph_agent.py @@ -101,6 +101,8 @@ def _load_all_tools(builder, config) -> list: }) _GO_VULN_DB_HINT_RE = re.compile(r"Vulnerable module \(Go vuln DB hint\): (\S+)") + _OBS_CONTEXT_MAX_TOKENS = 1200 + _OBS_CONTEXT_TRUNCATION_NOTE = "... (context truncated to fit token window)" @classmethod def _find_go_stdlib_candidate(cls, ecosystem: str, candidate_packages: list[dict], @@ -260,7 +262,8 @@ async def thought_node(self, state: AgentState) -> AgentState: active_prompt = state.get("runtime_prompt") messages = [SystemMessage(content=active_prompt)] + state["messages"] obs = state.get("observation", None) - context_block = self._build_observation_context(obs) + crit_context = state.get("critical_context") + context_block = self._build_observation_context(obs, crit_context) if context_block is None and obs is not None: context_block = "KNOWLEDGE:\n- No prior knowledge.\nLATEST FINDINGS:\n- No recent findings." if context_block: @@ -352,19 +355,66 @@ async def should_continue(self, state: AgentState) -> str: return "tool_node" @staticmethod - def _build_observation_context(obs) -> str | None: - """Format observation memory and recent findings into a context block.""" - if obs is None: + def _merge_unique_context_lines(dynamic_memory: list[str], static_context: list[str]) -> list[str]: + merged: list[str] = [] + seen: set[str] = set() + for line in [*dynamic_memory, *static_context]: + if not isinstance(line, str): + continue + normalized_line = line.strip() + if not normalized_line or normalized_line in seen: + continue + seen.add(normalized_line) + merged.append(normalized_line) + return merged + + def _build_observation_context(self, obs, crit_context) -> str | None: + """Format merged observation and critical context within token budget.""" + dynamic_memory = list(obs.memory) if obs is not None and obs.memory else [] + static_context = list(crit_context) if crit_context else [] + recent_findings = list(obs.results) if obs is not None and obs.results else [] + + merged_knowledge = self._merge_unique_context_lines(dynamic_memory, static_context) + if not merged_knowledge and not recent_findings: return None - memory_list = obs.memory if obs.memory else [] - recent_findings = obs.results if obs.results else [] - if not memory_list and not recent_findings: - return None - parts = [] - if memory_list: - parts.append("KNOWLEDGE:\n" + "\n".join(f"- {m}" for m in memory_list)) + + findings_block = None if recent_findings: - parts.append("LATEST FINDINGS:\n" + "\n".join(f"- {f}" for f in recent_findings)) + findings_block = "LATEST FINDINGS:\n" + "\n".join(f"- {finding}" for finding in recent_findings) + + # Reserve room for recent findings while capping merged knowledge growth. + knowledge_budget = self._OBS_CONTEXT_MAX_TOKENS + if findings_block: + knowledge_budget = max(150, self._OBS_CONTEXT_MAX_TOKENS - count_tokens(findings_block)) + + knowledge_header = "KNOWLEDGE:" + kept_knowledge: list[str] = [] + # Use running counter to avoid O(N²) re-tokenization on each iteration + running_tokens = count_tokens(knowledge_header + "\n") + for entry in merged_knowledge: + entry_line = f"- {entry}\n" + entry_tokens = count_tokens(entry_line) + if running_tokens + entry_tokens > knowledge_budget: + break + kept_knowledge.append(f"- {entry}") + running_tokens += entry_tokens + + if kept_knowledge and len(kept_knowledge) < len(merged_knowledge): + truncation_line = f"- {self._OBS_CONTEXT_TRUNCATION_NOTE}" + candidate_block = knowledge_header + "\n" + "\n".join([*kept_knowledge, truncation_line]) + if count_tokens(candidate_block) <= knowledge_budget: + kept_knowledge.append(truncation_line) + + parts = [] + if kept_knowledge: + parts.append(knowledge_header + "\n" + "\n".join(kept_knowledge)) + elif merged_knowledge: + # No entries fit in budget - show "no knowledge" rather than misleading truncation + parts.append(f"{knowledge_header}\n- No prior knowledge (budget exceeded).") + elif recent_findings: + parts.append("KNOWLEDGE:\n- No prior knowledge.") + if findings_block: + parts.append(findings_block) return "\n".join(parts) async def forced_finish_node(self, state: AgentState) -> AgentState: @@ -373,7 +423,8 @@ async def forced_finish_node(self, state: AgentState) -> AgentState: try: active_prompt = state.get("runtime_prompt") messages = [SystemMessage(content=active_prompt)] + state["messages"] - context_block = self._build_observation_context(state.get("observation", None)) + crit_context = state.get("critical_context",None) + context_block = self._build_observation_context(state.get("observation", None), crit_context) if context_block: messages.append(SystemMessage(content=context_block)) question = state.get("input", "") diff --git a/src/vuln_analysis/functions/code_understanding_agent.py b/src/vuln_analysis/functions/code_understanding_agent.py index 7368dcd28..2cd7e5075 100644 --- a/src/vuln_analysis/functions/code_understanding_agent.py +++ b/src/vuln_analysis/functions/code_understanding_agent.py @@ -179,7 +179,7 @@ async def pre_process_node(self, state: AgentState) -> AgentState: "ecosystem": ecosystem, "runtime_prompt": runtime_prompt, "is_reachability": "no", - "observation": Observation(memory=critical_context, results=[]), + "observation": Observation(memory=[], results=[]), "critical_context": critical_context, "app_package": selected_package, } diff --git a/src/vuln_analysis/functions/reachability_agent.py b/src/vuln_analysis/functions/reachability_agent.py index 152272e91..74aa0bc41 100644 --- a/src/vuln_analysis/functions/reachability_agent.py +++ b/src/vuln_analysis/functions/reachability_agent.py @@ -200,7 +200,7 @@ async def pre_process_node(self, state: AgentState) -> AgentState: "ecosystem": ecosystem, "runtime_prompt": runtime_prompt, "is_reachability": is_reachability, - "observation": Observation(memory=critical_context, results=[]), + "observation": Observation(memory=[], results=[]), "critical_context": critical_context, "app_package": app_package, } @@ -234,7 +234,8 @@ async def forced_finish_node(self, state: AgentState) -> AgentState: try: active_prompt = state.get("runtime_prompt") messages = [SystemMessage(content=active_prompt)] + state["messages"] - context_block = self._build_observation_context(state.get("observation", None)) + crit_context = state.get("critical_context",None) + context_block = self._build_observation_context(state.get("observation", None), crit_context) if context_block: messages.append(SystemMessage(content=context_block)) question = state.get("input", "") diff --git a/src/vuln_analysis/tools/import_usage_analyzer.py b/src/vuln_analysis/tools/import_usage_analyzer.py index f5be30aa0..0b43141d6 100644 --- a/src/vuln_analysis/tools/import_usage_analyzer.py +++ b/src/vuln_analysis/tools/import_usage_analyzer.py @@ -80,6 +80,9 @@ def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: s try: raw = searcher.doc(doc_address) file_path = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if file_path == ".git" or file_path.startswith(".git/") or "/.git/" in file_path: + continue content = raw["content"][0] except Exception as e: doc_errors += 1 diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 87247b67b..452e5393a 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -185,7 +185,11 @@ def search_index(self, query: str, top_k: int = 10, source_scope: list[str] | No for _, doc_id in results: raw = searcher.doc(doc_id) - doc = {"source": raw["file_path"][0], "content": raw["content"][0]} + source_path = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if source_path == ".git" or source_path.startswith(".git/") or "/.git/" in source_path: + continue + doc = {"source": source_path, "content": raw["content"][0]} if is_dependency_path(doc["source"]): dep_docs.append(doc) else: @@ -197,6 +201,9 @@ def search_index(self, query: str, top_k: int = 10, source_scope: list[str] | No for _, doc_id in wider: raw = searcher.doc(doc_id) src = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if src == ".git" or src.startswith(".git/") or "/.git/" in src: + continue if src in seen_sources: continue doc = {"source": src, "content": raw["content"][0]} @@ -266,11 +273,14 @@ def add_documents_from_code_path(self, doc_content = [] if use_langparser: + # Exclude internal/metadata directories from indexing + exclude_patterns = [".git/**", ".git", "**/.git/**", "**/__pycache__/**"] loader = GenericLoader.from_filesystem( code_path, glob="**/*", suffixes=include_extensions, + exclude=exclude_patterns, parser=LanguageParser(), ) docs = loader.load() @@ -281,8 +291,13 @@ def add_documents_from_code_path(self, doc_content = [(doc.metadata["source"], doc.page_content) for doc in docs] else: + # Directories to skip during traversal (internal/metadata dirs) + skip_dirs = {".git", "__pycache__", ".hg", ".svn"} + + for root, dirs, files in os.walk(code_path): + # Prune directories in-place to skip internal/metadata directories + dirs[:] = [d for d in dirs if d not in skip_dirs] - for root, _, files in os.walk(code_path): for file in files: if any(file.endswith(ext) for ext in include_extensions) or file in no_extension: file_path = os.path.join(root, file) diff --git a/src/vuln_analysis/utils/rpm_checker_prompts.py b/src/vuln_analysis/utils/rpm_checker_prompts.py index 6007e8ad8..80c75420e 100644 --- a/src/vuln_analysis/utils/rpm_checker_prompts.py +++ b/src/vuln_analysis/utils/rpm_checker_prompts.py @@ -1498,14 +1498,14 @@ def select_upstream_prompt_and_instructions(vulnerable_patterns: list[str]) -> t 3. Grep the TARGET ARCHITECTURE kernel config file for that CONFIG symbol Use the basename from "Target Architecture Config File" above (e.g. kernel-x86_64-rhel.config). - Example: "CONFIG_NF_TABLES=,kernel-x86_64-rhel.config" + Example: "CONFIG_NF_TABLES,kernel-x86_64-rhel.config" Result: CONFIG_NF_TABLES=m 4. Interpret the result: - CONFIG_X=y → Built into kernel (COMPILED) - CONFIG_X=m → Built as module (COMPILED) - CONFIG_X=n or "# CONFIG_X is not set" → NOT_COMPILED - - Symbol not found → Likely NOT_COMPILED (feature not configured) + - Symbol not found → NOT_COMPILED (feature effectively disabled/not configured) @@ -1540,7 +1540,7 @@ def select_upstream_prompt_and_instructions(vulnerable_patterns: list[str]) -> t 3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. 4. For kernel packages: FIRST grep the directory Makefile for CONFIG/obj-* lines, THEN grep the arch config file. 5. Makefile step: use pattern[,relative/path/Makefile] — e.g. "nf_tables,net/netfilter/Makefile" or "nvme-tcp,drivers/nvme/host/Makefile". -6. Config step: use CONFIG_SYMBOL=,config-basename — e.g. "CONFIG_NVME_TCP=,kernel-x86_64-rhel.config" (basename only, not full path). +6. Config step: use CONFIG_SYMBOL,config-basename — e.g. "CONFIG_NVME_TCP,kernel-x86_64-rhel.config" (basename only, not full path). 7. Do NOT search build logs for .c compilation - kernel uses make -s (silent). 8. Do NOT call the same tool with the same input twice. @@ -1554,11 +1554,11 @@ def select_upstream_prompt_and_instructions(vulnerable_patterns: list[str]) -> t -{{"thought": "Makefile shows obj-$(CONFIG_NF_TABLES) += nf_tables.o. Grep arch config for CONFIG_NF_TABLES value.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NF_TABLES=,kernel-x86_64-rhel.config", "reason": "Check if CONFIG_NF_TABLES is y/m (compiled) or n (not compiled)"}}, "final_answer": null}} +{{"thought": "Makefile shows obj-$(CONFIG_NF_TABLES) += nf_tables.o. Grep arch config for CONFIG_NF_TABLES value.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NF_TABLES,kernel-x86_64-rhel.config", "reason": "Check if CONFIG_NF_TABLES is y/m (compiled), '# CONFIG_* is not set', or missing (not compiled)"}}, "final_answer": null}} -{{"thought": "Makefile shows obj-$(CONFIG_NVME_TCP) += nvme-tcp.o and nvme-tcp-y += tcp.o. Check CONFIG_NVME_TCP in arch config.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NVME_TCP=,kernel-x86_64-rhel.config", "reason": "Confirm CONFIG_NVME_TCP=y/m/n for compilation verdict"}}, "final_answer": null}} +{{"thought": "Makefile shows obj-$(CONFIG_NVME_TCP) += nvme-tcp.o and nvme-tcp-y += tcp.o. Check CONFIG_NVME_TCP in arch config.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NVME_TCP,kernel-x86_64-rhel.config", "reason": "Confirm CONFIG_NVME_TCP=y/m or '# CONFIG_* is not set'/missing for compilation verdict"}}, "final_answer": null}} From 2846e5179173f0c7d989459ece8d52069f13d4f3 Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf Date: Sun, 28 Jun 2026 15:09:49 +0300 Subject: [PATCH 285/286] just a test --- try | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 try diff --git a/try b/try new file mode 100644 index 000000000..e69de29bb From 2ddbc0e86d5bb27cc1600b1d302bec964a50d80c Mon Sep 17 00:00:00 2001 From: Tamar Weisskopf Date: Sun, 28 Jun 2026 15:12:10 +0300 Subject: [PATCH 286/286] just a test --- src/vuln_analysis/functions/tests/test.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/vuln_analysis/functions/tests/test.py diff --git a/src/vuln_analysis/functions/tests/test.py b/src/vuln_analysis/functions/tests/test.py new file mode 100644 index 000000000..9e1d223e2 --- /dev/null +++ b/src/vuln_analysis/functions/tests/test.py @@ -0,0 +1 @@ +assert True \ No newline at end of file